tests: cache test directories (#4601)

* tests: ignore more hdfs tests

* tests: remove unused mark

* gha: don't limit the number of test workers

* cache created test dirs

.github/workflows/tests.yaml

@@ -31,7 +31,7 @@ jobs:
         git config --global user.email "[email protected]"
         git config --global user.name "DVC Tester"
     - name: run tests
-      run: python -m tests -n=2 --cov-report=xml
+      run: python -m tests --cov-report=xml
     - name: upload coverage report
       if: github.event.name == 'push'
       uses: codecov/[email protected]

tests/dir_helpers.py

@@ -253,12 +253,33 @@ class PosixTmpDir(TmpDir, pathlib.PurePosixPath):
     pass
 
 
+CACHE = {}
+
+
 @pytest.fixture(scope="session")
-def make_tmp_dir(tmp_path_factory, request):
-    def make(name, *, scm=False, dvc=False, **kwargs):
+def make_tmp_dir(tmp_path_factory, request, worker_id):
+    def make(name, *, scm=False, dvc=False, subdir=False):
+        from dvc.repo import Repo
+        from dvc.scm.git import Git
+        from dvc.utils.fs import fs_copy
+
+        cache = CACHE.get((scm, dvc, subdir))
+        if not cache:
+            cache = tmp_path_factory.mktemp("dvc-test-cache" + worker_id)
+            TmpDir(cache).init(scm=scm, dvc=dvc, subdir=subdir)
+            CACHE[(scm, dvc, subdir)] = os.fspath(cache)
         path = tmp_path_factory.mktemp(name) if isinstance(name, str) else name
+        for entry in os.listdir(cache):
+            # shutil.copytree's dirs_exist_ok is only available in >=3.8
+            fs_copy(os.path.join(cache, entry), os.path.join(path, entry))
         new_dir = TmpDir(path)
-        new_dir.init(scm=scm, dvc=dvc, **kwargs)
+        str_path = os.fspath(new_dir)
+        if dvc:
+            new_dir.dvc = Repo(str_path)
+        if scm:
+            new_dir.scm = (
+                new_dir.dvc.scm if hasattr(new_dir, "dvc") else Git(str_path)
+            )
         request.addfinalizer(new_dir.close)
         return new_dir
 

tests/func/test_api.py

@@ -84,10 +84,15 @@ def test_open(tmp_dir, dvc, remote):
                 "azure",
                 "gdrive",
                 "oss",
-                "hdfs",
                 "http",
             ]
         ],
+        pytest.param(
+            pytest.lazy_fixture("hdfs"),
+            marks=pytest.mark.xfail(
+                reason="https://github.com/iterative/dvc/issues/4418",
+            ),
+        ),
         pytest.param(
             pytest.lazy_fixture("ssh"),
             marks=pytest.mark.xfail(
@@ -136,17 +141,24 @@ def test_open_granular(tmp_dir, dvc, remote):
 @pytest.mark.parametrize(
     "remote",
     [
-        pytest.lazy_fixture(cloud)
-        for cloud in [
-            "real_s3",  # NOTE: moto's s3 fails in some tests
-            "gs",
-            "azure",
-            "gdrive",
-            "oss",
-            "ssh",
-            "hdfs",
-            "http",
-        ]
+        *[
+            pytest.lazy_fixture(cloud)
+            for cloud in [
+                "real_s3",  # NOTE: moto's s3 fails in some tests
+                "gs",
+                "azure",
+                "gdrive",
+                "oss",
+                "ssh",
+                "http",
+            ]
+        ],
+        pytest.param(
+            pytest.lazy_fixture("hdfs"),
+            marks=pytest.mark.xfail(
+                reason="https://github.com/iterative/dvc/issues/4418",
+            ),
+        ),
     ],
     indirect=True,
 )

tests/func/test_update.py

@@ -172,7 +172,12 @@ def test_update_before_and_after_dvc_init(tmp_dir, dvc, git_dir):
         pytest.lazy_fixture("local_cloud"),
         pytest.lazy_fixture("s3"),
         pytest.lazy_fixture("gs"),
-        pytest.lazy_fixture("hdfs"),
+        pytest.param(
+            pytest.lazy_fixture("hdfs"),
+            marks=pytest.mark.xfail(
+                reason="https://github.com/iterative/dvc/issues/4418",
+            ),
+        ),
         pytest.param(
             pytest.lazy_fixture("ssh"),
             marks=pytest.mark.skipif(

tests/unit/dependency/test_params.py

@@ -132,7 +132,6 @@ def test_get_hash_missing_param(tmp_dir, dvc):
         dep.get_hash()
 
 
-@pytest.mark.regression_4184
 @pytest.mark.parametrize("param_value", ["", "false", "[]", "{}", "null"])
 def test_params_with_false_values(tmp_dir, dvc, param_value):
     """These falsy params values should not ignored by `status` on loading."""