From 688da04d4a396503af1c0cd92072a39703a83e1c Mon Sep 17 00:00:00 2001
From: Ruslan Kuprieiev <kupruser@gmail.com>
Date: Wed, 23 Sep 2020 23:22:27 +0300
Subject: [PATCH] tests: cache test directories (#4601)

* tests: ignore more hdfs tests

* tests: remove unused mark

* gha: don't limit the number of test workers

* cache created test dirs
---
 .github/workflows/tests.yaml         |  2 +-
 tests/dir_helpers.py                 | 27 ++++++++++++++++++---
 tests/func/test_api.py               | 36 ++++++++++++++++++----------
 tests/func/test_update.py            |  7 +++++-
 tests/unit/dependency/test_params.py |  1 -
 5 files changed, 55 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
index 2ac73bbb9..5803d7f60 100644
--- a/.github/workflows/tests.yaml
+++ b/.github/workflows/tests.yaml
@@ -31,7 +31,7 @@ jobs:
         git config --global user.email "dvctester@example.com"
         git config --global user.name "DVC Tester"
     - name: run tests
-      run: python -m tests -n=2 --cov-report=xml
+      run: python -m tests --cov-report=xml
     - name: upload coverage report
       if: github.event.name == 'push'
       uses: codecov/codecov-action@v1.0.13
diff --git a/tests/dir_helpers.py b/tests/dir_helpers.py
index 30fdfd295..34e6dbf46 100644
--- a/tests/dir_helpers.py
+++ b/tests/dir_helpers.py
@@ -253,12 +253,33 @@ class PosixTmpDir(TmpDir, pathlib.PurePosixPath):
     pass
 
 
+CACHE = {}
+
+
 @pytest.fixture(scope="session")
-def make_tmp_dir(tmp_path_factory, request):
-    def make(name, *, scm=False, dvc=False, **kwargs):
+def make_tmp_dir(tmp_path_factory, request, worker_id):
+    def make(name, *, scm=False, dvc=False, subdir=False):
+        from dvc.repo import Repo
+        from dvc.scm.git import Git
+        from dvc.utils.fs import fs_copy
+
+        cache = CACHE.get((scm, dvc, subdir))
+        if not cache:
+            cache = tmp_path_factory.mktemp("dvc-test-cache" + worker_id)
+            TmpDir(cache).init(scm=scm, dvc=dvc, subdir=subdir)
+            CACHE[(scm, dvc, subdir)] = os.fspath(cache)
         path = tmp_path_factory.mktemp(name) if isinstance(name, str) else name
+        for entry in os.listdir(cache):
+            # shutil.copytree's dirs_exist_ok is only available in >=3.8
+            fs_copy(os.path.join(cache, entry), os.path.join(path, entry))
         new_dir = TmpDir(path)
-        new_dir.init(scm=scm, dvc=dvc, **kwargs)
+        str_path = os.fspath(new_dir)
+        if dvc:
+            new_dir.dvc = Repo(str_path)
+        if scm:
+            new_dir.scm = (
+                new_dir.dvc.scm if hasattr(new_dir, "dvc") else Git(str_path)
+            )
         request.addfinalizer(new_dir.close)
         return new_dir
 
diff --git a/tests/func/test_api.py b/tests/func/test_api.py
index 3f1579638..896a2fea7 100644
--- a/tests/func/test_api.py
+++ b/tests/func/test_api.py
@@ -84,10 +84,15 @@ def test_open(tmp_dir, dvc, remote):
                 "azure",
                 "gdrive",
                 "oss",
-                "hdfs",
                 "http",
             ]
         ],
+        pytest.param(
+            pytest.lazy_fixture("hdfs"),
+            marks=pytest.mark.xfail(
+                reason="https://github.com/iterative/dvc/issues/4418",
+            ),
+        ),
         pytest.param(
             pytest.lazy_fixture("ssh"),
             marks=pytest.mark.xfail(
@@ -136,17 +141,24 @@ def test_open_granular(tmp_dir, dvc, remote):
 @pytest.mark.parametrize(
     "remote",
     [
-        pytest.lazy_fixture(cloud)
-        for cloud in [
-            "real_s3",  # NOTE: moto's s3 fails in some tests
-            "gs",
-            "azure",
-            "gdrive",
-            "oss",
-            "ssh",
-            "hdfs",
-            "http",
-        ]
+        *[
+            pytest.lazy_fixture(cloud)
+            for cloud in [
+                "real_s3",  # NOTE: moto's s3 fails in some tests
+                "gs",
+                "azure",
+                "gdrive",
+                "oss",
+                "ssh",
+                "http",
+            ]
+        ],
+        pytest.param(
+            pytest.lazy_fixture("hdfs"),
+            marks=pytest.mark.xfail(
+                reason="https://github.com/iterative/dvc/issues/4418",
+            ),
+        ),
     ],
     indirect=True,
 )
diff --git a/tests/func/test_update.py b/tests/func/test_update.py
index 129660b7c..fbfe5406c 100644
--- a/tests/func/test_update.py
+++ b/tests/func/test_update.py
@@ -172,7 +172,12 @@ def test_update_before_and_after_dvc_init(tmp_dir, dvc, git_dir):
         pytest.lazy_fixture("local_cloud"),
         pytest.lazy_fixture("s3"),
         pytest.lazy_fixture("gs"),
-        pytest.lazy_fixture("hdfs"),
+        pytest.param(
+            pytest.lazy_fixture("hdfs"),
+            marks=pytest.mark.xfail(
+                reason="https://github.com/iterative/dvc/issues/4418",
+            ),
+        ),
         pytest.param(
             pytest.lazy_fixture("ssh"),
             marks=pytest.mark.skipif(
diff --git a/tests/unit/dependency/test_params.py b/tests/unit/dependency/test_params.py
index c8475f516..a1343a84b 100644
--- a/tests/unit/dependency/test_params.py
+++ b/tests/unit/dependency/test_params.py
@@ -132,7 +132,6 @@ def test_get_hash_missing_param(tmp_dir, dvc):
         dep.get_hash()
 
 
-@pytest.mark.regression_4184
 @pytest.mark.parametrize("param_value", ["", "false", "[]", "{}", "null"])
 def test_params_with_false_values(tmp_dir, dvc, param_value):
     """These falsy params values should not ignored by `status` on loading."""