Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

gha: enable hdfs tests (#4644)

Browse Source
This commit is contained in:
Ruslan Kuprieiev
2020-09-30 05:43:34 +03:00
committed by GitHub
parent ab22118cd4
commit d7a35b9852
7 changed files with 26 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/func/test_add.py
+3 -8
View File
@@ -208,14 +208,9 @@ def test_add_file_in_dir(tmp_dir, dvc):
),
(pytest.lazy_fixture("gs"), "md5", "8c7dd922ad47494fc02c388e12c00eac"),
(
pytest.param(
pytest.lazy_fixture("hdfs"),
"checksum",
"000002000000000000000000a86fe4d846edc1bf4c355cb6112f141e",
marks=pytest.mark.xfail(
reason="https://github.com/iterative/dvc/issues/4418"
),
)
pytest.lazy_fixture("hdfs"),
"checksum",
"000002000000000000000000a86fe4d846edc1bf4c355cb6112f141e",
),
],
indirect=["workspace"],
tests/func/test_api.py
+14 -41
View File
@@ -9,23 +9,8 @@ from dvc.path_info import URLInfo
from dvc.utils.fs import remove
from tests.unit.tree.test_repo import make_subrepo
cloud_names = [
"s3",
"gs",
"azure",
"gdrive",
"oss",
"ssh",
"http",
]
clouds = [pytest.lazy_fixture(cloud) for cloud in cloud_names] + [
pytest.param(
pytest.lazy_fixture("hdfs"),
marks=pytest.mark.xfail(
reason="https://github.com/iterative/dvc/issues/4418"
),
)
]
cloud_names = ["s3", "gs", "azure", "gdrive", "oss", "ssh", "http", "hdfs"]
clouds = [pytest.lazy_fixture(cloud) for cloud in cloud_names]
all_clouds = [pytest.lazy_fixture("local_cloud")] + clouds
# `lazy_fixture` is confusing pylint, pylint: disable=unused-argument
@@ -85,14 +70,9 @@ def test_open(tmp_dir, dvc, remote):
"gdrive",
"oss",
"http",
"hdfs",
]
],
pytest.param(
pytest.lazy_fixture("hdfs"),
marks=pytest.mark.xfail(
reason="https://github.com/iterative/dvc/issues/4418",
),
),
pytest.param(
pytest.lazy_fixture("ssh"),
marks=pytest.mark.xfail(
@@ -141,24 +121,17 @@ def test_open_granular(tmp_dir, dvc, remote):
@pytest.mark.parametrize(
"remote",
[
*[
pytest.lazy_fixture(cloud)
for cloud in [
"real_s3", # NOTE: moto's s3 fails in some tests
"gs",
"azure",
"gdrive",
"oss",
"ssh",
"http",
]
],
pytest.param(
pytest.lazy_fixture("hdfs"),
marks=pytest.mark.xfail(
reason="https://github.com/iterative/dvc/issues/4418",
),
),
pytest.lazy_fixture(cloud)
for cloud in [
"real_s3", # NOTE: moto's s3 fails in some tests
"gs",
"azure",
"gdrive",
"oss",
"ssh",
"http",
"hdfs",
]
],
indirect=True,
)
tests/func/test_gc.py
+1 -6
View File
@@ -354,12 +354,7 @@ def test_gc_not_collect_pipeline_tracked_files(tmp_dir, dvc, run_copy):
pytest.lazy_fixture("local_cloud"),
pytest.lazy_fixture("s3"),
pytest.lazy_fixture("gs"),
pytest.param(
pytest.lazy_fixture("hdfs"),
marks=pytest.mark.xfail(
reason="https://github.com/iterative/dvc/issues/4418"
),
),
pytest.lazy_fixture("hdfs"),
pytest.param(
pytest.lazy_fixture("ssh"),
marks=pytest.mark.skipif(
tests/func/test_import_url.py
+2 -10
View File
@@ -121,12 +121,7 @@ def test_import_url_with_no_exec(tmp_dir, dvc, erepo_dir):
pytest.lazy_fixture("local_cloud"),
pytest.lazy_fixture("s3"),
pytest.lazy_fixture("gs"),
pytest.param(
pytest.lazy_fixture("hdfs"),
marks=pytest.mark.xfail(
reason="https://github.com/iterative/dvc/issues/4418"
),
),
pytest.lazy_fixture("hdfs"),
pytest.param(
pytest.lazy_fixture("ssh"),
marks=pytest.mark.skipif(
@@ -164,13 +159,10 @@ def test_import_url(tmp_dir, dvc, workspace):
"dc24e1271084ee317ac3c2656fb8812b",
"b6dcab6ccd17ca0a8bf4a215a37d14cc.dir",
),
pytest.param(
(
pytest.lazy_fixture("hdfs"),
"ec0943f83357f702033c98e70b853c8c",
"e6dcd267966dc628d732874f94ef4280.dir",
marks=pytest.mark.xfail(
reason="https://github.com/iterative/dvc/issues/4418"
),
),
pytest.param(
pytest.lazy_fixture("ssh"),
tests/func/test_run_multistage.py
+4 -9
View File
@@ -386,15 +386,10 @@ def test_run_overwrite_preserves_meta_and_comment(tmp_dir, dvc, run_copy):
"37b51d194a7513e45b56f6524f2d51f2",
),
(
pytest.param(
pytest.lazy_fixture("hdfs"),
"checksum",
"0000020000000000000000003dba826b9be9c6a8e2f8310a770555c4",
"00000200000000000000000075433c81259d3c38e364b348af52e84d",
marks=pytest.mark.xfail(
reason="https://github.com/iterative/dvc/issues/4418"
),
)
pytest.lazy_fixture("hdfs"),
"checksum",
"0000020000000000000000003dba826b9be9c6a8e2f8310a770555c4",
"00000200000000000000000075433c81259d3c38e364b348af52e84d",
),
],
indirect=["workspace"],
tests/func/test_update.py
+1 -6
View File
@@ -172,12 +172,7 @@ def test_update_before_and_after_dvc_init(tmp_dir, dvc, git_dir):
pytest.lazy_fixture("local_cloud"),
pytest.lazy_fixture("s3"),
pytest.lazy_fixture("gs"),
pytest.param(
pytest.lazy_fixture("hdfs"),
marks=pytest.mark.xfail(
reason="https://github.com/iterative/dvc/issues/4418",
),
),
pytest.lazy_fixture("hdfs"),
pytest.param(
pytest.lazy_fixture("ssh"),
marks=pytest.mark.skipif(
tests/remotes/hdfs.py
+1 -4
View File
@@ -70,8 +70,6 @@ class HDFS(Base, URLInfo): # pylint: disable=abstract-method
@pytest.fixture(scope="session")
def hadoop():
import tarfile
import wget
from appdirs import user_cache_dir
@@ -100,8 +98,7 @@ def hadoop():
if not os.path.exists(tar):
wget.download(url, out=tar)
tar = tarfile.open(tar)
tar.extractall(dname)
assert os.system(f"tar -xvf {tar} -C {dname}") == 0
assert os.path.isdir(target)
os.makedirs(dname, exist_ok=True)