Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

tests: remove redundant remote tests

Browse Source 
With introduction of https://github.com/iterative/dvc-plugins,
all important tests are being ran there, so there is no need
for dvc repo to run them again.

This PR also removes some redundant/broken remote tests.
This commit is contained in:
Ruslan Kuprieiev
2021-12-03 22:45:41 +02:00
parent 024f06d672
commit b6f2a8e653
35 changed files with 128 additions and 2418 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/repo/experiments/__init__.py
+3 -1
View File
@@ -13,7 +13,6 @@ from funcy import cached_property, first
from dvc.dependency.param import MissingParamsError
from dvc.env import DVCLIVE_RESUME
from dvc.exceptions import DvcException
from dvc.stage.monitor import CheckpointKilledError
from dvc.utils import relpath
from .base import (
@@ -696,6 +695,8 @@ class Experiments:
dict mapping stash revs to the successfully executed experiments
for each stash rev.
"""
from dvc.stage.monitor import CheckpointKilledError
result: Dict[str, Dict[str, str]] = defaultdict(dict)
manager = Manager()
@@ -798,6 +799,7 @@ class Experiments:
@unlocked_repo
def _workspace_repro(self) -> Mapping[str, str]:
"""Run the most recently stashed experiment in the workspace."""
from dvc.stage.monitor import CheckpointKilledError
from dvc.utils.fs import makedirs
from .executor.base import BaseExecutor
dvc/testing/fixtures.py
+32 -9
View File
@@ -9,11 +9,15 @@ __all__ = [
"scm",
"dvc",
"make_cloud",
"make_local",
"cloud",
"local_cloud",
"make_remote",
"remote",
"local_remote",
"workspace",
"make_workspace",
"local_workspace",
"docker",
"docker_compose",
"docker_compose_project_name",
@@ -94,6 +98,14 @@ def dvc(tmp_dir):
yield _dvc
@pytest.fixture
def make_local(make_tmp_dir):
def _make_local():
return make_tmp_dir("local-cloud")
return _make_local
@pytest.fixture
def make_cloud(request):
def _make_cloud(typ):
@@ -104,9 +116,13 @@ def make_cloud(request):
@pytest.fixture
def cloud(make_cloud, request):
if not hasattr(request, "param"):
pytest.skip("no cloud specified")
return make_cloud(request.param)
typ = getattr(request, "param", "local")
return make_cloud(typ)
@pytest.fixture
def local_cloud(make_cloud):
return make_cloud("local")
@pytest.fixture
@@ -121,9 +137,13 @@ def make_remote(tmp_dir, dvc, make_cloud):
@pytest.fixture
def remote(make_remote, request):
if not hasattr(request, "param"):
pytest.skip("no remote specified")
return make_remote("upstream", typ=request.param)
typ = getattr(request, "param", "local")
return make_remote("upstream", typ=typ)
@pytest.fixture
def local_remote(make_remote):
return make_remote("upstream", typ="local")
@pytest.fixture
@@ -152,10 +172,13 @@ def make_workspace(tmp_dir, dvc, make_cloud):
@pytest.fixture
def workspace(make_workspace, request):
if not hasattr(request, "param"):
pytest.skip("no workspace specified")
typ = getattr(request, "param", "local")
return make_workspace("workspace", typ=typ)
return make_workspace("workspace", typ=request.param)
@pytest.fixture
def local_workspace(make_workspace):
return make_workspace("workspace", typ="local")
@pytest.fixture(scope="session")
dvc/testing/test_workspace.py
+24
View File
@@ -11,6 +11,14 @@ class TestImport:
assert (tmp_dir / "file").read_text() == "file"
assert dvc.status() == {}
@pytest.fixture
def stage_md5(self):
pytest.skip()
@pytest.fixture
def dir_md5(self):
pytest.skip()
def test_import_dir(self, tmp_dir, dvc, workspace, stage_md5, dir_md5):
from dvc.objects.db import ODBManager
@@ -51,6 +59,10 @@ class TestImport:
" path: dir\n"
)
@pytest.fixture
def is_object_storage(self):
pytest.skip()
def test_import_empty_dir(
self, tmp_dir, dvc, workspace, is_object_storage
):
@@ -72,6 +84,18 @@ class TestImport:
class TestAdd:
@pytest.fixture
def hash_name(self):
pytest.skip()
@pytest.fixture
def hash_value(self):
pytest.skip()
@pytest.fixture
def dir_hash_value(self):
pytest.skip()
def test_add(self, tmp_dir, dvc, workspace, hash_name, hash_value):
from dvc.stage.exceptions import StageExternalOutputsError
setup.cfg
+1
View File
@@ -115,6 +115,7 @@ terraform = tpi[ssh]>=2.1.0
tests =
%(terraform)s
wheel==0.37.0
dvc_ssh==0.0.1a0
# Test requirements
pytest==6.2.5
pytest-cov==3.0.0
tests/docker-compose.yml
-37
View File
@@ -1,43 +1,6 @@
---
version: '3.2'
services:
azurite:
image: mcr.microsoft.com/azure-storage/azurite:3.14.2
command: azurite -L -l /data --blobHost 0.0.0.0 --queueHost 0.0.0.0
ports:
- "10000"
motoserver:
image: motoserver/moto
ports:
- "5000"
oss:
image: rkuprieiev/oss-emulator
ports:
- "8880"
hdfs:
image: rkuprieiev/docker-hdfs
hostname: localhost
ports:
- "8020"
# NOTE: having this port as dynamic one will require modifying
# `dfs.datanode.address` in `hdfs-site.xml` and probably something
# else, so using default one for now.
- "50010:50010"
- "50075:50075"
- "50070"
openssh-server:
image: ghcr.io/linuxserver/openssh-server
environment:
- USER_NAME=user
- PUBLIC_KEY_FILE=/tmp/key
ports:
- 2222
volumes:
- ./remotes/user.key.pub:/tmp/key
git-server:
image: ghcr.io/linuxserver/openssh-server
environment:
tests/func/experiments/executor/test_ssh.py
+1 -1
View File
@@ -3,12 +3,12 @@ from contextlib import contextmanager
from functools import partial
import pytest
from dvc_ssh.tests.cloud import TEST_SSH_KEY_PATH, TEST_SSH_USER
from dvc.fs.ssh import SSHFileSystem
from dvc.repo.experiments.base import EXEC_HEAD, EXEC_MERGE
from dvc.repo.experiments.executor.ssh import SSHExecutor
from tests.func.machine.conftest import * # noqa, pylint: disable=wildcard-import
from tests.remotes.ssh import TEST_SSH_KEY_PATH, TEST_SSH_USER
@contextmanager
tests/func/test_add.py
+15 -86
View File
@@ -293,71 +293,17 @@ def test_add_filtered_files_in_dir(
class TestAddExternal(TestAdd):
@pytest.mark.parametrize(
"workspace, hash_name, hash_value",
[
(
"local",
"md5",
"8c7dd922ad47494fc02c388e12c00eac",
),
pytest.param(
"ssh",
"md5",
"8c7dd922ad47494fc02c388e12c00eac",
marks=pytest.mark.skipif(
os.name == "nt", reason="disabled on windows"
),
),
(
"s3",
"etag",
"8c7dd922ad47494fc02c388e12c00eac",
),
(
"hdfs",
"checksum",
"000002000000000000000000a86fe4d846edc1bf4c355cb6112f141e",
),
(
"webhdfs",
"checksum",
"000002000000000000000000a86fe4d846edc1bf4c355cb6112f141e00000000", # noqa: E501
),
],
indirect=["workspace"],
)
def test_add(self, tmp_dir, dvc, workspace, hash_name, hash_value):
super().test_add(tmp_dir, dvc, workspace, hash_name, hash_value)
@pytest.fixture
def hash_name(self):
return "md5"
@pytest.mark.parametrize(
"workspace, hash_name, dir_hash_value",
[
(
"local",
"md5",
"b6dcab6ccd17ca0a8bf4a215a37d14cc.dir",
),
pytest.param(
"ssh",
"md5",
"b6dcab6ccd17ca0a8bf4a215a37d14cc.dir",
marks=pytest.mark.skipif(
os.name == "nt", reason="disabled on windows"
),
),
(
"s3",
"etag",
"ec602a6ba97b2dd07bd6d2cd89674a60.dir",
),
],
indirect=["workspace"],
)
def test_add_dir(self, tmp_dir, dvc, workspace, hash_name, dir_hash_value):
super().test_add_dir(
tmp_dir, dvc, workspace, hash_name, dir_hash_value
)
@pytest.fixture
def hash_value(self):
return "8c7dd922ad47494fc02c388e12c00eac"
@pytest.fixture
def dir_hash_value(self):
return "b6dcab6ccd17ca0a8bf4a215a37d14cc.dir"
def test_add_external_relpath(tmp_dir, dvc, local_cloud):
@@ -994,10 +940,10 @@ def test_add_long_fname(tmp_dir, dvc):
assert (tmp_dir / "data").read_text() == {name: "foo"}
def test_add_to_remote(tmp_dir, dvc, local_cloud, local_remote):
local_cloud.gen("foo", "foo")
def test_add_to_remote(tmp_dir, dvc, remote, workspace):
workspace.gen("foo", "foo")
url = "remote://upstream/foo"
url = "remote://workspace/foo"
[stage] = dvc.add(url, to_remote=True)
assert not (tmp_dir / "foo").exists()
@@ -1009,14 +955,14 @@ def test_add_to_remote(tmp_dir, dvc, local_cloud, local_remote):
hash_info = stage.outs[0].hash_info
meta = stage.outs[0].meta
with open(
local_remote.hash_to_path(hash_info.value), encoding="utf-8"
remote.hash_to_path(hash_info.value), encoding="utf-8"
) as stream:
assert stream.read() == "foo"
assert meta.size == len("foo")
def test_add_to_remote_absolute(tmp_dir, make_tmp_dir, dvc, local_remote):
def test_add_to_remote_absolute(tmp_dir, make_tmp_dir, dvc, remote):
tmp_abs_dir = make_tmp_dir("abs")
tmp_foo = tmp_abs_dir / "foo"
tmp_foo.write_text("foo")
@@ -1132,23 +1078,6 @@ def test_add_to_cache_invalid_combinations(dvc, invalid_opt, kwargs):
dvc.add(out="bar", **kwargs)
@pytest.mark.parametrize(
"workspace",
[
"local",
"s3",
pytest.param("gs", marks=pytest.mark.needs_internet),
"hdfs",
pytest.param(
"ssh",
marks=pytest.mark.skipif(
os.name == "nt", reason="disabled on windows"
),
),
"http",
],
indirect=True,
)
def test_add_to_cache_from_remote(tmp_dir, dvc, workspace):
workspace.gen("foo", "foo")
tests/func/test_api.py
+7 -66
View File
@@ -5,35 +5,11 @@ from funcy import first, get_in
from dvc import api
from dvc.exceptions import FileMissingError, OutputNotFoundError
from dvc.testing.test_api import TestAPI as _TestAPI
from dvc.testing.test_api import TestAPI # noqa, pylint: disable=unused-import
from dvc.utils.fs import remove
from tests.unit.fs.test_repo import make_subrepo
clouds = [
"s3",
"gs",
"azure",
"gdrive",
"oss",
"ssh",
"http",
"hdfs",
"webdav",
"webhdfs",
]
all_clouds = ["local"] + clouds
# `lazy_fixture` is confusing pylint, pylint: disable=unused-argument
@pytest.mark.needs_internet
@pytest.mark.parametrize("remote", clouds, indirect=True)
class TestAPI(_TestAPI):
pass
@pytest.mark.needs_internet
@pytest.mark.parametrize("cloud", clouds, indirect=True)
def test_get_url_external(tmp_dir, erepo_dir, cloud):
erepo_dir.add_remote(config=cloud.config)
with erepo_dir.chdir():
@@ -55,22 +31,6 @@ def test_get_url_requires_dvc(tmp_dir, scm):
api.get_url("foo", repo=f"file://{tmp_dir}")
@pytest.mark.needs_internet
@pytest.mark.parametrize(
"cloud",
[
"s3",
"gs",
"azure",
"gdrive",
"oss",
"http",
"hdfs",
"ssh",
"webdav",
],
indirect=True,
)
def test_open_external(tmp_dir, erepo_dir, cloud):
erepo_dir.add_remote(config=cloud.config)
@@ -94,8 +54,6 @@ def test_open_external(tmp_dir, erepo_dir, cloud):
assert api.read("version", repo=repo_url, rev="branch") == "branchver"
@pytest.mark.needs_internet
@pytest.mark.parametrize("remote", all_clouds, indirect=True)
def test_open_granular(tmp_dir, dvc, remote):
tmp_dir.dvc_gen({"dir": {"foo": "foo-text"}})
dvc.push()
@@ -107,22 +65,6 @@ def test_open_granular(tmp_dir, dvc, remote):
assert fd.read() == "foo-text"
@pytest.mark.needs_internet
@pytest.mark.parametrize(
"remote",
[
"s3",
"gs",
"azure",
"gdrive",
"oss",
"ssh",
"http",
"hdfs",
"webdav",
],
indirect=True,
)
def test_missing(tmp_dir, dvc, remote):
tmp_dir.dvc_gen("foo", "foo")
@@ -174,7 +116,6 @@ def test_open_rev(tmp_dir, scm, dvc):
@pytest.mark.parametrize("as_external", [True, False])
@pytest.mark.parametrize("remote", ["ssh"], indirect=True)
@pytest.mark.parametrize(
"files, to_read",
[
@@ -227,22 +168,22 @@ def test_read_with_subrepos(tmp_dir, scm, local_cloud, local_repo):
)
def test_get_url_granular(tmp_dir, dvc, s3):
tmp_dir.add_remote(config=s3.config)
def test_get_url_granular(tmp_dir, dvc, cloud):
tmp_dir.add_remote(config=cloud.config)
tmp_dir.dvc_gen(
{"dir": {"foo": "foo", "bar": "bar", "nested": {"file": "file"}}}
)
expected_url = (s3 / "5f/c28ea78987408341668eba6525ebd1.dir").url
expected_url = (cloud / "5f" / "c28ea78987408341668eba6525ebd1.dir").url
assert api.get_url("dir") == expected_url
expected_url = (s3 / "ac/bd18db4cc2f85cedef654fccc4a4d8").url
expected_url = (cloud / "ac" / "bd18db4cc2f85cedef654fccc4a4d8").url
assert api.get_url("dir/foo") == expected_url
expected_url = (s3 / "37/b51d194a7513e45b56f6524f2d51f2").url
expected_url = (cloud / "37" / "b51d194a7513e45b56f6524f2d51f2").url
assert api.get_url("dir/bar") == expected_url
expected_url = (s3 / "8c/7dd922ad47494fc02c388e12c00eac").url
expected_url = (cloud / "8c" / "7dd922ad47494fc02c388e12c00eac").url
assert api.get_url(os.path.join("dir", "nested", "file")) == expected_url
tests/func/test_checkout.py
-6
View File
@@ -752,9 +752,6 @@ def test_checkout_recursive(tmp_dir, dvc):
}
@pytest.mark.parametrize(
"workspace", [pytest.lazy_fixture("s3")], indirect=True
)
def test_checkout_for_external_outputs(tmp_dir, dvc, workspace):
workspace.gen("foo", "foo")
@@ -844,9 +841,6 @@ def test_checkouts_for_pipeline_tracked_outs(tmp_dir, dvc, scm, run_copy):
assert set(dvc.checkout()["added"]) == {"bar", "ipsum"}
@pytest.mark.parametrize(
"workspace", [pytest.lazy_fixture("s3")], indirect=True
)
def test_checkout_external_modified_file(tmp_dir, dvc, scm, mocker, workspace):
# regression: happened when file in external output changed and checkout
# was attempted without force, dvc checks if it's present in its cache
tests/func/test_data_cloud.py
+3 -66
View File
@@ -9,35 +9,12 @@ import dvc as dvc_module
from dvc.external_repo import clean_repos
from dvc.main import main
from dvc.stage.exceptions import StageNotFound
from dvc.testing.test_remote import TestRemote as _TestRemote
from dvc.testing.test_remote import ( # noqa, pylint: disable=unused-import
TestRemote,
)
from dvc.utils.fs import remove
all_clouds = [
"s3",
"gs",
"azure",
"ssh",
"http",
"hdfs",
"webdav",
"webhdfs",
"oss",
"gdrive",
]
# Clouds that implement the general methods that can be tested
# for functional tests that require extensive apis (e.g traversing
# via walk_files())
full_clouds = ["s3", "gs", "azure", "ssh", "hdfs"]
@pytest.mark.parametrize("remote", all_clouds, indirect=True)
class TestRemote(_TestRemote):
pass
@pytest.mark.needs_internet
@pytest.mark.parametrize("remote", all_clouds, indirect=True)
def test_cloud_cli(tmp_dir, dvc, remote):
args = ["-v", "-j", "2"]
@@ -479,46 +456,6 @@ def test_pull_partial(tmp_dir, dvc, local_remote):
assert (tmp_dir / "foo").read_text() == {"bar": {"baz": "baz"}}
@pytest.mark.parametrize("remote", full_clouds, indirect=True)
def test_pull_00_prefix(tmp_dir, dvc, remote, monkeypatch):
# Related: https://github.com/iterative/dvc/issues/6089
fs_type = type(dvc.cloud.get_remote_odb("upstream").fs)
monkeypatch.setattr(fs_type, "_ALWAYS_TRAVERSE", True, raising=False)
monkeypatch.setattr(fs_type, "LIST_OBJECT_PAGE_SIZE", 256, raising=False)
# foo's md5 checksum is 00411460f7c92d2124a67ea0f4cb5f85
# bar's md5 checksum is 0000000018e6137ac2caab16074784a6
tmp_dir.dvc_gen({"foo": "363", "bar": "jk8ssl"})
dvc.push()
clean(["foo", "bar"], dvc)
stats = dvc.pull()
assert stats["fetched"] == 2
assert set(stats["added"]) == {"foo", "bar"}
@pytest.mark.parametrize("remote", full_clouds, indirect=True)
def test_pull_no_00_prefix(tmp_dir, dvc, remote, monkeypatch):
# Related: https://github.com/iterative/dvc/issues/6244
fs_type = type(dvc.cloud.get_remote_odb("upstream").fs)
monkeypatch.setattr(fs_type, "_ALWAYS_TRAVERSE", True, raising=False)
monkeypatch.setattr(fs_type, "LIST_OBJECT_PAGE_SIZE", 256, raising=False)
# foo's md5 checksum is 14ffd92a6cbf5f2f657067df0d5881a6
# bar's md5 checksum is 64020400f00960c0ef04052547b134b3
tmp_dir.dvc_gen({"foo": "dvc", "bar": "cml"})
dvc.push()
clean(["foo", "bar"], dvc)
stats = dvc.pull()
assert stats["fetched"] == 2
assert set(stats["added"]) == {"foo", "bar"}
def test_output_remote(tmp_dir, dvc, make_remote):
make_remote("default", default=True)
make_remote("for_foo", default=False)
tests/func/test_fs.py
+1 -149
View File
@@ -231,18 +231,6 @@ def test_walk_dont_ignore_subrepos(tmp_dir, scm, dvc):
assert set(get_dirs(next(scm_fs.walk(path)))) == {".dvc", "subdir"}
@pytest.mark.parametrize(
"cloud",
[
pytest.lazy_fixture("local_cloud"),
pytest.lazy_fixture("s3"),
pytest.param(
pytest.lazy_fixture("gs"), marks=pytest.mark.needs_internet
),
pytest.lazy_fixture("hdfs"),
pytest.lazy_fixture("http"),
],
)
def test_fs_getsize(dvc, cloud):
cloud.gen({"data": {"foo": "foo"}, "baz": "baz baz"})
cls, config, path = get_cloud_fs(dvc, **cloud.config)
@@ -252,21 +240,6 @@ def test_fs_getsize(dvc, cloud):
assert fs.getsize(fs.path.join(path, "data", "foo")) == 3
@pytest.mark.needs_internet
@pytest.mark.parametrize(
"cloud",
[
pytest.lazy_fixture("azure"),
pytest.lazy_fixture("gs"),
pytest.lazy_fixture("gdrive"),
pytest.lazy_fixture("hdfs"),
pytest.lazy_fixture("local_cloud"),
pytest.lazy_fixture("oss"),
pytest.lazy_fixture("s3"),
pytest.lazy_fixture("ssh"),
pytest.lazy_fixture("webhdfs"),
],
)
def test_fs_upload_fobj(dvc, tmp_dir, cloud):
tmp_dir.gen("foo", "foo")
cls, config, path = get_cloud_fs(dvc, **cloud.config)
@@ -283,75 +256,6 @@ def test_fs_upload_fobj(dvc, tmp_dir, cloud):
assert stream.read() == b"foo"
@pytest.mark.needs_internet
@pytest.mark.parametrize("cloud", [pytest.lazy_fixture("gdrive")])
def test_fs_ls(dvc, cloud):
cloud.gen(
{
"directory": {
"foo": "foo",
"bar": "bar",
"baz": {"quux": "quux", "egg": {"foo": "foo"}},
"empty": {},
}
}
)
cls, config, path = get_cloud_fs(dvc, **cloud.config)
fs = cls(**config)
path = os.path.join(path, "directory")
assert {
os.path.basename(file_key.rstrip("/")) for file_key in fs.ls(path)
} == {
"foo",
"bar",
"baz",
"empty",
}
assert set(fs.ls(fs.path.join(path, "empty"))) == set()
assert {
(detail["type"], os.path.basename(detail["name"].rstrip("/")))
for detail in fs.ls(fs.path.join(path, "baz"), detail=True)
} == {("file", "quux"), ("directory", "egg")}
@pytest.mark.parametrize(
"cloud",
[
pytest.lazy_fixture("azure"),
pytest.param(
pytest.lazy_fixture("gs"), marks=pytest.mark.needs_internet
),
],
)
def test_fs_fsspec_path_management(dvc, cloud):
cloud.gen({"foo": "foo", "data": {"bar": "bar", "baz": {"foo": "foo"}}})
cls, config, _ = get_cloud_fs(dvc, **cloud.config)
fs = cls(**config)
root = cloud.parents[len(cloud.parents) - 1]
bucket_details = fs.info(root.fs_path)
# special conditions: name always points to the bucket name
assert bucket_details["name"] == root.bucket
assert bucket_details["type"] == "directory"
data = cloud / "data"
data_details = fs.info(data.fs_path)
assert data_details["name"].rstrip("/") == data.fs_path
assert data_details["type"] == "directory"
@pytest.mark.needs_internet
@pytest.mark.parametrize(
"cloud",
[
pytest.lazy_fixture("azure"),
pytest.lazy_fixture("gs"),
pytest.lazy_fixture("s3"),
pytest.lazy_fixture("webdav"),
],
)
def test_fs_makedirs_on_upload_and_copy(dvc, cloud):
cls, config, _ = get_cloud_fs(dvc, **cloud.config)
fs = cls(**config)
@@ -362,28 +266,12 @@ def test_fs_makedirs_on_upload_and_copy(dvc, cloud):
assert fs.isdir((cloud / "dir").fs_path)
assert fs.exists((cloud / "dir" / "foo").fs_path)
fs.makedirs((cloud / "dir2").fs_path)
fs.copy((cloud / "dir" / "foo").fs_path, (cloud / "dir2" / "foo").fs_path)
assert fs.isdir((cloud / "dir2").fs_path)
assert fs.exists((cloud / "dir2" / "foo").fs_path)
@pytest.mark.needs_internet
@pytest.mark.parametrize(
"cloud",
[
pytest.lazy_fixture("azure"),
pytest.lazy_fixture("gs"),
pytest.lazy_fixture("gdrive"),
pytest.lazy_fixture("hdfs"),
pytest.lazy_fixture("local_cloud"),
pytest.lazy_fixture("oss"),
pytest.lazy_fixture("s3"),
pytest.lazy_fixture("ssh"),
pytest.lazy_fixture("webhdfs"),
pytest.lazy_fixture("webdav"),
pytest.lazy_fixture("http"),
],
)
def test_upload_callback(tmp_dir, dvc, cloud):
tmp_dir.gen("foo", "foo")
cls, config, _ = get_cloud_fs(dvc, **cloud.config)
@@ -401,23 +289,6 @@ def test_upload_callback(tmp_dir, dvc, cloud):
assert callback.value == expected_size
@pytest.mark.needs_internet
@pytest.mark.parametrize(
"cloud",
[
pytest.lazy_fixture("azure"),
pytest.lazy_fixture("gs"),
pytest.lazy_fixture("gdrive"),
pytest.lazy_fixture("hdfs"),
pytest.lazy_fixture("local_cloud"),
pytest.lazy_fixture("oss"),
pytest.lazy_fixture("s3"),
pytest.lazy_fixture("ssh"),
pytest.lazy_fixture("webhdfs"),
pytest.lazy_fixture("webdav"),
pytest.lazy_fixture("http"),
],
)
def test_download_callback(tmp_dir, dvc, cloud, local_cloud):
cls, config, _ = get_cloud_fs(dvc, **cloud.config)
fs = cls(**config)
@@ -438,25 +309,6 @@ def test_download_callback(tmp_dir, dvc, cloud, local_cloud):
assert (tmp_dir / "foo").read_text() == "foo"
@pytest.mark.needs_internet
@pytest.mark.parametrize(
"cloud",
[
pytest.lazy_fixture("azure"),
pytest.lazy_fixture("gs"),
pytest.lazy_fixture("hdfs"),
pytest.lazy_fixture("local_cloud"),
pytest.lazy_fixture("s3"),
pytest.param(
pytest.lazy_fixture("ssh"),
marks=pytest.mark.skipif(
os.name == "nt", reason="unsupported on Windows."
),
),
pytest.lazy_fixture("gdrive"),
pytest.lazy_fixture("webdav"),
],
)
def test_download_dir_callback(tmp_dir, dvc, cloud):
cls, config, _ = get_cloud_fs(dvc, **cloud.config)
fs = cls(**config)
tests/func/test_gc.py
-16
View File
@@ -343,22 +343,6 @@ def test_gc_not_collect_pipeline_tracked_files(tmp_dir, dvc, run_copy):
assert _count_files(dvc.odb.local.cache_dir) == 0
@pytest.mark.parametrize(
"workspace",
[
"local",
"s3",
"hdfs",
"webhdfs",
pytest.param(
"ssh",
marks=pytest.mark.skipif(
os.name == "nt", reason="disabled on windows"
),
),
],
indirect=True,
)
def test_gc_external_output(tmp_dir, dvc, workspace):
workspace.gen({"foo": "foo", "bar": "bar"})
tests/func/test_import_url.py
+9 -133
View File
@@ -119,108 +119,17 @@ def test_import_url_with_no_exec(tmp_dir, dvc, erepo_dir):
class TestImport(_TestImport):
@pytest.mark.parametrize(
"workspace",
[
"local",
"s3",
"azure",
pytest.param("gs", marks=pytest.mark.needs_internet),
"hdfs",
"webhdfs",
pytest.param(
"ssh",
marks=pytest.mark.skipif(
os.name == "nt", reason="disabled on windows"
),
),
"http",
],
indirect=True,
)
def test_import(self, tmp_dir, dvc, workspace):
super().test_import(tmp_dir, dvc, workspace)
@pytest.fixture
def stage_md5(self):
return "dc24e1271084ee317ac3c2656fb8812b"
@pytest.mark.parametrize(
"workspace, stage_md5, dir_md5",
[
(
"local",
"dc24e1271084ee317ac3c2656fb8812b",
"b6dcab6ccd17ca0a8bf4a215a37d14cc.dir",
),
(
"s3",
"2aa17f8daa26996b3f7a4cf8888ac9ac",
"ec602a6ba97b2dd07bd6d2cd89674a60.dir",
),
(
"azure",
None, # ETags for azure are not consistent with the actual
None, # content of the object, so they will change every time
# we create new objects. We'll skip stage content
# check for Azure.
),
(
"hdfs",
"ec0943f83357f702033c98e70b853c8c",
"e6dcd267966dc628d732874f94ef4280.dir",
),
pytest.param(
"ssh",
"dc24e1271084ee317ac3c2656fb8812b",
"b6dcab6ccd17ca0a8bf4a215a37d14cc.dir",
marks=pytest.mark.skipif(
os.name == "nt", reason="disabled on windows"
),
),
],
indirect=["workspace"],
)
def test_import_dir(self, tmp_dir, dvc, workspace, stage_md5, dir_md5):
super().test_import_dir(tmp_dir, dvc, workspace, stage_md5, dir_md5)
@pytest.fixture
def dir_md5(self):
return "b6dcab6ccd17ca0a8bf4a215a37d14cc.dir"
@pytest.mark.parametrize(
"workspace, is_object_storage",
[
("hdfs", False),
pytest.param(
"s3",
True,
marks=pytest.mark.xfail(
reason="https://github.com/iterative/dvc/issues/5521",
),
),
pytest.param(
"gs",
True,
marks=pytest.mark.xfail(
reason="https://github.com/iterative/dvc/issues/5521",
),
),
pytest.param(
"azure",
True,
marks=pytest.mark.xfail(
reason="https://github.com/iterative/dvc/issues/5521",
),
),
pytest.param(
"ssh",
False,
marks=pytest.mark.skipif(
os.name == "nt", reason="disabled on windows"
),
),
],
indirect=["workspace"],
)
def test_import_empty_dir(
self, tmp_dir, dvc, workspace, is_object_storage
):
super().test_import_empty_dir(
tmp_dir, dvc, workspace, is_object_storage
)
@pytest.fixture
def is_object_storage(self):
return False
def test_import_url_preserve_meta(tmp_dir, dvc):
@@ -260,23 +169,6 @@ def test_import_url_preserve_meta(tmp_dir, dvc):
)
@pytest.mark.parametrize(
"workspace",
[
"local",
"s3",
pytest.param("gs", marks=pytest.mark.needs_internet),
"hdfs",
pytest.param(
"ssh",
marks=pytest.mark.skipif(
os.name == "nt", reason="disabled on windows"
),
),
"http",
],
indirect=True,
)
def test_import_url_to_remote_single_file(
tmp_dir, dvc, workspace, local_remote
):
@@ -301,22 +193,6 @@ def test_import_url_to_remote_single_file(
assert stage.outs[0].meta.size == len("foo")
@pytest.mark.parametrize(
"workspace",
[
"local",
"s3",
pytest.param("gs", marks=pytest.mark.needs_internet),
"hdfs",
pytest.param(
"ssh",
marks=pytest.mark.skipif(
os.name == "nt", reason="disabled on windows"
),
),
],
indirect=True,
)
def test_import_url_to_remote_directory(tmp_dir, dvc, workspace, local_remote):
workspace.gen(
{
tests/func/test_merge_driver.py
-32
View File
@@ -114,38 +114,6 @@ def test_merge_conflict(tmp_dir, dvc, ancestor, our, their, error, caplog):
assert error in caplog.text
@pytest.mark.parametrize("workspace", ["ssh"], indirect=True)
def test_merge_different_output_types(tmp_dir, dvc, caplog, workspace):
(tmp_dir / "ancestor").touch()
(tmp_dir / "our").write_text(
"outs:\n- md5: f123456789.dir\n path: ssh://example.com/path\n"
)
(tmp_dir / "their").write_text(
"outs:\n- md5: f987654321.dir\n path: path\n"
)
assert (
main(
[
"git-hook",
"merge-driver",
"--ancestor",
"ancestor",
"--our",
"our",
"--their",
"their",
]
)
!= 0
)
error = "unable to auto-merge outputs of different types"
assert error in caplog.text
def test_merge_different_output_options(tmp_dir, dvc, caplog):
(tmp_dir / "ancestor").touch()
tests/func/test_remote.py
+7 -11
View File
@@ -11,7 +11,6 @@ from dvc.exceptions import DownloadError, RemoteCacheRequiredError, UploadError
from dvc.main import main
from dvc.utils.fs import remove
from tests.basic_env import TestDvc
from tests.remotes import Local
class TestRemote(TestDvc):
@@ -127,19 +126,16 @@ def test_show_default(dvc, capsys):
assert out == "foo\n"
class TestRemoteShouldHandleUppercaseRemoteName(TestDvc):
upper_case_remote_name = "UPPERCASEREMOTE"
def test_upper_case_remote(tmp_dir, dvc, local_cloud):
remote_name = "UPPERCASEREMOTE"
def test(self):
remote_url = Local.get_storagepath()
ret = main(["remote", "add", self.upper_case_remote_name, remote_url])
self.assertEqual(ret, 0)
tmp_dir.gen("foo", "foo")
ret = main(["add", self.FOO])
self.assertEqual(ret, 0)
ret = main(["remote", "add", remote_name, local_cloud.url])
assert ret == 0
ret = main(["push", "-r", self.upper_case_remote_name])
self.assertEqual(ret, 0)
ret = main(["push", "-r", remote_name])
assert ret == 0
def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
tests/func/test_run_multistage.py
+12 -39
View File
@@ -350,43 +350,16 @@ def test_run_overwrite_preserves_meta_and_comment(tmp_dir, dvc, run_copy):
)
@pytest.mark.parametrize(
"workspace, hash_name, foo_hash, bar_hash",
[
(
"local",
"md5",
"acbd18db4cc2f85cedef654fccc4a4d8",
"37b51d194a7513e45b56f6524f2d51f2",
),
pytest.param(
"ssh",
"md5",
"acbd18db4cc2f85cedef654fccc4a4d8",
"37b51d194a7513e45b56f6524f2d51f2",
marks=pytest.mark.skipif(
os.name == "nt", reason="disabled on windows"
),
),
(
"s3",
"etag",
"acbd18db4cc2f85cedef654fccc4a4d8",
"37b51d194a7513e45b56f6524f2d51f2",
),
(
"hdfs",
"checksum",
"0000020000000000000000003dba826b9be9c6a8e2f8310a770555c4",
"00000200000000000000000075433c81259d3c38e364b348af52e84d",
),
],
indirect=["workspace"],
)
def test_run_external_outputs(
tmp_dir, dvc, workspace, hash_name, foo_hash, bar_hash
tmp_dir,
dvc,
local_workspace,
):
workspace.gen("foo", "foo")
hash_name = "md5"
foo_hash = "acbd18db4cc2f85cedef654fccc4a4d8"
bar_hash = "37b51d194a7513e45b56f6524f2d51f2"
local_workspace.gen("foo", "foo")
dvc.run(
name="mystage",
cmd="mycmd",
@@ -408,7 +381,7 @@ def test_run_external_outputs(
assert (tmp_dir / "dvc.yaml").read_text() == dvc_yaml
assert not (tmp_dir / "dvc.lock").exists()
workspace.gen("bar", "bar")
local_workspace.gen("bar", "bar")
dvc.commit("dvc.yaml", force=True)
assert (tmp_dir / "dvc.yaml").read_text() == dvc_yaml
@@ -427,8 +400,8 @@ def test_run_external_outputs(
" size: 3\n"
)
assert (workspace / "foo").read_text() == "foo"
assert (workspace / "bar").read_text() == "bar"
assert (local_workspace / "foo").read_text() == "foo"
assert (local_workspace / "bar").read_text() == "bar"
assert (
workspace / "cache" / bar_hash[:2] / bar_hash[2:]
local_workspace / "cache" / bar_hash[:2] / bar_hash[2:]
).read_text() == "bar"
tests/func/test_scm.py
+1 -1
View File
@@ -336,10 +336,10 @@ def test_git_stash_clear(tmp_dir, scm, ref):
@pytest.mark.parametrize("server", [pytest.lazy_fixture("git_ssh")])
def test_git_ssh(tmp_dir, scm, server):
from dulwich.repo import Repo as DulwichRepo
from dvc_ssh.tests.cloud import TEST_SSH_KEY_PATH, TEST_SSH_USER
from sshfs import SSHFileSystem
from dvc.utils.fs import remove
from tests.remotes.ssh import TEST_SSH_KEY_PATH, TEST_SSH_USER
fs = SSHFileSystem(
host=server.host,
tests/func/test_update.py
+8 -52
View File
@@ -156,23 +156,6 @@ def test_update_before_and_after_dvc_init(tmp_dir, dvc, git_dir):
assert dvc.status([stage.path]) == {}
@pytest.mark.parametrize(
"workspace",
[
"local",
"s3",
pytest.param("gs", marks=pytest.mark.needs_internet),
"hdfs",
"webhdfs",
pytest.param(
"ssh",
marks=pytest.mark.skipif(
os.name == "nt", reason="disabled on windows"
),
),
],
indirect=True,
)
def test_update_import_url(tmp_dir, dvc, workspace):
workspace.gen("file", "file content")
@@ -321,16 +304,6 @@ def test_update_import_to_remote(tmp_dir, dvc, erepo_dir, local_remote):
dvc.update(stage.path, to_remote=True)
@pytest.mark.parametrize(
"workspace",
[
"local",
"s3",
pytest.param("gs", marks=pytest.mark.needs_internet),
"hdfs",
],
indirect=True,
)
def test_update_import_url_to_remote(tmp_dir, dvc, workspace, local_remote):
workspace.gen("foo", "foo")
stage = dvc.imp_url("remote://workspace/foo", to_remote=True)
@@ -342,15 +315,6 @@ def test_update_import_url_to_remote(tmp_dir, dvc, workspace, local_remote):
assert (tmp_dir / "foo").read_text() == "bar"
@pytest.mark.parametrize(
"workspace",
[
"s3",
pytest.param("gs", marks=pytest.mark.needs_internet),
"hdfs",
],
indirect=True,
)
def test_update_import_url_to_remote_directory(
mocker, tmp_dir, dvc, workspace, local_remote
):
@@ -371,16 +335,8 @@ def test_update_import_url_to_remote_directory(
}
)
download_file_mock = mocker.spy(
type(dvc.cloud.get_remote_odb("workspace-cache").fs), "download_file"
)
stage = dvc.update(stage.path, to_remote=True)
# 2 new hashes (foo2, baz2)
assert download_file_mock.mock.call_count == 2
# The new .dir hash will be transferred through MemFs, so
# we will not account the download_file() for it.
dvc.pull("data")
assert (tmp_dir / "data").read_text() == {
"foo": "foo",
@@ -395,12 +351,12 @@ def test_update_import_url_to_remote_directory(
def test_update_import_url_to_remote_directory_changed_contents(
tmp_dir, dvc, local_cloud, local_remote
tmp_dir, dvc, local_workspace, local_remote
):
local_cloud.gen({"data": {"foo": "foo", "bar": {"baz": "baz"}}})
stage = dvc.imp_url("remote://upstream/data", to_remote=True)
local_workspace.gen({"data": {"foo": "foo", "bar": {"baz": "baz"}}})
stage = dvc.imp_url("remote://workspace/data", to_remote=True)
local_cloud.gen(
local_workspace.gen(
{"data": {"foo": "not_foo", "foo2": "foo", "bar": {"baz2": "baz2"}}}
)
stage = dvc.update(stage.path, to_remote=True)
@@ -414,14 +370,14 @@ def test_update_import_url_to_remote_directory_changed_contents(
def test_update_import_url_to_remote_directory_same_hash(
tmp_dir, dvc, local_cloud, local_remote
tmp_dir, dvc, local_workspace, local_remote
):
local_cloud.gen(
local_workspace.gen(
{"data": {"foo": "foo", "bar": {"baz": "baz"}, "same": "same"}}
)
stage = dvc.imp_url("remote://upstream/data", to_remote=True)
stage = dvc.imp_url("remote://workspace/data", to_remote=True)
local_cloud.gen(
local_workspace.gen(
{"data": {"foo": "baz", "bar": {"baz": "foo"}, "same": "same"}}
)
stage = dvc.update(stage.path, to_remote=True)
tests/remotes/__init__.py
+3 -47
View File
@@ -1,54 +1,10 @@
from .azure import Azure, azure, azure_server, make_azure # noqa: F401
from .gdrive import ( # noqa: F401; noqa: F401
TEST_GDRIVE_REPO_BUCKET,
GDrive,
gdrive,
make_gdrive,
)
from .git_server import git_server, git_ssh # noqa: F401
from .gs import ( # noqa: F401; noqa: F401
GCP,
TEST_GCP_CREDS_FILE,
TEST_GCP_REPO_BUCKET,
gs,
make_gs,
)
from .hdfs import ( # noqa: F401
HDFS,
hadoop,
hdfs,
hdfs_server,
make_hdfs,
real_hdfs,
)
from .http import HTTP, http, http_server, make_http # noqa: F401
from .local import Local, local_cloud, local_remote, make_local # noqa: F401
from .oss import ( # noqa: F401
OSS,
TEST_OSS_REPO_BUCKET,
make_oss,
oss,
oss_server,
real_oss,
)
from .s3 import ( # noqa: F401
S3,
TEST_AWS_REPO_BUCKET,
make_s3,
real_s3,
s3,
s3_fake_creds_file,
s3_server,
)
from .ssh import ( # noqa: F401; noqa: F401
SSH,
from dvc_ssh.tests.fixtures import ( # noqa, pylint: disable=unused-import
make_ssh,
ssh,
ssh_connection,
ssh_server,
)
from .webdav import Webdav, make_webdav, webdav, webdav_server # noqa: F401
from .webhdfs import WebHDFS, make_webhdfs, webhdfs # noqa: F401
from .git_server import git_server, git_ssh # noqa: F401
TEST_REMOTE = "upstream"
TEST_CONFIG = {
tests/remotes/azure.py
-137
View File
@@ -1,137 +0,0 @@
# pylint:disable=abstract-method
import locale
import uuid
import pytest
from funcy import cached_property
from dvc.testing.cloud import Cloud
from dvc.testing.path_info import CloudURLInfo
TEST_AZURE_CONTAINER = "tests"
TEST_AZURE_CONNECTION_STRING = (
"DefaultEndpointsProtocol=http;"
"AccountName=devstoreaccount1;"
"AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSR"
"Z6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;"
"BlobEndpoint=http://127.0.0.1:{port}/devstoreaccount1;"
)
class Azure(Cloud, CloudURLInfo):
IS_OBJECT_STORAGE = True
CONNECTION_STRING = None
def __init__(self, url, **kwargs):
super().__init__(url)
self.opts = kwargs
@cached_property
def service_client(self):
# pylint: disable=no-name-in-module
from azure.storage.blob import BlobServiceClient
service_client = BlobServiceClient.from_connection_string(
self.CONNECTION_STRING
)
return service_client
@property
def blob_client(self):
return self.service_client.get_blob_client(self.bucket, self.path)
def mkdir(self, mode=0o777, parents=False, exist_ok=False):
assert mode == 0o777
assert parents
def write_bytes(self, contents):
self.blob_client.upload_blob(contents, overwrite=True)
def read_bytes(self):
stream = self.blob_client.download_blob()
return stream.readall()
def read_text(self, encoding=None, errors=None):
if not encoding:
encoding = locale.getpreferredencoding(False)
assert errors is None
return self.read_bytes().decode(encoding)
@property
def fs_path(self):
bucket = self.bucket.rstrip("/")
path = self.path.lstrip("/")
return f"{bucket}/{path}"
def is_file(self):
raise NotImplementedError
def is_dir(self):
raise NotImplementedError
def exists(self):
raise NotImplementedError
@property
def config(self):
return {"url": self.url, **self.opts}
@pytest.fixture(scope="session")
def azure_server(test_config, docker_compose, docker_services):
test_config.requires("azure")
from azure.core.exceptions import ( # pylint: disable=no-name-in-module
AzureError,
)
from azure.storage.blob import ( # pylint: disable=no-name-in-module
BlobServiceClient,
)
port = docker_services.port_for("azurite", 10000)
connection_string = TEST_AZURE_CONNECTION_STRING.format(port=port)
def _check():
try:
BlobServiceClient.from_connection_string(
connection_string
).list_containers()
return True
except AzureError:
return False
docker_services.wait_until_responsive(
timeout=60.0, pause=0.1, check=_check
)
Azure.CONNECTION_STRING = connection_string
return connection_string
@pytest.fixture
def make_azure(azure_server):
def _make_azure():
from azure.core.exceptions import ResourceNotFoundError
url = f"azure://{TEST_AZURE_CONTAINER}/{uuid.uuid4()}"
ret = Azure(url, connection_string=azure_server)
container = ret.service_client.get_container_client(
TEST_AZURE_CONTAINER
)
try: # verify that container exists
container.get_container_properties()
except ResourceNotFoundError:
container.create_container()
return ret
return _make_azure
@pytest.fixture
def azure(make_azure):
return make_azure()
tests/remotes/gdrive.py
-168
View File
@@ -1,168 +0,0 @@
# pylint:disable=abstract-method
import json
import os
import uuid
from functools import partialmethod
from urllib.parse import urlparse
import pytest
from funcy import cached_property, retry
from dvc.fs.gdrive import GDriveFileSystem
from dvc.testing.cloud import Cloud
from dvc.testing.path_info import CloudURLInfo
from dvc.utils import tmp_fname
TEST_GDRIVE_REPO_BUCKET = "root"
class GDriveURLInfo(CloudURLInfo):
def __init__(self, url):
super().__init__(url)
# GDrive URL host part is case sensitive,
# we are restoring it here.
p = urlparse(url)
self.host = p.netloc
assert self.netloc == self.host
# Normalize path. Important since we have a cache (path to ID)
# and don't want to deal with different variations of path in it.
import re
self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/"))
def _gdrive_retry(func):
def should_retry(exc):
from googleapiclient.errors import HttpError
if not isinstance(exc, HttpError):
return False
if 500 <= exc.resp.status < 600:
return True
if exc.resp.status == 403:
try:
reason = json.loads(exc.content)["error"]["errors"][0][
"reason"
]
except (ValueError, LookupError):
return False
return reason in ["userRateLimitExceeded", "rateLimitExceeded"]
# 16 tries, start at 0.5s, multiply by golden ratio, cap at 20s
return retry(
16,
timeout=lambda a: min(0.5 * 1.618 ** a, 20),
filter_errors=should_retry,
)(func)
class GDrive(Cloud, GDriveURLInfo):
@staticmethod
def should_test():
return bool(os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA))
@property
def config(self):
tmp_path = tmp_fname()
with open(tmp_path, "w", encoding="utf-8") as stream:
raw_credentials = os.getenv(
GDriveFileSystem.GDRIVE_CREDENTIALS_DATA
)
try:
credentials = json.loads(raw_credentials)
except ValueError:
credentials = {}
use_service_account = credentials.get("type") == "service_account"
stream.write(raw_credentials)
return {
"url": self.url,
"gdrive_service_account_json_file_path": tmp_path,
"gdrive_use_service_account": use_service_account,
}
@staticmethod
def _get_storagepath():
return TEST_GDRIVE_REPO_BUCKET + "/" + str(uuid.uuid4())
@staticmethod
def get_url():
# NOTE: `get_url` should always return new random url
return "gdrive://" + GDrive._get_storagepath()
@cached_property
def client(self):
try:
from gdrivefs import GoogleDriveFileSystem
except ImportError:
pytest.skip("gdrivefs is not installed")
return GoogleDriveFileSystem(
token="cache",
tokens_file=self.config["gdrive_service_account_json_file_path"],
service_account=self.config["gdrive_use_service_account"],
)
def is_file(self):
raise NotImplementedError
def is_dir(self):
raise NotImplementedError
def exists(self):
raise NotImplementedError
@_gdrive_retry
def mkdir(self, mode=0o777, parents=False, exist_ok=False):
try:
self.client.info(self.path)
except FileNotFoundError:
self.client.mkdir(self.path, parents=parents)
else:
if not exist_ok:
raise FileExistsError(self.path)
@_gdrive_retry
def write_bytes(self, contents):
with self.client.open(self.path, mode="wb") as stream:
stream.write(contents)
@_gdrive_retry
def _read(self, mode):
with self.client.open(self.path, mode=mode) as stream:
return stream.read()
read_text = partialmethod(_read, mode="r")
read_bytes = partialmethod(_read, mode="rb")
@pytest.fixture
def make_gdrive(test_config, make_tmp_dir):
test_config.requires("gdrive")
if not GDrive.should_test():
pytest.skip("no gdrive")
def _make_gdrive():
# NOTE: temporary workaround
tmp_dir = make_tmp_dir("gdrive", dvc=True)
ret = GDrive(GDrive.get_url())
fs = GDriveFileSystem(
gdrive_credentials_tmp_dir=tmp_dir.dvc.tmp_dir, **ret.config
)
fs.fs._gdrive_create_dir("root", fs.url)
yield ret
return _make_gdrive
@pytest.fixture
def gdrive(make_gdrive):
return make_gdrive()
tests/remotes/git_server.py
+1 -2
View File
@@ -1,6 +1,5 @@
import pytest
from .ssh import SSH, TEST_SSH_KEY_PATH, TEST_SSH_USER
from dvc_ssh.tests.cloud import SSH, TEST_SSH_KEY_PATH, TEST_SSH_USER
class GitSSH(SSH):
tests/remotes/gs.py
-105
View File
@@ -1,105 +0,0 @@
import locale
import os
import uuid
import pytest
from dvc.testing.cloud import Cloud
from dvc.testing.path_info import CloudURLInfo
from dvc.utils import env2bool
TEST_GCP_REPO_BUCKET = os.environ.get("DVC_TEST_GCP_REPO_BUCKET", "dvc-test")
TEST_GCP_CREDS_FILE = os.path.abspath(
os.environ.get(
"GOOGLE_APPLICATION_CREDENTIALS",
os.path.join("scripts", "ci", "gcp-creds.json"),
)
)
# Ensure that absolute path is used
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = TEST_GCP_CREDS_FILE
class GCP(Cloud, CloudURLInfo):
IS_OBJECT_STORAGE = True
@staticmethod
def should_test():
do_test = env2bool("DVC_TEST_GCP", undefined=None)
if do_test is not None:
return do_test
if not os.path.exists(TEST_GCP_CREDS_FILE):
return False
return True
@property
def config(self):
return {"url": self.url, "credentialpath": TEST_GCP_CREDS_FILE}
@staticmethod
def _get_storagepath():
return TEST_GCP_REPO_BUCKET + "/" + str(uuid.uuid4())
@staticmethod
def get_url():
return "gs://" + GCP._get_storagepath()
@property
def _gc(self):
from google.cloud.storage import Client
return Client.from_service_account_json(TEST_GCP_CREDS_FILE)
@property
def _bucket(self):
return self._gc.bucket(self.bucket)
@property
def _blob(self):
return self._bucket.blob(self.path)
def is_file(self):
if self.path.endswith("/"):
return False
return self._blob.exists()
def is_dir(self):
dir_path = self / ""
return bool(list(self._bucket.list_blobs(prefix=dir_path.path)))
def exists(self):
return self.is_file() or self.is_dir()
def mkdir(self, mode=0o777, parents=False, exist_ok=False):
assert mode == 0o777
assert parents
def write_bytes(self, contents):
assert isinstance(contents, bytes)
self._blob.upload_from_string(contents)
def read_bytes(self):
return self._blob.download_as_string()
def read_text(self, encoding=None, errors=None):
if not encoding:
encoding = locale.getpreferredencoding(False)
assert errors is None
return self.read_bytes().decode(encoding)
@pytest.fixture
def make_gs(test_config):
test_config.requires("gs")
if not GCP.should_test():
pytest.skip("no gs")
yield GCP(GCP.get_url())
@pytest.fixture
def gs(make_gs):
return make_gs()
tests/remotes/hdfs.py
-311
View File
@@ -1,311 +0,0 @@
import locale
import os
import platform
import subprocess
import uuid
from contextlib import contextmanager
from pathlib import Path
from tempfile import TemporaryDirectory
from urllib.parse import urlparse
import pytest
from dvc.testing.cloud import Cloud
from dvc.testing.path_info import URLInfo
_hdfs_root = TemporaryDirectory()
class HDFS(Cloud, URLInfo): # pylint: disable=abstract-method
@contextmanager
def _hdfs(self):
import pyarrow.fs
conn = pyarrow.fs.HadoopFileSystem(self.host, self.port)
yield conn
@property
def config(self):
return {"url": self.url}
def is_file(self):
with self._hdfs() as _hdfs:
import pyarrow.fs
file_info = _hdfs.get_file_info(self.path)
return file_info.type == pyarrow.fs.FileType.File
def is_dir(self):
with self._hdfs() as _hdfs:
import pyarrow.fs
file_info = _hdfs.get_file_info(self.path)
return file_info.type == pyarrow.fs.FileType.Directory
def exists(self):
with self._hdfs() as _hdfs:
import pyarrow.fs
file_info = _hdfs.get_file_info(self.path)
return file_info.type != pyarrow.fs.FileType.NotFound
def mkdir(self, mode=0o777, parents=False, exist_ok=False):
assert mode == 0o777
assert parents
with self._hdfs() as _hdfs:
# NOTE: fs.create_dir creates parents by default
_hdfs.create_dir(self.path)
def write_bytes(self, contents):
with self._hdfs() as _hdfs:
with _hdfs.open_output_stream(self.path) as fobj:
fobj.write(contents)
def read_bytes(self):
with self._hdfs() as _hdfs:
with _hdfs.open_input_stream(self.path) as fobj:
return fobj.read()
def read_text(self, encoding=None, errors=None):
if not encoding:
encoding = locale.getpreferredencoding(False)
assert errors is None
return self.read_bytes().decode(encoding)
@property
def fs_path(self):
return self.path.lstrip("/")
@pytest.fixture(scope="session")
def hadoop(test_config):
test_config.requires("real_hdfs")
pytest.importorskip("pyarrow.fs")
if platform.system() != "Linux":
pytest.skip("only supported on Linux")
import wget
from appdirs import user_cache_dir
hadoop_name = "hadoop-2.7.2.tar.gz"
java_name = "openjdk-7u75-b13-linux-x64-18_dec_2014.tar.gz"
base_url = "https://s3-us-east-2.amazonaws.com/dvc-public/dvc-test/"
hadoop_url = base_url + hadoop_name
java_url = base_url + java_name
(cache_dir,) = (user_cache_dir("dvc-test", "iterative"),)
dname = os.path.join(cache_dir, "hdfs")
java_tar = os.path.join(dname, java_name)
hadoop_tar = os.path.join(dname, hadoop_name)
java_home = os.path.join(dname, "java-se-7u75-ri")
hadoop_home = os.path.join(dname, "hadoop-2.7.2")
def _get(url, tar, target):
if os.path.isdir(target):
return
if not os.path.exists(tar):
wget.download(url, out=tar)
assert os.system(f"tar -xvf {tar} -C {dname}") == 0
assert os.path.isdir(target)
os.makedirs(dname, exist_ok=True)
_get(hadoop_url, hadoop_tar, hadoop_home)
_get(java_url, java_tar, java_home)
os.environ["JAVA_HOME"] = java_home
os.environ["HADOOP_HOME"] = hadoop_home
os.environ["PATH"] += f":{hadoop_home}/bin:{hadoop_home}/sbin"
# NOTE: must set CLASSPATH to connect using pyarrow.fs.HadoopFileSystem
result = subprocess.run(
[f"{hadoop_home}/bin/hdfs", "classpath", "--glob"],
universal_newlines=True,
stdout=subprocess.PIPE,
check=False,
)
os.environ["CLASSPATH"] = result.stdout
@pytest.fixture(scope="session")
def hdfs_server(hadoop, docker_compose, docker_services):
import pyarrow.fs
port = docker_services.port_for("hdfs", 8020)
web_port = docker_services.port_for("hdfs", 50070)
def _check():
try:
# NOTE: just connecting or even opening something is not enough,
# we need to make sure that we are able to write something.
conn = pyarrow.fs.HadoopFileSystem("hdfs://127.0.0.1", port)
with conn.open_output_stream(str(uuid.uuid4())) as fobj:
fobj.write(b"test")
return True
except (pyarrow.ArrowException, OSError):
import traceback
traceback.print_exc()
return False
try:
docker_services.wait_until_responsive(
timeout=30.0, pause=5, check=_check
)
except Exception: # pylint: disable=broad-except
pytest.skip("couldn't start hdfs server")
return {"hdfs": port, "webhdfs": web_port}
@pytest.fixture
def real_hdfs(hdfs_server):
port = hdfs_server["hdfs"]
url = f"hdfs://127.0.0.1:{port}/{uuid.uuid4()}"
yield HDFS(url)
def md5md5crc32c(path):
# https://github.com/colinmarc/hdfs/blob/f2f512db170db82ad41590c4ba3b7718b13317d2/file_reader.go#L76
import hashlib
from crc32c import crc32c # pylint: disable=no-name-in-module
# dfs.bytes-per-checksum = 512, default on hadoop 2.7
bytes_per_checksum = 512
padded = 32
total = 0
md5md5 = hashlib.md5()
with open(path, "rb") as fobj:
while True:
block = fobj.read(bytes_per_checksum)
if not block:
break
crc_int = crc32c(block)
# NOTE: hdfs is big-endian
crc_bytes = crc_int.to_bytes(
(crc_int.bit_length() + 7) // 8, "big"
)
md5 = hashlib.md5(crc_bytes).digest()
total += len(md5)
if padded < total:
padded *= 2
md5md5.update(md5)
md5md5.update(b"\0" * (padded - total))
return "000002000000000000000000" + md5md5.hexdigest()
def hadoop_fs_checksum(_, path):
parsed = urlparse(path)
return md5md5crc32c(Path(_hdfs_root.name) / parsed.path.lstrip("/"))
class FakeHadoopFileSystem:
def __init__(self, *args, **kwargs):
from pyarrow.fs import LocalFileSystem
self._root = Path(_hdfs_root.name)
self._fs = LocalFileSystem()
def _path(self, path):
from pyarrow.fs import FileSelector
if isinstance(path, FileSelector):
return FileSelector(
os.fspath(self._root / path.base_dir.lstrip("/")),
path.allow_not_found,
path.recursive,
)
if isinstance(path, list):
return [self._path(sub_path) for sub_path in path]
return os.fspath(self._root / path.lstrip("/"))
def create_dir(self, path, **kwargs):
return self._fs.create_dir(self._path(path), **kwargs)
def open_input_stream(self, path, **kwargs):
return self._fs.open_input_stream(self._path(path), **kwargs)
def open_output_stream(self, path, **kwargs):
import posixpath
# NOTE: HadoopFileSystem.open_output_stream creates directories
# automatically.
self.create_dir(posixpath.dirname(path))
return self._fs.open_output_stream(self._path(path), **kwargs)
def get_file_info(self, path, **kwargs):
from pyarrow.fs import FileInfo
entries = self._fs.get_file_info(self._path(path), **kwargs)
if isinstance(entries, FileInfo):
ret = self._adjust_entry(entries)
else:
assert isinstance(entries, list)
ret = list(map(self._adjust_entry, entries))
# import pdb; pdb.set_trace()
return ret
def _adjust_entry(self, entry):
import posixpath
from pyarrow.fs import FileInfo
mocked_path = os.path.relpath(entry.path, self._root)
mocked_parts = mocked_path.split(os.path.sep)
return FileInfo(
path=posixpath.join(*mocked_parts),
type=entry.type,
mtime=entry.mtime,
size=entry.size,
)
def move(self, from_path, to_path):
self._fs.move(self._path(from_path), self._path(to_path))
def delete_file(self, path):
self._fs.delete_file(self._path(path))
@pytest.fixture
def make_hdfs(test_config, mocker):
# Windows might not have Visual C++ Redistributable for Visual Studio
# 2015 installed, which will result in the following error:
# "The pyarrow installation is not built with support for
# 'HadoopFileSystem'"
test_config.requires("hdfs")
pytest.importorskip("pyarrow.fs")
mocker.patch("pyarrow.fs._not_imported", [])
mocker.patch(
"pyarrow.fs.HadoopFileSystem", FakeHadoopFileSystem, create=True
)
mocker.patch("dvc.fs.hdfs.HDFSFileSystem._checksum", hadoop_fs_checksum)
def _make_hdfs():
url = f"hdfs://example.com:12345/{uuid.uuid4()}"
return HDFS(url)
return _make_hdfs
@pytest.fixture
def hdfs(make_hdfs):
return make_hdfs()
tests/remotes/http.py
-68
View File
@@ -1,68 +0,0 @@
# pylint:disable=abstract-method
import os
import uuid
import pytest
import requests
from dvc.testing.cloud import Cloud
from dvc.testing.path_info import HTTPURLInfo
class HTTP(Cloud, HTTPURLInfo):
@staticmethod
def get_url(port): # pylint: disable=arguments-differ
dname = str(uuid.uuid4())
return f"http://127.0.0.1:{port}/{dname}"
def mkdir(self, mode=0o777, parents=False, exist_ok=False):
assert mode == 0o777
assert parents
def write_bytes(self, contents):
assert isinstance(contents, bytes)
response = requests.post(self.url, data=contents)
assert response.status_code == 200
@property
def config(self):
return {"url": self.url}
@property
def fs_path(self):
return self.url
def exists(self):
raise NotImplementedError
def is_dir(self):
raise NotImplementedError
def is_file(self):
raise NotImplementedError
def read_bytes(self):
raise NotImplementedError
@pytest.fixture(scope="session")
def http_server(test_config, tmp_path_factory):
from tests.utils.httpd import StaticFileServer
test_config.requires("http")
directory = os.fspath(tmp_path_factory.mktemp("http"))
with StaticFileServer(directory=directory) as httpd:
yield httpd
@pytest.fixture
def make_http(http_server):
def _make_http():
return HTTP(HTTP.get_url(http_server.server_port))
return _make_http
@pytest.fixture
def http(make_http):
return make_http()
tests/remotes/local.py
-34
View File
@@ -1,34 +0,0 @@
# pylint: disable=cyclic-import,disable=abstract-method
import pytest
from dvc.testing.cloud import Cloud
from tests.basic_env import TestDvc
class Local(Cloud):
@staticmethod
def get_storagepath():
return TestDvc.mkdtemp()
@staticmethod
def get_url():
return Local.get_storagepath()
@pytest.fixture
def make_local(make_tmp_dir):
def _make_local():
return make_tmp_dir("local-cloud")
return _make_local
@pytest.fixture
def local_cloud(make_local):
return make_local()
@pytest.fixture
def local_remote(tmp_dir, dvc, local_cloud):
tmp_dir.add_remote(config=local_cloud.config)
yield local_cloud
tests/remotes/oss.py
-131
View File
@@ -1,131 +0,0 @@
# pylint:disable=abstract-method
import os
import uuid
import pytest
from dvc.testing.cloud import Cloud
from dvc.testing.path_info import CloudURLInfo
from dvc.utils import env2bool
TEST_OSS_REPO_BUCKET = "dvc-test-github"
EMULATOR_OSS_ENDPOINT = "127.0.0.1:{port}"
EMULATOR_OSS_ACCESS_KEY_ID = "AccessKeyID"
EMULATOR_OSS_ACCESS_KEY_SECRET = "AccessKeySecret"
class OSS(Cloud, CloudURLInfo):
IS_OBJECT_STORAGE = True
@staticmethod
def get_url():
return f"oss://{TEST_OSS_REPO_BUCKET}/{uuid.uuid4()}"
@staticmethod
def should_test():
do_test = env2bool("DVC_TEST_OSS", undefined=None)
if do_test is not None:
return do_test
if os.getenv("OSS_ACCESS_KEY_ID") and os.getenv(
"OSS_ACCESS_KEY_SECRET"
):
return True
return False
@property
def config(self):
return {
"url": self.url,
"oss_key_id": os.environ.get("OSS_ACCESS_KEY_ID"),
"oss_key_secret": os.environ.get("OSS_ACCESS_KEY_SECRET"),
"oss_endpoint": os.environ.get("OSS_ENDPOINT"),
}
def is_file(self):
raise NotImplementedError
def is_dir(self):
raise NotImplementedError
def exists(self):
raise NotImplementedError
def mkdir(self, mode=0o777, parents=False, exist_ok=False):
raise NotImplementedError
def write_bytes(self, contents):
raise NotImplementedError
def read_bytes(self):
raise NotImplementedError
@pytest.fixture(scope="session")
def oss_server(test_config, docker_compose, docker_services):
import oss2
test_config.requires("oss")
port = docker_services.port_for("oss", 8880)
endpoint = EMULATOR_OSS_ENDPOINT.format(port=port)
def _check():
try:
auth = oss2.Auth(
EMULATOR_OSS_ACCESS_KEY_ID, EMULATOR_OSS_ACCESS_KEY_SECRET
)
oss2.Bucket(auth, endpoint, "mybucket").get_bucket_info()
return True
except oss2.exceptions.NoSuchBucket:
return True
except oss2.exceptions.OssError:
return False
docker_services.wait_until_responsive(timeout=30.0, pause=5, check=_check)
return endpoint
@pytest.fixture
def make_oss(real_oss):
def _make_oss():
import oss2
ret = real_oss
auth = oss2.Auth(
ret.config["oss_key_id"], ret.config["oss_key_secret"]
)
bucket = oss2.Bucket(
auth, ret.config["oss_endpoint"], TEST_OSS_REPO_BUCKET
)
try:
bucket.get_bucket_info()
except oss2.exceptions.NoSuchBucket:
bucket.create_bucket(
oss2.BUCKET_ACL_PUBLIC_READ,
oss2.models.BucketCreateConfig(
oss2.BUCKET_STORAGE_CLASS_STANDARD
),
)
return ret
return _make_oss
@pytest.fixture
def oss(make_oss):
return make_oss()
@pytest.fixture
def real_oss(test_config):
test_config.requires("oss")
if not OSS.should_test():
pytest.skip("no real OSS")
url = OSS.get_url()
return OSS(url)
tests/remotes/s3.py
-173
View File
@@ -1,173 +0,0 @@
import locale
import os
import uuid
import pytest
from funcy import cached_property
from dvc.testing.cloud import Cloud
from dvc.testing.path_info import CloudURLInfo
from dvc.utils import env2bool
TEST_AWS_REPO_BUCKET = os.environ.get("DVC_TEST_AWS_REPO_BUCKET", "dvc-temp")
TEST_AWS_ENDPOINT_URL = "http://127.0.0.1:{port}/"
class S3(Cloud, CloudURLInfo):
IS_OBJECT_STORAGE = True
TEST_AWS_ENDPOINT_URL = None
@property
def config(self):
return {"url": self.url, "endpointurl": self.TEST_AWS_ENDPOINT_URL}
@staticmethod
def should_test():
do_test = env2bool("DVC_TEST_AWS", undefined=None)
if do_test is not None:
return do_test
if os.getenv("AWS_ACCESS_KEY_ID") and os.getenv(
"AWS_SECRET_ACCESS_KEY"
):
return True
return False
@staticmethod
def _get_storagepath():
return (
TEST_AWS_REPO_BUCKET
+ "/"
+ "dvc_test_caches"
+ "/"
+ str(uuid.uuid4())
)
@staticmethod
def get_url():
return "s3://" + S3._get_storagepath()
@cached_property
def _s3(self):
import boto3
return boto3.client("s3", endpoint_url=self.config["endpointurl"])
def is_file(self):
from botocore.exceptions import ClientError
if self.path.endswith("/"):
return False
try:
self._s3.head_object(Bucket=self.bucket, Key=self.path)
except ClientError as exc:
if exc.response["Error"]["Code"] != "404":
raise
return False
return True
def is_dir(self):
path = (self / "").path
resp = self._s3.list_objects(Bucket=self.bucket, Prefix=path)
return bool(resp.get("Contents"))
def exists(self):
return self.is_file() or self.is_dir()
def mkdir(self, mode=0o777, parents=False, exist_ok=False):
assert mode == 0o777
assert parents
def write_bytes(self, contents):
self._s3.put_object(Bucket=self.bucket, Key=self.path, Body=contents)
def read_bytes(self):
data = self._s3.get_object(Bucket=self.bucket, Key=self.path)
return data["Body"].read()
def read_text(self, encoding=None, errors=None):
if not encoding:
encoding = locale.getpreferredencoding(False)
assert errors is None
return self.read_bytes().decode(encoding)
@property
def fs_path(self):
return self.bucket + "/" + self.path.lstrip("/")
@pytest.fixture
def s3_fake_creds_file(monkeypatch):
# https://github.com/spulec/moto#other-caveats
import pathlib
aws_dir = pathlib.Path("~").expanduser() / ".aws"
aws_dir.mkdir(exist_ok=True)
aws_creds = aws_dir / "credentials"
initially_exists = aws_creds.exists()
if not initially_exists:
aws_creds.touch()
try:
with monkeypatch.context() as m:
m.setenv("AWS_ACCESS_KEY_ID", "testing")
m.setenv("AWS_SECRET_ACCESS_KEY", "testing")
m.setenv("AWS_SECURITY_TOKEN", "testing")
m.setenv("AWS_SESSION_TOKEN", "testing")
yield
finally:
if aws_creds.exists() and not initially_exists:
aws_creds.unlink()
@pytest.fixture(scope="session")
def s3_server(test_config, docker_compose, docker_services):
import requests
test_config.requires("s3")
port = docker_services.port_for("motoserver", 5000)
endpoint_url = TEST_AWS_ENDPOINT_URL.format(port=port)
def _check():
try:
r = requests.get(endpoint_url)
return r.ok
except requests.RequestException:
return False
docker_services.wait_until_responsive(
timeout=60.0, pause=0.1, check=_check
)
S3.TEST_AWS_ENDPOINT_URL = endpoint_url
return endpoint_url
@pytest.fixture
def make_s3(test_config, s3_server, s3_fake_creds_file):
test_config.requires("s3")
def _make_s3():
cloud = S3(S3.get_url())
cloud._s3.create_bucket(Bucket=TEST_AWS_REPO_BUCKET)
return cloud
return _make_s3
@pytest.fixture
def s3(make_s3):
return make_s3()
@pytest.fixture
def real_s3():
if not S3.should_test():
pytest.skip("no real s3")
yield S3(S3.get_url())
tests/remotes/ssh.py
-135
View File
@@ -1,135 +0,0 @@
import locale
import os
import uuid
import pytest
from funcy import cached_property
from dvc.testing.cloud import Cloud
from dvc.testing.path_info import URLInfo
TEST_SSH_USER = "user"
TEST_SSH_KEY_PATH = os.path.join(
os.path.abspath(os.path.dirname(__file__)), f"{TEST_SSH_USER}.key"
)
class SSH(Cloud, URLInfo):
@staticmethod
def get_url(host, port): # pylint: disable=arguments-differ
return f"ssh://{host}:{port}/tmp/data/{uuid.uuid4()}"
@property
def config(self):
return {
"url": self.url,
"user": TEST_SSH_USER,
"keyfile": TEST_SSH_KEY_PATH,
}
@cached_property
def _ssh(self):
from sshfs import SSHFileSystem
return SSHFileSystem(
host=self.host,
port=self.port,
username=TEST_SSH_USER,
client_keys=[TEST_SSH_KEY_PATH],
)
def is_file(self):
return self._ssh.isfile(self.path)
def is_dir(self):
return self._ssh.isdir(self.path)
def exists(self):
return self._ssh.exists(self.path)
def mkdir(self, mode=0o777, parents=False, exist_ok=False):
assert mode == 0o777
assert parents
self._ssh.makedirs(self.path, exist_ok=exist_ok)
def write_bytes(self, contents):
assert isinstance(contents, bytes)
with self._ssh.open(self.path, "wb") as fobj:
fobj.write(contents)
def read_bytes(self):
with self._ssh.open(self.path, "rb") as fobj:
return fobj.read()
def read_text(self, encoding=None, errors=None):
if not encoding:
encoding = locale.getpreferredencoding(False)
assert errors is None
return self.read_bytes().decode(encoding)
@property
def fs_path(self):
return self.path
@pytest.fixture(scope="session")
def ssh_server(test_config, docker_compose, docker_services):
import asyncssh
from sshfs import SSHFileSystem
test_config.requires("ssh")
conn_info = {
"host": "127.0.0.1",
"port": docker_services.port_for("openssh-server", 2222),
}
def get_fs():
return SSHFileSystem(
**conn_info,
username=TEST_SSH_USER,
client_keys=[TEST_SSH_KEY_PATH],
)
def _check():
try:
get_fs().exists("/")
except asyncssh.Error:
return False
else:
return True
docker_services.wait_until_responsive(timeout=30.0, pause=1, check=_check)
return conn_info
@pytest.fixture
def ssh_connection(ssh_server):
from sshfs import SSHFileSystem
yield SSHFileSystem(
host=ssh_server["host"],
port=ssh_server["port"],
username=TEST_SSH_USER,
client_files=[TEST_SSH_KEY_PATH],
)
@pytest.fixture
def make_ssh(ssh_server, monkeypatch):
def _make_ssh():
from dvc.fs.ssh import SSHFileSystem
# NOTE: see http://github.com/iterative/dvc/pull/3501
monkeypatch.setattr(SSHFileSystem, "CAN_TRAVERSE", False)
url = SSH(SSH.get_url(**ssh_server))
url.mkdir(exist_ok=True, parents=True)
return url
return _make_ssh
@pytest.fixture
def ssh(make_ssh):
return make_ssh()
tests/remotes/webdav.py
-98
View File
@@ -1,98 +0,0 @@
# pylint:disable=abstract-method
import os
from wsgiref.simple_server import make_server
import pytest
from funcy import cached_property, first
from wsgidav.wsgidav_app import WsgiDAVApp
from dvc.testing.cloud import Cloud
from dvc.testing.path_info import WebDAVURLInfo
from tests.utils.httpd import run_server_on_thread
AUTH = {"user1": {"password": "password1"}}
class Webdav(Cloud, WebDAVURLInfo):
@staticmethod
def get_url(port): # pylint: disable=arguments-differ
return f"webdav://localhost:{port}"
@property
def config(self):
user, secrets = first(AUTH.items())
url = f"webdav://{self.netloc}{self._spath}"
return {"url": url, "user": user, **secrets}
@cached_property
def client(self):
from webdav4.client import Client
user, secrets = first(AUTH.items())
return Client(
self.replace(path="").url, auth=(user, secrets["password"])
)
def mkdir(self, mode=0o777, parents=False, exist_ok=False):
assert mode == 0o777
parent_dirs = list(reversed(self.parents))[1:] if parents else []
for d in parent_dirs + [self]:
path = d.path # pylint: disable=no-member
if not self.client.exists(path):
self.client.mkdir(path)
def write_bytes(self, contents):
from io import BytesIO
self.client.upload_fileobj(BytesIO(contents), self.path)
@property
def fs_path(self):
return self.path.lstrip("/")
def exists(self):
raise NotImplementedError
def is_dir(self):
raise NotImplementedError
def is_file(self):
raise NotImplementedError
def read_bytes(self):
raise NotImplementedError
@pytest.fixture
def webdav_server(test_config, tmp_path_factory):
test_config.requires("webdav")
host, port = "localhost", 0
directory = os.fspath(tmp_path_factory.mktemp("http"))
dirmap = {"/": directory}
app = WsgiDAVApp(
{
"host": host,
"port": port,
"provider_mapping": dirmap,
"simple_dc": {"user_mapping": {"*": AUTH}},
}
)
server = make_server(host, port, app)
with run_server_on_thread(server) as httpd:
yield httpd
@pytest.fixture
def make_webdav(webdav_server):
def _make_webdav():
url = Webdav.get_url(webdav_server.server_port)
return Webdav(url)
return _make_webdav
@pytest.fixture
def webdav(make_webdav):
return make_webdav()
tests/remotes/webhdfs.py
-78
View File
@@ -1,78 +0,0 @@
import locale
import uuid
from contextlib import contextmanager
import pytest
from dvc.testing.cloud import Cloud
from dvc.testing.path_info import CloudURLInfo
class WebHDFS(Cloud, CloudURLInfo): # pylint: disable=abstract-method
@contextmanager
def _webhdfs(self):
from hdfs import InsecureClient
client = InsecureClient(
f"http://{self.host}:{self.port}", self.user, root="/"
)
yield client
def is_file(self):
with self._webhdfs() as _hdfs:
return _hdfs.status(self.path)["type"] == "FILE"
def is_dir(self):
with self._webhdfs() as _hdfs:
return _hdfs.status(self.path)["type"] == "DIRECTORY"
def exists(self):
with self._webhdfs() as _hdfs:
return _hdfs.status(self.path, strict=False) is not None
def mkdir(self, mode=0o777, parents=False, exist_ok=False):
assert mode == 0o777
assert parents
with self._webhdfs() as _hdfs:
# NOTE: hdfs.makekdirs always creates parents
_hdfs.makedirs(self.path, permission=mode)
def write_bytes(self, contents):
with self._webhdfs() as _hdfs:
with _hdfs.write(self.path, overwrite=True) as writer:
writer.write(contents)
def read_bytes(self):
with self._webhdfs() as _hdfs:
with _hdfs.read(self.path) as reader:
return reader.read()
def read_text(self, encoding=None, errors=None):
if not encoding:
encoding = locale.getpreferredencoding(False)
assert errors is None
return self.read_bytes().decode(encoding)
@property
def config(self):
return {"url": self.url}
@property
def fs_path(self):
return "/" + self.path.lstrip("/")
@pytest.fixture
def make_webhdfs(hdfs_server):
def _make_webhdfs():
port = hdfs_server["webhdfs"]
url = f"webhdfs://127.0.0.1:{port}/{uuid.uuid4()}"
yield WebHDFS(url)
return _make_webhdfs
@pytest.fixture
def webhdfs(make_webhdfs):
return make_webhdfs()
tests/unit/fs/test_s3.py
-80
View File
@@ -1,7 +1,4 @@
import importlib
import os
import sys
import textwrap
import pytest
@@ -139,80 +136,3 @@ def test_key_id_and_secret(dvc):
assert fs.fs_args["key"] == key_id
assert fs.fs_args["secret"] == key_secret
assert fs.fs_args["token"] == session_token
KB = 1024
MB = KB ** 2
GB = KB ** 3
def test_s3_aws_config(tmp_dir, dvc, s3, monkeypatch):
config_directory = tmp_dir / ".aws"
config_directory.mkdir()
(config_directory / "config").write_text(
textwrap.dedent(
"""\
[default]
s3 =
max_concurrent_requests = 20000
max_queue_size = 1000
multipart_threshold = 1000KiB
multipart_chunksize = 64MB
use_accelerate_endpoint = true
addressing_style = path
"""
)
)
if sys.platform == "win32":
var = "USERPROFILE"
else:
var = "HOME"
with monkeypatch.context() as m:
m.setenv(var, str(tmp_dir))
# Fresh import to see the effects of changing HOME variable
s3_mod = importlib.reload(sys.modules[S3FileSystem.__module__])
fs = s3_mod.S3FileSystem(**s3.config)
importlib.reload(sys.modules[S3FileSystem.__module__])
s3_config = fs.fs_args["config_kwargs"]["s3"]
assert s3_config["use_accelerate_endpoint"]
assert s3_config["addressing_style"] == "path"
transfer_config = fs._transfer_config
assert transfer_config.max_io_queue_size == 1000
assert transfer_config.multipart_chunksize == 64 * MB
assert transfer_config.multipart_threshold == 1000 * KB
assert transfer_config.max_request_concurrency == 20000
def test_s3_aws_config_different_profile(tmp_dir, dvc, s3, monkeypatch):
config_file = tmp_dir / "aws_config.ini"
config_file.write_text(
textwrap.dedent(
"""\
[default]
extra = keys
s3 =
addressing_style = auto
use_accelerate_endpoint = true
multipart_threshold = ThisIsNotGoingToBeCasted!
[profile dev]
some_extra = keys
s3 =
addressing_style = virtual
multipart_threshold = 2GiB
"""
)
)
monkeypatch.setenv("AWS_CONFIG_FILE", config_file)
fs = S3FileSystem(profile="dev", **s3.config)
s3_config = fs.fs_args["config_kwargs"]["s3"]
assert s3_config["addressing_style"] == "virtual"
transfer_config = fs._transfer_config
assert transfer_config.multipart_threshold == 2 * GB
tests/unit/remote/test_http.py
-10
View File
@@ -1,20 +1,10 @@
import ssl
import pytest
from mock import patch
from dvc.fs.http import HTTPFileSystem
def test_download_fails_on_error_code(tmp_dir, http):
fs = HTTPFileSystem(**http.config)
with pytest.raises(FileNotFoundError):
fs.download_file(
(http / "missing.txt").fs_path, (tmp_dir / "missing.txt").fs_path
)
def test_public_auth_method(dvc):
config = {
"url": "http://example.com/",
tests/unit/remote/test_remote_tree.py
-45
View File
@@ -1,45 +0,0 @@
import pytest
from dvc.fs.s3 import S3FileSystem
from dvc.objects.db import _get_odb
remotes = [pytest.lazy_fixture(fix) for fix in ["gs", "s3"]]
FILE_WITH_CONTENTS = {
"data1.txt": "",
# "empty_dir/": "",
"empty_file": "",
"foo": "foo",
"data/alice": "alice",
"data/alpha": "alpha",
"data/subdir-file.txt": "subdir",
"data/subdir/1": "1",
"data/subdir/2": "2",
"data/subdir/3": "3",
# "data/subdir/empty_dir/": "",
"data/subdir/empty_file": "",
}
@pytest.mark.parametrize("cloud", [pytest.lazy_fixture("s3")])
def test_copy_preserve_etag_across_buckets(cloud, dvc):
cloud.gen(FILE_WITH_CONTENTS)
rem = _get_odb(dvc, cloud.config)
s3 = rem.fs.s3
s3.create_bucket(Bucket="another")
config = cloud.config.copy()
config["url"] = "s3://another"
config["region"] = "us-east-1"
another = S3FileSystem(**config)
from_info = rem.fs.path.join(rem.fs_path, "foo")
to_info = "another/foo"
rem.fs.copy(from_info, to_info)
from_hash = rem.fs.info(from_info)["ETag"].strip('"')
to_hash = another.info(to_info)["ETag"].strip('"')
assert from_hash == to_hash
tests/utils/httpd.py
-91
View File
@@ -1,91 +0,0 @@
import hashlib
import os
import threading
from contextlib import contextmanager
from http import HTTPStatus
from http.server import HTTPServer
from RangeHTTPServer import RangeRequestHandler
class TestRequestHandler(RangeRequestHandler):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def end_headers(self):
# RangeRequestHandler only sends Accept-Ranges header if Range header
# is present, see https://github.com/danvk/RangeHTTPServer/issues/23
if not self.headers.get("Range"):
self.send_header("Accept-Ranges", "bytes")
# Add a checksum header
file = self.translate_path(self.path)
if not os.path.isdir(file) and os.path.exists(file):
with open(file, encoding="utf-8") as fd:
encoded_text = fd.read().encode("utf8")
checksum = hashlib.md5(encoded_text).hexdigest()
self.send_header("Content-MD5", checksum)
RangeRequestHandler.end_headers(self)
def _chunks(self):
while True:
data = self.rfile.readline(65537)
chunk_size = int(data[:-2], 16)
if chunk_size == 0:
return
data = self.rfile.read(chunk_size)
yield data
self.rfile.read(2)
def do_POST(self):
chunked = self.headers.get("Transfer-Encoding", "") == "chunked"
path = self.translate_path(self.path)
try:
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "wb") as fd:
if chunked:
for chunk in self._chunks():
fd.write(chunk)
else:
size = int(self.headers.get("Content-Length", 0))
fd.write(self.rfile.read(size))
except OSError as e:
self.send_error(HTTPStatus.INTERNAL_SERVER_ERROR, str(e))
self.send_response(HTTPStatus.OK)
self.end_headers()
@contextmanager
def run_server_on_thread(server):
thread = threading.Thread(target=server.serve_forever)
thread.daemon = True
thread.start()
yield server
server.socket.close()
server.shutdown()
server.server_close()
class StaticFileServer:
_lock = threading.Lock()
def __init__(self, directory):
from functools import partial
addr = ("localhost", 0)
req = partial(TestRequestHandler, directory=directory)
server = HTTPServer(addr, req)
self.runner = run_server_on_thread(server)
# pylint: disable=no-member
def __enter__(self):
self._lock.acquire()
return self.runner.__enter__()
def __exit__(self, *args):
self.runner.__exit__(*args)
self._lock.release()