diff --git a/dvc/repo/experiments/__init__.py b/dvc/repo/experiments/__init__.py index 0982b1e0f..e2a6da324 100644 --- a/dvc/repo/experiments/__init__.py +++ b/dvc/repo/experiments/__init__.py @@ -13,7 +13,6 @@ from funcy import cached_property, first from dvc.dependency.param import MissingParamsError from dvc.env import DVCLIVE_RESUME from dvc.exceptions import DvcException -from dvc.stage.monitor import CheckpointKilledError from dvc.utils import relpath from .base import ( @@ -696,6 +695,8 @@ class Experiments: dict mapping stash revs to the successfully executed experiments for each stash rev. """ + from dvc.stage.monitor import CheckpointKilledError + result: Dict[str, Dict[str, str]] = defaultdict(dict) manager = Manager() @@ -798,6 +799,7 @@ class Experiments: @unlocked_repo def _workspace_repro(self) -> Mapping[str, str]: """Run the most recently stashed experiment in the workspace.""" + from dvc.stage.monitor import CheckpointKilledError from dvc.utils.fs import makedirs from .executor.base import BaseExecutor diff --git a/dvc/testing/fixtures.py b/dvc/testing/fixtures.py index 3816848ec..b3bedacce 100644 --- a/dvc/testing/fixtures.py +++ b/dvc/testing/fixtures.py @@ -9,11 +9,15 @@ __all__ = [ "scm", "dvc", "make_cloud", + "make_local", "cloud", + "local_cloud", "make_remote", "remote", + "local_remote", "workspace", "make_workspace", + "local_workspace", "docker", "docker_compose", "docker_compose_project_name", @@ -94,6 +98,14 @@ def dvc(tmp_dir): yield _dvc +@pytest.fixture +def make_local(make_tmp_dir): + def _make_local(): + return make_tmp_dir("local-cloud") + + return _make_local + + @pytest.fixture def make_cloud(request): def _make_cloud(typ): @@ -104,9 +116,13 @@ def make_cloud(request): @pytest.fixture def cloud(make_cloud, request): - if not hasattr(request, "param"): - pytest.skip("no cloud specified") - return make_cloud(request.param) + typ = getattr(request, "param", "local") + return make_cloud(typ) + + +@pytest.fixture +def local_cloud(make_cloud): + return make_cloud("local") @pytest.fixture @@ -121,9 +137,13 @@ def make_remote(tmp_dir, dvc, make_cloud): @pytest.fixture def remote(make_remote, request): - if not hasattr(request, "param"): - pytest.skip("no remote specified") - return make_remote("upstream", typ=request.param) + typ = getattr(request, "param", "local") + return make_remote("upstream", typ=typ) + + +@pytest.fixture +def local_remote(make_remote): + return make_remote("upstream", typ="local") @pytest.fixture @@ -152,10 +172,13 @@ def make_workspace(tmp_dir, dvc, make_cloud): @pytest.fixture def workspace(make_workspace, request): - if not hasattr(request, "param"): - pytest.skip("no workspace specified") + typ = getattr(request, "param", "local") + return make_workspace("workspace", typ=typ) - return make_workspace("workspace", typ=request.param) + +@pytest.fixture +def local_workspace(make_workspace): + return make_workspace("workspace", typ="local") @pytest.fixture(scope="session") diff --git a/dvc/testing/test_workspace.py b/dvc/testing/test_workspace.py index d555fbaf9..0151bc76f 100644 --- a/dvc/testing/test_workspace.py +++ b/dvc/testing/test_workspace.py @@ -11,6 +11,14 @@ class TestImport: assert (tmp_dir / "file").read_text() == "file" assert dvc.status() == {} + @pytest.fixture + def stage_md5(self): + pytest.skip() + + @pytest.fixture + def dir_md5(self): + pytest.skip() + def test_import_dir(self, tmp_dir, dvc, workspace, stage_md5, dir_md5): from dvc.objects.db import ODBManager @@ -51,6 +59,10 @@ class TestImport: " path: dir\n" ) + @pytest.fixture + def is_object_storage(self): + pytest.skip() + def test_import_empty_dir( self, tmp_dir, dvc, workspace, is_object_storage ): @@ -72,6 +84,18 @@ class TestImport: class TestAdd: + @pytest.fixture + def hash_name(self): + pytest.skip() + + @pytest.fixture + def hash_value(self): + pytest.skip() + + @pytest.fixture + def dir_hash_value(self): + pytest.skip() + def test_add(self, tmp_dir, dvc, workspace, hash_name, hash_value): from dvc.stage.exceptions import StageExternalOutputsError diff --git a/setup.cfg b/setup.cfg index b071a9edb..2eddadf1f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -115,6 +115,7 @@ terraform = tpi[ssh]>=2.1.0 tests = %(terraform)s wheel==0.37.0 + dvc_ssh==0.0.1a0 # Test requirements pytest==6.2.5 pytest-cov==3.0.0 diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 2aa6b3a11..51233e820 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -1,43 +1,6 @@ --- version: '3.2' services: - azurite: - image: mcr.microsoft.com/azure-storage/azurite:3.14.2 - command: azurite -L -l /data --blobHost 0.0.0.0 --queueHost 0.0.0.0 - ports: - - "10000" - - motoserver: - image: motoserver/moto - ports: - - "5000" - - oss: - image: rkuprieiev/oss-emulator - ports: - - "8880" - hdfs: - image: rkuprieiev/docker-hdfs - hostname: localhost - ports: - - "8020" - # NOTE: having this port as dynamic one will require modifying - # `dfs.datanode.address` in `hdfs-site.xml` and probably something - # else, so using default one for now. - - "50010:50010" - - "50075:50075" - - "50070" - - openssh-server: - image: ghcr.io/linuxserver/openssh-server - environment: - - USER_NAME=user - - PUBLIC_KEY_FILE=/tmp/key - ports: - - 2222 - volumes: - - ./remotes/user.key.pub:/tmp/key - git-server: image: ghcr.io/linuxserver/openssh-server environment: diff --git a/tests/func/experiments/executor/test_ssh.py b/tests/func/experiments/executor/test_ssh.py index 3ec6a2216..bc29f8b94 100644 --- a/tests/func/experiments/executor/test_ssh.py +++ b/tests/func/experiments/executor/test_ssh.py @@ -3,12 +3,12 @@ from contextlib import contextmanager from functools import partial import pytest +from dvc_ssh.tests.cloud import TEST_SSH_KEY_PATH, TEST_SSH_USER from dvc.fs.ssh import SSHFileSystem from dvc.repo.experiments.base import EXEC_HEAD, EXEC_MERGE from dvc.repo.experiments.executor.ssh import SSHExecutor from tests.func.machine.conftest import * # noqa, pylint: disable=wildcard-import -from tests.remotes.ssh import TEST_SSH_KEY_PATH, TEST_SSH_USER @contextmanager diff --git a/tests/func/test_add.py b/tests/func/test_add.py index 4e61d4f16..deb4cedbb 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -293,71 +293,17 @@ def test_add_filtered_files_in_dir( class TestAddExternal(TestAdd): - @pytest.mark.parametrize( - "workspace, hash_name, hash_value", - [ - ( - "local", - "md5", - "8c7dd922ad47494fc02c388e12c00eac", - ), - pytest.param( - "ssh", - "md5", - "8c7dd922ad47494fc02c388e12c00eac", - marks=pytest.mark.skipif( - os.name == "nt", reason="disabled on windows" - ), - ), - ( - "s3", - "etag", - "8c7dd922ad47494fc02c388e12c00eac", - ), - ( - "hdfs", - "checksum", - "000002000000000000000000a86fe4d846edc1bf4c355cb6112f141e", - ), - ( - "webhdfs", - "checksum", - "000002000000000000000000a86fe4d846edc1bf4c355cb6112f141e00000000", # noqa: E501 - ), - ], - indirect=["workspace"], - ) - def test_add(self, tmp_dir, dvc, workspace, hash_name, hash_value): - super().test_add(tmp_dir, dvc, workspace, hash_name, hash_value) + @pytest.fixture + def hash_name(self): + return "md5" - @pytest.mark.parametrize( - "workspace, hash_name, dir_hash_value", - [ - ( - "local", - "md5", - "b6dcab6ccd17ca0a8bf4a215a37d14cc.dir", - ), - pytest.param( - "ssh", - "md5", - "b6dcab6ccd17ca0a8bf4a215a37d14cc.dir", - marks=pytest.mark.skipif( - os.name == "nt", reason="disabled on windows" - ), - ), - ( - "s3", - "etag", - "ec602a6ba97b2dd07bd6d2cd89674a60.dir", - ), - ], - indirect=["workspace"], - ) - def test_add_dir(self, tmp_dir, dvc, workspace, hash_name, dir_hash_value): - super().test_add_dir( - tmp_dir, dvc, workspace, hash_name, dir_hash_value - ) + @pytest.fixture + def hash_value(self): + return "8c7dd922ad47494fc02c388e12c00eac" + + @pytest.fixture + def dir_hash_value(self): + return "b6dcab6ccd17ca0a8bf4a215a37d14cc.dir" def test_add_external_relpath(tmp_dir, dvc, local_cloud): @@ -994,10 +940,10 @@ def test_add_long_fname(tmp_dir, dvc): assert (tmp_dir / "data").read_text() == {name: "foo"} -def test_add_to_remote(tmp_dir, dvc, local_cloud, local_remote): - local_cloud.gen("foo", "foo") +def test_add_to_remote(tmp_dir, dvc, remote, workspace): + workspace.gen("foo", "foo") - url = "remote://upstream/foo" + url = "remote://workspace/foo" [stage] = dvc.add(url, to_remote=True) assert not (tmp_dir / "foo").exists() @@ -1009,14 +955,14 @@ def test_add_to_remote(tmp_dir, dvc, local_cloud, local_remote): hash_info = stage.outs[0].hash_info meta = stage.outs[0].meta with open( - local_remote.hash_to_path(hash_info.value), encoding="utf-8" + remote.hash_to_path(hash_info.value), encoding="utf-8" ) as stream: assert stream.read() == "foo" assert meta.size == len("foo") -def test_add_to_remote_absolute(tmp_dir, make_tmp_dir, dvc, local_remote): +def test_add_to_remote_absolute(tmp_dir, make_tmp_dir, dvc, remote): tmp_abs_dir = make_tmp_dir("abs") tmp_foo = tmp_abs_dir / "foo" tmp_foo.write_text("foo") @@ -1132,23 +1078,6 @@ def test_add_to_cache_invalid_combinations(dvc, invalid_opt, kwargs): dvc.add(out="bar", **kwargs) -@pytest.mark.parametrize( - "workspace", - [ - "local", - "s3", - pytest.param("gs", marks=pytest.mark.needs_internet), - "hdfs", - pytest.param( - "ssh", - marks=pytest.mark.skipif( - os.name == "nt", reason="disabled on windows" - ), - ), - "http", - ], - indirect=True, -) def test_add_to_cache_from_remote(tmp_dir, dvc, workspace): workspace.gen("foo", "foo") diff --git a/tests/func/test_api.py b/tests/func/test_api.py index 6ab103044..997b8a710 100644 --- a/tests/func/test_api.py +++ b/tests/func/test_api.py @@ -5,35 +5,11 @@ from funcy import first, get_in from dvc import api from dvc.exceptions import FileMissingError, OutputNotFoundError -from dvc.testing.test_api import TestAPI as _TestAPI +from dvc.testing.test_api import TestAPI # noqa, pylint: disable=unused-import from dvc.utils.fs import remove from tests.unit.fs.test_repo import make_subrepo -clouds = [ - "s3", - "gs", - "azure", - "gdrive", - "oss", - "ssh", - "http", - "hdfs", - "webdav", - "webhdfs", -] -all_clouds = ["local"] + clouds -# `lazy_fixture` is confusing pylint, pylint: disable=unused-argument - - -@pytest.mark.needs_internet -@pytest.mark.parametrize("remote", clouds, indirect=True) -class TestAPI(_TestAPI): - pass - - -@pytest.mark.needs_internet -@pytest.mark.parametrize("cloud", clouds, indirect=True) def test_get_url_external(tmp_dir, erepo_dir, cloud): erepo_dir.add_remote(config=cloud.config) with erepo_dir.chdir(): @@ -55,22 +31,6 @@ def test_get_url_requires_dvc(tmp_dir, scm): api.get_url("foo", repo=f"file://{tmp_dir}") -@pytest.mark.needs_internet -@pytest.mark.parametrize( - "cloud", - [ - "s3", - "gs", - "azure", - "gdrive", - "oss", - "http", - "hdfs", - "ssh", - "webdav", - ], - indirect=True, -) def test_open_external(tmp_dir, erepo_dir, cloud): erepo_dir.add_remote(config=cloud.config) @@ -94,8 +54,6 @@ def test_open_external(tmp_dir, erepo_dir, cloud): assert api.read("version", repo=repo_url, rev="branch") == "branchver" -@pytest.mark.needs_internet -@pytest.mark.parametrize("remote", all_clouds, indirect=True) def test_open_granular(tmp_dir, dvc, remote): tmp_dir.dvc_gen({"dir": {"foo": "foo-text"}}) dvc.push() @@ -107,22 +65,6 @@ def test_open_granular(tmp_dir, dvc, remote): assert fd.read() == "foo-text" -@pytest.mark.needs_internet -@pytest.mark.parametrize( - "remote", - [ - "s3", - "gs", - "azure", - "gdrive", - "oss", - "ssh", - "http", - "hdfs", - "webdav", - ], - indirect=True, -) def test_missing(tmp_dir, dvc, remote): tmp_dir.dvc_gen("foo", "foo") @@ -174,7 +116,6 @@ def test_open_rev(tmp_dir, scm, dvc): @pytest.mark.parametrize("as_external", [True, False]) -@pytest.mark.parametrize("remote", ["ssh"], indirect=True) @pytest.mark.parametrize( "files, to_read", [ @@ -227,22 +168,22 @@ def test_read_with_subrepos(tmp_dir, scm, local_cloud, local_repo): ) -def test_get_url_granular(tmp_dir, dvc, s3): - tmp_dir.add_remote(config=s3.config) +def test_get_url_granular(tmp_dir, dvc, cloud): + tmp_dir.add_remote(config=cloud.config) tmp_dir.dvc_gen( {"dir": {"foo": "foo", "bar": "bar", "nested": {"file": "file"}}} ) - expected_url = (s3 / "5f/c28ea78987408341668eba6525ebd1.dir").url + expected_url = (cloud / "5f" / "c28ea78987408341668eba6525ebd1.dir").url assert api.get_url("dir") == expected_url - expected_url = (s3 / "ac/bd18db4cc2f85cedef654fccc4a4d8").url + expected_url = (cloud / "ac" / "bd18db4cc2f85cedef654fccc4a4d8").url assert api.get_url("dir/foo") == expected_url - expected_url = (s3 / "37/b51d194a7513e45b56f6524f2d51f2").url + expected_url = (cloud / "37" / "b51d194a7513e45b56f6524f2d51f2").url assert api.get_url("dir/bar") == expected_url - expected_url = (s3 / "8c/7dd922ad47494fc02c388e12c00eac").url + expected_url = (cloud / "8c" / "7dd922ad47494fc02c388e12c00eac").url assert api.get_url(os.path.join("dir", "nested", "file")) == expected_url diff --git a/tests/func/test_checkout.py b/tests/func/test_checkout.py index 8787363a0..ffd30232e 100644 --- a/tests/func/test_checkout.py +++ b/tests/func/test_checkout.py @@ -752,9 +752,6 @@ def test_checkout_recursive(tmp_dir, dvc): } -@pytest.mark.parametrize( - "workspace", [pytest.lazy_fixture("s3")], indirect=True -) def test_checkout_for_external_outputs(tmp_dir, dvc, workspace): workspace.gen("foo", "foo") @@ -844,9 +841,6 @@ def test_checkouts_for_pipeline_tracked_outs(tmp_dir, dvc, scm, run_copy): assert set(dvc.checkout()["added"]) == {"bar", "ipsum"} -@pytest.mark.parametrize( - "workspace", [pytest.lazy_fixture("s3")], indirect=True -) def test_checkout_external_modified_file(tmp_dir, dvc, scm, mocker, workspace): # regression: happened when file in external output changed and checkout # was attempted without force, dvc checks if it's present in its cache diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index 39bca1e9e..08311ef8f 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -9,35 +9,12 @@ import dvc as dvc_module from dvc.external_repo import clean_repos from dvc.main import main from dvc.stage.exceptions import StageNotFound -from dvc.testing.test_remote import TestRemote as _TestRemote +from dvc.testing.test_remote import ( # noqa, pylint: disable=unused-import + TestRemote, +) from dvc.utils.fs import remove -all_clouds = [ - "s3", - "gs", - "azure", - "ssh", - "http", - "hdfs", - "webdav", - "webhdfs", - "oss", - "gdrive", -] -# Clouds that implement the general methods that can be tested -# for functional tests that require extensive apis (e.g traversing -# via walk_files()) -full_clouds = ["s3", "gs", "azure", "ssh", "hdfs"] - - -@pytest.mark.parametrize("remote", all_clouds, indirect=True) -class TestRemote(_TestRemote): - pass - - -@pytest.mark.needs_internet -@pytest.mark.parametrize("remote", all_clouds, indirect=True) def test_cloud_cli(tmp_dir, dvc, remote): args = ["-v", "-j", "2"] @@ -479,46 +456,6 @@ def test_pull_partial(tmp_dir, dvc, local_remote): assert (tmp_dir / "foo").read_text() == {"bar": {"baz": "baz"}} -@pytest.mark.parametrize("remote", full_clouds, indirect=True) -def test_pull_00_prefix(tmp_dir, dvc, remote, monkeypatch): - # Related: https://github.com/iterative/dvc/issues/6089 - - fs_type = type(dvc.cloud.get_remote_odb("upstream").fs) - monkeypatch.setattr(fs_type, "_ALWAYS_TRAVERSE", True, raising=False) - monkeypatch.setattr(fs_type, "LIST_OBJECT_PAGE_SIZE", 256, raising=False) - - # foo's md5 checksum is 00411460f7c92d2124a67ea0f4cb5f85 - # bar's md5 checksum is 0000000018e6137ac2caab16074784a6 - tmp_dir.dvc_gen({"foo": "363", "bar": "jk8ssl"}) - - dvc.push() - clean(["foo", "bar"], dvc) - - stats = dvc.pull() - assert stats["fetched"] == 2 - assert set(stats["added"]) == {"foo", "bar"} - - -@pytest.mark.parametrize("remote", full_clouds, indirect=True) -def test_pull_no_00_prefix(tmp_dir, dvc, remote, monkeypatch): - # Related: https://github.com/iterative/dvc/issues/6244 - - fs_type = type(dvc.cloud.get_remote_odb("upstream").fs) - monkeypatch.setattr(fs_type, "_ALWAYS_TRAVERSE", True, raising=False) - monkeypatch.setattr(fs_type, "LIST_OBJECT_PAGE_SIZE", 256, raising=False) - - # foo's md5 checksum is 14ffd92a6cbf5f2f657067df0d5881a6 - # bar's md5 checksum is 64020400f00960c0ef04052547b134b3 - tmp_dir.dvc_gen({"foo": "dvc", "bar": "cml"}) - - dvc.push() - clean(["foo", "bar"], dvc) - - stats = dvc.pull() - assert stats["fetched"] == 2 - assert set(stats["added"]) == {"foo", "bar"} - - def test_output_remote(tmp_dir, dvc, make_remote): make_remote("default", default=True) make_remote("for_foo", default=False) diff --git a/tests/func/test_fs.py b/tests/func/test_fs.py index 2c9871c68..9e6d7e46c 100644 --- a/tests/func/test_fs.py +++ b/tests/func/test_fs.py @@ -231,18 +231,6 @@ def test_walk_dont_ignore_subrepos(tmp_dir, scm, dvc): assert set(get_dirs(next(scm_fs.walk(path)))) == {".dvc", "subdir"} -@pytest.mark.parametrize( - "cloud", - [ - pytest.lazy_fixture("local_cloud"), - pytest.lazy_fixture("s3"), - pytest.param( - pytest.lazy_fixture("gs"), marks=pytest.mark.needs_internet - ), - pytest.lazy_fixture("hdfs"), - pytest.lazy_fixture("http"), - ], -) def test_fs_getsize(dvc, cloud): cloud.gen({"data": {"foo": "foo"}, "baz": "baz baz"}) cls, config, path = get_cloud_fs(dvc, **cloud.config) @@ -252,21 +240,6 @@ def test_fs_getsize(dvc, cloud): assert fs.getsize(fs.path.join(path, "data", "foo")) == 3 -@pytest.mark.needs_internet -@pytest.mark.parametrize( - "cloud", - [ - pytest.lazy_fixture("azure"), - pytest.lazy_fixture("gs"), - pytest.lazy_fixture("gdrive"), - pytest.lazy_fixture("hdfs"), - pytest.lazy_fixture("local_cloud"), - pytest.lazy_fixture("oss"), - pytest.lazy_fixture("s3"), - pytest.lazy_fixture("ssh"), - pytest.lazy_fixture("webhdfs"), - ], -) def test_fs_upload_fobj(dvc, tmp_dir, cloud): tmp_dir.gen("foo", "foo") cls, config, path = get_cloud_fs(dvc, **cloud.config) @@ -283,75 +256,6 @@ def test_fs_upload_fobj(dvc, tmp_dir, cloud): assert stream.read() == b"foo" -@pytest.mark.needs_internet -@pytest.mark.parametrize("cloud", [pytest.lazy_fixture("gdrive")]) -def test_fs_ls(dvc, cloud): - cloud.gen( - { - "directory": { - "foo": "foo", - "bar": "bar", - "baz": {"quux": "quux", "egg": {"foo": "foo"}}, - "empty": {}, - } - } - ) - cls, config, path = get_cloud_fs(dvc, **cloud.config) - fs = cls(**config) - path = os.path.join(path, "directory") - - assert { - os.path.basename(file_key.rstrip("/")) for file_key in fs.ls(path) - } == { - "foo", - "bar", - "baz", - "empty", - } - assert set(fs.ls(fs.path.join(path, "empty"))) == set() - assert { - (detail["type"], os.path.basename(detail["name"].rstrip("/"))) - for detail in fs.ls(fs.path.join(path, "baz"), detail=True) - } == {("file", "quux"), ("directory", "egg")} - - -@pytest.mark.parametrize( - "cloud", - [ - pytest.lazy_fixture("azure"), - pytest.param( - pytest.lazy_fixture("gs"), marks=pytest.mark.needs_internet - ), - ], -) -def test_fs_fsspec_path_management(dvc, cloud): - cloud.gen({"foo": "foo", "data": {"bar": "bar", "baz": {"foo": "foo"}}}) - cls, config, _ = get_cloud_fs(dvc, **cloud.config) - fs = cls(**config) - - root = cloud.parents[len(cloud.parents) - 1] - bucket_details = fs.info(root.fs_path) - - # special conditions: name always points to the bucket name - assert bucket_details["name"] == root.bucket - assert bucket_details["type"] == "directory" - - data = cloud / "data" - data_details = fs.info(data.fs_path) - assert data_details["name"].rstrip("/") == data.fs_path - assert data_details["type"] == "directory" - - -@pytest.mark.needs_internet -@pytest.mark.parametrize( - "cloud", - [ - pytest.lazy_fixture("azure"), - pytest.lazy_fixture("gs"), - pytest.lazy_fixture("s3"), - pytest.lazy_fixture("webdav"), - ], -) def test_fs_makedirs_on_upload_and_copy(dvc, cloud): cls, config, _ = get_cloud_fs(dvc, **cloud.config) fs = cls(**config) @@ -362,28 +266,12 @@ def test_fs_makedirs_on_upload_and_copy(dvc, cloud): assert fs.isdir((cloud / "dir").fs_path) assert fs.exists((cloud / "dir" / "foo").fs_path) + fs.makedirs((cloud / "dir2").fs_path) fs.copy((cloud / "dir" / "foo").fs_path, (cloud / "dir2" / "foo").fs_path) assert fs.isdir((cloud / "dir2").fs_path) assert fs.exists((cloud / "dir2" / "foo").fs_path) -@pytest.mark.needs_internet -@pytest.mark.parametrize( - "cloud", - [ - pytest.lazy_fixture("azure"), - pytest.lazy_fixture("gs"), - pytest.lazy_fixture("gdrive"), - pytest.lazy_fixture("hdfs"), - pytest.lazy_fixture("local_cloud"), - pytest.lazy_fixture("oss"), - pytest.lazy_fixture("s3"), - pytest.lazy_fixture("ssh"), - pytest.lazy_fixture("webhdfs"), - pytest.lazy_fixture("webdav"), - pytest.lazy_fixture("http"), - ], -) def test_upload_callback(tmp_dir, dvc, cloud): tmp_dir.gen("foo", "foo") cls, config, _ = get_cloud_fs(dvc, **cloud.config) @@ -401,23 +289,6 @@ def test_upload_callback(tmp_dir, dvc, cloud): assert callback.value == expected_size -@pytest.mark.needs_internet -@pytest.mark.parametrize( - "cloud", - [ - pytest.lazy_fixture("azure"), - pytest.lazy_fixture("gs"), - pytest.lazy_fixture("gdrive"), - pytest.lazy_fixture("hdfs"), - pytest.lazy_fixture("local_cloud"), - pytest.lazy_fixture("oss"), - pytest.lazy_fixture("s3"), - pytest.lazy_fixture("ssh"), - pytest.lazy_fixture("webhdfs"), - pytest.lazy_fixture("webdav"), - pytest.lazy_fixture("http"), - ], -) def test_download_callback(tmp_dir, dvc, cloud, local_cloud): cls, config, _ = get_cloud_fs(dvc, **cloud.config) fs = cls(**config) @@ -438,25 +309,6 @@ def test_download_callback(tmp_dir, dvc, cloud, local_cloud): assert (tmp_dir / "foo").read_text() == "foo" -@pytest.mark.needs_internet -@pytest.mark.parametrize( - "cloud", - [ - pytest.lazy_fixture("azure"), - pytest.lazy_fixture("gs"), - pytest.lazy_fixture("hdfs"), - pytest.lazy_fixture("local_cloud"), - pytest.lazy_fixture("s3"), - pytest.param( - pytest.lazy_fixture("ssh"), - marks=pytest.mark.skipif( - os.name == "nt", reason="unsupported on Windows." - ), - ), - pytest.lazy_fixture("gdrive"), - pytest.lazy_fixture("webdav"), - ], -) def test_download_dir_callback(tmp_dir, dvc, cloud): cls, config, _ = get_cloud_fs(dvc, **cloud.config) fs = cls(**config) diff --git a/tests/func/test_gc.py b/tests/func/test_gc.py index 20d8d485e..1e0885f45 100644 --- a/tests/func/test_gc.py +++ b/tests/func/test_gc.py @@ -343,22 +343,6 @@ def test_gc_not_collect_pipeline_tracked_files(tmp_dir, dvc, run_copy): assert _count_files(dvc.odb.local.cache_dir) == 0 -@pytest.mark.parametrize( - "workspace", - [ - "local", - "s3", - "hdfs", - "webhdfs", - pytest.param( - "ssh", - marks=pytest.mark.skipif( - os.name == "nt", reason="disabled on windows" - ), - ), - ], - indirect=True, -) def test_gc_external_output(tmp_dir, dvc, workspace): workspace.gen({"foo": "foo", "bar": "bar"}) diff --git a/tests/func/test_import_url.py b/tests/func/test_import_url.py index e87fdd030..153046d0e 100644 --- a/tests/func/test_import_url.py +++ b/tests/func/test_import_url.py @@ -119,108 +119,17 @@ def test_import_url_with_no_exec(tmp_dir, dvc, erepo_dir): class TestImport(_TestImport): - @pytest.mark.parametrize( - "workspace", - [ - "local", - "s3", - "azure", - pytest.param("gs", marks=pytest.mark.needs_internet), - "hdfs", - "webhdfs", - pytest.param( - "ssh", - marks=pytest.mark.skipif( - os.name == "nt", reason="disabled on windows" - ), - ), - "http", - ], - indirect=True, - ) - def test_import(self, tmp_dir, dvc, workspace): - super().test_import(tmp_dir, dvc, workspace) + @pytest.fixture + def stage_md5(self): + return "dc24e1271084ee317ac3c2656fb8812b" - @pytest.mark.parametrize( - "workspace, stage_md5, dir_md5", - [ - ( - "local", - "dc24e1271084ee317ac3c2656fb8812b", - "b6dcab6ccd17ca0a8bf4a215a37d14cc.dir", - ), - ( - "s3", - "2aa17f8daa26996b3f7a4cf8888ac9ac", - "ec602a6ba97b2dd07bd6d2cd89674a60.dir", - ), - ( - "azure", - None, # ETags for azure are not consistent with the actual - None, # content of the object, so they will change every time - # we create new objects. We'll skip stage content - # check for Azure. - ), - ( - "hdfs", - "ec0943f83357f702033c98e70b853c8c", - "e6dcd267966dc628d732874f94ef4280.dir", - ), - pytest.param( - "ssh", - "dc24e1271084ee317ac3c2656fb8812b", - "b6dcab6ccd17ca0a8bf4a215a37d14cc.dir", - marks=pytest.mark.skipif( - os.name == "nt", reason="disabled on windows" - ), - ), - ], - indirect=["workspace"], - ) - def test_import_dir(self, tmp_dir, dvc, workspace, stage_md5, dir_md5): - super().test_import_dir(tmp_dir, dvc, workspace, stage_md5, dir_md5) + @pytest.fixture + def dir_md5(self): + return "b6dcab6ccd17ca0a8bf4a215a37d14cc.dir" - @pytest.mark.parametrize( - "workspace, is_object_storage", - [ - ("hdfs", False), - pytest.param( - "s3", - True, - marks=pytest.mark.xfail( - reason="https://github.com/iterative/dvc/issues/5521", - ), - ), - pytest.param( - "gs", - True, - marks=pytest.mark.xfail( - reason="https://github.com/iterative/dvc/issues/5521", - ), - ), - pytest.param( - "azure", - True, - marks=pytest.mark.xfail( - reason="https://github.com/iterative/dvc/issues/5521", - ), - ), - pytest.param( - "ssh", - False, - marks=pytest.mark.skipif( - os.name == "nt", reason="disabled on windows" - ), - ), - ], - indirect=["workspace"], - ) - def test_import_empty_dir( - self, tmp_dir, dvc, workspace, is_object_storage - ): - super().test_import_empty_dir( - tmp_dir, dvc, workspace, is_object_storage - ) + @pytest.fixture + def is_object_storage(self): + return False def test_import_url_preserve_meta(tmp_dir, dvc): @@ -260,23 +169,6 @@ def test_import_url_preserve_meta(tmp_dir, dvc): ) -@pytest.mark.parametrize( - "workspace", - [ - "local", - "s3", - pytest.param("gs", marks=pytest.mark.needs_internet), - "hdfs", - pytest.param( - "ssh", - marks=pytest.mark.skipif( - os.name == "nt", reason="disabled on windows" - ), - ), - "http", - ], - indirect=True, -) def test_import_url_to_remote_single_file( tmp_dir, dvc, workspace, local_remote ): @@ -301,22 +193,6 @@ def test_import_url_to_remote_single_file( assert stage.outs[0].meta.size == len("foo") -@pytest.mark.parametrize( - "workspace", - [ - "local", - "s3", - pytest.param("gs", marks=pytest.mark.needs_internet), - "hdfs", - pytest.param( - "ssh", - marks=pytest.mark.skipif( - os.name == "nt", reason="disabled on windows" - ), - ), - ], - indirect=True, -) def test_import_url_to_remote_directory(tmp_dir, dvc, workspace, local_remote): workspace.gen( { diff --git a/tests/func/test_merge_driver.py b/tests/func/test_merge_driver.py index 79a5f96d5..eeae1f612 100644 --- a/tests/func/test_merge_driver.py +++ b/tests/func/test_merge_driver.py @@ -114,38 +114,6 @@ def test_merge_conflict(tmp_dir, dvc, ancestor, our, their, error, caplog): assert error in caplog.text -@pytest.mark.parametrize("workspace", ["ssh"], indirect=True) -def test_merge_different_output_types(tmp_dir, dvc, caplog, workspace): - (tmp_dir / "ancestor").touch() - - (tmp_dir / "our").write_text( - "outs:\n- md5: f123456789.dir\n path: ssh://example.com/path\n" - ) - - (tmp_dir / "their").write_text( - "outs:\n- md5: f987654321.dir\n path: path\n" - ) - - assert ( - main( - [ - "git-hook", - "merge-driver", - "--ancestor", - "ancestor", - "--our", - "our", - "--their", - "their", - ] - ) - != 0 - ) - - error = "unable to auto-merge outputs of different types" - assert error in caplog.text - - def test_merge_different_output_options(tmp_dir, dvc, caplog): (tmp_dir / "ancestor").touch() diff --git a/tests/func/test_remote.py b/tests/func/test_remote.py index c677e9660..c82fc038b 100644 --- a/tests/func/test_remote.py +++ b/tests/func/test_remote.py @@ -11,7 +11,6 @@ from dvc.exceptions import DownloadError, RemoteCacheRequiredError, UploadError from dvc.main import main from dvc.utils.fs import remove from tests.basic_env import TestDvc -from tests.remotes import Local class TestRemote(TestDvc): @@ -127,19 +126,16 @@ def test_show_default(dvc, capsys): assert out == "foo\n" -class TestRemoteShouldHandleUppercaseRemoteName(TestDvc): - upper_case_remote_name = "UPPERCASEREMOTE" +def test_upper_case_remote(tmp_dir, dvc, local_cloud): + remote_name = "UPPERCASEREMOTE" - def test(self): - remote_url = Local.get_storagepath() - ret = main(["remote", "add", self.upper_case_remote_name, remote_url]) - self.assertEqual(ret, 0) + tmp_dir.gen("foo", "foo") - ret = main(["add", self.FOO]) - self.assertEqual(ret, 0) + ret = main(["remote", "add", remote_name, local_cloud.url]) + assert ret == 0 - ret = main(["push", "-r", self.upper_case_remote_name]) - self.assertEqual(ret, 0) + ret = main(["push", "-r", remote_name]) + assert ret == 0 def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc): diff --git a/tests/func/test_run_multistage.py b/tests/func/test_run_multistage.py index 28eafe050..f83b7e18b 100644 --- a/tests/func/test_run_multistage.py +++ b/tests/func/test_run_multistage.py @@ -350,43 +350,16 @@ def test_run_overwrite_preserves_meta_and_comment(tmp_dir, dvc, run_copy): ) -@pytest.mark.parametrize( - "workspace, hash_name, foo_hash, bar_hash", - [ - ( - "local", - "md5", - "acbd18db4cc2f85cedef654fccc4a4d8", - "37b51d194a7513e45b56f6524f2d51f2", - ), - pytest.param( - "ssh", - "md5", - "acbd18db4cc2f85cedef654fccc4a4d8", - "37b51d194a7513e45b56f6524f2d51f2", - marks=pytest.mark.skipif( - os.name == "nt", reason="disabled on windows" - ), - ), - ( - "s3", - "etag", - "acbd18db4cc2f85cedef654fccc4a4d8", - "37b51d194a7513e45b56f6524f2d51f2", - ), - ( - "hdfs", - "checksum", - "0000020000000000000000003dba826b9be9c6a8e2f8310a770555c4", - "00000200000000000000000075433c81259d3c38e364b348af52e84d", - ), - ], - indirect=["workspace"], -) def test_run_external_outputs( - tmp_dir, dvc, workspace, hash_name, foo_hash, bar_hash + tmp_dir, + dvc, + local_workspace, ): - workspace.gen("foo", "foo") + hash_name = "md5" + foo_hash = "acbd18db4cc2f85cedef654fccc4a4d8" + bar_hash = "37b51d194a7513e45b56f6524f2d51f2" + + local_workspace.gen("foo", "foo") dvc.run( name="mystage", cmd="mycmd", @@ -408,7 +381,7 @@ def test_run_external_outputs( assert (tmp_dir / "dvc.yaml").read_text() == dvc_yaml assert not (tmp_dir / "dvc.lock").exists() - workspace.gen("bar", "bar") + local_workspace.gen("bar", "bar") dvc.commit("dvc.yaml", force=True) assert (tmp_dir / "dvc.yaml").read_text() == dvc_yaml @@ -427,8 +400,8 @@ def test_run_external_outputs( " size: 3\n" ) - assert (workspace / "foo").read_text() == "foo" - assert (workspace / "bar").read_text() == "bar" + assert (local_workspace / "foo").read_text() == "foo" + assert (local_workspace / "bar").read_text() == "bar" assert ( - workspace / "cache" / bar_hash[:2] / bar_hash[2:] + local_workspace / "cache" / bar_hash[:2] / bar_hash[2:] ).read_text() == "bar" diff --git a/tests/func/test_scm.py b/tests/func/test_scm.py index 2b3ca92e4..74d209280 100644 --- a/tests/func/test_scm.py +++ b/tests/func/test_scm.py @@ -336,10 +336,10 @@ def test_git_stash_clear(tmp_dir, scm, ref): @pytest.mark.parametrize("server", [pytest.lazy_fixture("git_ssh")]) def test_git_ssh(tmp_dir, scm, server): from dulwich.repo import Repo as DulwichRepo + from dvc_ssh.tests.cloud import TEST_SSH_KEY_PATH, TEST_SSH_USER from sshfs import SSHFileSystem from dvc.utils.fs import remove - from tests.remotes.ssh import TEST_SSH_KEY_PATH, TEST_SSH_USER fs = SSHFileSystem( host=server.host, diff --git a/tests/func/test_update.py b/tests/func/test_update.py index 42977f19e..a305327c5 100644 --- a/tests/func/test_update.py +++ b/tests/func/test_update.py @@ -156,23 +156,6 @@ def test_update_before_and_after_dvc_init(tmp_dir, dvc, git_dir): assert dvc.status([stage.path]) == {} -@pytest.mark.parametrize( - "workspace", - [ - "local", - "s3", - pytest.param("gs", marks=pytest.mark.needs_internet), - "hdfs", - "webhdfs", - pytest.param( - "ssh", - marks=pytest.mark.skipif( - os.name == "nt", reason="disabled on windows" - ), - ), - ], - indirect=True, -) def test_update_import_url(tmp_dir, dvc, workspace): workspace.gen("file", "file content") @@ -321,16 +304,6 @@ def test_update_import_to_remote(tmp_dir, dvc, erepo_dir, local_remote): dvc.update(stage.path, to_remote=True) -@pytest.mark.parametrize( - "workspace", - [ - "local", - "s3", - pytest.param("gs", marks=pytest.mark.needs_internet), - "hdfs", - ], - indirect=True, -) def test_update_import_url_to_remote(tmp_dir, dvc, workspace, local_remote): workspace.gen("foo", "foo") stage = dvc.imp_url("remote://workspace/foo", to_remote=True) @@ -342,15 +315,6 @@ def test_update_import_url_to_remote(tmp_dir, dvc, workspace, local_remote): assert (tmp_dir / "foo").read_text() == "bar" -@pytest.mark.parametrize( - "workspace", - [ - "s3", - pytest.param("gs", marks=pytest.mark.needs_internet), - "hdfs", - ], - indirect=True, -) def test_update_import_url_to_remote_directory( mocker, tmp_dir, dvc, workspace, local_remote ): @@ -371,16 +335,8 @@ def test_update_import_url_to_remote_directory( } ) - download_file_mock = mocker.spy( - type(dvc.cloud.get_remote_odb("workspace-cache").fs), "download_file" - ) stage = dvc.update(stage.path, to_remote=True) - # 2 new hashes (foo2, baz2) - assert download_file_mock.mock.call_count == 2 - # The new .dir hash will be transferred through MemFs, so - # we will not account the download_file() for it. - dvc.pull("data") assert (tmp_dir / "data").read_text() == { "foo": "foo", @@ -395,12 +351,12 @@ def test_update_import_url_to_remote_directory( def test_update_import_url_to_remote_directory_changed_contents( - tmp_dir, dvc, local_cloud, local_remote + tmp_dir, dvc, local_workspace, local_remote ): - local_cloud.gen({"data": {"foo": "foo", "bar": {"baz": "baz"}}}) - stage = dvc.imp_url("remote://upstream/data", to_remote=True) + local_workspace.gen({"data": {"foo": "foo", "bar": {"baz": "baz"}}}) + stage = dvc.imp_url("remote://workspace/data", to_remote=True) - local_cloud.gen( + local_workspace.gen( {"data": {"foo": "not_foo", "foo2": "foo", "bar": {"baz2": "baz2"}}} ) stage = dvc.update(stage.path, to_remote=True) @@ -414,14 +370,14 @@ def test_update_import_url_to_remote_directory_changed_contents( def test_update_import_url_to_remote_directory_same_hash( - tmp_dir, dvc, local_cloud, local_remote + tmp_dir, dvc, local_workspace, local_remote ): - local_cloud.gen( + local_workspace.gen( {"data": {"foo": "foo", "bar": {"baz": "baz"}, "same": "same"}} ) - stage = dvc.imp_url("remote://upstream/data", to_remote=True) + stage = dvc.imp_url("remote://workspace/data", to_remote=True) - local_cloud.gen( + local_workspace.gen( {"data": {"foo": "baz", "bar": {"baz": "foo"}, "same": "same"}} ) stage = dvc.update(stage.path, to_remote=True) diff --git a/tests/remotes/__init__.py b/tests/remotes/__init__.py index 9a368a9b0..02c1b15ed 100644 --- a/tests/remotes/__init__.py +++ b/tests/remotes/__init__.py @@ -1,54 +1,10 @@ -from .azure import Azure, azure, azure_server, make_azure # noqa: F401 -from .gdrive import ( # noqa: F401; noqa: F401 - TEST_GDRIVE_REPO_BUCKET, - GDrive, - gdrive, - make_gdrive, -) -from .git_server import git_server, git_ssh # noqa: F401 -from .gs import ( # noqa: F401; noqa: F401 - GCP, - TEST_GCP_CREDS_FILE, - TEST_GCP_REPO_BUCKET, - gs, - make_gs, -) -from .hdfs import ( # noqa: F401 - HDFS, - hadoop, - hdfs, - hdfs_server, - make_hdfs, - real_hdfs, -) -from .http import HTTP, http, http_server, make_http # noqa: F401 -from .local import Local, local_cloud, local_remote, make_local # noqa: F401 -from .oss import ( # noqa: F401 - OSS, - TEST_OSS_REPO_BUCKET, - make_oss, - oss, - oss_server, - real_oss, -) -from .s3 import ( # noqa: F401 - S3, - TEST_AWS_REPO_BUCKET, - make_s3, - real_s3, - s3, - s3_fake_creds_file, - s3_server, -) -from .ssh import ( # noqa: F401; noqa: F401 - SSH, +from dvc_ssh.tests.fixtures import ( # noqa, pylint: disable=unused-import make_ssh, ssh, - ssh_connection, ssh_server, ) -from .webdav import Webdav, make_webdav, webdav, webdav_server # noqa: F401 -from .webhdfs import WebHDFS, make_webhdfs, webhdfs # noqa: F401 + +from .git_server import git_server, git_ssh # noqa: F401 TEST_REMOTE = "upstream" TEST_CONFIG = { diff --git a/tests/remotes/azure.py b/tests/remotes/azure.py deleted file mode 100644 index ce93d1a3e..000000000 --- a/tests/remotes/azure.py +++ /dev/null @@ -1,137 +0,0 @@ -# pylint:disable=abstract-method - -import locale -import uuid - -import pytest -from funcy import cached_property - -from dvc.testing.cloud import Cloud -from dvc.testing.path_info import CloudURLInfo - -TEST_AZURE_CONTAINER = "tests" -TEST_AZURE_CONNECTION_STRING = ( - "DefaultEndpointsProtocol=http;" - "AccountName=devstoreaccount1;" - "AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSR" - "Z6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;" - "BlobEndpoint=http://127.0.0.1:{port}/devstoreaccount1;" -) - - -class Azure(Cloud, CloudURLInfo): - - IS_OBJECT_STORAGE = True - CONNECTION_STRING = None - - def __init__(self, url, **kwargs): - super().__init__(url) - self.opts = kwargs - - @cached_property - def service_client(self): - # pylint: disable=no-name-in-module - from azure.storage.blob import BlobServiceClient - - service_client = BlobServiceClient.from_connection_string( - self.CONNECTION_STRING - ) - - return service_client - - @property - def blob_client(self): - return self.service_client.get_blob_client(self.bucket, self.path) - - def mkdir(self, mode=0o777, parents=False, exist_ok=False): - assert mode == 0o777 - assert parents - - def write_bytes(self, contents): - self.blob_client.upload_blob(contents, overwrite=True) - - def read_bytes(self): - stream = self.blob_client.download_blob() - return stream.readall() - - def read_text(self, encoding=None, errors=None): - if not encoding: - encoding = locale.getpreferredencoding(False) - assert errors is None - return self.read_bytes().decode(encoding) - - @property - def fs_path(self): - bucket = self.bucket.rstrip("/") - path = self.path.lstrip("/") - return f"{bucket}/{path}" - - def is_file(self): - raise NotImplementedError - - def is_dir(self): - raise NotImplementedError - - def exists(self): - raise NotImplementedError - - @property - def config(self): - return {"url": self.url, **self.opts} - - -@pytest.fixture(scope="session") -def azure_server(test_config, docker_compose, docker_services): - test_config.requires("azure") - - from azure.core.exceptions import ( # pylint: disable=no-name-in-module - AzureError, - ) - from azure.storage.blob import ( # pylint: disable=no-name-in-module - BlobServiceClient, - ) - - port = docker_services.port_for("azurite", 10000) - connection_string = TEST_AZURE_CONNECTION_STRING.format(port=port) - - def _check(): - try: - BlobServiceClient.from_connection_string( - connection_string - ).list_containers() - return True - except AzureError: - return False - - docker_services.wait_until_responsive( - timeout=60.0, pause=0.1, check=_check - ) - - Azure.CONNECTION_STRING = connection_string - return connection_string - - -@pytest.fixture -def make_azure(azure_server): - def _make_azure(): - from azure.core.exceptions import ResourceNotFoundError - - url = f"azure://{TEST_AZURE_CONTAINER}/{uuid.uuid4()}" - ret = Azure(url, connection_string=azure_server) - - container = ret.service_client.get_container_client( - TEST_AZURE_CONTAINER - ) - try: # verify that container exists - container.get_container_properties() - except ResourceNotFoundError: - container.create_container() - - return ret - - return _make_azure - - -@pytest.fixture -def azure(make_azure): - return make_azure() diff --git a/tests/remotes/gdrive.py b/tests/remotes/gdrive.py deleted file mode 100644 index c4b454b45..000000000 --- a/tests/remotes/gdrive.py +++ /dev/null @@ -1,168 +0,0 @@ -# pylint:disable=abstract-method -import json -import os -import uuid -from functools import partialmethod -from urllib.parse import urlparse - -import pytest -from funcy import cached_property, retry - -from dvc.fs.gdrive import GDriveFileSystem -from dvc.testing.cloud import Cloud -from dvc.testing.path_info import CloudURLInfo -from dvc.utils import tmp_fname - -TEST_GDRIVE_REPO_BUCKET = "root" - - -class GDriveURLInfo(CloudURLInfo): - def __init__(self, url): - super().__init__(url) - - # GDrive URL host part is case sensitive, - # we are restoring it here. - p = urlparse(url) - self.host = p.netloc - assert self.netloc == self.host - - # Normalize path. Important since we have a cache (path to ID) - # and don't want to deal with different variations of path in it. - import re - - self._spath = re.sub("/{2,}", "/", self._spath.rstrip("/")) - - -def _gdrive_retry(func): - def should_retry(exc): - from googleapiclient.errors import HttpError - - if not isinstance(exc, HttpError): - return False - - if 500 <= exc.resp.status < 600: - return True - - if exc.resp.status == 403: - try: - reason = json.loads(exc.content)["error"]["errors"][0][ - "reason" - ] - except (ValueError, LookupError): - return False - - return reason in ["userRateLimitExceeded", "rateLimitExceeded"] - - # 16 tries, start at 0.5s, multiply by golden ratio, cap at 20s - return retry( - 16, - timeout=lambda a: min(0.5 * 1.618 ** a, 20), - filter_errors=should_retry, - )(func) - - -class GDrive(Cloud, GDriveURLInfo): - @staticmethod - def should_test(): - return bool(os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA)) - - @property - def config(self): - tmp_path = tmp_fname() - with open(tmp_path, "w", encoding="utf-8") as stream: - raw_credentials = os.getenv( - GDriveFileSystem.GDRIVE_CREDENTIALS_DATA - ) - try: - credentials = json.loads(raw_credentials) - except ValueError: - credentials = {} - - use_service_account = credentials.get("type") == "service_account" - stream.write(raw_credentials) - - return { - "url": self.url, - "gdrive_service_account_json_file_path": tmp_path, - "gdrive_use_service_account": use_service_account, - } - - @staticmethod - def _get_storagepath(): - return TEST_GDRIVE_REPO_BUCKET + "/" + str(uuid.uuid4()) - - @staticmethod - def get_url(): - # NOTE: `get_url` should always return new random url - return "gdrive://" + GDrive._get_storagepath() - - @cached_property - def client(self): - try: - from gdrivefs import GoogleDriveFileSystem - except ImportError: - pytest.skip("gdrivefs is not installed") - - return GoogleDriveFileSystem( - token="cache", - tokens_file=self.config["gdrive_service_account_json_file_path"], - service_account=self.config["gdrive_use_service_account"], - ) - - def is_file(self): - raise NotImplementedError - - def is_dir(self): - raise NotImplementedError - - def exists(self): - raise NotImplementedError - - @_gdrive_retry - def mkdir(self, mode=0o777, parents=False, exist_ok=False): - try: - self.client.info(self.path) - except FileNotFoundError: - self.client.mkdir(self.path, parents=parents) - else: - if not exist_ok: - raise FileExistsError(self.path) - - @_gdrive_retry - def write_bytes(self, contents): - with self.client.open(self.path, mode="wb") as stream: - stream.write(contents) - - @_gdrive_retry - def _read(self, mode): - with self.client.open(self.path, mode=mode) as stream: - return stream.read() - - read_text = partialmethod(_read, mode="r") - read_bytes = partialmethod(_read, mode="rb") - - -@pytest.fixture -def make_gdrive(test_config, make_tmp_dir): - test_config.requires("gdrive") - - if not GDrive.should_test(): - pytest.skip("no gdrive") - - def _make_gdrive(): - # NOTE: temporary workaround - tmp_dir = make_tmp_dir("gdrive", dvc=True) - - ret = GDrive(GDrive.get_url()) - fs = GDriveFileSystem( - gdrive_credentials_tmp_dir=tmp_dir.dvc.tmp_dir, **ret.config - ) - fs.fs._gdrive_create_dir("root", fs.url) - yield ret - - return _make_gdrive - - -@pytest.fixture -def gdrive(make_gdrive): - return make_gdrive() diff --git a/tests/remotes/git_server.py b/tests/remotes/git_server.py index 13f470266..a8596d6f8 100644 --- a/tests/remotes/git_server.py +++ b/tests/remotes/git_server.py @@ -1,6 +1,5 @@ import pytest - -from .ssh import SSH, TEST_SSH_KEY_PATH, TEST_SSH_USER +from dvc_ssh.tests.cloud import SSH, TEST_SSH_KEY_PATH, TEST_SSH_USER class GitSSH(SSH): diff --git a/tests/remotes/gs.py b/tests/remotes/gs.py deleted file mode 100644 index 7c20b38cb..000000000 --- a/tests/remotes/gs.py +++ /dev/null @@ -1,105 +0,0 @@ -import locale -import os -import uuid - -import pytest - -from dvc.testing.cloud import Cloud -from dvc.testing.path_info import CloudURLInfo -from dvc.utils import env2bool - -TEST_GCP_REPO_BUCKET = os.environ.get("DVC_TEST_GCP_REPO_BUCKET", "dvc-test") - -TEST_GCP_CREDS_FILE = os.path.abspath( - os.environ.get( - "GOOGLE_APPLICATION_CREDENTIALS", - os.path.join("scripts", "ci", "gcp-creds.json"), - ) -) -# Ensure that absolute path is used -os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = TEST_GCP_CREDS_FILE - - -class GCP(Cloud, CloudURLInfo): - - IS_OBJECT_STORAGE = True - - @staticmethod - def should_test(): - do_test = env2bool("DVC_TEST_GCP", undefined=None) - if do_test is not None: - return do_test - - if not os.path.exists(TEST_GCP_CREDS_FILE): - return False - - return True - - @property - def config(self): - return {"url": self.url, "credentialpath": TEST_GCP_CREDS_FILE} - - @staticmethod - def _get_storagepath(): - return TEST_GCP_REPO_BUCKET + "/" + str(uuid.uuid4()) - - @staticmethod - def get_url(): - return "gs://" + GCP._get_storagepath() - - @property - def _gc(self): - from google.cloud.storage import Client - - return Client.from_service_account_json(TEST_GCP_CREDS_FILE) - - @property - def _bucket(self): - return self._gc.bucket(self.bucket) - - @property - def _blob(self): - return self._bucket.blob(self.path) - - def is_file(self): - if self.path.endswith("/"): - return False - - return self._blob.exists() - - def is_dir(self): - dir_path = self / "" - return bool(list(self._bucket.list_blobs(prefix=dir_path.path))) - - def exists(self): - return self.is_file() or self.is_dir() - - def mkdir(self, mode=0o777, parents=False, exist_ok=False): - assert mode == 0o777 - assert parents - - def write_bytes(self, contents): - assert isinstance(contents, bytes) - self._blob.upload_from_string(contents) - - def read_bytes(self): - return self._blob.download_as_string() - - def read_text(self, encoding=None, errors=None): - if not encoding: - encoding = locale.getpreferredencoding(False) - assert errors is None - return self.read_bytes().decode(encoding) - - -@pytest.fixture -def make_gs(test_config): - test_config.requires("gs") - if not GCP.should_test(): - pytest.skip("no gs") - yield GCP(GCP.get_url()) - - -@pytest.fixture -def gs(make_gs): - return make_gs() diff --git a/tests/remotes/hdfs.py b/tests/remotes/hdfs.py deleted file mode 100644 index 192a75470..000000000 --- a/tests/remotes/hdfs.py +++ /dev/null @@ -1,311 +0,0 @@ -import locale -import os -import platform -import subprocess -import uuid -from contextlib import contextmanager -from pathlib import Path -from tempfile import TemporaryDirectory -from urllib.parse import urlparse - -import pytest - -from dvc.testing.cloud import Cloud -from dvc.testing.path_info import URLInfo - -_hdfs_root = TemporaryDirectory() - - -class HDFS(Cloud, URLInfo): # pylint: disable=abstract-method - @contextmanager - def _hdfs(self): - import pyarrow.fs - - conn = pyarrow.fs.HadoopFileSystem(self.host, self.port) - yield conn - - @property - def config(self): - return {"url": self.url} - - def is_file(self): - with self._hdfs() as _hdfs: - import pyarrow.fs - - file_info = _hdfs.get_file_info(self.path) - return file_info.type == pyarrow.fs.FileType.File - - def is_dir(self): - with self._hdfs() as _hdfs: - import pyarrow.fs - - file_info = _hdfs.get_file_info(self.path) - return file_info.type == pyarrow.fs.FileType.Directory - - def exists(self): - with self._hdfs() as _hdfs: - import pyarrow.fs - - file_info = _hdfs.get_file_info(self.path) - return file_info.type != pyarrow.fs.FileType.NotFound - - def mkdir(self, mode=0o777, parents=False, exist_ok=False): - assert mode == 0o777 - assert parents - - with self._hdfs() as _hdfs: - # NOTE: fs.create_dir creates parents by default - _hdfs.create_dir(self.path) - - def write_bytes(self, contents): - with self._hdfs() as _hdfs: - with _hdfs.open_output_stream(self.path) as fobj: - fobj.write(contents) - - def read_bytes(self): - with self._hdfs() as _hdfs: - with _hdfs.open_input_stream(self.path) as fobj: - return fobj.read() - - def read_text(self, encoding=None, errors=None): - if not encoding: - encoding = locale.getpreferredencoding(False) - assert errors is None - return self.read_bytes().decode(encoding) - - @property - def fs_path(self): - return self.path.lstrip("/") - - -@pytest.fixture(scope="session") -def hadoop(test_config): - test_config.requires("real_hdfs") - pytest.importorskip("pyarrow.fs") - if platform.system() != "Linux": - pytest.skip("only supported on Linux") - - import wget - from appdirs import user_cache_dir - - hadoop_name = "hadoop-2.7.2.tar.gz" - java_name = "openjdk-7u75-b13-linux-x64-18_dec_2014.tar.gz" - - base_url = "https://s3-us-east-2.amazonaws.com/dvc-public/dvc-test/" - hadoop_url = base_url + hadoop_name - java_url = base_url + java_name - - (cache_dir,) = (user_cache_dir("dvc-test", "iterative"),) - dname = os.path.join(cache_dir, "hdfs") - - java_tar = os.path.join(dname, java_name) - hadoop_tar = os.path.join(dname, hadoop_name) - - java_home = os.path.join(dname, "java-se-7u75-ri") - hadoop_home = os.path.join(dname, "hadoop-2.7.2") - - def _get(url, tar, target): - if os.path.isdir(target): - return - - if not os.path.exists(tar): - wget.download(url, out=tar) - assert os.system(f"tar -xvf {tar} -C {dname}") == 0 - assert os.path.isdir(target) - - os.makedirs(dname, exist_ok=True) - _get(hadoop_url, hadoop_tar, hadoop_home) - _get(java_url, java_tar, java_home) - - os.environ["JAVA_HOME"] = java_home - os.environ["HADOOP_HOME"] = hadoop_home - os.environ["PATH"] += f":{hadoop_home}/bin:{hadoop_home}/sbin" - - # NOTE: must set CLASSPATH to connect using pyarrow.fs.HadoopFileSystem - result = subprocess.run( - [f"{hadoop_home}/bin/hdfs", "classpath", "--glob"], - universal_newlines=True, - stdout=subprocess.PIPE, - check=False, - ) - os.environ["CLASSPATH"] = result.stdout - - -@pytest.fixture(scope="session") -def hdfs_server(hadoop, docker_compose, docker_services): - import pyarrow.fs - - port = docker_services.port_for("hdfs", 8020) - web_port = docker_services.port_for("hdfs", 50070) - - def _check(): - try: - # NOTE: just connecting or even opening something is not enough, - # we need to make sure that we are able to write something. - conn = pyarrow.fs.HadoopFileSystem("hdfs://127.0.0.1", port) - with conn.open_output_stream(str(uuid.uuid4())) as fobj: - fobj.write(b"test") - return True - except (pyarrow.ArrowException, OSError): - import traceback - - traceback.print_exc() - return False - - try: - docker_services.wait_until_responsive( - timeout=30.0, pause=5, check=_check - ) - except Exception: # pylint: disable=broad-except - pytest.skip("couldn't start hdfs server") - - return {"hdfs": port, "webhdfs": web_port} - - -@pytest.fixture -def real_hdfs(hdfs_server): - port = hdfs_server["hdfs"] - url = f"hdfs://127.0.0.1:{port}/{uuid.uuid4()}" - yield HDFS(url) - - -def md5md5crc32c(path): - # https://github.com/colinmarc/hdfs/blob/f2f512db170db82ad41590c4ba3b7718b13317d2/file_reader.go#L76 - import hashlib - - from crc32c import crc32c # pylint: disable=no-name-in-module - - # dfs.bytes-per-checksum = 512, default on hadoop 2.7 - bytes_per_checksum = 512 - padded = 32 - total = 0 - - md5md5 = hashlib.md5() - - with open(path, "rb") as fobj: - while True: - block = fobj.read(bytes_per_checksum) - if not block: - break - - crc_int = crc32c(block) - - # NOTE: hdfs is big-endian - crc_bytes = crc_int.to_bytes( - (crc_int.bit_length() + 7) // 8, "big" - ) - - md5 = hashlib.md5(crc_bytes).digest() - - total += len(md5) - if padded < total: - padded *= 2 - - md5md5.update(md5) - - md5md5.update(b"\0" * (padded - total)) - return "000002000000000000000000" + md5md5.hexdigest() - - -def hadoop_fs_checksum(_, path): - parsed = urlparse(path) - - return md5md5crc32c(Path(_hdfs_root.name) / parsed.path.lstrip("/")) - - -class FakeHadoopFileSystem: - def __init__(self, *args, **kwargs): - from pyarrow.fs import LocalFileSystem - - self._root = Path(_hdfs_root.name) - self._fs = LocalFileSystem() - - def _path(self, path): - from pyarrow.fs import FileSelector - - if isinstance(path, FileSelector): - return FileSelector( - os.fspath(self._root / path.base_dir.lstrip("/")), - path.allow_not_found, - path.recursive, - ) - if isinstance(path, list): - return [self._path(sub_path) for sub_path in path] - - return os.fspath(self._root / path.lstrip("/")) - - def create_dir(self, path, **kwargs): - return self._fs.create_dir(self._path(path), **kwargs) - - def open_input_stream(self, path, **kwargs): - return self._fs.open_input_stream(self._path(path), **kwargs) - - def open_output_stream(self, path, **kwargs): - import posixpath - - # NOTE: HadoopFileSystem.open_output_stream creates directories - # automatically. - self.create_dir(posixpath.dirname(path)) - return self._fs.open_output_stream(self._path(path), **kwargs) - - def get_file_info(self, path, **kwargs): - from pyarrow.fs import FileInfo - - entries = self._fs.get_file_info(self._path(path), **kwargs) - if isinstance(entries, FileInfo): - ret = self._adjust_entry(entries) - else: - assert isinstance(entries, list) - ret = list(map(self._adjust_entry, entries)) - - # import pdb; pdb.set_trace() - - return ret - - def _adjust_entry(self, entry): - import posixpath - - from pyarrow.fs import FileInfo - - mocked_path = os.path.relpath(entry.path, self._root) - mocked_parts = mocked_path.split(os.path.sep) - return FileInfo( - path=posixpath.join(*mocked_parts), - type=entry.type, - mtime=entry.mtime, - size=entry.size, - ) - - def move(self, from_path, to_path): - self._fs.move(self._path(from_path), self._path(to_path)) - - def delete_file(self, path): - self._fs.delete_file(self._path(path)) - - -@pytest.fixture -def make_hdfs(test_config, mocker): - # Windows might not have Visual C++ Redistributable for Visual Studio - # 2015 installed, which will result in the following error: - # "The pyarrow installation is not built with support for - # 'HadoopFileSystem'" - test_config.requires("hdfs") - pytest.importorskip("pyarrow.fs") - - mocker.patch("pyarrow.fs._not_imported", []) - mocker.patch( - "pyarrow.fs.HadoopFileSystem", FakeHadoopFileSystem, create=True - ) - - mocker.patch("dvc.fs.hdfs.HDFSFileSystem._checksum", hadoop_fs_checksum) - - def _make_hdfs(): - url = f"hdfs://example.com:12345/{uuid.uuid4()}" - return HDFS(url) - - return _make_hdfs - - -@pytest.fixture -def hdfs(make_hdfs): - return make_hdfs() diff --git a/tests/remotes/http.py b/tests/remotes/http.py deleted file mode 100644 index 8440ad26c..000000000 --- a/tests/remotes/http.py +++ /dev/null @@ -1,68 +0,0 @@ -# pylint:disable=abstract-method -import os -import uuid - -import pytest -import requests - -from dvc.testing.cloud import Cloud -from dvc.testing.path_info import HTTPURLInfo - - -class HTTP(Cloud, HTTPURLInfo): - @staticmethod - def get_url(port): # pylint: disable=arguments-differ - dname = str(uuid.uuid4()) - return f"http://127.0.0.1:{port}/{dname}" - - def mkdir(self, mode=0o777, parents=False, exist_ok=False): - assert mode == 0o777 - assert parents - - def write_bytes(self, contents): - assert isinstance(contents, bytes) - response = requests.post(self.url, data=contents) - assert response.status_code == 200 - - @property - def config(self): - return {"url": self.url} - - @property - def fs_path(self): - return self.url - - def exists(self): - raise NotImplementedError - - def is_dir(self): - raise NotImplementedError - - def is_file(self): - raise NotImplementedError - - def read_bytes(self): - raise NotImplementedError - - -@pytest.fixture(scope="session") -def http_server(test_config, tmp_path_factory): - from tests.utils.httpd import StaticFileServer - - test_config.requires("http") - directory = os.fspath(tmp_path_factory.mktemp("http")) - with StaticFileServer(directory=directory) as httpd: - yield httpd - - -@pytest.fixture -def make_http(http_server): - def _make_http(): - return HTTP(HTTP.get_url(http_server.server_port)) - - return _make_http - - -@pytest.fixture -def http(make_http): - return make_http() diff --git a/tests/remotes/local.py b/tests/remotes/local.py deleted file mode 100644 index 7b163d6b9..000000000 --- a/tests/remotes/local.py +++ /dev/null @@ -1,34 +0,0 @@ -# pylint: disable=cyclic-import,disable=abstract-method -import pytest - -from dvc.testing.cloud import Cloud -from tests.basic_env import TestDvc - - -class Local(Cloud): - @staticmethod - def get_storagepath(): - return TestDvc.mkdtemp() - - @staticmethod - def get_url(): - return Local.get_storagepath() - - -@pytest.fixture -def make_local(make_tmp_dir): - def _make_local(): - return make_tmp_dir("local-cloud") - - return _make_local - - -@pytest.fixture -def local_cloud(make_local): - return make_local() - - -@pytest.fixture -def local_remote(tmp_dir, dvc, local_cloud): - tmp_dir.add_remote(config=local_cloud.config) - yield local_cloud diff --git a/tests/remotes/oss.py b/tests/remotes/oss.py deleted file mode 100644 index d395722c2..000000000 --- a/tests/remotes/oss.py +++ /dev/null @@ -1,131 +0,0 @@ -# pylint:disable=abstract-method -import os -import uuid - -import pytest - -from dvc.testing.cloud import Cloud -from dvc.testing.path_info import CloudURLInfo -from dvc.utils import env2bool - -TEST_OSS_REPO_BUCKET = "dvc-test-github" -EMULATOR_OSS_ENDPOINT = "127.0.0.1:{port}" -EMULATOR_OSS_ACCESS_KEY_ID = "AccessKeyID" -EMULATOR_OSS_ACCESS_KEY_SECRET = "AccessKeySecret" - - -class OSS(Cloud, CloudURLInfo): - - IS_OBJECT_STORAGE = True - - @staticmethod - def get_url(): - return f"oss://{TEST_OSS_REPO_BUCKET}/{uuid.uuid4()}" - - @staticmethod - def should_test(): - do_test = env2bool("DVC_TEST_OSS", undefined=None) - if do_test is not None: - return do_test - - if os.getenv("OSS_ACCESS_KEY_ID") and os.getenv( - "OSS_ACCESS_KEY_SECRET" - ): - return True - - return False - - @property - def config(self): - return { - "url": self.url, - "oss_key_id": os.environ.get("OSS_ACCESS_KEY_ID"), - "oss_key_secret": os.environ.get("OSS_ACCESS_KEY_SECRET"), - "oss_endpoint": os.environ.get("OSS_ENDPOINT"), - } - - def is_file(self): - raise NotImplementedError - - def is_dir(self): - raise NotImplementedError - - def exists(self): - raise NotImplementedError - - def mkdir(self, mode=0o777, parents=False, exist_ok=False): - raise NotImplementedError - - def write_bytes(self, contents): - raise NotImplementedError - - def read_bytes(self): - raise NotImplementedError - - -@pytest.fixture(scope="session") -def oss_server(test_config, docker_compose, docker_services): - import oss2 - - test_config.requires("oss") - port = docker_services.port_for("oss", 8880) - endpoint = EMULATOR_OSS_ENDPOINT.format(port=port) - - def _check(): - try: - auth = oss2.Auth( - EMULATOR_OSS_ACCESS_KEY_ID, EMULATOR_OSS_ACCESS_KEY_SECRET - ) - oss2.Bucket(auth, endpoint, "mybucket").get_bucket_info() - return True - except oss2.exceptions.NoSuchBucket: - return True - except oss2.exceptions.OssError: - return False - - docker_services.wait_until_responsive(timeout=30.0, pause=5, check=_check) - - return endpoint - - -@pytest.fixture -def make_oss(real_oss): - def _make_oss(): - import oss2 - - ret = real_oss - - auth = oss2.Auth( - ret.config["oss_key_id"], ret.config["oss_key_secret"] - ) - bucket = oss2.Bucket( - auth, ret.config["oss_endpoint"], TEST_OSS_REPO_BUCKET - ) - try: - bucket.get_bucket_info() - except oss2.exceptions.NoSuchBucket: - bucket.create_bucket( - oss2.BUCKET_ACL_PUBLIC_READ, - oss2.models.BucketCreateConfig( - oss2.BUCKET_STORAGE_CLASS_STANDARD - ), - ) - - return ret - - return _make_oss - - -@pytest.fixture -def oss(make_oss): - return make_oss() - - -@pytest.fixture -def real_oss(test_config): - test_config.requires("oss") - if not OSS.should_test(): - pytest.skip("no real OSS") - - url = OSS.get_url() - return OSS(url) diff --git a/tests/remotes/s3.py b/tests/remotes/s3.py deleted file mode 100644 index 13208d79c..000000000 --- a/tests/remotes/s3.py +++ /dev/null @@ -1,173 +0,0 @@ -import locale -import os -import uuid - -import pytest -from funcy import cached_property - -from dvc.testing.cloud import Cloud -from dvc.testing.path_info import CloudURLInfo -from dvc.utils import env2bool - -TEST_AWS_REPO_BUCKET = os.environ.get("DVC_TEST_AWS_REPO_BUCKET", "dvc-temp") -TEST_AWS_ENDPOINT_URL = "http://127.0.0.1:{port}/" - - -class S3(Cloud, CloudURLInfo): - - IS_OBJECT_STORAGE = True - TEST_AWS_ENDPOINT_URL = None - - @property - def config(self): - return {"url": self.url, "endpointurl": self.TEST_AWS_ENDPOINT_URL} - - @staticmethod - def should_test(): - do_test = env2bool("DVC_TEST_AWS", undefined=None) - if do_test is not None: - return do_test - - if os.getenv("AWS_ACCESS_KEY_ID") and os.getenv( - "AWS_SECRET_ACCESS_KEY" - ): - return True - - return False - - @staticmethod - def _get_storagepath(): - return ( - TEST_AWS_REPO_BUCKET - + "/" - + "dvc_test_caches" - + "/" - + str(uuid.uuid4()) - ) - - @staticmethod - def get_url(): - return "s3://" + S3._get_storagepath() - - @cached_property - def _s3(self): - import boto3 - - return boto3.client("s3", endpoint_url=self.config["endpointurl"]) - - def is_file(self): - from botocore.exceptions import ClientError - - if self.path.endswith("/"): - return False - - try: - self._s3.head_object(Bucket=self.bucket, Key=self.path) - except ClientError as exc: - if exc.response["Error"]["Code"] != "404": - raise - return False - - return True - - def is_dir(self): - path = (self / "").path - resp = self._s3.list_objects(Bucket=self.bucket, Prefix=path) - return bool(resp.get("Contents")) - - def exists(self): - return self.is_file() or self.is_dir() - - def mkdir(self, mode=0o777, parents=False, exist_ok=False): - assert mode == 0o777 - assert parents - - def write_bytes(self, contents): - self._s3.put_object(Bucket=self.bucket, Key=self.path, Body=contents) - - def read_bytes(self): - data = self._s3.get_object(Bucket=self.bucket, Key=self.path) - return data["Body"].read() - - def read_text(self, encoding=None, errors=None): - if not encoding: - encoding = locale.getpreferredencoding(False) - assert errors is None - return self.read_bytes().decode(encoding) - - @property - def fs_path(self): - return self.bucket + "/" + self.path.lstrip("/") - - -@pytest.fixture -def s3_fake_creds_file(monkeypatch): - # https://github.com/spulec/moto#other-caveats - import pathlib - - aws_dir = pathlib.Path("~").expanduser() / ".aws" - aws_dir.mkdir(exist_ok=True) - - aws_creds = aws_dir / "credentials" - initially_exists = aws_creds.exists() - - if not initially_exists: - aws_creds.touch() - - try: - with monkeypatch.context() as m: - m.setenv("AWS_ACCESS_KEY_ID", "testing") - m.setenv("AWS_SECRET_ACCESS_KEY", "testing") - m.setenv("AWS_SECURITY_TOKEN", "testing") - m.setenv("AWS_SESSION_TOKEN", "testing") - yield - finally: - if aws_creds.exists() and not initially_exists: - aws_creds.unlink() - - -@pytest.fixture(scope="session") -def s3_server(test_config, docker_compose, docker_services): - import requests - - test_config.requires("s3") - - port = docker_services.port_for("motoserver", 5000) - endpoint_url = TEST_AWS_ENDPOINT_URL.format(port=port) - - def _check(): - try: - r = requests.get(endpoint_url) - return r.ok - except requests.RequestException: - return False - - docker_services.wait_until_responsive( - timeout=60.0, pause=0.1, check=_check - ) - S3.TEST_AWS_ENDPOINT_URL = endpoint_url - return endpoint_url - - -@pytest.fixture -def make_s3(test_config, s3_server, s3_fake_creds_file): - test_config.requires("s3") - - def _make_s3(): - cloud = S3(S3.get_url()) - cloud._s3.create_bucket(Bucket=TEST_AWS_REPO_BUCKET) - return cloud - - return _make_s3 - - -@pytest.fixture -def s3(make_s3): - return make_s3() - - -@pytest.fixture -def real_s3(): - if not S3.should_test(): - pytest.skip("no real s3") - yield S3(S3.get_url()) diff --git a/tests/remotes/ssh.py b/tests/remotes/ssh.py deleted file mode 100644 index 1b3f51e13..000000000 --- a/tests/remotes/ssh.py +++ /dev/null @@ -1,135 +0,0 @@ -import locale -import os -import uuid - -import pytest -from funcy import cached_property - -from dvc.testing.cloud import Cloud -from dvc.testing.path_info import URLInfo - -TEST_SSH_USER = "user" -TEST_SSH_KEY_PATH = os.path.join( - os.path.abspath(os.path.dirname(__file__)), f"{TEST_SSH_USER}.key" -) - - -class SSH(Cloud, URLInfo): - @staticmethod - def get_url(host, port): # pylint: disable=arguments-differ - return f"ssh://{host}:{port}/tmp/data/{uuid.uuid4()}" - - @property - def config(self): - return { - "url": self.url, - "user": TEST_SSH_USER, - "keyfile": TEST_SSH_KEY_PATH, - } - - @cached_property - def _ssh(self): - from sshfs import SSHFileSystem - - return SSHFileSystem( - host=self.host, - port=self.port, - username=TEST_SSH_USER, - client_keys=[TEST_SSH_KEY_PATH], - ) - - def is_file(self): - return self._ssh.isfile(self.path) - - def is_dir(self): - return self._ssh.isdir(self.path) - - def exists(self): - return self._ssh.exists(self.path) - - def mkdir(self, mode=0o777, parents=False, exist_ok=False): - assert mode == 0o777 - assert parents - - self._ssh.makedirs(self.path, exist_ok=exist_ok) - - def write_bytes(self, contents): - assert isinstance(contents, bytes) - with self._ssh.open(self.path, "wb") as fobj: - fobj.write(contents) - - def read_bytes(self): - with self._ssh.open(self.path, "rb") as fobj: - return fobj.read() - - def read_text(self, encoding=None, errors=None): - if not encoding: - encoding = locale.getpreferredencoding(False) - assert errors is None - return self.read_bytes().decode(encoding) - - @property - def fs_path(self): - return self.path - - -@pytest.fixture(scope="session") -def ssh_server(test_config, docker_compose, docker_services): - import asyncssh - from sshfs import SSHFileSystem - - test_config.requires("ssh") - conn_info = { - "host": "127.0.0.1", - "port": docker_services.port_for("openssh-server", 2222), - } - - def get_fs(): - return SSHFileSystem( - **conn_info, - username=TEST_SSH_USER, - client_keys=[TEST_SSH_KEY_PATH], - ) - - def _check(): - try: - get_fs().exists("/") - except asyncssh.Error: - return False - else: - return True - - docker_services.wait_until_responsive(timeout=30.0, pause=1, check=_check) - return conn_info - - -@pytest.fixture -def ssh_connection(ssh_server): - from sshfs import SSHFileSystem - - yield SSHFileSystem( - host=ssh_server["host"], - port=ssh_server["port"], - username=TEST_SSH_USER, - client_files=[TEST_SSH_KEY_PATH], - ) - - -@pytest.fixture -def make_ssh(ssh_server, monkeypatch): - def _make_ssh(): - from dvc.fs.ssh import SSHFileSystem - - # NOTE: see http://github.com/iterative/dvc/pull/3501 - monkeypatch.setattr(SSHFileSystem, "CAN_TRAVERSE", False) - - url = SSH(SSH.get_url(**ssh_server)) - url.mkdir(exist_ok=True, parents=True) - return url - - return _make_ssh - - -@pytest.fixture -def ssh(make_ssh): - return make_ssh() diff --git a/tests/remotes/webdav.py b/tests/remotes/webdav.py deleted file mode 100644 index 483670d1b..000000000 --- a/tests/remotes/webdav.py +++ /dev/null @@ -1,98 +0,0 @@ -# pylint:disable=abstract-method -import os -from wsgiref.simple_server import make_server - -import pytest -from funcy import cached_property, first -from wsgidav.wsgidav_app import WsgiDAVApp - -from dvc.testing.cloud import Cloud -from dvc.testing.path_info import WebDAVURLInfo -from tests.utils.httpd import run_server_on_thread - -AUTH = {"user1": {"password": "password1"}} - - -class Webdav(Cloud, WebDAVURLInfo): - @staticmethod - def get_url(port): # pylint: disable=arguments-differ - return f"webdav://localhost:{port}" - - @property - def config(self): - user, secrets = first(AUTH.items()) - url = f"webdav://{self.netloc}{self._spath}" - return {"url": url, "user": user, **secrets} - - @cached_property - def client(self): - from webdav4.client import Client - - user, secrets = first(AUTH.items()) - return Client( - self.replace(path="").url, auth=(user, secrets["password"]) - ) - - def mkdir(self, mode=0o777, parents=False, exist_ok=False): - assert mode == 0o777 - parent_dirs = list(reversed(self.parents))[1:] if parents else [] - for d in parent_dirs + [self]: - path = d.path # pylint: disable=no-member - if not self.client.exists(path): - self.client.mkdir(path) - - def write_bytes(self, contents): - from io import BytesIO - - self.client.upload_fileobj(BytesIO(contents), self.path) - - @property - def fs_path(self): - return self.path.lstrip("/") - - def exists(self): - raise NotImplementedError - - def is_dir(self): - raise NotImplementedError - - def is_file(self): - raise NotImplementedError - - def read_bytes(self): - raise NotImplementedError - - -@pytest.fixture -def webdav_server(test_config, tmp_path_factory): - test_config.requires("webdav") - - host, port = "localhost", 0 - directory = os.fspath(tmp_path_factory.mktemp("http")) - dirmap = {"/": directory} - - app = WsgiDAVApp( - { - "host": host, - "port": port, - "provider_mapping": dirmap, - "simple_dc": {"user_mapping": {"*": AUTH}}, - } - ) - server = make_server(host, port, app) - with run_server_on_thread(server) as httpd: - yield httpd - - -@pytest.fixture -def make_webdav(webdav_server): - def _make_webdav(): - url = Webdav.get_url(webdav_server.server_port) - return Webdav(url) - - return _make_webdav - - -@pytest.fixture -def webdav(make_webdav): - return make_webdav() diff --git a/tests/remotes/webhdfs.py b/tests/remotes/webhdfs.py deleted file mode 100644 index 7a4cd33d7..000000000 --- a/tests/remotes/webhdfs.py +++ /dev/null @@ -1,78 +0,0 @@ -import locale -import uuid -from contextlib import contextmanager - -import pytest - -from dvc.testing.cloud import Cloud -from dvc.testing.path_info import CloudURLInfo - - -class WebHDFS(Cloud, CloudURLInfo): # pylint: disable=abstract-method - @contextmanager - def _webhdfs(self): - from hdfs import InsecureClient - - client = InsecureClient( - f"http://{self.host}:{self.port}", self.user, root="/" - ) - yield client - - def is_file(self): - with self._webhdfs() as _hdfs: - return _hdfs.status(self.path)["type"] == "FILE" - - def is_dir(self): - with self._webhdfs() as _hdfs: - return _hdfs.status(self.path)["type"] == "DIRECTORY" - - def exists(self): - with self._webhdfs() as _hdfs: - return _hdfs.status(self.path, strict=False) is not None - - def mkdir(self, mode=0o777, parents=False, exist_ok=False): - assert mode == 0o777 - assert parents - - with self._webhdfs() as _hdfs: - # NOTE: hdfs.makekdirs always creates parents - _hdfs.makedirs(self.path, permission=mode) - - def write_bytes(self, contents): - with self._webhdfs() as _hdfs: - with _hdfs.write(self.path, overwrite=True) as writer: - writer.write(contents) - - def read_bytes(self): - with self._webhdfs() as _hdfs: - with _hdfs.read(self.path) as reader: - return reader.read() - - def read_text(self, encoding=None, errors=None): - if not encoding: - encoding = locale.getpreferredencoding(False) - assert errors is None - return self.read_bytes().decode(encoding) - - @property - def config(self): - return {"url": self.url} - - @property - def fs_path(self): - return "/" + self.path.lstrip("/") - - -@pytest.fixture -def make_webhdfs(hdfs_server): - def _make_webhdfs(): - port = hdfs_server["webhdfs"] - url = f"webhdfs://127.0.0.1:{port}/{uuid.uuid4()}" - yield WebHDFS(url) - - return _make_webhdfs - - -@pytest.fixture -def webhdfs(make_webhdfs): - return make_webhdfs() diff --git a/tests/unit/fs/test_s3.py b/tests/unit/fs/test_s3.py index d5195848d..d2a8bda78 100644 --- a/tests/unit/fs/test_s3.py +++ b/tests/unit/fs/test_s3.py @@ -1,7 +1,4 @@ -import importlib import os -import sys -import textwrap import pytest @@ -139,80 +136,3 @@ def test_key_id_and_secret(dvc): assert fs.fs_args["key"] == key_id assert fs.fs_args["secret"] == key_secret assert fs.fs_args["token"] == session_token - - -KB = 1024 -MB = KB ** 2 -GB = KB ** 3 - - -def test_s3_aws_config(tmp_dir, dvc, s3, monkeypatch): - config_directory = tmp_dir / ".aws" - config_directory.mkdir() - (config_directory / "config").write_text( - textwrap.dedent( - """\ - [default] - s3 = - max_concurrent_requests = 20000 - max_queue_size = 1000 - multipart_threshold = 1000KiB - multipart_chunksize = 64MB - use_accelerate_endpoint = true - addressing_style = path - """ - ) - ) - - if sys.platform == "win32": - var = "USERPROFILE" - else: - var = "HOME" - - with monkeypatch.context() as m: - m.setenv(var, str(tmp_dir)) - # Fresh import to see the effects of changing HOME variable - s3_mod = importlib.reload(sys.modules[S3FileSystem.__module__]) - fs = s3_mod.S3FileSystem(**s3.config) - - importlib.reload(sys.modules[S3FileSystem.__module__]) - - s3_config = fs.fs_args["config_kwargs"]["s3"] - assert s3_config["use_accelerate_endpoint"] - assert s3_config["addressing_style"] == "path" - - transfer_config = fs._transfer_config - assert transfer_config.max_io_queue_size == 1000 - assert transfer_config.multipart_chunksize == 64 * MB - assert transfer_config.multipart_threshold == 1000 * KB - assert transfer_config.max_request_concurrency == 20000 - - -def test_s3_aws_config_different_profile(tmp_dir, dvc, s3, monkeypatch): - config_file = tmp_dir / "aws_config.ini" - config_file.write_text( - textwrap.dedent( - """\ - [default] - extra = keys - s3 = - addressing_style = auto - use_accelerate_endpoint = true - multipart_threshold = ThisIsNotGoingToBeCasted! - [profile dev] - some_extra = keys - s3 = - addressing_style = virtual - multipart_threshold = 2GiB - """ - ) - ) - monkeypatch.setenv("AWS_CONFIG_FILE", config_file) - - fs = S3FileSystem(profile="dev", **s3.config) - - s3_config = fs.fs_args["config_kwargs"]["s3"] - assert s3_config["addressing_style"] == "virtual" - - transfer_config = fs._transfer_config - assert transfer_config.multipart_threshold == 2 * GB diff --git a/tests/unit/remote/test_http.py b/tests/unit/remote/test_http.py index 52cd578ab..effa53377 100644 --- a/tests/unit/remote/test_http.py +++ b/tests/unit/remote/test_http.py @@ -1,20 +1,10 @@ import ssl -import pytest from mock import patch from dvc.fs.http import HTTPFileSystem -def test_download_fails_on_error_code(tmp_dir, http): - fs = HTTPFileSystem(**http.config) - - with pytest.raises(FileNotFoundError): - fs.download_file( - (http / "missing.txt").fs_path, (tmp_dir / "missing.txt").fs_path - ) - - def test_public_auth_method(dvc): config = { "url": "http://example.com/", diff --git a/tests/unit/remote/test_remote_tree.py b/tests/unit/remote/test_remote_tree.py deleted file mode 100644 index c1b941a0b..000000000 --- a/tests/unit/remote/test_remote_tree.py +++ /dev/null @@ -1,45 +0,0 @@ -import pytest - -from dvc.fs.s3 import S3FileSystem -from dvc.objects.db import _get_odb - -remotes = [pytest.lazy_fixture(fix) for fix in ["gs", "s3"]] - -FILE_WITH_CONTENTS = { - "data1.txt": "", - # "empty_dir/": "", - "empty_file": "", - "foo": "foo", - "data/alice": "alice", - "data/alpha": "alpha", - "data/subdir-file.txt": "subdir", - "data/subdir/1": "1", - "data/subdir/2": "2", - "data/subdir/3": "3", - # "data/subdir/empty_dir/": "", - "data/subdir/empty_file": "", -} - - -@pytest.mark.parametrize("cloud", [pytest.lazy_fixture("s3")]) -def test_copy_preserve_etag_across_buckets(cloud, dvc): - cloud.gen(FILE_WITH_CONTENTS) - rem = _get_odb(dvc, cloud.config) - s3 = rem.fs.s3 - s3.create_bucket(Bucket="another") - - config = cloud.config.copy() - config["url"] = "s3://another" - config["region"] = "us-east-1" - - another = S3FileSystem(**config) - - from_info = rem.fs.path.join(rem.fs_path, "foo") - to_info = "another/foo" - - rem.fs.copy(from_info, to_info) - - from_hash = rem.fs.info(from_info)["ETag"].strip('"') - to_hash = another.info(to_info)["ETag"].strip('"') - - assert from_hash == to_hash diff --git a/tests/utils/httpd.py b/tests/utils/httpd.py deleted file mode 100644 index 8ca526df6..000000000 --- a/tests/utils/httpd.py +++ /dev/null @@ -1,91 +0,0 @@ -import hashlib -import os -import threading -from contextlib import contextmanager -from http import HTTPStatus -from http.server import HTTPServer - -from RangeHTTPServer import RangeRequestHandler - - -class TestRequestHandler(RangeRequestHandler): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def end_headers(self): - # RangeRequestHandler only sends Accept-Ranges header if Range header - # is present, see https://github.com/danvk/RangeHTTPServer/issues/23 - if not self.headers.get("Range"): - self.send_header("Accept-Ranges", "bytes") - - # Add a checksum header - file = self.translate_path(self.path) - - if not os.path.isdir(file) and os.path.exists(file): - with open(file, encoding="utf-8") as fd: - encoded_text = fd.read().encode("utf8") - checksum = hashlib.md5(encoded_text).hexdigest() - self.send_header("Content-MD5", checksum) - - RangeRequestHandler.end_headers(self) - - def _chunks(self): - while True: - data = self.rfile.readline(65537) - chunk_size = int(data[:-2], 16) - if chunk_size == 0: - return - data = self.rfile.read(chunk_size) - yield data - self.rfile.read(2) - - def do_POST(self): - chunked = self.headers.get("Transfer-Encoding", "") == "chunked" - path = self.translate_path(self.path) - try: - os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, "wb") as fd: - if chunked: - for chunk in self._chunks(): - fd.write(chunk) - else: - size = int(self.headers.get("Content-Length", 0)) - fd.write(self.rfile.read(size)) - except OSError as e: - self.send_error(HTTPStatus.INTERNAL_SERVER_ERROR, str(e)) - self.send_response(HTTPStatus.OK) - self.end_headers() - - -@contextmanager -def run_server_on_thread(server): - thread = threading.Thread(target=server.serve_forever) - thread.daemon = True - thread.start() - - yield server - - server.socket.close() - server.shutdown() - server.server_close() - - -class StaticFileServer: - _lock = threading.Lock() - - def __init__(self, directory): - from functools import partial - - addr = ("localhost", 0) - req = partial(TestRequestHandler, directory=directory) - server = HTTPServer(addr, req) - self.runner = run_server_on_thread(server) - - # pylint: disable=no-member - def __enter__(self): - self._lock.acquire() - return self.runner.__enter__() - - def __exit__(self, *args): - self.runner.__exit__(*args) - self._lock.release()