diff --git a/dvc/testing/__init__.py b/dvc/testing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/remotes/base.py b/dvc/testing/cloud.py similarity index 77% rename from tests/remotes/base.py rename to dvc/testing/cloud.py index ca432f643..8aa01ec6d 100644 --- a/tests/remotes/base.py +++ b/dvc/testing/cloud.py @@ -1,26 +1,26 @@ import locale import pathlib - -from funcy import cached_property - -from .path_info import URLInfo +from abc import ABC, abstractmethod -class Base(URLInfo): - +class Cloud(ABC): IS_OBJECT_STORAGE = False + @abstractmethod def is_file(self): - raise NotImplementedError + pass + @abstractmethod def is_dir(self): - raise NotImplementedError + pass + @abstractmethod def exists(self): - raise NotImplementedError + pass + @abstractmethod def mkdir(self, mode=0o777, parents=False, exist_ok=False): - raise NotImplementedError + pass def write_text(self, contents, encoding=None, errors=None): if not encoding: @@ -28,14 +28,19 @@ class Base(URLInfo): assert errors is None self.write_bytes(contents.encode(encoding)) + @abstractmethod def write_bytes(self, contents): raise NotImplementedError def read_text(self, encoding=None, errors=None): - raise NotImplementedError + if not encoding: + encoding = locale.getpreferredencoding(False) + assert errors is None + return self.read_bytes().decode(encoding) + @abstractmethod def read_bytes(self): - raise NotImplementedError + pass def _gen(self, struct, prefix=None): for name, contents in struct.items(): @@ -71,6 +76,7 @@ class Base(URLInfo): def get_url(): raise NotImplementedError - @cached_property + @property + @abstractmethod def config(self): - return {"url": self.url} + pass diff --git a/dvc/testing/conftest.py b/dvc/testing/conftest.py new file mode 100644 index 000000000..12c22cdf0 --- /dev/null +++ b/dvc/testing/conftest.py @@ -0,0 +1 @@ +from .fixtures import * # noqa, pylint: disable=wildcard-import diff --git a/dvc/testing/fixtures.py b/dvc/testing/fixtures.py new file mode 100644 index 000000000..3816848ec --- /dev/null +++ b/dvc/testing/fixtures.py @@ -0,0 +1,207 @@ +import os +import subprocess + +import pytest + +__all__ = [ + "make_tmp_dir", + "tmp_dir", + "scm", + "dvc", + "make_cloud", + "cloud", + "make_remote", + "remote", + "workspace", + "make_workspace", + "docker", + "docker_compose", + "docker_compose_project_name", + "docker_services", +] + +CACHE = {} # type: ignore + + +def _fs_copy(src, dst, ignore=None): + import shutil + + if os.path.isdir(src): + shutil.copytree(src, dst, ignore=ignore) + else: + shutil.copy2(src, dst) + + +@pytest.fixture(scope="session") +def make_tmp_dir(tmp_path_factory, request, worker_id): + def make( + name, *, scm=False, dvc=False, subdir=False + ): # pylint: disable=W0621 + from shutil import ignore_patterns + + from scmrepo.git import Git + + from dvc.repo import Repo + + from .tmp_dir import TmpDir + + cache = CACHE.get((scm, dvc, subdir)) + if not cache: + cache = tmp_path_factory.mktemp("dvc-test-cache" + worker_id) + TmpDir(cache).init(scm=scm, dvc=dvc, subdir=subdir) + CACHE[(scm, dvc, subdir)] = os.fspath(cache) + path = tmp_path_factory.mktemp(name) if isinstance(name, str) else name + + # ignore sqlite files from .dvc/tmp. We might not be closing the cache + # connection resulting in PermissionErrors in Windows. + ignore = ignore_patterns("cache.db*") + for entry in os.listdir(cache): + # shutil.copytree's dirs_exist_ok is only available in >=3.8 + _fs_copy( + os.path.join(cache, entry), + os.path.join(path, entry), + ignore=ignore, + ) + new_dir = TmpDir(path) + str_path = os.fspath(new_dir) + if dvc: + new_dir.dvc = Repo(str_path) + if scm: + new_dir.scm = ( + new_dir.dvc.scm if hasattr(new_dir, "dvc") else Git(str_path) + ) + request.addfinalizer(new_dir.close) + return new_dir + + return make + + +@pytest.fixture +def tmp_dir(tmp_path, make_tmp_dir, request, monkeypatch): + monkeypatch.chdir(tmp_path) + fixtures = request.fixturenames + return make_tmp_dir(tmp_path, scm="scm" in fixtures, dvc="dvc" in fixtures) + + +@pytest.fixture +def scm(tmp_dir): + return tmp_dir.scm + + +@pytest.fixture +def dvc(tmp_dir): + with tmp_dir.dvc as _dvc: + yield _dvc + + +@pytest.fixture +def make_cloud(request): + def _make_cloud(typ): + return request.getfixturevalue(f"make_{typ}")() + + return _make_cloud + + +@pytest.fixture +def cloud(make_cloud, request): + if not hasattr(request, "param"): + pytest.skip("no cloud specified") + return make_cloud(request.param) + + +@pytest.fixture +def make_remote(tmp_dir, dvc, make_cloud): + def _make_remote(name, typ="local", **kwargs): + cloud = make_cloud(typ) # pylint: disable=W0621 + tmp_dir.add_remote(name=name, config=cloud.config, **kwargs) + return cloud + + return _make_remote + + +@pytest.fixture +def remote(make_remote, request): + if not hasattr(request, "param"): + pytest.skip("no remote specified") + return make_remote("upstream", typ=request.param) + + +@pytest.fixture +def make_workspace(tmp_dir, dvc, make_cloud): + def _make_workspace(name, typ="local"): + from dvc.objects.db import ODBManager + + cloud = make_cloud(typ) # pylint: disable=W0621 + + tmp_dir.add_remote(name=name, config=cloud.config, default=False) + tmp_dir.add_remote( + name=f"{name}-cache", url="remote://workspace/cache", default=False + ) + + scheme = getattr(cloud, "scheme", "local") + if scheme != "http": + with dvc.config.edit() as conf: + conf["cache"][scheme] = f"{name}-cache" + + dvc.odb = ODBManager(dvc) + + return cloud + + return _make_workspace + + +@pytest.fixture +def workspace(make_workspace, request): + if not hasattr(request, "param"): + pytest.skip("no workspace specified") + + return make_workspace("workspace", typ=request.param) + + +@pytest.fixture(scope="session") +def docker(): + # See https://travis-ci.community/t/docker-linux-containers-on-windows/301 + if os.environ.get("CI") and os.name == "nt": + pytest.skip("disabled for Windows on Github Actions") + + try: + subprocess.check_output("docker ps", shell=True) + except (subprocess.CalledProcessError, OSError): + pytest.skip("no docker installed") + + +@pytest.fixture(scope="session") +def docker_compose(docker): + try: + subprocess.check_output("docker-compose version", shell=True) + except (subprocess.CalledProcessError, OSError): + pytest.skip("no docker-compose installed") + + +@pytest.fixture(scope="session") +def docker_compose_project_name(): + return "pytest-dvc-test" + + +@pytest.fixture(scope="session") +def docker_services( + docker_compose_file, docker_compose_project_name, tmp_path_factory +): + # overriding `docker_services` fixture from `pytest_docker` plugin to + # only launch docker images once. + + from filelock import FileLock + from pytest_docker.plugin import DockerComposeExecutor, Services + + executor = DockerComposeExecutor( + docker_compose_file, docker_compose_project_name + ) + + # making sure we don't accidentally launch docker-compose in parallel, + # as it might result in network conflicts. Inspired by: + # https://github.com/pytest-dev/pytest-xdist#making-session-scoped-fixtures-execute-only-once + lockfile = tmp_path_factory.getbasetemp().parent / "docker-compose.lock" + with FileLock(str(lockfile)): # pylint:disable=abstract-class-instantiated + executor.execute("up --build -d") + + return Services(executor) diff --git a/tests/remotes/path_info.py b/dvc/testing/path_info.py similarity index 100% rename from tests/remotes/path_info.py rename to dvc/testing/path_info.py diff --git a/dvc/testing/test_api.py b/dvc/testing/test_api.py new file mode 100644 index 000000000..c842b5c17 --- /dev/null +++ b/dvc/testing/test_api.py @@ -0,0 +1,20 @@ +from dvc import api +from dvc.utils.fs import remove + + +class TestAPI: + def test_get_url(self, tmp_dir, dvc, remote): + tmp_dir.dvc_gen("foo", "foo") + + expected_url = (remote / "ac/bd18db4cc2f85cedef654fccc4a4d8").url + assert api.get_url("foo") == expected_url + + def test_open(self, tmp_dir, dvc, remote): + tmp_dir.dvc_gen("foo", "foo-text") + dvc.push() + + # Remove cache to force download + remove(dvc.odb.local.cache_dir) + + with api.open("foo") as fd: + assert fd.read() == "foo-text" diff --git a/dvc/testing/test_remote.py b/dvc/testing/test_remote.py new file mode 100644 index 000000000..cb311b7e3 --- /dev/null +++ b/dvc/testing/test_remote.py @@ -0,0 +1,92 @@ +import os + +from dvc.utils.fs import move, remove + + +class TestRemote: + def test(self, tmp_dir, dvc, remote): # pylint: disable=W0613 + (stage,) = tmp_dir.dvc_gen("foo", "foo") + out = stage.outs[0] + cache = out.cache_path + foo_hash = out.hash_info + foo_hashes = out.get_used_objs().get(None, set()) + + (stage_dir,) = tmp_dir.dvc_gen( + { + "data_dir": { + "data_sub_dir": {"data_sub": "data_sub"}, + "data": "data", + "empty": "", + } + } + ) + + out_dir = stage_dir.outs[0] + cache_dir = out_dir.cache_path + dir_hash = out_dir.hash_info + dir_hashes = {dir_hash} | {oid for _, _, oid in out_dir.obj} + + def _check_status(status, **kwargs): + for key in ("ok", "missing", "new", "deleted"): + expected = kwargs.get(key, set()) + assert expected == set(getattr(status, key)) + + # Check status + status = dvc.cloud.status(foo_hashes) + _check_status(status, new={foo_hash}) + + status_dir = dvc.cloud.status(dir_hashes) + _check_status(status_dir, new=dir_hashes) + + # Move cache and check status + # See issue https://github.com/iterative/dvc/issues/4383 for details + backup_dir = dvc.odb.local.cache_dir + ".backup" + move(dvc.odb.local.cache_dir, backup_dir) + status = dvc.cloud.status(foo_hashes) + _check_status(status, missing={foo_hash}) + + status_dir = dvc.cloud.status(dir_hashes) + _check_status(status_dir, missing=dir_hashes) + + # Restore original cache: + remove(dvc.odb.local.cache_dir) + move(backup_dir, dvc.odb.local.cache_dir) + + # Push and check status + dvc.cloud.push(foo_hashes) + assert os.path.exists(cache) + assert os.path.isfile(cache) + + dvc.cloud.push(dir_hashes) + assert os.path.isfile(cache_dir) + + status = dvc.cloud.status(foo_hashes) + _check_status(status, ok={foo_hash}) + + status_dir = dvc.cloud.status(dir_hashes) + _check_status(status_dir, ok=dir_hashes) + + # Remove and check status + remove(dvc.odb.local.cache_dir) + + status = dvc.cloud.status(foo_hashes) + _check_status(status, deleted={foo_hash}) + + status_dir = dvc.cloud.status(dir_hashes) + _check_status(status_dir, deleted=dir_hashes) + + # Pull and check status + dvc.cloud.pull(foo_hashes) + assert os.path.exists(cache) + assert os.path.isfile(cache) + with open(cache, encoding="utf-8") as fd: + assert fd.read() == "foo" + + dvc.cloud.pull(dir_hashes) + assert os.path.isfile(cache_dir) + + status = dvc.cloud.status(foo_hashes) + _check_status(status, ok={foo_hash}) + + status_dir = dvc.cloud.status(dir_hashes) + _check_status(status_dir, ok=dir_hashes) diff --git a/dvc/testing/test_workspace.py b/dvc/testing/test_workspace.py new file mode 100644 index 000000000..d555fbaf9 --- /dev/null +++ b/dvc/testing/test_workspace.py @@ -0,0 +1,112 @@ +import os + +import pytest + + +class TestImport: + def test_import(self, tmp_dir, dvc, workspace): + workspace.gen("file", "file") + assert not (tmp_dir / "file").exists() # sanity check + dvc.imp_url("remote://workspace/file") + assert (tmp_dir / "file").read_text() == "file" + assert dvc.status() == {} + + def test_import_dir(self, tmp_dir, dvc, workspace, stage_md5, dir_md5): + from dvc.objects.db import ODBManager + + workspace.gen( + {"dir": {"file": "file", "subdir": {"subfile": "subfile"}}} + ) + + # remove external cache to make sure that we don't need it + # to import dirs + with dvc.config.edit() as conf: + del conf["cache"] + dvc.odb = ODBManager(dvc) + + assert not (tmp_dir / "dir").exists() # sanity check + dvc.imp_url("remote://workspace/dir") + assert set(os.listdir(tmp_dir / "dir")) == {"file", "subdir"} + assert (tmp_dir / "dir" / "file").read_text() == "file" + assert list(os.listdir(tmp_dir / "dir" / "subdir")) == ["subfile"] + assert ( + tmp_dir / "dir" / "subdir" / "subfile" + ).read_text() == "subfile" + + assert dvc.status() == {} + + if stage_md5 is not None and dir_md5 is not None: + assert (tmp_dir / "dir.dvc").read_text() == ( + f"md5: {stage_md5}\n" + "frozen: true\n" + "deps:\n" + f"- md5: {dir_md5}\n" + " size: 11\n" + " nfiles: 2\n" + " path: remote://workspace/dir\n" + "outs:\n" + "- md5: b6dcab6ccd17ca0a8bf4a215a37d14cc.dir\n" + " size: 11\n" + " nfiles: 2\n" + " path: dir\n" + ) + + def test_import_empty_dir( + self, tmp_dir, dvc, workspace, is_object_storage + ): + # prefix based storage services (e.g s3) doesn't have the real concept + # of directories. So instead we create an empty file that ends with a + # trailing slash in order to actually support this operation + if is_object_storage: + contents = "" + else: + contents = {} + + workspace.gen({"empty_dir/": contents}) + + dvc.imp_url("remote://workspace/empty_dir/") + + empty_dir = tmp_dir / "empty_dir" + assert empty_dir.is_dir() + assert tuple(empty_dir.iterdir()) == () + + +class TestAdd: + def test_add(self, tmp_dir, dvc, workspace, hash_name, hash_value): + from dvc.stage.exceptions import StageExternalOutputsError + + workspace.gen("file", "file") + + with pytest.raises(StageExternalOutputsError): + dvc.add(workspace.url) + + dvc.add("remote://workspace/file") + assert (tmp_dir / "file.dvc").read_text() == ( + "outs:\n" + f"- {hash_name}: {hash_value}\n" + " size: 4\n" + " path: remote://workspace/file\n" + ) + assert (workspace / "file").read_text() == "file" + assert ( + workspace / "cache" / hash_value[:2] / hash_value[2:] + ).read_text() == "file" + + assert dvc.status() == {} + + def test_add_dir(self, tmp_dir, dvc, workspace, hash_name, dir_hash_value): + workspace.gen( + {"dir": {"file": "file", "subdir": {"subfile": "subfile"}}} + ) + + dvc.add("remote://workspace/dir") + assert (tmp_dir / "dir.dvc").read_text() == ( + "outs:\n" + f"- {hash_name}: {dir_hash_value}\n" + " size: 11\n" + " nfiles: 2\n" + " path: remote://workspace/dir\n" + ) + assert ( + workspace / "cache" / dir_hash_value[:2] / dir_hash_value[2:] + ).is_file() diff --git a/dvc/testing/tmp_dir.py b/dvc/testing/tmp_dir.py new file mode 100644 index 000000000..4cfcbd926 --- /dev/null +++ b/dvc/testing/tmp_dir.py @@ -0,0 +1,288 @@ +""" +The goal of this module is making dvc functional tests setup a breeze. This +includes a temporary dir, initializing git and DVC repos and bootstrapping some +file structure. + +The cornerstone of these fixtures is `tmp_dir`, which creates a temporary dir +and changes path to it, it might be combined with `scm` and `dvc` to initialize +empty git and DVC repos. `tmp_dir` returns a Path instance, which should save +you from using `open()`, `os` and `os.path` utils many times: + + (tmp_dir / "some_file").write_text("some text") + # ... + assert "some text" == (tmp_dir / "some_file").read_text() + assert (tmp_dir / "some_file").exists() + +Additionally it provides `.gen()`, `.scm_gen()` and `.dvc_gen()` methods to +bootstrap a required file structure in a single call: + + # Generate a dir with files + tmp_dir.gen({"dir": {"file": "file text", "second_file": "..."}}) + + # Generate a single file, dirs will be created along the way + tmp_dir.gen("dir/file", "file text") + + # Generate + git add + tmp_dir.scm_gen({"file1": "...", ...}) + + # Generate + git add + git commit + tmp_dir.scm_gen({"file1": "...", ...}, commit="add files") + + # Generate + dvc add + tmp_dir.dvc_gen({"file1": "...", ...}) + + # Generate + dvc add + git commit -am "..." + # This commits stages to git not the generated files. + tmp_dir.dvc_gen({"file1": "...", ...}, commit="add files") + +Making it easier to bootstrap things has a supergoal of incentivizing a move +from global repo template to creating everything inplace, which: + + - makes all path references local to test, enhancing readability + - allows using telling filenames, e.g. "git_tracked_file" instead of "foo" + - does not create unnecessary files +""" + +# pylint: disable=redefined-outer-name, attribute-defined-outside-init + +import os +import pathlib +import sys +from contextlib import contextmanager +from functools import partialmethod + +from funcy import lmap, retry + +from dvc.utils import serialize +from dvc.utils.fs import makedirs + + +class TmpDir(pathlib.Path): + scheme = "local" + + @property + def fs_path(self): + return str(self) + + @property + def url(self): + return self.fs_path + + @property + def config(self): + return {"url": self.url} + + def __new__(cls, *args, **kwargs): + if cls is TmpDir: + cls = ( # pylint: disable=self-cls-assignment + WindowsTmpDir if os.name == "nt" else PosixTmpDir + ) + # init parameter and `_init` method has been removed in Python 3.10. + kw = {"init": False} if sys.version_info < (3, 10) else {} + self = cls._from_parts( # pylint: disable=unexpected-keyword-arg + args, **kw + ) + if not self._flavour.is_supported: + raise NotImplementedError( + f"cannot instantiate {cls.__name__!r} on your system" + ) + if sys.version_info < (3, 10): + self._init() # pylint: disable=no-member + return self + + def init(self, *, scm=False, dvc=False, subdir=False): + from scmrepo.git import Git + + from dvc.repo import Repo + + assert not scm or not hasattr(self, "scm") + assert not dvc or not hasattr(self, "dvc") + + str_path = os.fspath(self) + + if scm: + git_init(str_path) + if dvc: + self.dvc = Repo.init( + str_path, + no_scm=not scm and not hasattr(self, "scm"), + subdir=subdir, + ) + if scm: + self.scm = self.dvc.scm if hasattr(self, "dvc") else Git(str_path) + if dvc and hasattr(self, "scm"): + self.scm.commit("init dvc") + + def close(self): + if hasattr(self, "scm"): + self.scm.close() + if hasattr(self, "dvc"): + self.dvc.close() + + def _require(self, name): + if not hasattr(self, name): + raise TypeError( + "Can't use {name} for this temporary dir. " + 'Did you forget to use "{name}" fixture?'.format(name=name) + ) + + # Bootstrapping methods + def gen(self, struct, text=""): + if isinstance(struct, (str, bytes, pathlib.PurePath)): + struct = {struct: text} + + return self._gen(struct) + + def _gen(self, struct, prefix=None): + paths = [] + for name, contents in struct.items(): + path = (prefix or self) / name + + if isinstance(contents, dict): + if not contents: + makedirs(path, exist_ok=True) + else: + self._gen(contents, prefix=path) + else: + makedirs(path.parent, exist_ok=True) + if isinstance(contents, bytes): + path.write_bytes(contents) + else: + path.write_text(contents, encoding="utf-8") + paths.append(path) + return paths + + def dvc_gen(self, struct, text="", commit=None): + paths = self.gen(struct, text) + return self.dvc_add(paths, commit=commit) + + def scm_gen(self, struct, text="", commit=None): + paths = self.gen(struct, text) + return self.scm_add(paths, commit=commit) + + def commit(self, output_paths, msg): + def to_gitignore(stage_path): + from scmrepo.git import Git + + return os.path.join(os.path.dirname(stage_path), Git.GITIGNORE) + + gitignores = [ + to_gitignore(s) + for s in output_paths + if os.path.exists(to_gitignore(s)) + ] + return self.scm_add(output_paths + gitignores, commit=msg) + + def dvc_add(self, filenames, commit=None): + self._require("dvc") + filenames = _coerce_filenames(filenames) + + stages = self.dvc.add(filenames) + if commit: + self.commit([s.path for s in stages], msg=commit) + return stages + + def scm_add(self, filenames, commit=None): + self._require("scm") + filenames = _coerce_filenames(filenames) + self.scm.add(filenames) + if commit: + self.scm.commit(commit) + + def add_remote( + self, *, url=None, config=None, name="upstream", default=True + ): + self._require("dvc") + + assert bool(url) ^ bool(config) + + if url: + config = {"url": url} + + with self.dvc.config.edit() as conf: + conf["remote"][name] = config + if default: + conf["core"]["remote"] = name + + if hasattr(self, "scm"): + self.scm.add(self.dvc.config.files["repo"]) + self.scm.commit(f"add '{name}' remote") + + return url or config["url"] + + # contexts + @contextmanager + def chdir(self): + old = os.getcwd() + try: + os.chdir(self) + yield + finally: + os.chdir(old) + + @contextmanager + def branch(self, name, new=False): + self._require("scm") + old = self.scm.active_branch() + try: + self.scm.checkout(name, create_new=new) + yield + finally: + self.scm.checkout(old) + + def read_text(self, *args, **kwargs): # pylint: disable=signature-differs + # NOTE: on windows we'll get PermissionError instead of + # IsADirectoryError when we try to `open` a directory, so we can't + # rely on exception flow control + if self.is_dir(): + return { + path.name: path.read_text(*args, **kwargs) + for path in self.iterdir() + } + return super().read_text(*args, encoding="utf-8", **kwargs) + + def hash_to_path(self, hash_): + return str(self / hash_[0:2] / hash_[2:]) + + def dump(self, *args, **kwargs): + return serialize.DUMPERS[self.suffix](self, *args, **kwargs) + + def parse(self, *args, **kwargs): + return serialize.LOADERS[self.suffix](self, *args, **kwargs) + + def modify(self, *args, **kwargs): + return serialize.MODIFIERS[self.suffix](self, *args, **kwargs) + + load_yaml = partialmethod(serialize.load_yaml) + dump_yaml = partialmethod(serialize.dump_yaml) + load_json = partialmethod(serialize.load_json) + dump_json = partialmethod(serialize.dump_json) + load_toml = partialmethod(serialize.load_toml) + dump_toml = partialmethod(serialize.dump_toml) + + +def _coerce_filenames(filenames): + if isinstance(filenames, (str, bytes, pathlib.PurePath)): + filenames = [filenames] + return lmap(os.fspath, filenames) + + +class WindowsTmpDir(TmpDir, pathlib.PureWindowsPath): + pass + + +class PosixTmpDir(TmpDir, pathlib.PurePosixPath): + pass + + +def git_init(path): + from git import Repo + from git.exc import GitCommandNotFound + + # NOTE: handles EAGAIN error on BSD systems (osx in our case). + # Otherwise when running tests you might get this exception: + # + # GitCommandNotFound: Cmd('git') not found due to: + # OSError('[Errno 35] Resource temporarily unavailable') + git = retry(5, GitCommandNotFound)(Repo.init)(path) + git.close() diff --git a/setup.cfg b/setup.cfg index 41a89e169..b071a9edb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -158,6 +158,8 @@ tests = [options.packages.find] exclude = + dvc.testing + dvc.testing.* tests tests.* diff --git a/tests/conftest.py b/tests/conftest.py index c08c370bc..f3ce802cb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,6 +3,8 @@ from contextlib import suppress import pytest +from dvc.testing.fixtures import * # noqa, pylint: disable=wildcard-import + from .dir_helpers import * # noqa, pylint: disable=wildcard-import from .remotes import * # noqa, pylint: disable=wildcard-import from .utils.scriptify import scriptify diff --git a/tests/dir_helpers.py b/tests/dir_helpers.py index 0a3873365..b1da1b3d4 100644 --- a/tests/dir_helpers.py +++ b/tests/dir_helpers.py @@ -47,23 +47,14 @@ from global repo template to creating everything inplace, which: import os import pathlib -import sys -from contextlib import contextmanager -from functools import partialmethod from textwrap import dedent import pytest -from funcy import lmap, retry from dvc.logger import disable_other_loggers -from dvc.utils import serialize -from dvc.utils.fs import makedirs +from dvc.testing.tmp_dir import git_init __all__ = [ - "make_tmp_dir", - "tmp_dir", - "scm", - "dvc", "run_copy", "run_head", "erepo_dir", @@ -78,299 +69,6 @@ __all__ = [ disable_other_loggers() -class TmpDir(pathlib.Path): - scheme = "local" - - @property - def fs_path(self): - return str(self) - - def __new__(cls, *args, **kwargs): - if cls is TmpDir: - cls = ( # pylint: disable=self-cls-assignment - WindowsTmpDir if os.name == "nt" else PosixTmpDir - ) - # init parameter and `_init` method has been removed in Python 3.10. - kw = {"init": False} if sys.version_info < (3, 10) else {} - self = cls._from_parts( # pylint: disable=unexpected-keyword-arg - args, **kw - ) - if not self._flavour.is_supported: - raise NotImplementedError( - f"cannot instantiate {cls.__name__!r} on your system" - ) - if sys.version_info < (3, 10): - self._init() # pylint: disable=no-member - return self - - def init(self, *, scm=False, dvc=False, subdir=False): - from scmrepo.git import Git - - from dvc.repo import Repo - - assert not scm or not hasattr(self, "scm") - assert not dvc or not hasattr(self, "dvc") - - str_path = os.fspath(self) - - if scm: - git_init(str_path) - if dvc: - self.dvc = Repo.init( - str_path, - no_scm=not scm and not hasattr(self, "scm"), - subdir=subdir, - ) - if scm: - self.scm = self.dvc.scm if hasattr(self, "dvc") else Git(str_path) - if dvc and hasattr(self, "scm"): - self.scm.commit("init dvc") - - def close(self): - if hasattr(self, "scm"): - self.scm.close() - if hasattr(self, "dvc"): - self.dvc.close() - - def _require(self, name): - if not hasattr(self, name): - raise TypeError( - "Can't use {name} for this temporary dir. " - 'Did you forget to use "{name}" fixture?'.format(name=name) - ) - - # Bootstrapping methods - def gen(self, struct, text=""): - if isinstance(struct, (str, bytes, pathlib.PurePath)): - struct = {struct: text} - - return self._gen(struct) - - def _gen(self, struct, prefix=None): - paths = [] - for name, contents in struct.items(): - path = (prefix or self) / name - - if isinstance(contents, dict): - if not contents: - makedirs(path, exist_ok=True) - else: - self._gen(contents, prefix=path) - else: - makedirs(path.parent, exist_ok=True) - if isinstance(contents, bytes): - path.write_bytes(contents) - else: - path.write_text(contents, encoding="utf-8") - paths.append(path) - return paths - - def dvc_gen(self, struct, text="", commit=None): - paths = self.gen(struct, text) - return self.dvc_add(paths, commit=commit) - - def scm_gen(self, struct, text="", commit=None): - paths = self.gen(struct, text) - return self.scm_add(paths, commit=commit) - - def commit(self, output_paths, msg): - def to_gitignore(stage_path): - from dvc.scm import Git - - return os.path.join(os.path.dirname(stage_path), Git.GITIGNORE) - - gitignores = [ - to_gitignore(s) - for s in output_paths - if os.path.exists(to_gitignore(s)) - ] - return self.scm_add(output_paths + gitignores, commit=msg) - - def dvc_add(self, filenames, commit=None): - self._require("dvc") - filenames = _coerce_filenames(filenames) - - stages = self.dvc.add(filenames) - if commit: - self.commit([s.path for s in stages], msg=commit) - return stages - - def scm_add(self, filenames, commit=None): - self._require("scm") - filenames = _coerce_filenames(filenames) - self.scm.add(filenames) - if commit: - self.scm.commit(commit) - - def add_remote( - self, *, url=None, config=None, name="upstream", default=True - ): - self._require("dvc") - - assert bool(url) ^ bool(config) - - if url: - config = {"url": url} - - with self.dvc.config.edit() as conf: - conf["remote"][name] = config - if default: - conf["core"]["remote"] = name - - if hasattr(self, "scm"): - self.scm.add(self.dvc.config.files["repo"]) - self.scm.commit(f"add '{name}' remote") - - return url or config["url"] - - # contexts - @contextmanager - def chdir(self): - old = os.getcwd() - try: - os.chdir(self) - yield - finally: - os.chdir(old) - - @contextmanager - def branch(self, name, new=False): - self._require("scm") - old = self.scm.active_branch() - try: - self.scm.checkout(name, create_new=new) - yield - finally: - self.scm.checkout(old) - - def read_text(self, *args, **kwargs): # pylint: disable=signature-differs - # NOTE: on windows we'll get PermissionError instead of - # IsADirectoryError when we try to `open` a directory, so we can't - # rely on exception flow control - if self.is_dir(): - return { - path.name: path.read_text(*args, **kwargs) - for path in self.iterdir() - } - return super().read_text(*args, encoding="utf-8", **kwargs) - - def hash_to_path(self, hash_): - return str(self / hash_[0:2] / hash_[2:]) - - def dump(self, *args, **kwargs): - return serialize.DUMPERS[self.suffix](self, *args, **kwargs) - - def parse(self, *args, **kwargs): - return serialize.LOADERS[self.suffix](self, *args, **kwargs) - - def modify(self, *args, **kwargs): - return serialize.MODIFIERS[self.suffix](self, *args, **kwargs) - - load_yaml = partialmethod(serialize.load_yaml) - dump_yaml = partialmethod(serialize.dump_yaml) - load_json = partialmethod(serialize.load_json) - dump_json = partialmethod(serialize.dump_json) - load_toml = partialmethod(serialize.load_toml) - dump_toml = partialmethod(serialize.dump_toml) - - -def _coerce_filenames(filenames): - if isinstance(filenames, (str, bytes, pathlib.PurePath)): - filenames = [filenames] - return lmap(os.fspath, filenames) - - -class WindowsTmpDir(TmpDir, pathlib.PureWindowsPath): - pass - - -class PosixTmpDir(TmpDir, pathlib.PurePosixPath): - pass - - -CACHE = {} - - -def _fs_copy(src, dst, ignore=None): - import shutil - - if os.path.isdir(src): - shutil.copytree(src, dst, ignore=ignore) - else: - shutil.copy2(src, dst) - - -@pytest.fixture(scope="session") -def make_tmp_dir(tmp_path_factory, request, worker_id): - def make(name, *, scm=False, dvc=False, subdir=False): - from shutil import ignore_patterns - - from scmrepo.git import Git - - from dvc.repo import Repo - - cache = CACHE.get((scm, dvc, subdir)) - if not cache: - cache = tmp_path_factory.mktemp("dvc-test-cache" + worker_id) - TmpDir(cache).init(scm=scm, dvc=dvc, subdir=subdir) - CACHE[(scm, dvc, subdir)] = os.fspath(cache) - path = tmp_path_factory.mktemp(name) if isinstance(name, str) else name - - # ignore sqlite files from .dvc/tmp. We might not be closing the cache - # connection resulting in PermissionErrors in Windows. - ignore = ignore_patterns("cache.db*") - for entry in os.listdir(cache): - # shutil.copytree's dirs_exist_ok is only available in >=3.8 - _fs_copy( - os.path.join(cache, entry), - os.path.join(path, entry), - ignore=ignore, - ) - new_dir = TmpDir(path) - str_path = os.fspath(new_dir) - if dvc: - new_dir.dvc = Repo(str_path) - if scm: - new_dir.scm = ( - new_dir.dvc.scm if hasattr(new_dir, "dvc") else Git(str_path) - ) - request.addfinalizer(new_dir.close) - return new_dir - - return make - - -@pytest.fixture -def tmp_dir(tmp_path, make_tmp_dir, request, monkeypatch): - monkeypatch.chdir(tmp_path) - fixtures = request.fixturenames - return make_tmp_dir(tmp_path, scm="scm" in fixtures, dvc="dvc" in fixtures) - - -@pytest.fixture -def scm(tmp_dir): - return tmp_dir.scm - - -@pytest.fixture -def dvc(tmp_dir): - with tmp_dir.dvc as _dvc: - yield _dvc - - -def git_init(path): - from git import Repo - from git.exc import GitCommandNotFound - - # NOTE: handles EAGAIN error on BSD systems (osx in our case). - # Otherwise when running tests you might get this exception: - # - # GitCommandNotFound: Cmd('git') not found due to: - # OSError('[Errno 35] Resource temporarily unavailable') - git = retry(5, GitCommandNotFound)(Repo.init)(path) - git.close() - - @pytest.fixture def run_copy(tmp_dir, dvc): tmp_dir.gen( @@ -443,14 +141,19 @@ def git_dir(make_tmp_dir): return path +class GitRemote: + def __init__(self, tmp_dir, name, url): + self.tmp_dir = tmp_dir + self.remote = name + self.url = url + + @pytest.fixture def git_upstream(tmp_dir, erepo_dir, git_dir, request): remote = erepo_dir if "dvc" in request.fixturenames else git_dir url = "file://{}".format(remote.resolve().as_posix()) tmp_dir.scm.gitpython.repo.create_remote("upstream", url) - remote.remote = "upstream" - remote.url = url - return remote + return GitRemote(remote, "upstream", url) @pytest.fixture @@ -458,6 +161,4 @@ def git_downstream(tmp_dir, erepo_dir, git_dir, request): remote = erepo_dir if "dvc" in request.fixturenames else git_dir url = "file://{}".format(tmp_dir.resolve().as_posix()) remote.scm.gitpython.repo.create_remote("upstream", url) - remote.remote = "upstream" - remote.url = url - return remote + return GitRemote(remote, "upstream", url) diff --git a/tests/func/experiments/test_checkpoints.py b/tests/func/experiments/test_checkpoints.py index e82d35749..3535a1e72 100644 --- a/tests/func/experiments/test_checkpoints.py +++ b/tests/func/experiments/test_checkpoints.py @@ -219,7 +219,7 @@ def test_auto_push_during_iterations( ): # set up remote repo remote = git_upstream.url if use_url else git_upstream.remote - git_upstream.scm.fetch_refspecs(str(tmp_dir), ["master:master"]) + git_upstream.tmp_dir.scm.fetch_refspecs(str(tmp_dir), ["master:master"]) monkeypatch.setenv(DVC_EXP_GIT_REMOTE, remote) auto_push_spy = mocker.spy(BaseExecutor, "_auto_push") @@ -233,7 +233,7 @@ def test_auto_push_during_iterations( assert (tmp_dir / "foo").read_text() == "4" exp = first(results) ref_info = first(exp_refs_by_rev(scm, exp)) - assert git_upstream.scm.get_ref(str(ref_info)) == exp + assert git_upstream.tmp_dir.scm.get_ref(str(ref_info)) == exp assert auto_push_spy.call_count == 2 assert auto_push_spy.call_args[0][2] == remote diff --git a/tests/func/experiments/test_remote.py b/tests/func/experiments/test_remote.py index e988b3eb3..47bce63e8 100644 --- a/tests/func/experiments/test_remote.py +++ b/tests/func/experiments/test_remote.py @@ -20,30 +20,30 @@ def test_push(tmp_dir, scm, dvc, git_upstream, exp_stage, use_url): ref_info = first(exp_refs_by_rev(scm, exp)) dvc.experiments.push(remote, ref_info.name) - assert git_upstream.scm.get_ref(str(ref_info)) == exp + assert git_upstream.tmp_dir.scm.get_ref(str(ref_info)) == exp - git_upstream.scm.remove_ref(str(ref_info)) + git_upstream.tmp_dir.scm.remove_ref(str(ref_info)) dvc.experiments.push(remote, str(ref_info)) - assert git_upstream.scm.get_ref(str(ref_info)) == exp + assert git_upstream.tmp_dir.scm.get_ref(str(ref_info)) == exp def test_push_diverged(tmp_dir, scm, dvc, git_upstream, exp_stage): - git_upstream.scm_gen("foo", "foo", commit="init") - remote_rev = git_upstream.scm.get_rev() + git_upstream.tmp_dir.scm_gen("foo", "foo", commit="init") + remote_rev = git_upstream.tmp_dir.scm.get_rev() results = dvc.experiments.run(exp_stage.addressing, params=["foo=2"]) exp = first(results) ref_info = first(exp_refs_by_rev(scm, exp)) - git_upstream.scm.set_ref(str(ref_info), remote_rev) + git_upstream.tmp_dir.scm.set_ref(str(ref_info), remote_rev) with pytest.raises(DvcException): dvc.experiments.push(git_upstream.remote, ref_info.name) - assert git_upstream.scm.get_ref(str(ref_info)) == remote_rev + assert git_upstream.tmp_dir.scm.get_ref(str(ref_info)) == remote_rev dvc.experiments.push(git_upstream.remote, ref_info.name, force=True) - assert git_upstream.scm.get_ref(str(ref_info)) == exp + assert git_upstream.tmp_dir.scm.get_ref(str(ref_info)) == exp def test_push_checkpoint(tmp_dir, scm, dvc, git_upstream, checkpoint_stage): @@ -54,7 +54,7 @@ def test_push_checkpoint(tmp_dir, scm, dvc, git_upstream, checkpoint_stage): ref_info_a = first(exp_refs_by_rev(scm, exp_a)) dvc.experiments.push(git_upstream.remote, ref_info_a.name, force=True) - assert git_upstream.scm.get_ref(str(ref_info_a)) == exp_a + assert git_upstream.tmp_dir.scm.get_ref(str(ref_info_a)) == exp_a results = dvc.experiments.run( checkpoint_stage.addressing, checkpoint_resume=exp_a @@ -65,7 +65,7 @@ def test_push_checkpoint(tmp_dir, scm, dvc, git_upstream, checkpoint_stage): tmp_dir.scm_gen("new", "new", commit="new") dvc.experiments.push(git_upstream.remote, ref_info_b.name, force=True) - assert git_upstream.scm.get_ref(str(ref_info_b)) == exp_b + assert git_upstream.tmp_dir.scm.get_ref(str(ref_info_b)) == exp_b def test_push_ambiguous_name(tmp_dir, scm, dvc, git_upstream, exp_stage): @@ -87,7 +87,7 @@ def test_push_ambiguous_name(tmp_dir, scm, dvc, git_upstream, exp_stage): ref_info_b = first(exp_refs_by_rev(scm, exp_b)) dvc.experiments.push(remote, "foo") - assert git_upstream.scm.get_ref(str(ref_info_b)) == exp_b + assert git_upstream.tmp_dir.scm.get_ref(str(ref_info_b)) == exp_b tmp_dir.scm_gen("new", "new 2", commit="new 2") @@ -95,7 +95,7 @@ def test_push_ambiguous_name(tmp_dir, scm, dvc, git_upstream, exp_stage): dvc.experiments.push(remote, "foo") dvc.experiments.push(remote, str(ref_info_a)) - assert git_upstream.scm.get_ref(str(ref_info_a)) == exp_a + assert git_upstream.tmp_dir.scm.get_ref(str(ref_info_a)) == exp_a @pytest.mark.parametrize("use_url", [True, False]) @@ -117,8 +117,8 @@ def test_list_remote(tmp_dir, scm, dvc, git_downstream, exp_stage, use_url): remote = git_downstream.url if use_url else git_downstream.remote - assert git_downstream.scm.get_ref("HEAD") != scm.get_ref("HEAD") - downstream_exp = git_downstream.dvc.experiments + assert git_downstream.tmp_dir.scm.get_ref("HEAD") != scm.get_ref("HEAD") + downstream_exp = git_downstream.tmp_dir.dvc.experiments assert downstream_exp.ls(git_remote=remote) == {} exp_list = downstream_exp.ls(rev=baseline_a, git_remote=remote) @@ -138,7 +138,7 @@ def test_pull(tmp_dir, scm, dvc, git_downstream, exp_stage, use_url): from dvc.exceptions import InvalidArgumentError remote = git_downstream.url if use_url else git_downstream.remote - downstream_exp = git_downstream.dvc.experiments + downstream_exp = git_downstream.tmp_dir.dvc.experiments with pytest.raises(InvalidArgumentError): downstream_exp.pull(remote, "foo") @@ -147,31 +147,31 @@ def test_pull(tmp_dir, scm, dvc, git_downstream, exp_stage, use_url): ref_info = first(exp_refs_by_rev(scm, exp)) downstream_exp.pull(remote, ref_info.name) - assert git_downstream.scm.get_ref(str(ref_info)) == exp + assert git_downstream.tmp_dir.scm.get_ref(str(ref_info)) == exp - git_downstream.scm.remove_ref(str(ref_info)) + git_downstream.tmp_dir.scm.remove_ref(str(ref_info)) downstream_exp.pull(remote, str(ref_info)) - assert git_downstream.scm.get_ref(str(ref_info)) == exp + assert git_downstream.tmp_dir.scm.get_ref(str(ref_info)) == exp def test_pull_diverged(tmp_dir, scm, dvc, git_downstream, exp_stage): - git_downstream.scm_gen("foo", "foo", commit="init") - remote_rev = git_downstream.scm.get_rev() + git_downstream.tmp_dir.scm_gen("foo", "foo", commit="init") + remote_rev = git_downstream.tmp_dir.scm.get_rev() results = dvc.experiments.run(exp_stage.addressing, params=["foo=2"]) exp = first(results) ref_info = first(exp_refs_by_rev(scm, exp)) - git_downstream.scm.set_ref(str(ref_info), remote_rev) + git_downstream.tmp_dir.scm.set_ref(str(ref_info), remote_rev) - downstream_exp = git_downstream.dvc.experiments + downstream_exp = git_downstream.tmp_dir.dvc.experiments with pytest.raises(DvcException): downstream_exp.pull(git_downstream.remote, ref_info.name) - assert git_downstream.scm.get_ref(str(ref_info)) == remote_rev + assert git_downstream.tmp_dir.scm.get_ref(str(ref_info)) == remote_rev downstream_exp.pull(git_downstream.remote, ref_info.name, force=True) - assert git_downstream.scm.get_ref(str(ref_info)) == exp + assert git_downstream.tmp_dir.scm.get_ref(str(ref_info)) == exp def test_pull_checkpoint(tmp_dir, scm, dvc, git_downstream, checkpoint_stage): @@ -181,9 +181,9 @@ def test_pull_checkpoint(tmp_dir, scm, dvc, git_downstream, checkpoint_stage): exp_a = first(results) ref_info_a = first(exp_refs_by_rev(scm, exp_a)) - downstream_exp = git_downstream.dvc.experiments + downstream_exp = git_downstream.tmp_dir.dvc.experiments downstream_exp.pull(git_downstream.remote, ref_info_a.name, force=True) - assert git_downstream.scm.get_ref(str(ref_info_a)) == exp_a + assert git_downstream.tmp_dir.scm.get_ref(str(ref_info_a)) == exp_a results = dvc.experiments.run( checkpoint_stage.addressing, checkpoint_resume=exp_a @@ -192,7 +192,7 @@ def test_pull_checkpoint(tmp_dir, scm, dvc, git_downstream, checkpoint_stage): ref_info_b = first(exp_refs_by_rev(scm, exp_b)) downstream_exp.pull(git_downstream.remote, ref_info_b.name, force=True) - assert git_downstream.scm.get_ref(str(ref_info_b)) == exp_b + assert git_downstream.tmp_dir.scm.get_ref(str(ref_info_b)) == exp_b def test_pull_ambiguous_name(tmp_dir, scm, dvc, git_downstream, exp_stage): @@ -212,16 +212,16 @@ def test_pull_ambiguous_name(tmp_dir, scm, dvc, git_downstream, exp_stage): ref_info_b = first(exp_refs_by_rev(scm, exp_b)) remote = git_downstream.remote - downstream_exp = git_downstream.dvc.experiments + downstream_exp = git_downstream.tmp_dir.dvc.experiments with pytest.raises(InvalidArgumentError): downstream_exp.pull(remote, "foo") downstream_exp.pull(remote, str(ref_info_b)) - assert git_downstream.scm.get_ref(str(ref_info_b)) == exp_b + assert git_downstream.tmp_dir.scm.get_ref(str(ref_info_b)) == exp_b - with git_downstream.scm.detach_head(ref_info_a.baseline_sha): + with git_downstream.tmp_dir.scm.detach_head(ref_info_a.baseline_sha): downstream_exp.pull(remote, "foo") - assert git_downstream.scm.get_ref(str(ref_info_a)) == exp_a + assert git_downstream.tmp_dir.scm.get_ref(str(ref_info_a)) == exp_a def test_push_pull_cache( diff --git a/tests/func/experiments/test_remove.py b/tests/func/experiments/test_remove.py index f240785d4..30fd7d9e9 100644 --- a/tests/func/experiments/test_remove.py +++ b/tests/func/experiments/test_remove.py @@ -107,12 +107,14 @@ def test_remove_remote(tmp_dir, scm, dvc, exp_stage, git_upstream, use_url): ref_info = first(exp_refs_by_rev(scm, exp)) ref_info_list.append(ref_info) dvc.experiments.push(remote, ref_info.name) - assert git_upstream.scm.get_ref(str(ref_info)) == exp + assert git_upstream.tmp_dir.scm.get_ref(str(ref_info)) == exp dvc.experiments.remove( remote=remote, exp_names=[str(ref_info_list[0]), ref_info_list[1].name] ) - assert git_upstream.scm.get_ref(str(ref_info_list[0])) is None - assert git_upstream.scm.get_ref(str(ref_info_list[1])) is None - assert git_upstream.scm.get_ref(str(ref_info_list[2])) == exp_list[2] + assert git_upstream.tmp_dir.scm.get_ref(str(ref_info_list[0])) is None + assert git_upstream.tmp_dir.scm.get_ref(str(ref_info_list[1])) is None + assert ( + git_upstream.tmp_dir.scm.get_ref(str(ref_info_list[2])) == exp_list[2] + ) diff --git a/tests/func/test_add.py b/tests/func/test_add.py index 76fa17b36..4e61d4f16 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -34,6 +34,7 @@ from dvc.stage.exceptions import ( StagePathNotFoundError, ) from dvc.system import System +from dvc.testing.test_workspace import TestAdd from dvc.utils import LARGE_DIR_SIZE, file_md5, relpath from dvc.utils.fs import path_isin from dvc.utils.serialize import YAMLFileCorruptedError, load_yaml @@ -291,59 +292,72 @@ def test_add_filtered_files_in_dir( assert stage.outs[0].def_path in expected_def_paths -@pytest.mark.parametrize( - "workspace, hash_name, hash_value", - [ - ( - pytest.lazy_fixture("local_cloud"), - "md5", - "8c7dd922ad47494fc02c388e12c00eac", - ), - pytest.param( - pytest.lazy_fixture("ssh"), - "md5", - "8c7dd922ad47494fc02c388e12c00eac", - marks=pytest.mark.skipif( - os.name == "nt", reason="disabled on windows" +class TestAddExternal(TestAdd): + @pytest.mark.parametrize( + "workspace, hash_name, hash_value", + [ + ( + "local", + "md5", + "8c7dd922ad47494fc02c388e12c00eac", ), - ), - ( - pytest.lazy_fixture("s3"), - "etag", - "8c7dd922ad47494fc02c388e12c00eac", - ), - ( - pytest.lazy_fixture("hdfs"), - "checksum", - "000002000000000000000000a86fe4d846edc1bf4c355cb6112f141e", - ), - ( - pytest.lazy_fixture("webhdfs"), - "checksum", - "000002000000000000000000a86fe4d846edc1bf4c355cb6112f141e00000000", - ), - ], - indirect=["workspace"], -) -def test_add_external_file(tmp_dir, dvc, workspace, hash_name, hash_value): - workspace.gen("file", "file") - - with pytest.raises(StageExternalOutputsError): - dvc.add(workspace.url) - - dvc.add("remote://workspace/file") - assert (tmp_dir / "file.dvc").read_text() == ( - "outs:\n" - f"- {hash_name}: {hash_value}\n" - " size: 4\n" - " path: remote://workspace/file\n" + pytest.param( + "ssh", + "md5", + "8c7dd922ad47494fc02c388e12c00eac", + marks=pytest.mark.skipif( + os.name == "nt", reason="disabled on windows" + ), + ), + ( + "s3", + "etag", + "8c7dd922ad47494fc02c388e12c00eac", + ), + ( + "hdfs", + "checksum", + "000002000000000000000000a86fe4d846edc1bf4c355cb6112f141e", + ), + ( + "webhdfs", + "checksum", + "000002000000000000000000a86fe4d846edc1bf4c355cb6112f141e00000000", # noqa: E501 + ), + ], + indirect=["workspace"], ) - assert (workspace / "file").read_text() == "file" - assert ( - workspace / "cache" / hash_value[:2] / hash_value[2:] - ).read_text() == "file" + def test_add(self, tmp_dir, dvc, workspace, hash_name, hash_value): + super().test_add(tmp_dir, dvc, workspace, hash_name, hash_value) - assert dvc.status() == {} + @pytest.mark.parametrize( + "workspace, hash_name, dir_hash_value", + [ + ( + "local", + "md5", + "b6dcab6ccd17ca0a8bf4a215a37d14cc.dir", + ), + pytest.param( + "ssh", + "md5", + "b6dcab6ccd17ca0a8bf4a215a37d14cc.dir", + marks=pytest.mark.skipif( + os.name == "nt", reason="disabled on windows" + ), + ), + ( + "s3", + "etag", + "ec602a6ba97b2dd07bd6d2cd89674a60.dir", + ), + ], + indirect=["workspace"], + ) + def test_add_dir(self, tmp_dir, dvc, workspace, hash_name, dir_hash_value): + super().test_add_dir( + tmp_dir, dvc, workspace, hash_name, dir_hash_value + ) def test_add_external_relpath(tmp_dir, dvc, local_cloud): @@ -364,44 +378,6 @@ def test_add_external_relpath(tmp_dir, dvc, local_cloud): assert dvc.status() == {} -@pytest.mark.parametrize( - "workspace, hash_name, hash_value", - [ - ( - pytest.lazy_fixture("local_cloud"), - "md5", - "b6dcab6ccd17ca0a8bf4a215a37d14cc.dir", - ), - pytest.param( - pytest.lazy_fixture("ssh"), - "md5", - "b6dcab6ccd17ca0a8bf4a215a37d14cc.dir", - marks=pytest.mark.skipif( - os.name == "nt", reason="disabled on windows" - ), - ), - ( - pytest.lazy_fixture("s3"), - "etag", - "ec602a6ba97b2dd07bd6d2cd89674a60.dir", - ), - ], - indirect=["workspace"], -) -def test_add_external_dir(tmp_dir, dvc, workspace, hash_name, hash_value): - workspace.gen({"dir": {"file": "file", "subdir": {"subfile": "subfile"}}}) - - dvc.add("remote://workspace/dir") - assert (tmp_dir / "dir.dvc").read_text() == ( - "outs:\n" - f"- {hash_name}: {hash_value}\n" - " size: 11\n" - " nfiles: 2\n" - " path: remote://workspace/dir\n" - ) - assert (workspace / "cache" / hash_value[:2] / hash_value[2:]).is_file() - - class TestAddLocalRemoteFile(TestDvc): def test(self): """ @@ -1159,19 +1135,17 @@ def test_add_to_cache_invalid_combinations(dvc, invalid_opt, kwargs): @pytest.mark.parametrize( "workspace", [ - pytest.lazy_fixture("local_cloud"), - pytest.lazy_fixture("s3"), + "local", + "s3", + pytest.param("gs", marks=pytest.mark.needs_internet), + "hdfs", pytest.param( - pytest.lazy_fixture("gs"), marks=pytest.mark.needs_internet - ), - pytest.lazy_fixture("hdfs"), - pytest.param( - pytest.lazy_fixture("ssh"), + "ssh", marks=pytest.mark.skipif( os.name == "nt", reason="disabled on windows" ), ), - pytest.lazy_fixture("http"), + "http", ], indirect=True, ) diff --git a/tests/func/test_api.py b/tests/func/test_api.py index c0c6b7194..6ab103044 100644 --- a/tests/func/test_api.py +++ b/tests/func/test_api.py @@ -5,10 +5,11 @@ from funcy import first, get_in from dvc import api from dvc.exceptions import FileMissingError, OutputNotFoundError +from dvc.testing.test_api import TestAPI as _TestAPI from dvc.utils.fs import remove from tests.unit.fs.test_repo import make_subrepo -cloud_names = [ +clouds = [ "s3", "gs", "azure", @@ -20,23 +21,19 @@ cloud_names = [ "webdav", "webhdfs", ] -clouds = [pytest.lazy_fixture(cloud) for cloud in cloud_names] -all_clouds = [pytest.lazy_fixture("local_cloud")] + clouds +all_clouds = ["local"] + clouds # `lazy_fixture` is confusing pylint, pylint: disable=unused-argument @pytest.mark.needs_internet @pytest.mark.parametrize("remote", clouds, indirect=True) -def test_get_url(tmp_dir, dvc, remote): - tmp_dir.dvc_gen("foo", "foo") - - expected_url = (remote / "ac/bd18db4cc2f85cedef654fccc4a4d8").url - assert api.get_url("foo") == expected_url +class TestAPI(_TestAPI): + pass @pytest.mark.needs_internet -@pytest.mark.parametrize("cloud", clouds) +@pytest.mark.parametrize("cloud", clouds, indirect=True) def test_get_url_external(tmp_dir, erepo_dir, cloud): erepo_dir.add_remote(config=cloud.config) with erepo_dir.chdir(): @@ -58,36 +55,21 @@ def test_get_url_requires_dvc(tmp_dir, scm): api.get_url("foo", repo=f"file://{tmp_dir}") -@pytest.mark.needs_internet -@pytest.mark.parametrize("remote", all_clouds, indirect=True) -def test_open(tmp_dir, dvc, remote): - tmp_dir.dvc_gen("foo", "foo-text") - dvc.push() - - # Remove cache to force download - remove(dvc.odb.local.cache_dir) - - with api.open("foo") as fd: - assert fd.read() == "foo-text" - - @pytest.mark.needs_internet @pytest.mark.parametrize( "cloud", [ - pytest.lazy_fixture(cloud) - for cloud in [ - "s3", - "gs", - "azure", - "gdrive", - "oss", - "http", - "hdfs", - "ssh", - "webdav", - ] + "s3", + "gs", + "azure", + "gdrive", + "oss", + "http", + "hdfs", + "ssh", + "webdav", ], + indirect=True, ) def test_open_external(tmp_dir, erepo_dir, cloud): erepo_dir.add_remote(config=cloud.config) @@ -129,18 +111,15 @@ def test_open_granular(tmp_dir, dvc, remote): @pytest.mark.parametrize( "remote", [ - pytest.lazy_fixture(cloud) - for cloud in [ - "s3", - "gs", - "azure", - "gdrive", - "oss", - "ssh", - "http", - "hdfs", - "webdav", - ] + "s3", + "gs", + "azure", + "gdrive", + "oss", + "ssh", + "http", + "hdfs", + "webdav", ], indirect=True, ) @@ -195,7 +174,7 @@ def test_open_rev(tmp_dir, scm, dvc): @pytest.mark.parametrize("as_external", [True, False]) -@pytest.mark.parametrize("remote", [pytest.lazy_fixture("ssh")], indirect=True) +@pytest.mark.parametrize("remote", ["ssh"], indirect=True) @pytest.mark.parametrize( "files, to_read", [ diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index 94b0e17fa..39bca1e9e 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -9,120 +9,31 @@ import dvc as dvc_module from dvc.external_repo import clean_repos from dvc.main import main from dvc.stage.exceptions import StageNotFound -from dvc.utils.fs import move, remove +from dvc.testing.test_remote import TestRemote as _TestRemote +from dvc.utils.fs import remove all_clouds = [ - pytest.lazy_fixture(cloud) - for cloud in [ - "s3", - "gs", - "azure", - "ssh", - "http", - "hdfs", - "webdav", - "webhdfs", - "oss", - "gdrive", - ] + "s3", + "gs", + "azure", + "ssh", + "http", + "hdfs", + "webdav", + "webhdfs", + "oss", + "gdrive", ] # Clouds that implement the general methods that can be tested # for functional tests that require extensive apis (e.g traversing # via walk_files()) -full_clouds = [ - pytest.lazy_fixture(cloud) - for cloud in ["s3", "gs", "azure", "ssh", "hdfs"] -] +full_clouds = ["s3", "gs", "azure", "ssh", "hdfs"] -@pytest.mark.needs_internet @pytest.mark.parametrize("remote", all_clouds, indirect=True) -def test_cloud(tmp_dir, dvc, remote): # pylint:disable=unused-argument - (stage,) = tmp_dir.dvc_gen("foo", "foo") - out = stage.outs[0] - cache = out.cache_path - foo_hash = out.hash_info - foo_hashes = out.get_used_objs().get(None, set()) - - (stage_dir,) = tmp_dir.dvc_gen( - { - "data_dir": { - "data_sub_dir": {"data_sub": "data_sub"}, - "data": "data", - "empty": "", - } - } - ) - out_dir = stage_dir.outs[0] - cache_dir = out_dir.cache_path - dir_hash = out_dir.hash_info - dir_hashes = {dir_hash} | {oid for _, _, oid in out_dir.obj} - - def _check_status(status, **kwargs): - for key in ("ok", "missing", "new", "deleted"): - expected = kwargs.get(key, set()) - assert expected == set(getattr(status, key)) - - # Check status - status = dvc.cloud.status(foo_hashes) - _check_status(status, new={foo_hash}) - - status_dir = dvc.cloud.status(dir_hashes) - _check_status(status_dir, new=dir_hashes) - - # Move cache and check status - # See issue https://github.com/iterative/dvc/issues/4383 for details - backup_dir = dvc.odb.local.cache_dir + ".backup" - move(dvc.odb.local.cache_dir, backup_dir) - status = dvc.cloud.status(foo_hashes) - _check_status(status, missing={foo_hash}) - - status_dir = dvc.cloud.status(dir_hashes) - _check_status(status_dir, missing=dir_hashes) - - # Restore original cache: - remove(dvc.odb.local.cache_dir) - move(backup_dir, dvc.odb.local.cache_dir) - - # Push and check status - dvc.cloud.push(foo_hashes) - assert os.path.exists(cache) - assert os.path.isfile(cache) - - dvc.cloud.push(dir_hashes) - assert os.path.isfile(cache_dir) - - status = dvc.cloud.status(foo_hashes) - _check_status(status, ok={foo_hash}) - - status_dir = dvc.cloud.status(dir_hashes) - _check_status(status_dir, ok=dir_hashes) - - # Remove and check status - remove(dvc.odb.local.cache_dir) - - status = dvc.cloud.status(foo_hashes) - _check_status(status, deleted={foo_hash}) - - status_dir = dvc.cloud.status(dir_hashes) - _check_status(status_dir, deleted=dir_hashes) - - # Pull and check status - dvc.cloud.pull(foo_hashes) - assert os.path.exists(cache) - assert os.path.isfile(cache) - with open(cache, encoding="utf-8") as fd: - assert fd.read() == "foo" - - dvc.cloud.pull(dir_hashes) - assert os.path.isfile(cache_dir) - - status = dvc.cloud.status(foo_hashes) - _check_status(status, ok={foo_hash}) - - status_dir = dvc.cloud.status(dir_hashes) - _check_status(status_dir, ok=dir_hashes) +class TestRemote(_TestRemote): + pass @pytest.mark.needs_internet @@ -448,7 +359,7 @@ def test_pipeline_file_target_ops(tmp_dir, dvc, run_copy, local_remote): assert set(dvc.pull()["added"]) == set(outs) # clean everything in remote and push - from tests.dir_helpers import TmpDir + from dvc.testing.tmp_dir import TmpDir clean(TmpDir(path).iterdir()) dvc.push(["dvc.yaml:copy-ipsum-baz"]) diff --git a/tests/func/test_gc.py b/tests/func/test_gc.py index 129851556..20d8d485e 100644 --- a/tests/func/test_gc.py +++ b/tests/func/test_gc.py @@ -346,12 +346,12 @@ def test_gc_not_collect_pipeline_tracked_files(tmp_dir, dvc, run_copy): @pytest.mark.parametrize( "workspace", [ - pytest.lazy_fixture("local_cloud"), - pytest.lazy_fixture("s3"), - pytest.lazy_fixture("hdfs"), - pytest.lazy_fixture("webhdfs"), + "local", + "s3", + "hdfs", + "webhdfs", pytest.param( - pytest.lazy_fixture("ssh"), + "ssh", marks=pytest.mark.skipif( os.name == "nt", reason="disabled on windows" ), diff --git a/tests/func/test_ignore.py b/tests/func/test_ignore.py index 1185d5af6..cd17afb9c 100644 --- a/tests/func/test_ignore.py +++ b/tests/func/test_ignore.py @@ -9,9 +9,9 @@ from dvc.ignore import DvcIgnore, DvcIgnorePatterns from dvc.output import OutputIsIgnoredError from dvc.pathspec_math import PatternInfo, merge_patterns from dvc.repo import Repo +from dvc.testing.tmp_dir import TmpDir from dvc.types import List from dvc.utils.fs import get_mtime_and_size -from tests.dir_helpers import TmpDir def _to_pattern_info_list(str_list: List): diff --git a/tests/func/test_import_url.py b/tests/func/test_import_url.py index 5628c3b59..e87fdd030 100644 --- a/tests/func/test_import_url.py +++ b/tests/func/test_import_url.py @@ -8,8 +8,8 @@ import pytest from dvc.dependency.base import DependencyDoesNotExistError from dvc.exceptions import InvalidArgumentError from dvc.main import main -from dvc.objects.db import ODBManager from dvc.stage import Stage +from dvc.testing.test_workspace import TestImport as _TestImport from dvc.utils.fs import makedirs from tests.basic_env import TestDvc @@ -118,102 +118,108 @@ def test_import_url_with_no_exec(tmp_dir, dvc, erepo_dir): assert not dst.exists() -@pytest.mark.parametrize( - "workspace", - [ - pytest.lazy_fixture("local_cloud"), - pytest.lazy_fixture("s3"), - pytest.lazy_fixture("azure"), - pytest.param( - pytest.lazy_fixture("gs"), marks=pytest.mark.needs_internet - ), - pytest.lazy_fixture("hdfs"), - pytest.lazy_fixture("webhdfs"), - pytest.param( - pytest.lazy_fixture("ssh"), - marks=pytest.mark.skipif( - os.name == "nt", reason="disabled on windows" +class TestImport(_TestImport): + @pytest.mark.parametrize( + "workspace", + [ + "local", + "s3", + "azure", + pytest.param("gs", marks=pytest.mark.needs_internet), + "hdfs", + "webhdfs", + pytest.param( + "ssh", + marks=pytest.mark.skipif( + os.name == "nt", reason="disabled on windows" + ), ), - ), - pytest.lazy_fixture("http"), - ], - indirect=True, -) -def test_import_url(tmp_dir, dvc, workspace): - workspace.gen("file", "file") - assert not (tmp_dir / "file").exists() # sanity check - dvc.imp_url("remote://workspace/file") - assert (tmp_dir / "file").read_text() == "file" + "http", + ], + indirect=True, + ) + def test_import(self, tmp_dir, dvc, workspace): + super().test_import(tmp_dir, dvc, workspace) - assert dvc.status() == {} - - -@pytest.mark.parametrize( - "workspace, stage_md5, dir_md5", - [ - ( - pytest.lazy_fixture("local_cloud"), - "dc24e1271084ee317ac3c2656fb8812b", - "b6dcab6ccd17ca0a8bf4a215a37d14cc.dir", - ), - ( - pytest.lazy_fixture("s3"), - "2aa17f8daa26996b3f7a4cf8888ac9ac", - "ec602a6ba97b2dd07bd6d2cd89674a60.dir", - ), - ( - pytest.lazy_fixture("azure"), - None, # ETags for azure are not consistent with the actual - None, # content of the object, so they will change every time - # we create new objects. We'll skip stage content check for Azure. - ), - ( - pytest.lazy_fixture("hdfs"), - "ec0943f83357f702033c98e70b853c8c", - "e6dcd267966dc628d732874f94ef4280.dir", - ), - pytest.param( - pytest.lazy_fixture("ssh"), - "dc24e1271084ee317ac3c2656fb8812b", - "b6dcab6ccd17ca0a8bf4a215a37d14cc.dir", - marks=pytest.mark.skipif( - os.name == "nt", reason="disabled on windows" + @pytest.mark.parametrize( + "workspace, stage_md5, dir_md5", + [ + ( + "local", + "dc24e1271084ee317ac3c2656fb8812b", + "b6dcab6ccd17ca0a8bf4a215a37d14cc.dir", ), - ), - ], - indirect=["workspace"], -) -def test_import_url_dir(tmp_dir, dvc, workspace, stage_md5, dir_md5): - workspace.gen({"dir": {"file": "file", "subdir": {"subfile": "subfile"}}}) + ( + "s3", + "2aa17f8daa26996b3f7a4cf8888ac9ac", + "ec602a6ba97b2dd07bd6d2cd89674a60.dir", + ), + ( + "azure", + None, # ETags for azure are not consistent with the actual + None, # content of the object, so they will change every time + # we create new objects. We'll skip stage content + # check for Azure. + ), + ( + "hdfs", + "ec0943f83357f702033c98e70b853c8c", + "e6dcd267966dc628d732874f94ef4280.dir", + ), + pytest.param( + "ssh", + "dc24e1271084ee317ac3c2656fb8812b", + "b6dcab6ccd17ca0a8bf4a215a37d14cc.dir", + marks=pytest.mark.skipif( + os.name == "nt", reason="disabled on windows" + ), + ), + ], + indirect=["workspace"], + ) + def test_import_dir(self, tmp_dir, dvc, workspace, stage_md5, dir_md5): + super().test_import_dir(tmp_dir, dvc, workspace, stage_md5, dir_md5) - # remove external cache to make sure that we don't need it to import dirs - with dvc.config.edit() as conf: - del conf["cache"] - dvc.odb = ODBManager(dvc) - - assert not (tmp_dir / "dir").exists() # sanity check - dvc.imp_url("remote://workspace/dir") - assert set(os.listdir(tmp_dir / "dir")) == {"file", "subdir"} - assert (tmp_dir / "dir" / "file").read_text() == "file" - assert list(os.listdir(tmp_dir / "dir" / "subdir")) == ["subfile"] - assert (tmp_dir / "dir" / "subdir" / "subfile").read_text() == "subfile" - - assert dvc.status() == {} - - if stage_md5 is not None and dir_md5 is not None: - assert (tmp_dir / "dir.dvc").read_text() == ( - f"md5: {stage_md5}\n" - "frozen: true\n" - "deps:\n" - f"- md5: {dir_md5}\n" - " size: 11\n" - " nfiles: 2\n" - " path: remote://workspace/dir\n" - "outs:\n" - "- md5: b6dcab6ccd17ca0a8bf4a215a37d14cc.dir\n" - " size: 11\n" - " nfiles: 2\n" - " path: dir\n" + @pytest.mark.parametrize( + "workspace, is_object_storage", + [ + ("hdfs", False), + pytest.param( + "s3", + True, + marks=pytest.mark.xfail( + reason="https://github.com/iterative/dvc/issues/5521", + ), + ), + pytest.param( + "gs", + True, + marks=pytest.mark.xfail( + reason="https://github.com/iterative/dvc/issues/5521", + ), + ), + pytest.param( + "azure", + True, + marks=pytest.mark.xfail( + reason="https://github.com/iterative/dvc/issues/5521", + ), + ), + pytest.param( + "ssh", + False, + marks=pytest.mark.skipif( + os.name == "nt", reason="disabled on windows" + ), + ), + ], + indirect=["workspace"], + ) + def test_import_empty_dir( + self, tmp_dir, dvc, workspace, is_object_storage + ): + super().test_import_empty_dir( + tmp_dir, dvc, workspace, is_object_storage ) @@ -257,19 +263,17 @@ def test_import_url_preserve_meta(tmp_dir, dvc): @pytest.mark.parametrize( "workspace", [ - pytest.lazy_fixture("local_cloud"), - pytest.lazy_fixture("s3"), + "local", + "s3", + pytest.param("gs", marks=pytest.mark.needs_internet), + "hdfs", pytest.param( - pytest.lazy_fixture("gs"), marks=pytest.mark.needs_internet - ), - pytest.lazy_fixture("hdfs"), - pytest.param( - pytest.lazy_fixture("ssh"), + "ssh", marks=pytest.mark.skipif( os.name == "nt", reason="disabled on windows" ), ), - pytest.lazy_fixture("http"), + "http", ], indirect=True, ) @@ -300,14 +304,12 @@ def test_import_url_to_remote_single_file( @pytest.mark.parametrize( "workspace", [ - pytest.lazy_fixture("local_cloud"), - pytest.lazy_fixture("s3"), + "local", + "s3", + pytest.param("gs", marks=pytest.mark.needs_internet), + "hdfs", pytest.param( - pytest.lazy_fixture("gs"), marks=pytest.mark.needs_internet - ), - pytest.lazy_fixture("hdfs"), - pytest.param( - pytest.lazy_fixture("ssh"), + "ssh", marks=pytest.mark.skipif( os.name == "nt", reason="disabled on windows" ), @@ -381,40 +383,6 @@ empty_xfail = pytest.mark.xfail( ) -@pytest.mark.parametrize( - "workspace", - [ - pytest.lazy_fixture("hdfs"), - pytest.param(pytest.lazy_fixture("s3"), marks=empty_xfail), - pytest.param(pytest.lazy_fixture("gs"), marks=empty_xfail), - pytest.param(pytest.lazy_fixture("azure"), marks=empty_xfail), - pytest.param( - pytest.lazy_fixture("ssh"), - marks=pytest.mark.skipif( - os.name == "nt", reason="disabled on windows" - ), - ), - ], - indirect=True, -) -def test_import_url_empty_directory(tmp_dir, dvc, workspace): - # prefix based storage services (e.g s3) doesn't have the real concept - # of directories. So instead we create an empty file that ends with a - # trailing slash in order to actually support this operation - if workspace.IS_OBJECT_STORAGE: - contents = "" - else: - contents = {} - - workspace.gen({"empty_dir/": contents}) - - dvc.imp_url("remote://workspace/empty_dir/") - - empty_dir = tmp_dir / "empty_dir" - assert empty_dir.is_dir() - assert tuple(empty_dir.iterdir()) == () - - def test_import_url_to_remote_status(tmp_dir, dvc, local_cloud, local_remote): local_cloud.gen("foo", "foo") diff --git a/tests/func/test_merge_driver.py b/tests/func/test_merge_driver.py index 873954779..79a5f96d5 100644 --- a/tests/func/test_merge_driver.py +++ b/tests/func/test_merge_driver.py @@ -114,9 +114,7 @@ def test_merge_conflict(tmp_dir, dvc, ancestor, our, their, error, caplog): assert error in caplog.text -@pytest.mark.parametrize( - "workspace", [pytest.lazy_fixture("ssh")], indirect=True -) +@pytest.mark.parametrize("workspace", ["ssh"], indirect=True) def test_merge_different_output_types(tmp_dir, dvc, caplog, workspace): (tmp_dir / "ancestor").touch() diff --git a/tests/func/test_run_multistage.py b/tests/func/test_run_multistage.py index a34e085ac..28eafe050 100644 --- a/tests/func/test_run_multistage.py +++ b/tests/func/test_run_multistage.py @@ -354,13 +354,13 @@ def test_run_overwrite_preserves_meta_and_comment(tmp_dir, dvc, run_copy): "workspace, hash_name, foo_hash, bar_hash", [ ( - pytest.lazy_fixture("local_cloud"), + "local", "md5", "acbd18db4cc2f85cedef654fccc4a4d8", "37b51d194a7513e45b56f6524f2d51f2", ), pytest.param( - pytest.lazy_fixture("ssh"), + "ssh", "md5", "acbd18db4cc2f85cedef654fccc4a4d8", "37b51d194a7513e45b56f6524f2d51f2", @@ -369,13 +369,13 @@ def test_run_overwrite_preserves_meta_and_comment(tmp_dir, dvc, run_copy): ), ), ( - pytest.lazy_fixture("s3"), + "s3", "etag", "acbd18db4cc2f85cedef654fccc4a4d8", "37b51d194a7513e45b56f6524f2d51f2", ), ( - pytest.lazy_fixture("hdfs"), + "hdfs", "checksum", "0000020000000000000000003dba826b9be9c6a8e2f8310a770555c4", "00000200000000000000000075433c81259d3c38e364b348af52e84d", diff --git a/tests/func/test_update.py b/tests/func/test_update.py index 84405cd32..42977f19e 100644 --- a/tests/func/test_update.py +++ b/tests/func/test_update.py @@ -159,15 +159,13 @@ def test_update_before_and_after_dvc_init(tmp_dir, dvc, git_dir): @pytest.mark.parametrize( "workspace", [ - pytest.lazy_fixture("local_cloud"), - pytest.lazy_fixture("s3"), + "local", + "s3", + pytest.param("gs", marks=pytest.mark.needs_internet), + "hdfs", + "webhdfs", pytest.param( - pytest.lazy_fixture("gs"), marks=pytest.mark.needs_internet - ), - pytest.lazy_fixture("hdfs"), - pytest.lazy_fixture("webhdfs"), - pytest.param( - pytest.lazy_fixture("ssh"), + "ssh", marks=pytest.mark.skipif( os.name == "nt", reason="disabled on windows" ), @@ -326,12 +324,10 @@ def test_update_import_to_remote(tmp_dir, dvc, erepo_dir, local_remote): @pytest.mark.parametrize( "workspace", [ - pytest.lazy_fixture("local_cloud"), - pytest.lazy_fixture("s3"), - pytest.param( - pytest.lazy_fixture("gs"), marks=pytest.mark.needs_internet - ), - pytest.lazy_fixture("hdfs"), + "local", + "s3", + pytest.param("gs", marks=pytest.mark.needs_internet), + "hdfs", ], indirect=True, ) @@ -349,11 +345,9 @@ def test_update_import_url_to_remote(tmp_dir, dvc, workspace, local_remote): @pytest.mark.parametrize( "workspace", [ - pytest.lazy_fixture("s3"), - pytest.param( - pytest.lazy_fixture("gs"), marks=pytest.mark.needs_internet - ), - pytest.lazy_fixture("hdfs"), + "s3", + pytest.param("gs", marks=pytest.mark.needs_internet), + "hdfs", ], indirect=True, ) @@ -378,7 +372,7 @@ def test_update_import_url_to_remote_directory( ) download_file_mock = mocker.spy( - type(dvc.cloud.get_remote_odb("cache").fs), "download_file" + type(dvc.cloud.get_remote_odb("workspace-cache").fs), "download_file" ) stage = dvc.update(stage.path, to_remote=True) diff --git a/tests/remotes/__init__.py b/tests/remotes/__init__.py index 4e9b426a0..9a368a9b0 100644 --- a/tests/remotes/__init__.py +++ b/tests/remotes/__init__.py @@ -1,12 +1,9 @@ -import subprocess - -import pytest - -from .azure import Azure, azure, azure_server # noqa: F401 +from .azure import Azure, azure, azure_server, make_azure # noqa: F401 from .gdrive import ( # noqa: F401; noqa: F401 TEST_GDRIVE_REPO_BUCKET, GDrive, gdrive, + make_gdrive, ) from .git_server import git_server, git_ssh # noqa: F401 from .gs import ( # noqa: F401; noqa: F401 @@ -14,13 +11,22 @@ from .gs import ( # noqa: F401; noqa: F401 TEST_GCP_CREDS_FILE, TEST_GCP_REPO_BUCKET, gs, + make_gs, ) -from .hdfs import HDFS, hadoop, hdfs, hdfs_server, real_hdfs # noqa: F401 -from .http import HTTP, http, http_server # noqa: F401 +from .hdfs import ( # noqa: F401 + HDFS, + hadoop, + hdfs, + hdfs_server, + make_hdfs, + real_hdfs, +) +from .http import HTTP, http, http_server, make_http # noqa: F401 from .local import Local, local_cloud, local_remote, make_local # noqa: F401 from .oss import ( # noqa: F401 OSS, TEST_OSS_REPO_BUCKET, + make_oss, oss, oss_server, real_oss, @@ -28,14 +34,21 @@ from .oss import ( # noqa: F401 from .s3 import ( # noqa: F401 S3, TEST_AWS_REPO_BUCKET, + make_s3, real_s3, s3, s3_fake_creds_file, s3_server, ) -from .ssh import SSH, ssh, ssh_connection, ssh_server # noqa: F401; noqa: F401 -from .webdav import Webdav, webdav, webdav_server # noqa: F401 -from .webhdfs import WebHDFS, webhdfs # noqa: F401 +from .ssh import ( # noqa: F401; noqa: F401 + SSH, + make_ssh, + ssh, + ssh_connection, + ssh_server, +) +from .webdav import Webdav, make_webdav, webdav, webdav_server # noqa: F401 +from .webhdfs import WebHDFS, make_webhdfs, webhdfs # noqa: F401 TEST_REMOTE = "upstream" TEST_CONFIG = { @@ -43,103 +56,3 @@ TEST_CONFIG = { "core": {"remote": TEST_REMOTE}, "remote": {TEST_REMOTE: {"url": ""}}, } - - -@pytest.fixture(scope="session") -def docker(): - import os - - # See https://travis-ci.community/t/docker-linux-containers-on-windows/301 - if os.environ.get("CI") and os.name == "nt": - pytest.skip("disabled for Windows on Github Actions") - - try: - subprocess.check_output("docker ps", shell=True) - except (subprocess.CalledProcessError, OSError): - pytest.skip("no docker installed") - - -@pytest.fixture(scope="session") -def docker_compose(docker): - try: - subprocess.check_output("docker-compose version", shell=True) - except (subprocess.CalledProcessError, OSError): - pytest.skip("no docker-compose installed") - - -@pytest.fixture(scope="session") -def docker_compose_project_name(): - return "pytest-dvc-test" - - -@pytest.fixture(scope="session") -def docker_services( - docker_compose_file, docker_compose_project_name, tmp_path_factory -): - # overriding `docker_services` fixture from `pytest_docker` plugin to - # only launch docker images once. - - from filelock import FileLock - from pytest_docker.plugin import DockerComposeExecutor, Services - - executor = DockerComposeExecutor( - docker_compose_file, docker_compose_project_name - ) - - # making sure we don't accidentally launch docker-compose in parallel, - # as it might result in network conflicts. Inspired by: - # https://github.com/pytest-dev/pytest-xdist#making-session-scoped-fixtures-execute-only-once - lockfile = tmp_path_factory.getbasetemp().parent / "docker-compose.lock" - with FileLock(str(lockfile)): # pylint:disable=abstract-class-instantiated - executor.execute("up --build -d") - - return Services(executor) - - -@pytest.fixture -def make_cloud(request): - def _make_cloud(typ): - return request.getfixturevalue(f"make_{typ}")() - - return _make_cloud - - -@pytest.fixture -def make_remote(tmp_dir, dvc, make_cloud): - def _make_remote(name, typ="local", **kwargs): - cloud = make_cloud(typ) - tmp_dir.add_remote(name=name, config=cloud.config, **kwargs) - return cloud - - return _make_remote - - -@pytest.fixture -def remote(tmp_dir, dvc, request): - cloud = request.param - assert cloud - tmp_dir.add_remote(config=cloud.config) - yield cloud - - -@pytest.fixture -def workspace(tmp_dir, dvc, request): - from dvc.objects.db import ODBManager - - cloud = request.param - - assert cloud - - tmp_dir.add_remote(name="workspace", config=cloud.config, default=False) - tmp_dir.add_remote( - name="cache", url="remote://workspace/cache", default=False - ) - - scheme = getattr(cloud, "scheme", "local") - if scheme != "http": - with dvc.config.edit() as conf: - conf["cache"][scheme] = "cache" - - dvc.odb = ODBManager(dvc) - - return cloud diff --git a/tests/remotes/azure.py b/tests/remotes/azure.py index 151514611..ce93d1a3e 100644 --- a/tests/remotes/azure.py +++ b/tests/remotes/azure.py @@ -6,8 +6,8 @@ import uuid import pytest from funcy import cached_property -from .base import Base -from .path_info import CloudURLInfo +from dvc.testing.cloud import Cloud +from dvc.testing.path_info import CloudURLInfo TEST_AZURE_CONTAINER = "tests" TEST_AZURE_CONNECTION_STRING = ( @@ -19,11 +19,15 @@ TEST_AZURE_CONNECTION_STRING = ( ) -class Azure(Base, CloudURLInfo): +class Azure(Cloud, CloudURLInfo): IS_OBJECT_STORAGE = True CONNECTION_STRING = None + def __init__(self, url, **kwargs): + super().__init__(url) + self.opts = kwargs + @cached_property def service_client(self): # pylint: disable=no-name-in-module @@ -62,6 +66,19 @@ class Azure(Base, CloudURLInfo): path = self.path.lstrip("/") return f"{bucket}/{path}" + def is_file(self): + raise NotImplementedError + + def is_dir(self): + raise NotImplementedError + + def exists(self): + raise NotImplementedError + + @property + def config(self): + return {"url": self.url, **self.opts} + @pytest.fixture(scope="session") def azure_server(test_config, docker_compose, docker_services): @@ -95,17 +112,26 @@ def azure_server(test_config, docker_compose, docker_services): @pytest.fixture -def azure(azure_server): - from azure.core.exceptions import ResourceNotFoundError +def make_azure(azure_server): + def _make_azure(): + from azure.core.exceptions import ResourceNotFoundError - url = f"azure://{TEST_AZURE_CONTAINER}/{uuid.uuid4()}" - ret = Azure(url) - ret.config = {"url": url, "connection_string": azure_server} + url = f"azure://{TEST_AZURE_CONTAINER}/{uuid.uuid4()}" + ret = Azure(url, connection_string=azure_server) - container = ret.service_client.get_container_client(TEST_AZURE_CONTAINER) - try: # verify that container exists - container.get_container_properties() - except ResourceNotFoundError: - container.create_container() + container = ret.service_client.get_container_client( + TEST_AZURE_CONTAINER + ) + try: # verify that container exists + container.get_container_properties() + except ResourceNotFoundError: + container.create_container() - return ret + return ret + + return _make_azure + + +@pytest.fixture +def azure(make_azure): + return make_azure() diff --git a/tests/remotes/gdrive.py b/tests/remotes/gdrive.py index 4d48f2529..c4b454b45 100644 --- a/tests/remotes/gdrive.py +++ b/tests/remotes/gdrive.py @@ -9,11 +9,10 @@ import pytest from funcy import cached_property, retry from dvc.fs.gdrive import GDriveFileSystem +from dvc.testing.cloud import Cloud +from dvc.testing.path_info import CloudURLInfo from dvc.utils import tmp_fname -from .base import Base -from .path_info import CloudURLInfo - TEST_GDRIVE_REPO_BUCKET = "root" @@ -62,12 +61,12 @@ def _gdrive_retry(func): )(func) -class GDrive(Base, GDriveURLInfo): +class GDrive(Cloud, GDriveURLInfo): @staticmethod def should_test(): return bool(os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA)) - @cached_property + @property def config(self): tmp_path = tmp_fname() with open(tmp_path, "w", encoding="utf-8") as stream: @@ -110,6 +109,15 @@ class GDrive(Base, GDriveURLInfo): service_account=self.config["gdrive_use_service_account"], ) + def is_file(self): + raise NotImplementedError + + def is_dir(self): + raise NotImplementedError + + def exists(self): + raise NotImplementedError + @_gdrive_retry def mkdir(self, mode=0o777, parents=False, exist_ok=False): try: @@ -135,17 +143,26 @@ class GDrive(Base, GDriveURLInfo): @pytest.fixture -def gdrive(test_config, make_tmp_dir): +def make_gdrive(test_config, make_tmp_dir): test_config.requires("gdrive") + if not GDrive.should_test(): pytest.skip("no gdrive") - # NOTE: temporary workaround - tmp_dir = make_tmp_dir("gdrive", dvc=True) + def _make_gdrive(): + # NOTE: temporary workaround + tmp_dir = make_tmp_dir("gdrive", dvc=True) - ret = GDrive(GDrive.get_url()) - fs = GDriveFileSystem( - gdrive_credentials_tmp_dir=tmp_dir.dvc.tmp_dir, **ret.config - ) - fs.fs._gdrive_create_dir("root", fs.url) - yield ret + ret = GDrive(GDrive.get_url()) + fs = GDriveFileSystem( + gdrive_credentials_tmp_dir=tmp_dir.dvc.tmp_dir, **ret.config + ) + fs.fs._gdrive_create_dir("root", fs.url) + yield ret + + return _make_gdrive + + +@pytest.fixture +def gdrive(make_gdrive): + return make_gdrive() diff --git a/tests/remotes/gs.py b/tests/remotes/gs.py index 4ecbd26fe..7c20b38cb 100644 --- a/tests/remotes/gs.py +++ b/tests/remotes/gs.py @@ -3,13 +3,11 @@ import os import uuid import pytest -from funcy import cached_property +from dvc.testing.cloud import Cloud +from dvc.testing.path_info import CloudURLInfo from dvc.utils import env2bool -from .base import Base -from .path_info import CloudURLInfo - TEST_GCP_REPO_BUCKET = os.environ.get("DVC_TEST_GCP_REPO_BUCKET", "dvc-test") TEST_GCP_CREDS_FILE = os.path.abspath( @@ -22,7 +20,7 @@ TEST_GCP_CREDS_FILE = os.path.abspath( os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = TEST_GCP_CREDS_FILE -class GCP(Base, CloudURLInfo): +class GCP(Cloud, CloudURLInfo): IS_OBJECT_STORAGE = True @@ -37,7 +35,7 @@ class GCP(Base, CloudURLInfo): return True - @cached_property + @property def config(self): return {"url": self.url, "credentialpath": TEST_GCP_CREDS_FILE} @@ -95,8 +93,13 @@ class GCP(Base, CloudURLInfo): @pytest.fixture -def gs(test_config): +def make_gs(test_config): test_config.requires("gs") if not GCP.should_test(): pytest.skip("no gs") yield GCP(GCP.get_url()) + + +@pytest.fixture +def gs(make_gs): + return make_gs() diff --git a/tests/remotes/hdfs.py b/tests/remotes/hdfs.py index 1c993d28c..192a75470 100644 --- a/tests/remotes/hdfs.py +++ b/tests/remotes/hdfs.py @@ -10,13 +10,13 @@ from urllib.parse import urlparse import pytest -from .base import Base -from .path_info import URLInfo +from dvc.testing.cloud import Cloud +from dvc.testing.path_info import URLInfo _hdfs_root = TemporaryDirectory() -class HDFS(Base, URLInfo): # pylint: disable=abstract-method +class HDFS(Cloud, URLInfo): # pylint: disable=abstract-method @contextmanager def _hdfs(self): import pyarrow.fs @@ -24,6 +24,10 @@ class HDFS(Base, URLInfo): # pylint: disable=abstract-method conn = pyarrow.fs.HadoopFileSystem(self.host, self.port) yield conn + @property + def config(self): + return {"url": self.url} + def is_file(self): with self._hdfs() as _hdfs: import pyarrow.fs @@ -280,7 +284,7 @@ class FakeHadoopFileSystem: @pytest.fixture -def hdfs(test_config, mocker): +def make_hdfs(test_config, mocker): # Windows might not have Visual C++ Redistributable for Visual Studio # 2015 installed, which will result in the following error: # "The pyarrow installation is not built with support for @@ -295,5 +299,13 @@ def hdfs(test_config, mocker): mocker.patch("dvc.fs.hdfs.HDFSFileSystem._checksum", hadoop_fs_checksum) - url = f"hdfs://example.com:12345/{uuid.uuid4()}" - yield HDFS(url) + def _make_hdfs(): + url = f"hdfs://example.com:12345/{uuid.uuid4()}" + return HDFS(url) + + return _make_hdfs + + +@pytest.fixture +def hdfs(make_hdfs): + return make_hdfs() diff --git a/tests/remotes/http.py b/tests/remotes/http.py index 537bfed26..8440ad26c 100644 --- a/tests/remotes/http.py +++ b/tests/remotes/http.py @@ -5,11 +5,11 @@ import uuid import pytest import requests -from .base import Base -from .path_info import HTTPURLInfo +from dvc.testing.cloud import Cloud +from dvc.testing.path_info import HTTPURLInfo -class HTTP(Base, HTTPURLInfo): +class HTTP(Cloud, HTTPURLInfo): @staticmethod def get_url(port): # pylint: disable=arguments-differ dname = str(uuid.uuid4()) @@ -24,10 +24,26 @@ class HTTP(Base, HTTPURLInfo): response = requests.post(self.url, data=contents) assert response.status_code == 200 + @property + def config(self): + return {"url": self.url} + @property def fs_path(self): return self.url + def exists(self): + raise NotImplementedError + + def is_dir(self): + raise NotImplementedError + + def is_file(self): + raise NotImplementedError + + def read_bytes(self): + raise NotImplementedError + @pytest.fixture(scope="session") def http_server(test_config, tmp_path_factory): @@ -40,5 +56,13 @@ def http_server(test_config, tmp_path_factory): @pytest.fixture -def http(http_server): - yield HTTP(HTTP.get_url(http_server.server_port)) +def make_http(http_server): + def _make_http(): + return HTTP(HTTP.get_url(http_server.server_port)) + + return _make_http + + +@pytest.fixture +def http(make_http): + return make_http() diff --git a/tests/remotes/local.py b/tests/remotes/local.py index 869efb8a3..7b163d6b9 100644 --- a/tests/remotes/local.py +++ b/tests/remotes/local.py @@ -1,12 +1,11 @@ # pylint: disable=cyclic-import,disable=abstract-method import pytest +from dvc.testing.cloud import Cloud from tests.basic_env import TestDvc -from .base import Base - -class Local(Base): +class Local(Cloud): @staticmethod def get_storagepath(): return TestDvc.mkdtemp() @@ -19,10 +18,7 @@ class Local(Base): @pytest.fixture def make_local(make_tmp_dir): def _make_local(): - ret = make_tmp_dir("local-cloud") - ret.url = str(ret) - ret.config = {"url": ret.url} - return ret + return make_tmp_dir("local-cloud") return _make_local diff --git a/tests/remotes/oss.py b/tests/remotes/oss.py index 2b548f430..d395722c2 100644 --- a/tests/remotes/oss.py +++ b/tests/remotes/oss.py @@ -4,18 +4,17 @@ import uuid import pytest +from dvc.testing.cloud import Cloud +from dvc.testing.path_info import CloudURLInfo from dvc.utils import env2bool -from .base import Base -from .path_info import CloudURLInfo - TEST_OSS_REPO_BUCKET = "dvc-test-github" EMULATOR_OSS_ENDPOINT = "127.0.0.1:{port}" EMULATOR_OSS_ACCESS_KEY_ID = "AccessKeyID" EMULATOR_OSS_ACCESS_KEY_SECRET = "AccessKeySecret" -class OSS(Base, CloudURLInfo): +class OSS(Cloud, CloudURLInfo): IS_OBJECT_STORAGE = True @@ -36,6 +35,33 @@ class OSS(Base, CloudURLInfo): return False + @property + def config(self): + return { + "url": self.url, + "oss_key_id": os.environ.get("OSS_ACCESS_KEY_ID"), + "oss_key_secret": os.environ.get("OSS_ACCESS_KEY_SECRET"), + "oss_endpoint": os.environ.get("OSS_ENDPOINT"), + } + + def is_file(self): + raise NotImplementedError + + def is_dir(self): + raise NotImplementedError + + def exists(self): + raise NotImplementedError + + def mkdir(self, mode=0o777, parents=False, exist_ok=False): + raise NotImplementedError + + def write_bytes(self, contents): + raise NotImplementedError + + def read_bytes(self): + raise NotImplementedError + @pytest.fixture(scope="session") def oss_server(test_config, docker_compose, docker_services): @@ -63,24 +89,36 @@ def oss_server(test_config, docker_compose, docker_services): @pytest.fixture -def oss(real_oss): - import oss2 +def make_oss(real_oss): + def _make_oss(): + import oss2 - ret = real_oss + ret = real_oss - auth = oss2.Auth(ret.config["oss_key_id"], ret.config["oss_key_secret"]) - bucket = oss2.Bucket( - auth, ret.config["oss_endpoint"], TEST_OSS_REPO_BUCKET - ) - try: - bucket.get_bucket_info() - except oss2.exceptions.NoSuchBucket: - bucket.create_bucket( - oss2.BUCKET_ACL_PUBLIC_READ, - oss2.models.BucketCreateConfig(oss2.BUCKET_STORAGE_CLASS_STANDARD), + auth = oss2.Auth( + ret.config["oss_key_id"], ret.config["oss_key_secret"] ) + bucket = oss2.Bucket( + auth, ret.config["oss_endpoint"], TEST_OSS_REPO_BUCKET + ) + try: + bucket.get_bucket_info() + except oss2.exceptions.NoSuchBucket: + bucket.create_bucket( + oss2.BUCKET_ACL_PUBLIC_READ, + oss2.models.BucketCreateConfig( + oss2.BUCKET_STORAGE_CLASS_STANDARD + ), + ) - return ret + return ret + + return _make_oss + + +@pytest.fixture +def oss(make_oss): + return make_oss() @pytest.fixture @@ -90,11 +128,4 @@ def real_oss(test_config): pytest.skip("no real OSS") url = OSS.get_url() - ret = OSS(url) - ret.config = { - "url": url, - "oss_key_id": os.environ.get("OSS_ACCESS_KEY_ID"), - "oss_key_secret": os.environ.get("OSS_ACCESS_KEY_SECRET"), - "oss_endpoint": os.environ.get("OSS_ENDPOINT"), - } - return ret + return OSS(url) diff --git a/tests/remotes/s3.py b/tests/remotes/s3.py index 6ccb37333..13208d79c 100644 --- a/tests/remotes/s3.py +++ b/tests/remotes/s3.py @@ -5,21 +5,20 @@ import uuid import pytest from funcy import cached_property +from dvc.testing.cloud import Cloud +from dvc.testing.path_info import CloudURLInfo from dvc.utils import env2bool -from .base import Base -from .path_info import CloudURLInfo - TEST_AWS_REPO_BUCKET = os.environ.get("DVC_TEST_AWS_REPO_BUCKET", "dvc-temp") TEST_AWS_ENDPOINT_URL = "http://127.0.0.1:{port}/" -class S3(Base, CloudURLInfo): +class S3(Cloud, CloudURLInfo): IS_OBJECT_STORAGE = True TEST_AWS_ENDPOINT_URL = None - @cached_property + @property def config(self): return {"url": self.url, "endpointurl": self.TEST_AWS_ENDPOINT_URL} @@ -151,11 +150,20 @@ def s3_server(test_config, docker_compose, docker_services): @pytest.fixture -def s3(test_config, s3_server, s3_fake_creds_file): +def make_s3(test_config, s3_server, s3_fake_creds_file): test_config.requires("s3") - workspace = S3(S3.get_url()) - workspace._s3.create_bucket(Bucket=TEST_AWS_REPO_BUCKET) - yield workspace + + def _make_s3(): + cloud = S3(S3.get_url()) + cloud._s3.create_bucket(Bucket=TEST_AWS_REPO_BUCKET) + return cloud + + return _make_s3 + + +@pytest.fixture +def s3(make_s3): + return make_s3() @pytest.fixture diff --git a/tests/remotes/ssh.py b/tests/remotes/ssh.py index 9dca302de..1b3f51e13 100644 --- a/tests/remotes/ssh.py +++ b/tests/remotes/ssh.py @@ -5,8 +5,8 @@ import uuid import pytest from funcy import cached_property -from .base import Base -from .path_info import URLInfo +from dvc.testing.cloud import Cloud +from dvc.testing.path_info import URLInfo TEST_SSH_USER = "user" TEST_SSH_KEY_PATH = os.path.join( @@ -14,12 +14,12 @@ TEST_SSH_KEY_PATH = os.path.join( ) -class SSH(Base, URLInfo): +class SSH(Cloud, URLInfo): @staticmethod def get_url(host, port): # pylint: disable=arguments-differ return f"ssh://{host}:{port}/tmp/data/{uuid.uuid4()}" - @cached_property + @property def config(self): return { "url": self.url, @@ -116,12 +116,20 @@ def ssh_connection(ssh_server): @pytest.fixture -def ssh(ssh_server, monkeypatch): - from dvc.fs.ssh import SSHFileSystem +def make_ssh(ssh_server, monkeypatch): + def _make_ssh(): + from dvc.fs.ssh import SSHFileSystem - # NOTE: see http://github.com/iterative/dvc/pull/3501 - monkeypatch.setattr(SSHFileSystem, "CAN_TRAVERSE", False) + # NOTE: see http://github.com/iterative/dvc/pull/3501 + monkeypatch.setattr(SSHFileSystem, "CAN_TRAVERSE", False) - url = SSH(SSH.get_url(**ssh_server)) - url.mkdir(exist_ok=True, parents=True) - return url + url = SSH(SSH.get_url(**ssh_server)) + url.mkdir(exist_ok=True, parents=True) + return url + + return _make_ssh + + +@pytest.fixture +def ssh(make_ssh): + return make_ssh() diff --git a/tests/remotes/webdav.py b/tests/remotes/webdav.py index 91ed12858..483670d1b 100644 --- a/tests/remotes/webdav.py +++ b/tests/remotes/webdav.py @@ -6,19 +6,24 @@ import pytest from funcy import cached_property, first from wsgidav.wsgidav_app import WsgiDAVApp +from dvc.testing.cloud import Cloud +from dvc.testing.path_info import WebDAVURLInfo from tests.utils.httpd import run_server_on_thread -from .base import Base -from .path_info import WebDAVURLInfo - AUTH = {"user1": {"password": "password1"}} -class Webdav(Base, WebDAVURLInfo): +class Webdav(Cloud, WebDAVURLInfo): @staticmethod def get_url(port): # pylint: disable=arguments-differ return f"webdav://localhost:{port}" + @property + def config(self): + user, secrets = first(AUTH.items()) + url = f"webdav://{self.netloc}{self._spath}" + return {"url": url, "user": user, **secrets} + @cached_property def client(self): from webdav4.client import Client @@ -45,6 +50,18 @@ class Webdav(Base, WebDAVURLInfo): def fs_path(self): return self.path.lstrip("/") + def exists(self): + raise NotImplementedError + + def is_dir(self): + raise NotImplementedError + + def is_file(self): + raise NotImplementedError + + def read_bytes(self): + raise NotImplementedError + @pytest.fixture def webdav_server(test_config, tmp_path_factory): @@ -68,9 +85,14 @@ def webdav_server(test_config, tmp_path_factory): @pytest.fixture -def webdav(webdav_server): - url = Webdav.get_url(webdav_server.server_port) - ret = Webdav(url) - user, secrets = first(AUTH.items()) - ret.config = {"url": url, "user": user, **secrets} - yield ret +def make_webdav(webdav_server): + def _make_webdav(): + url = Webdav.get_url(webdav_server.server_port) + return Webdav(url) + + return _make_webdav + + +@pytest.fixture +def webdav(make_webdav): + return make_webdav() diff --git a/tests/remotes/webhdfs.py b/tests/remotes/webhdfs.py index b65e60ad5..7a4cd33d7 100644 --- a/tests/remotes/webhdfs.py +++ b/tests/remotes/webhdfs.py @@ -4,11 +4,11 @@ from contextlib import contextmanager import pytest -from .base import Base -from .path_info import CloudURLInfo +from dvc.testing.cloud import Cloud +from dvc.testing.path_info import CloudURLInfo -class WebHDFS(Base, CloudURLInfo): # pylint: disable=abstract-method +class WebHDFS(Cloud, CloudURLInfo): # pylint: disable=abstract-method @contextmanager def _webhdfs(self): from hdfs import InsecureClient @@ -54,13 +54,25 @@ class WebHDFS(Base, CloudURLInfo): # pylint: disable=abstract-method assert errors is None return self.read_bytes().decode(encoding) + @property + def config(self): + return {"url": self.url} + @property def fs_path(self): return "/" + self.path.lstrip("/") @pytest.fixture -def webhdfs(hdfs_server): - port = hdfs_server["webhdfs"] - url = f"webhdfs://127.0.0.1:{port}/{uuid.uuid4()}" - yield WebHDFS(url) +def make_webhdfs(hdfs_server): + def _make_webhdfs(): + port = hdfs_server["webhdfs"] + url = f"webhdfs://127.0.0.1:{port}/{uuid.uuid4()}" + yield WebHDFS(url) + + return _make_webhdfs + + +@pytest.fixture +def webhdfs(make_webhdfs): + return make_webhdfs()