mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
4cc06339c8
* Clean up remotes's exps fix #6006 1. add a new argument `--git-remote` to `dvc exp remove` 2. support remote a special remote exp 3. add tests for it 4. fix #6421 5. add a test for #6421. 6. extract some functions from the `dvc pull`, `dvc push`, `dvc remove` to utils. 7. add a tests for this new util function `resolve_exp_ref` 8. add `__init__.py` to some test package for the pytest fail Co-authored-by: Jorge Orpinel <[email protected]> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Dave Berenbaum <[email protected]>
432 lines
12 KiB
Python
432 lines
12 KiB
Python
"""
|
|
The goal of this module is making dvc functional tests setup a breeze. This
|
|
includes a temporary dir, initializing git and DVC repos and bootstrapping some
|
|
file structure.
|
|
|
|
The cornerstone of these fixtures is `tmp_dir`, which creates a temporary dir
|
|
and changes path to it, it might be combined with `scm` and `dvc` to initialize
|
|
empty git and DVC repos. `tmp_dir` returns a Path instance, which should save
|
|
you from using `open()`, `os` and `os.path` utils many times:
|
|
|
|
(tmp_dir / "some_file").write_text("some text")
|
|
# ...
|
|
assert "some text" == (tmp_dir / "some_file").read_text()
|
|
assert (tmp_dir / "some_file").exists()
|
|
|
|
Additionally it provides `.gen()`, `.scm_gen()` and `.dvc_gen()` methods to
|
|
bootstrap a required file structure in a single call:
|
|
|
|
# Generate a dir with files
|
|
tmp_dir.gen({"dir": {"file": "file text", "second_file": "..."}})
|
|
|
|
# Generate a single file, dirs will be created along the way
|
|
tmp_dir.gen("dir/file", "file text")
|
|
|
|
# Generate + git add
|
|
tmp_dir.scm_gen({"file1": "...", ...})
|
|
|
|
# Generate + git add + git commit
|
|
tmp_dir.scm_gen({"file1": "...", ...}, commit="add files")
|
|
|
|
# Generate + dvc add
|
|
tmp_dir.dvc_gen({"file1": "...", ...})
|
|
|
|
# Generate + dvc add + git commit -am "..."
|
|
# This commits stages to git not the generated files.
|
|
tmp_dir.dvc_gen({"file1": "...", ...}, commit="add files")
|
|
|
|
Making it easier to bootstrap things has a supergoal of incentivizing a move
|
|
from global repo template to creating everything inplace, which:
|
|
|
|
- makes all path references local to test, enhancing readability
|
|
- allows using telling filenames, e.g. "git_tracked_file" instead of "foo"
|
|
- does not create unnecessary files
|
|
"""
|
|
|
|
# pylint: disable=redefined-outer-name, attribute-defined-outside-init
|
|
|
|
import os
|
|
import pathlib
|
|
import sys
|
|
from contextlib import contextmanager
|
|
from textwrap import dedent
|
|
|
|
import pytest
|
|
from funcy import lmap, retry
|
|
|
|
from dvc.logger import disable_other_loggers
|
|
from dvc.utils.fs import makedirs
|
|
|
|
__all__ = [
|
|
"make_tmp_dir",
|
|
"tmp_dir",
|
|
"scm",
|
|
"dvc",
|
|
"run_copy",
|
|
"run_head",
|
|
"erepo_dir",
|
|
"git_dir",
|
|
"git_init",
|
|
"git_upstream",
|
|
"git_downstream",
|
|
]
|
|
|
|
|
|
# see https://github.com/iterative/dvc/issues/3167
|
|
disable_other_loggers()
|
|
|
|
|
|
class TmpDir(pathlib.Path):
|
|
scheme = "local"
|
|
|
|
def __new__(cls, *args, **kwargs):
|
|
if cls is TmpDir:
|
|
cls = ( # pylint: disable=self-cls-assignment
|
|
WindowsTmpDir if os.name == "nt" else PosixTmpDir
|
|
)
|
|
# init parameter and `_init` method has been removed in Python 3.10.
|
|
kw = {"init": False} if sys.version_info < (3, 10) else {}
|
|
self = cls._from_parts( # pylint: disable=unexpected-keyword-arg
|
|
args, **kw
|
|
)
|
|
if not self._flavour.is_supported:
|
|
raise NotImplementedError(
|
|
f"cannot instantiate {cls.__name__!r} on your system"
|
|
)
|
|
if sys.version_info < (3, 10):
|
|
self._init() # pylint: disable=no-member
|
|
return self
|
|
|
|
def init(self, *, scm=False, dvc=False, subdir=False):
|
|
from dvc.repo import Repo
|
|
from dvc.scm.git import Git
|
|
|
|
assert not scm or not hasattr(self, "scm")
|
|
assert not dvc or not hasattr(self, "dvc")
|
|
|
|
str_path = os.fspath(self)
|
|
|
|
if scm:
|
|
git_init(str_path)
|
|
if dvc:
|
|
self.dvc = Repo.init(
|
|
str_path,
|
|
no_scm=not scm and not hasattr(self, "scm"),
|
|
subdir=subdir,
|
|
)
|
|
if scm:
|
|
self.scm = self.dvc.scm if hasattr(self, "dvc") else Git(str_path)
|
|
if dvc and hasattr(self, "scm"):
|
|
self.scm.commit("init dvc")
|
|
|
|
def close(self):
|
|
if hasattr(self, "scm"):
|
|
self.scm.close()
|
|
if hasattr(self, "dvc"):
|
|
self.dvc.close()
|
|
|
|
def _require(self, name):
|
|
if not hasattr(self, name):
|
|
raise TypeError(
|
|
"Can't use {name} for this temporary dir. "
|
|
'Did you forget to use "{name}" fixture?'.format(name=name)
|
|
)
|
|
|
|
# Bootstrapping methods
|
|
def gen(self, struct, text=""):
|
|
if isinstance(struct, (str, bytes, pathlib.PurePath)):
|
|
struct = {struct: text}
|
|
|
|
return self._gen(struct)
|
|
|
|
def _gen(self, struct, prefix=None):
|
|
paths = []
|
|
for name, contents in struct.items():
|
|
path = (prefix or self) / name
|
|
|
|
if isinstance(contents, dict):
|
|
if not contents:
|
|
makedirs(path, exist_ok=True)
|
|
else:
|
|
self._gen(contents, prefix=path)
|
|
else:
|
|
makedirs(path.parent, exist_ok=True)
|
|
if isinstance(contents, bytes):
|
|
path.write_bytes(contents)
|
|
else:
|
|
path.write_text(contents, encoding="utf-8")
|
|
paths.append(path)
|
|
return paths
|
|
|
|
def dvc_gen(self, struct, text="", commit=None):
|
|
paths = self.gen(struct, text)
|
|
return self.dvc_add(paths, commit=commit)
|
|
|
|
def scm_gen(self, struct, text="", commit=None):
|
|
paths = self.gen(struct, text)
|
|
return self.scm_add(paths, commit=commit)
|
|
|
|
def commit(self, output_paths, msg):
|
|
def to_gitignore(stage_path):
|
|
from dvc.scm import Git
|
|
|
|
return os.path.join(os.path.dirname(stage_path), Git.GITIGNORE)
|
|
|
|
gitignores = [
|
|
to_gitignore(s)
|
|
for s in output_paths
|
|
if os.path.exists(to_gitignore(s))
|
|
]
|
|
return self.scm_add(output_paths + gitignores, commit=msg)
|
|
|
|
def dvc_add(self, filenames, commit=None):
|
|
self._require("dvc")
|
|
filenames = _coerce_filenames(filenames)
|
|
|
|
stages = self.dvc.add(filenames)
|
|
if commit:
|
|
self.commit([s.path for s in stages], msg=commit)
|
|
return stages
|
|
|
|
def scm_add(self, filenames, commit=None):
|
|
self._require("scm")
|
|
filenames = _coerce_filenames(filenames)
|
|
self.scm.add(filenames)
|
|
if commit:
|
|
self.scm.commit(commit)
|
|
|
|
def add_remote(
|
|
self, *, url=None, config=None, name="upstream", default=True
|
|
):
|
|
self._require("dvc")
|
|
|
|
assert bool(url) ^ bool(config)
|
|
|
|
if url:
|
|
config = {"url": url}
|
|
|
|
with self.dvc.config.edit() as conf:
|
|
conf["remote"][name] = config
|
|
if default:
|
|
conf["core"]["remote"] = name
|
|
|
|
if hasattr(self, "scm"):
|
|
self.scm.add(self.dvc.config.files["repo"])
|
|
self.scm.commit(f"add '{name}' remote")
|
|
|
|
return url or config["url"]
|
|
|
|
# contexts
|
|
@contextmanager
|
|
def chdir(self):
|
|
old = os.getcwd()
|
|
try:
|
|
os.chdir(self)
|
|
yield
|
|
finally:
|
|
os.chdir(old)
|
|
|
|
@contextmanager
|
|
def branch(self, name, new=False):
|
|
self._require("scm")
|
|
old = self.scm.active_branch()
|
|
try:
|
|
self.scm.checkout(name, create_new=new)
|
|
yield
|
|
finally:
|
|
self.scm.checkout(old)
|
|
|
|
def read_text(self, *args, **kwargs): # pylint: disable=signature-differs
|
|
# NOTE: on windows we'll get PermissionError instead of
|
|
# IsADirectoryError when we try to `open` a directory, so we can't
|
|
# rely on exception flow control
|
|
if self.is_dir():
|
|
return {
|
|
path.name: path.read_text(*args, **kwargs)
|
|
for path in self.iterdir()
|
|
}
|
|
return super().read_text(*args, **kwargs)
|
|
|
|
def hash_to_path_info(self, hash_):
|
|
return self / hash_[0:2] / hash_[2:]
|
|
|
|
|
|
def _coerce_filenames(filenames):
|
|
if isinstance(filenames, (str, bytes, pathlib.PurePath)):
|
|
filenames = [filenames]
|
|
return lmap(os.fspath, filenames)
|
|
|
|
|
|
class WindowsTmpDir(TmpDir, pathlib.PureWindowsPath):
|
|
pass
|
|
|
|
|
|
class PosixTmpDir(TmpDir, pathlib.PurePosixPath):
|
|
pass
|
|
|
|
|
|
CACHE = {}
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def make_tmp_dir(tmp_path_factory, request, worker_id):
|
|
def make(name, *, scm=False, dvc=False, subdir=False):
|
|
from shutil import ignore_patterns
|
|
|
|
from dvc.repo import Repo
|
|
from dvc.scm.git import Git
|
|
from dvc.utils.fs import fs_copy
|
|
|
|
cache = CACHE.get((scm, dvc, subdir))
|
|
if not cache:
|
|
cache = tmp_path_factory.mktemp("dvc-test-cache" + worker_id)
|
|
TmpDir(cache).init(scm=scm, dvc=dvc, subdir=subdir)
|
|
CACHE[(scm, dvc, subdir)] = os.fspath(cache)
|
|
path = tmp_path_factory.mktemp(name) if isinstance(name, str) else name
|
|
|
|
# ignore sqlite files from .dvc/tmp. We might not be closing the cache
|
|
# connection resulting in PermissionErrors in Windows.
|
|
ignore = ignore_patterns("cache.db*")
|
|
for entry in os.listdir(cache):
|
|
# shutil.copytree's dirs_exist_ok is only available in >=3.8
|
|
fs_copy(
|
|
os.path.join(cache, entry),
|
|
os.path.join(path, entry),
|
|
ignore=ignore,
|
|
)
|
|
new_dir = TmpDir(path)
|
|
str_path = os.fspath(new_dir)
|
|
if dvc:
|
|
new_dir.dvc = Repo(str_path)
|
|
if scm:
|
|
new_dir.scm = (
|
|
new_dir.dvc.scm if hasattr(new_dir, "dvc") else Git(str_path)
|
|
)
|
|
request.addfinalizer(new_dir.close)
|
|
return new_dir
|
|
|
|
return make
|
|
|
|
|
|
@pytest.fixture
|
|
def tmp_dir(tmp_path, make_tmp_dir, request, monkeypatch):
|
|
monkeypatch.chdir(tmp_path)
|
|
fixtures = request.fixturenames
|
|
return make_tmp_dir(tmp_path, scm="scm" in fixtures, dvc="dvc" in fixtures)
|
|
|
|
|
|
@pytest.fixture
|
|
def scm(tmp_dir):
|
|
return tmp_dir.scm
|
|
|
|
|
|
@pytest.fixture
|
|
def dvc(tmp_dir):
|
|
with tmp_dir.dvc as _dvc:
|
|
yield _dvc
|
|
|
|
|
|
def git_init(path):
|
|
from git import Repo
|
|
from git.exc import GitCommandNotFound
|
|
|
|
# NOTE: handles EAGAIN error on BSD systems (osx in our case).
|
|
# Otherwise when running tests you might get this exception:
|
|
#
|
|
# GitCommandNotFound: Cmd('git') not found due to:
|
|
# OSError('[Errno 35] Resource temporarily unavailable')
|
|
git = retry(5, GitCommandNotFound)(Repo.init)(path)
|
|
git.close()
|
|
|
|
|
|
@pytest.fixture
|
|
def run_copy(tmp_dir, dvc):
|
|
tmp_dir.gen(
|
|
"copy.py",
|
|
(
|
|
"import sys, shutil, os\n"
|
|
"shutil.copyfile(sys.argv[1], sys.argv[2]) "
|
|
"if os.path.isfile(sys.argv[1]) "
|
|
"else shutil.copytree(sys.argv[1], sys.argv[2])"
|
|
),
|
|
)
|
|
|
|
def run_copy(src, dst, **run_kwargs):
|
|
wdir = pathlib.Path(run_kwargs.get("wdir", "."))
|
|
wdir = pathlib.Path("../" * len(wdir.parts))
|
|
script_path = wdir / "copy.py"
|
|
|
|
return dvc.run(
|
|
cmd=f"python {script_path} {src} {dst}",
|
|
outs=[dst],
|
|
deps=[src, f"{script_path}"],
|
|
**run_kwargs,
|
|
)
|
|
|
|
return run_copy
|
|
|
|
|
|
@pytest.fixture
|
|
def run_head(tmp_dir, dvc):
|
|
"""Output first line of each file to different file with '-1' appended.
|
|
Useful for tracking multiple outputs/dependencies which are not a copy
|
|
of each others.
|
|
"""
|
|
tmp_dir.gen(
|
|
{
|
|
"head.py": dedent(
|
|
"""
|
|
import sys
|
|
for file in sys.argv[1:]:
|
|
with open(file) as f, open(file +"-1","w+") as w:
|
|
w.write(f.readline())
|
|
"""
|
|
)
|
|
}
|
|
)
|
|
script = os.path.abspath(tmp_dir / "head.py")
|
|
|
|
def run(*args, **run_kwargs):
|
|
return dvc.run(
|
|
**{
|
|
"cmd": "python {} {}".format(script, " ".join(args)),
|
|
"outs": [dep + "-1" for dep in args],
|
|
"deps": list(args),
|
|
**run_kwargs,
|
|
}
|
|
)
|
|
|
|
return run
|
|
|
|
|
|
@pytest.fixture
|
|
def erepo_dir(make_tmp_dir):
|
|
return make_tmp_dir("erepo", scm=True, dvc=True)
|
|
|
|
|
|
@pytest.fixture
|
|
def git_dir(make_tmp_dir):
|
|
path = make_tmp_dir("git-erepo", scm=True)
|
|
path.scm.commit("init repo")
|
|
return path
|
|
|
|
|
|
@pytest.fixture
|
|
def git_upstream(tmp_dir, erepo_dir, git_dir, request):
|
|
remote = erepo_dir if "dvc" in request.fixturenames else git_dir
|
|
url = "file://{}".format(remote.resolve().as_posix())
|
|
tmp_dir.scm.gitpython.repo.create_remote("upstream", url)
|
|
remote.remote = "upstream"
|
|
remote.url = url
|
|
return remote
|
|
|
|
|
|
@pytest.fixture
|
|
def git_downstream(tmp_dir, erepo_dir, git_dir, request):
|
|
remote = erepo_dir if "dvc" in request.fixturenames else git_dir
|
|
url = "file://{}".format(tmp_dir.resolve().as_posix())
|
|
remote.scm.gitpython.repo.create_remote("upstream", url)
|
|
remote.remote = "upstream"
|
|
remote.url = url
|
|
return remote
|