mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
Saugat Pachhai
8d1934d494
* Allow ignoring subrepos in dvcignore
I implemented this inside dvcignore, and it
should not require "internal" APIs hack as we discussed before.
Right now, `ignore_subrepos` will only work for the paths of subrepo,
not anything inside it. And the API user is more or less only
RepoTree, so should be safe (we switch DvcTree and repo.tree based
on path prefixes, so this is only required during `walk`, so as it
shows the repo that were previously dvcignored and so that we could
switch the trees later during walk.
So, if dir is `dir -> repo`
So, `tree.walk("dir")` will return `[]`,
whereas `tree.walk("dir", ignore_subrepos=False)` will return `["repo"]`
But, successive walk will just return `("repo", [], [])`, and could
just be ignored as a side-effect.
* Fix tests
359 lines
9.9 KiB
Python
359 lines
9.9 KiB
Python
"""
|
|
The goal of this module is making dvc functional tests setup a breeze. This
|
|
includes a temporary dir, initializing git and DVC repos and bootstrapping some
|
|
file structure.
|
|
|
|
The cornerstone of these fixtures is `tmp_dir`, which creates a temporary dir
|
|
and changes path to it, it might be combined with `scm` and `dvc` to initialize
|
|
empty git and DVC repos. `tmp_dir` returns a Path instance, which should save
|
|
you from using `open()`, `os` and `os.path` utils many times:
|
|
|
|
(tmp_dir / "some_file").write_text("some text")
|
|
# ...
|
|
assert "some text" == (tmp_dir / "some_file").read_text()
|
|
assert (tmp_dir / "some_file").exists()
|
|
|
|
Additionally it provides `.gen()`, `.scm_gen()` and `.dvc_gen()` methods to
|
|
bootstrap a required file structure in a single call:
|
|
|
|
# Generate a dir with files
|
|
tmp_dir.gen({"dir": {"file": "file text", "second_file": "..."}})
|
|
|
|
# Generate a single file, dirs will be created along the way
|
|
tmp_dir.gen("dir/file", "file text")
|
|
|
|
# Generate + git add
|
|
tmp_dir.scm_gen({"file1": "...", ...})
|
|
|
|
# Generate + git add + git commit
|
|
tmp_dir.scm_gen({"file1": "...", ...}, commit="add files")
|
|
|
|
# Generate + dvc add
|
|
tmp_dir.dvc_gen({"file1": "...", ...})
|
|
|
|
# Generate + dvc add + git commit -am "..."
|
|
# This commits stages to git not the generated files.
|
|
tmp_dir.dvc_gen({"file1": "...", ...}, commit="add files")
|
|
|
|
Making it easier to bootstrap things has a supergoal of incentivizing a move
|
|
from global repo template to creating everything inplace, which:
|
|
|
|
- makes all path references local to test, enhancing readability
|
|
- allows using telling filenames, e.g. "git_tracked_file" instead of "foo"
|
|
- does not create unnecessary files
|
|
"""
|
|
|
|
# pylint: disable=redefined-outer-name, attribute-defined-outside-init
|
|
|
|
import os
|
|
import pathlib
|
|
from contextlib import contextmanager
|
|
from textwrap import dedent
|
|
|
|
import pytest
|
|
from funcy import lmap, retry
|
|
|
|
from dvc.logger import disable_other_loggers
|
|
from dvc.utils.fs import makedirs
|
|
|
|
__all__ = [
|
|
"make_tmp_dir",
|
|
"tmp_dir",
|
|
"scm",
|
|
"dvc",
|
|
"run_copy",
|
|
"run_head",
|
|
"erepo_dir",
|
|
"git_dir",
|
|
"git_init",
|
|
]
|
|
|
|
|
|
# see https://github.com/iterative/dvc/issues/3167
|
|
disable_other_loggers()
|
|
|
|
|
|
class TmpDir(pathlib.Path):
|
|
scheme = "local"
|
|
|
|
def __new__(cls, *args, **kwargs):
|
|
if cls is TmpDir:
|
|
cls = ( # pylint: disable=self-cls-assignment
|
|
WindowsTmpDir if os.name == "nt" else PosixTmpDir
|
|
)
|
|
self = cls._from_parts(args, init=False)
|
|
if not self._flavour.is_supported:
|
|
raise NotImplementedError(
|
|
f"cannot instantiate {cls.__name__!r} on your system"
|
|
)
|
|
self._init()
|
|
return self
|
|
|
|
def init(self, *, scm=False, dvc=False, subdir=False):
|
|
from dvc.repo import Repo
|
|
from dvc.scm.git import Git
|
|
|
|
assert not scm or not hasattr(self, "scm")
|
|
assert not dvc or not hasattr(self, "dvc")
|
|
|
|
str_path = os.fspath(self)
|
|
|
|
if scm:
|
|
git_init(str_path)
|
|
if dvc:
|
|
self.dvc = Repo.init(
|
|
str_path,
|
|
no_scm=not scm and not hasattr(self, "scm"),
|
|
subdir=subdir,
|
|
)
|
|
if scm:
|
|
self.scm = self.dvc.scm if hasattr(self, "dvc") else Git(str_path)
|
|
if dvc and hasattr(self, "scm"):
|
|
self.scm.commit("init dvc")
|
|
|
|
def close(self):
|
|
if hasattr(self, "scm"):
|
|
self.scm.close()
|
|
|
|
def _require(self, name):
|
|
if not hasattr(self, name):
|
|
raise TypeError(
|
|
"Can't use {name} for this temporary dir. "
|
|
'Did you forget to use "{name}" fixture?'.format(name=name)
|
|
)
|
|
|
|
# Bootstrapping methods
|
|
def gen(self, struct, text=""):
|
|
if isinstance(struct, (str, bytes, pathlib.PurePath)):
|
|
struct = {struct: text}
|
|
|
|
return self._gen(struct)
|
|
|
|
def _gen(self, struct, prefix=None):
|
|
paths = []
|
|
for name, contents in struct.items():
|
|
path = (prefix or self) / name
|
|
|
|
if isinstance(contents, dict):
|
|
if not contents:
|
|
makedirs(path, exist_ok=True)
|
|
else:
|
|
self._gen(contents, prefix=path)
|
|
else:
|
|
makedirs(path.parent, exist_ok=True)
|
|
if isinstance(contents, bytes):
|
|
path.write_bytes(contents)
|
|
else:
|
|
path.write_text(contents, encoding="utf-8")
|
|
paths.append(path)
|
|
return paths
|
|
|
|
def dvc_gen(self, struct, text="", commit=None):
|
|
paths = self.gen(struct, text)
|
|
return self.dvc_add(paths, commit=commit)
|
|
|
|
def scm_gen(self, struct, text="", commit=None):
|
|
paths = self.gen(struct, text)
|
|
return self.scm_add(paths, commit=commit)
|
|
|
|
def dvc_add(self, filenames, commit=None):
|
|
self._require("dvc")
|
|
filenames = _coerce_filenames(filenames)
|
|
|
|
stages = self.dvc.add(filenames)
|
|
if commit:
|
|
stage_paths = [s.path for s in stages]
|
|
|
|
def to_gitignore(stage_path):
|
|
from dvc.scm import Git
|
|
|
|
return os.path.join(os.path.dirname(stage_path), Git.GITIGNORE)
|
|
|
|
gitignores = [
|
|
to_gitignore(s)
|
|
for s in stage_paths
|
|
if os.path.exists(to_gitignore(s))
|
|
]
|
|
self.scm_add(stage_paths + gitignores, commit=commit)
|
|
|
|
return stages
|
|
|
|
def scm_add(self, filenames, commit=None):
|
|
self._require("scm")
|
|
filenames = _coerce_filenames(filenames)
|
|
|
|
self.scm.add(filenames)
|
|
if commit:
|
|
self.scm.commit(commit)
|
|
|
|
def add_remote(
|
|
self, *, url=None, config=None, name="upstream", default=True
|
|
):
|
|
self._require("dvc")
|
|
|
|
assert bool(url) ^ bool(config)
|
|
|
|
if url:
|
|
config = {"url": url}
|
|
|
|
with self.dvc.config.edit() as conf:
|
|
conf["remote"][name] = config
|
|
if default:
|
|
conf["core"]["remote"] = name
|
|
|
|
if hasattr(self, "scm"):
|
|
self.scm.add(self.dvc.config.files["repo"])
|
|
self.scm.commit(f"add '{name}' remote")
|
|
|
|
return url or config["url"]
|
|
|
|
# contexts
|
|
@contextmanager
|
|
def chdir(self):
|
|
old = os.getcwd()
|
|
try:
|
|
os.chdir(self)
|
|
yield
|
|
finally:
|
|
os.chdir(old)
|
|
|
|
@contextmanager
|
|
def branch(self, name, new=False):
|
|
self._require("scm")
|
|
old = self.scm.active_branch()
|
|
try:
|
|
self.scm.checkout(name, create_new=new)
|
|
yield
|
|
finally:
|
|
self.scm.checkout(old)
|
|
|
|
def read_text(self, *args, **kwargs): # pylint: disable=signature-differs
|
|
# NOTE: on windows we'll get PermissionError instead of
|
|
# IsADirectoryError when we try to `open` a directory, so we can't
|
|
# rely on exception flow control
|
|
if self.is_dir():
|
|
return {
|
|
path.name: path.read_text(*args, **kwargs)
|
|
for path in self.iterdir()
|
|
}
|
|
return super().read_text(*args, **kwargs)
|
|
|
|
|
|
def _coerce_filenames(filenames):
|
|
if isinstance(filenames, (str, bytes, pathlib.PurePath)):
|
|
filenames = [filenames]
|
|
return lmap(os.fspath, filenames)
|
|
|
|
|
|
class WindowsTmpDir(TmpDir, pathlib.PureWindowsPath):
|
|
pass
|
|
|
|
|
|
class PosixTmpDir(TmpDir, pathlib.PurePosixPath):
|
|
pass
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def make_tmp_dir(tmp_path_factory, request):
|
|
def make(name, *, scm=False, dvc=False, **kwargs):
|
|
path = tmp_path_factory.mktemp(name) if isinstance(name, str) else name
|
|
new_dir = TmpDir(path)
|
|
new_dir.init(scm=scm, dvc=dvc, **kwargs)
|
|
request.addfinalizer(new_dir.close)
|
|
return new_dir
|
|
|
|
return make
|
|
|
|
|
|
@pytest.fixture
|
|
def tmp_dir(tmp_path, make_tmp_dir, request, monkeypatch):
|
|
monkeypatch.chdir(tmp_path)
|
|
fixtures = request.fixturenames
|
|
return make_tmp_dir(tmp_path, scm="scm" in fixtures, dvc="dvc" in fixtures)
|
|
|
|
|
|
@pytest.fixture
|
|
def scm(tmp_dir):
|
|
return tmp_dir.scm
|
|
|
|
|
|
@pytest.fixture
|
|
def dvc(tmp_dir):
|
|
return tmp_dir.dvc
|
|
|
|
|
|
def git_init(path):
|
|
from git import Repo
|
|
from git.exc import GitCommandNotFound
|
|
|
|
# NOTE: handles EAGAIN error on BSD systems (osx in our case).
|
|
# Otherwise when running tests you might get this exception:
|
|
#
|
|
# GitCommandNotFound: Cmd('git') not found due to:
|
|
# OSError('[Errno 35] Resource temporarily unavailable')
|
|
git = retry(5, GitCommandNotFound)(Repo.init)(path)
|
|
git.close()
|
|
|
|
|
|
@pytest.fixture
|
|
def run_copy(tmp_dir, dvc):
|
|
tmp_dir.gen(
|
|
"copy.py",
|
|
"import sys, shutil\nshutil.copyfile(sys.argv[1], sys.argv[2])",
|
|
)
|
|
|
|
def run_copy(src, dst, **run_kwargs):
|
|
return dvc.run(
|
|
cmd=f"python copy.py {src} {dst}",
|
|
outs=[dst],
|
|
deps=[src, "copy.py"],
|
|
**run_kwargs,
|
|
)
|
|
|
|
return run_copy
|
|
|
|
|
|
@pytest.fixture
|
|
def run_head(tmp_dir, dvc):
|
|
"""Output first line of each file to different file with '-1' appended.
|
|
Useful for tracking multiple outputs/dependencies which are not a copy
|
|
of each others.
|
|
"""
|
|
tmp_dir.gen(
|
|
{
|
|
"head.py": dedent(
|
|
"""
|
|
import sys
|
|
for file in sys.argv[1:]:
|
|
with open(file) as f, open(file +"-1","w+") as w:
|
|
w.write(f.readline())
|
|
"""
|
|
)
|
|
}
|
|
)
|
|
script = os.path.abspath(tmp_dir / "head.py")
|
|
|
|
def run(*args, **run_kwargs):
|
|
return dvc.run(
|
|
**{
|
|
"cmd": "python {} {}".format(script, " ".join(args)),
|
|
"outs": [dep + "-1" for dep in args],
|
|
"deps": list(args),
|
|
**run_kwargs,
|
|
}
|
|
)
|
|
|
|
return run
|
|
|
|
|
|
@pytest.fixture
|
|
def erepo_dir(make_tmp_dir):
|
|
return make_tmp_dir("erepo", scm=True, dvc=True)
|
|
|
|
|
|
@pytest.fixture
|
|
def git_dir(make_tmp_dir):
|
|
path = make_tmp_dir("git-erepo", scm=True)
|
|
path.scm.commit("init repo")
|
|
return path
|