mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
Merge pull request #2913 from Suor/migrate-test-ignore
test: migrate test_ignore
This commit is contained in:
Ruslan Kuprieiev
@@ -6,7 +6,7 @@ import pytest
|
||||
from funcy.py3 import lmap, retry
|
||||
|
||||
from dvc.utils import makedirs
|
||||
from dvc.utils.compat import basestring, is_py2, pathlib, fspath, fspath_py35
|
||||
from dvc.utils.compat import basestring, pathlib, fspath, fspath_py35, bytes
|
||||
|
||||
|
||||
__all__ = ["tmp_dir", "scm", "dvc", "repo_template", "run_copy", "erepo_dir"]
|
||||
@@ -55,13 +55,16 @@ class TmpDir(pathlib.Path):
|
||||
path = (prefix or self) / name
|
||||
|
||||
if isinstance(contents, dict):
|
||||
self._gen(contents, prefix=path)
|
||||
if not contents:
|
||||
makedirs(path, exist_ok=True)
|
||||
else:
|
||||
self._gen(contents, prefix=path)
|
||||
else:
|
||||
makedirs(path.parent, exist_ok=True)
|
||||
if is_py2 and isinstance(contents, str):
|
||||
if isinstance(contents, bytes):
|
||||
path.write_bytes(contents)
|
||||
else:
|
||||
path.write_text(contents)
|
||||
path.write_text(contents, encoding="utf-8")
|
||||
|
||||
def dvc_gen(self, struct, text="", commit=None):
|
||||
paths = self.gen(struct, text)
|
||||
|
||||
+77
-142
@@ -1,186 +1,121 @@
|
||||
import itertools
|
||||
# encoding: utf-8
|
||||
from __future__ import unicode_literals
|
||||
import os
|
||||
import shutil
|
||||
|
||||
import pytest
|
||||
|
||||
from dvc.exceptions import DvcIgnoreInCollectedDirError
|
||||
from dvc.ignore import DvcIgnore
|
||||
from dvc.ignore import DvcIgnoreDirs
|
||||
from dvc.ignore import DvcIgnoreFilter
|
||||
from dvc.ignore import DvcIgnorePatterns
|
||||
from dvc.ignore import DvcIgnore, DvcIgnoreDirs, DvcIgnorePatterns
|
||||
from dvc.scm.tree import WorkingTree
|
||||
from dvc.utils.compat import cast_bytes
|
||||
from dvc.utils import walk_files
|
||||
from dvc.utils.compat import fspath
|
||||
from dvc.utils.fs import get_mtime_and_size
|
||||
from tests.basic_env import TestDvc
|
||||
|
||||
from tests.utils import to_posixpath
|
||||
|
||||
|
||||
class TestDvcIgnore(TestDvc):
|
||||
def setUp(self):
|
||||
super(TestDvcIgnore, self).setUp()
|
||||
def test_ignore(tmp_dir, dvc, monkeypatch):
|
||||
tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}})
|
||||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored")
|
||||
|
||||
def _get_all_paths(self):
|
||||
assert _files_set("dir", dvc.dvcignore) == {"dir/other"}
|
||||
|
||||
paths = []
|
||||
for root, dirs, files in self.dvc.tree.walk(
|
||||
self.dvc.root_dir, dvcignore=self.dvc.dvcignore
|
||||
):
|
||||
for dname in dirs:
|
||||
paths.append(os.path.join(root, dname))
|
||||
|
||||
for fname in files:
|
||||
paths.append(os.path.join(root, fname))
|
||||
|
||||
return paths
|
||||
|
||||
def test_ignore_in_child_dir(self):
|
||||
ignore_file = os.path.join(self.dvc.root_dir, DvcIgnore.DVCIGNORE_FILE)
|
||||
with open(ignore_file, "w") as fobj:
|
||||
fobj.write("data_dir/data")
|
||||
|
||||
forbidden_path = os.path.join(self.dvc.root_dir, self.DATA)
|
||||
all_paths = self._get_all_paths()
|
||||
|
||||
self.assertNotIn(forbidden_path, all_paths)
|
||||
|
||||
def test_ignore_in_child_dir_unicode(self):
|
||||
ignore_file = os.path.join(self.dvc.root_dir, DvcIgnore.DVCIGNORE_FILE)
|
||||
with open(ignore_file, "wb") as fobj:
|
||||
fobj.write(cast_bytes(self.UNICODE, "utf-8"))
|
||||
|
||||
forbidden_path = os.path.join(self.dvc.root_dir, self.UNICODE)
|
||||
all_paths = self._get_all_paths()
|
||||
|
||||
self.assertNotIn(forbidden_path, all_paths)
|
||||
|
||||
def test_ignore_in_parent_dir(self):
|
||||
ignore_file = os.path.join(self.dvc.root_dir, DvcIgnore.DVCIGNORE_FILE)
|
||||
with open(ignore_file, "w") as fobj:
|
||||
fobj.write("data_dir/data")
|
||||
|
||||
os.chdir(self.DATA_DIR)
|
||||
|
||||
forbidden_path = os.path.join(self.dvc.root_dir, self.DATA)
|
||||
all_paths = self._get_all_paths()
|
||||
|
||||
self.assertNotIn(forbidden_path, all_paths)
|
||||
monkeypatch.chdir("dir")
|
||||
assert _files_set(".", dvc.dvcignore) == {"./other"}
|
||||
|
||||
|
||||
def test_metadata_unchanged_when_moving_ignored_file(dvc_repo, repo_dir):
|
||||
new_data_path = repo_dir.DATA_SUB + "_new"
|
||||
def test_ignore_unicode(tmp_dir, dvc):
|
||||
tmp_dir.gen({"dir": {"other": "text"}})
|
||||
# Path() doesn't handle unicode paths in Windows/Python 2
|
||||
# I don't know to debug it further, waiting till Python 2 EOL
|
||||
with open("dir/тест", "wb") as fd:
|
||||
fd.write("проверка".encode("utf-8"))
|
||||
|
||||
ignore_file = os.path.join(dvc_repo.root_dir, DvcIgnore.DVCIGNORE_FILE)
|
||||
repo_dir.create(
|
||||
ignore_file,
|
||||
"\n".join(
|
||||
[to_posixpath(repo_dir.DATA_SUB), to_posixpath(new_data_path)]
|
||||
),
|
||||
)
|
||||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/тест")
|
||||
|
||||
mtime_sig, size = get_mtime_and_size(repo_dir.DATA_DIR, dvc_repo.dvcignore)
|
||||
|
||||
shutil.move(repo_dir.DATA_SUB, new_data_path)
|
||||
|
||||
new_mtime_sig, new_size = get_mtime_and_size(
|
||||
repo_dir.DATA_DIR, dvc_repo.dvcignore
|
||||
)
|
||||
|
||||
assert new_mtime_sig == mtime_sig
|
||||
assert new_size == size
|
||||
assert _files_set("dir", dvc.dvcignore) == {"dir/other"}
|
||||
|
||||
|
||||
def test_mtime_changed_when_moving_non_ignored_file(dvc_repo, repo_dir):
|
||||
new_data_path = repo_dir.DATA_SUB + "_new"
|
||||
mtime, size = get_mtime_and_size(repo_dir.DATA_DIR, dvc_repo.dvcignore)
|
||||
def test_rename_ignored_file(tmp_dir, dvc):
|
||||
tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}})
|
||||
|
||||
shutil.move(repo_dir.DATA_SUB, new_data_path)
|
||||
new_mtime, new_size = get_mtime_and_size(
|
||||
repo_dir.DATA_DIR, dvc_repo.dvcignore
|
||||
)
|
||||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "ignored*")
|
||||
mtime, size = get_mtime_and_size("dir", dvc.dvcignore)
|
||||
|
||||
assert new_mtime != mtime
|
||||
assert new_size == size
|
||||
shutil.move("dir/ignored", "dir/ignored_new")
|
||||
new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore)
|
||||
|
||||
assert new_mtime == mtime and new_size == size
|
||||
|
||||
|
||||
def test_metadata_unchanged_on_ignored_file_deletion(dvc_repo, repo_dir):
|
||||
ignore_file = os.path.join(dvc_repo.root_dir, DvcIgnore.DVCIGNORE_FILE)
|
||||
repo_dir.create(ignore_file, to_posixpath(repo_dir.DATA_SUB))
|
||||
def test_rename_file(tmp_dir, dvc):
|
||||
tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
|
||||
mtime, size = get_mtime_and_size("dir", dvc.dvcignore)
|
||||
|
||||
mtime, size = get_mtime_and_size(repo_dir.DATA_DIR, dvc_repo.dvcignore)
|
||||
shutil.move("dir/foo", "dir/foo_new")
|
||||
new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore)
|
||||
|
||||
os.remove(repo_dir.DATA_SUB)
|
||||
new_mtime, new_size = get_mtime_and_size(
|
||||
repo_dir.DATA_DIR, dvc_repo.dvcignore
|
||||
)
|
||||
|
||||
assert new_mtime == mtime
|
||||
assert new_size == size
|
||||
assert new_mtime != mtime and new_size == size
|
||||
|
||||
|
||||
def test_metadata_changed_on_non_ignored_file_deletion(dvc_repo, repo_dir):
|
||||
mtime, size = get_mtime_and_size(repo_dir.DATA_DIR, dvc_repo.dvcignore)
|
||||
def test_remove_ignored_file(tmp_dir, dvc):
|
||||
tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}})
|
||||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored")
|
||||
|
||||
os.remove(repo_dir.DATA_SUB)
|
||||
new_mtime_sig, new_size = get_mtime_and_size(
|
||||
repo_dir.DATA_DIR, dvc_repo.dvcignore
|
||||
)
|
||||
mtime, size = get_mtime_and_size("dir", dvc.dvcignore)
|
||||
|
||||
assert new_mtime_sig != mtime
|
||||
assert new_size != size
|
||||
os.remove("dir/ignored")
|
||||
new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore)
|
||||
|
||||
assert new_mtime == mtime and new_size == size
|
||||
|
||||
|
||||
def test_should_raise_on_dvcignore_in_out_dir(dvc_repo, repo_dir):
|
||||
ignore_file = os.path.join(repo_dir.DATA_DIR, DvcIgnore.DVCIGNORE_FILE)
|
||||
repo_dir.create(ignore_file, "")
|
||||
def test_remove_file(tmp_dir, dvc):
|
||||
tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
|
||||
mtime, size = get_mtime_and_size("dir", dvc.dvcignore)
|
||||
|
||||
os.remove("dir/foo")
|
||||
new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore)
|
||||
|
||||
assert new_mtime != mtime and new_size != size
|
||||
|
||||
|
||||
def test_dvcignore_in_out_dir(tmp_dir, dvc):
|
||||
tmp_dir.gen({"dir": {"foo": "foo", DvcIgnore.DVCIGNORE_FILE: ""}})
|
||||
|
||||
with pytest.raises(DvcIgnoreInCollectedDirError):
|
||||
dvc_repo.add(repo_dir.DATA_DIR)
|
||||
dvc.add("dir")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dname", [TestDvc.DATA_DIR, TestDvc.DATA_SUB_DIR])
|
||||
def test_ignore_collecting_dvcignores(repo_dir, dname):
|
||||
top_ignore_file = os.path.join(
|
||||
repo_dir.root_dir, os.path.dirname(dname), DvcIgnore.DVCIGNORE_FILE
|
||||
)
|
||||
repo_dir.create(top_ignore_file, os.path.basename(dname))
|
||||
# @efiop why do we need to parametrize this?
|
||||
@pytest.mark.parametrize("dname", ["dir", "dir/subdir"])
|
||||
def test_ignore_collecting_dvcignores(tmp_dir, dvc, dname):
|
||||
tmp_dir.gen({"dir": {"subdir": {}}})
|
||||
|
||||
ignore_file = os.path.join(
|
||||
repo_dir.root_dir, dname, DvcIgnore.DVCIGNORE_FILE
|
||||
)
|
||||
repo_dir.create(ignore_file, repo_dir.FOO)
|
||||
top_ignore_file = (tmp_dir / dname).with_name(DvcIgnore.DVCIGNORE_FILE)
|
||||
top_ignore_file.write_text(os.path.basename(dname))
|
||||
|
||||
assert DvcIgnoreFilter(
|
||||
repo_dir.root_dir, WorkingTree(repo_dir.root_dir)
|
||||
).ignores == {
|
||||
ignore_file = tmp_dir / dname / DvcIgnore.DVCIGNORE_FILE
|
||||
ignore_file.write_text("foo")
|
||||
|
||||
assert dvc.dvcignore.ignores == {
|
||||
DvcIgnoreDirs([".git", ".hg", ".dvc"]),
|
||||
DvcIgnorePatterns(top_ignore_file, WorkingTree(repo_dir.root_dir)),
|
||||
DvcIgnorePatterns(fspath(top_ignore_file), WorkingTree(dvc.root_dir)),
|
||||
}
|
||||
|
||||
|
||||
def test_ignore_on_branch(git, dvc_repo, repo_dir):
|
||||
dvc_repo.add(repo_dir.DATA_DIR)
|
||||
dvc_repo.scm.commit("add data dir")
|
||||
def test_ignore_on_branch(tmp_dir, scm, dvc):
|
||||
tmp_dir.scm_gen({"foo": "foo", "bar": "bar"}, commit="add files")
|
||||
|
||||
branch_name = "branch_one"
|
||||
dvc_repo.scm.checkout(branch_name, create_new=True)
|
||||
scm.checkout("branch", create_new=True)
|
||||
tmp_dir.scm_gen(DvcIgnore.DVCIGNORE_FILE, "foo", commit="add ignore")
|
||||
|
||||
repo_dir.create(DvcIgnore.DVCIGNORE_FILE, to_posixpath(repo_dir.DATA_SUB))
|
||||
dvc_repo.scm.add([DvcIgnore.DVCIGNORE_FILE])
|
||||
git.index.commit("add ignore")
|
||||
scm.checkout("master")
|
||||
assert _files_set(".", dvc.dvcignore) == {"./foo", "./bar"}
|
||||
|
||||
dvc_repo.scm.checkout("master")
|
||||
dvc.tree = scm.get_tree("branch")
|
||||
assert _files_set(".", dvc.dvcignore) == {"./bar"}
|
||||
|
||||
git_tree = dvc_repo.scm.get_tree(branch_name)
|
||||
branch_data_files = set(
|
||||
itertools.chain.from_iterable(
|
||||
[
|
||||
files
|
||||
for _, _, files in dvc_repo.tree.walk(
|
||||
repo_dir.DATA_DIR,
|
||||
dvcignore=DvcIgnoreFilter(repo_dir.root_dir, git_tree),
|
||||
)
|
||||
]
|
||||
)
|
||||
)
|
||||
assert branch_data_files == {"data"}
|
||||
|
||||
def _files_set(root, dvcignore):
|
||||
return {to_posixpath(f) for f in walk_files(root, dvcignore)}
|
||||
|
||||
Reference in New Issue
Block a user