Files
dvc/tests/func/test_ignore.py
T
Ruslan Kuprieiev 625b20833e dvc: get rid of CleanTree (#4221)
* dvcignore: move file/dir/parent checks into DvcIgnoreFilter

* dvc: get rid of CleanTree

CleanTree is a very awkward wrapper that spreads the context across
the codebase and results in unexpected behaviour when using it.

This PR starts moving dvcignore-related logic into the trees themselves
(it makes a lot of sense, kinda like `state`) so they could deal with
it however they like.

There are at least two temporary ugly parts about this PR:

1) dvcignore is used by individual trees and not packed into
tree/base.py yet;

2) `dvcignore_root` argument. This one is caused by the dvcignore trying
to collect everything topdown starting from the certain root dir. What
it should do instead is for a certain path that is being checked look up
the tree through the parents until it finds repo root (.dvc dir) and
then stop. That would handle subrepos as well. At the same time we need
to leverage existing dvcignore trie structure to cache those results.
2020-07-16 14:22:56 +03:00

367 lines
11 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import os
import shutil
import pytest
from dvc.exceptions import DvcIgnoreInCollectedDirError
from dvc.ignore import (
DvcIgnore,
DvcIgnoreDirs,
DvcIgnorePatterns,
DvcIgnorePatternsTrie,
DvcIgnoreRepo,
)
from dvc.path_info import PathInfo
from dvc.repo import Repo
from dvc.tree.local import LocalRemoteTree
from dvc.utils import relpath
from dvc.utils.fs import get_mtime_and_size
from tests.dir_helpers import TmpDir
def test_ignore(tmp_dir, dvc, monkeypatch):
tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}})
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored")
dvc.tree.__dict__.pop("dvcignore", None)
path = PathInfo(tmp_dir)
assert set(dvc.tree.walk_files(path / "dir")) == {path / "dir" / "other"}
def test_ignore_unicode(tmp_dir, dvc):
tmp_dir.gen({"dir": {"other": "text", "тест": "проверка"}})
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/тест")
dvc.tree.__dict__.pop("dvcignore", None)
path = PathInfo(tmp_dir)
assert set(dvc.tree.walk_files(path / "dir")) == {path / "dir" / "other"}
def test_rename_ignored_file(tmp_dir, dvc):
tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}})
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "ignored*")
dvc.tree.__dict__.pop("dvcignore", None)
mtime, size = get_mtime_and_size("dir", dvc.tree)
shutil.move("dir/ignored", "dir/ignored_new")
new_mtime, new_size = get_mtime_and_size("dir", dvc.tree)
assert new_mtime == mtime and new_size == size
def test_rename_file(tmp_dir, dvc):
tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
mtime, size = get_mtime_and_size("dir", dvc.tree)
shutil.move("dir/foo", "dir/foo_new")
new_mtime, new_size = get_mtime_and_size("dir", dvc.tree)
assert new_mtime != mtime and new_size == size
def test_remove_ignored_file(tmp_dir, dvc):
tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}})
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored")
dvc.tree.__dict__.pop("dvcignore", None)
mtime, size = get_mtime_and_size("dir", dvc.tree)
os.remove("dir/ignored")
new_mtime, new_size = get_mtime_and_size("dir", dvc.tree)
assert new_mtime == mtime and new_size == size
def test_remove_file(tmp_dir, dvc):
tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
mtime, size = get_mtime_and_size("dir", dvc.tree)
os.remove("dir/foo")
new_mtime, new_size = get_mtime_and_size("dir", dvc.tree)
assert new_mtime != mtime and new_size != size
def test_dvcignore_in_out_dir(tmp_dir, dvc):
tmp_dir.gen({"dir": {"foo": "foo", DvcIgnore.DVCIGNORE_FILE: ""}})
with pytest.raises(DvcIgnoreInCollectedDirError):
dvc.add("dir")
@pytest.mark.parametrize("dname", ["dir", "dir/subdir"])
def test_ignore_collecting_dvcignores(tmp_dir, dvc, dname):
tmp_dir.gen({"dir": {"subdir": {}}})
top_ignore_file = (tmp_dir / dname).with_name(DvcIgnore.DVCIGNORE_FILE)
top_ignore_file.write_text(os.path.basename(dname))
dvc.tree.__dict__.pop("dvcignore", None)
ignore_file = tmp_dir / dname / DvcIgnore.DVCIGNORE_FILE
ignore_file.write_text("foo")
assert len(dvc.tree.dvcignore.ignores) == 3
assert DvcIgnoreDirs([".git", ".hg", ".dvc"]) in dvc.tree.dvcignore.ignores
ignore_pattern_trie = None
for ignore in dvc.tree.dvcignore.ignores:
if isinstance(ignore, DvcIgnorePatternsTrie):
ignore_pattern_trie = ignore
assert ignore_pattern_trie is not None
assert (
DvcIgnorePatterns.from_files(
os.fspath(top_ignore_file),
LocalRemoteTree(None, {"url": dvc.root_dir}),
)
== ignore_pattern_trie[os.fspath(ignore_file)]
)
assert any(
i for i in dvc.tree.dvcignore.ignores if isinstance(i, DvcIgnoreRepo)
)
def test_ignore_on_branch(tmp_dir, scm, dvc):
tmp_dir.scm_gen({"foo": "foo", "bar": "bar"}, commit="add files")
with tmp_dir.branch("branch", new=True):
tmp_dir.scm_gen(DvcIgnore.DVCIGNORE_FILE, "foo", commit="add ignore")
dvc.tree.__dict__.pop("dvcignore", None)
path = PathInfo(tmp_dir)
assert set(dvc.tree.walk_files(path)) == {
path / "foo",
path / "bar",
}
dvc.tree = scm.get_tree("branch", use_dvcignore=True)
assert set(dvc.tree.walk_files(path)) == {
os.fspath(path / DvcIgnore.DVCIGNORE_FILE),
os.fspath(path / "bar"),
}
def test_match_nested(tmp_dir, dvc):
tmp_dir.gen(
{
".dvcignore": "*.backup\ntmp",
"foo": "foo",
"tmp": "...",
"dir": {"x.backup": "x backup", "tmp": "content"},
}
)
dvc.tree.__dict__.pop("dvcignore", None)
result = {os.fspath(os.path.normpath(f)) for f in dvc.tree.walk_files(".")}
assert result == {".dvcignore", "foo"}
def test_ignore_external(tmp_dir, scm, dvc, tmp_path_factory):
tmp_dir.gen(".dvcignore", "*.backup\ntmp")
ext_dir = TmpDir(os.fspath(tmp_path_factory.mktemp("external_dir")))
ext_dir.gen({"y.backup": "y", "tmp": "ext tmp"})
result = {relpath(f, ext_dir) for f in dvc.tree.walk_files(ext_dir)}
assert result == {"y.backup", "tmp"}
def test_ignore_subrepo(tmp_dir, scm, dvc):
tmp_dir.gen({".dvcignore": "foo", "subdir": {"foo": "foo"}})
scm.add([".dvcignore"])
scm.commit("init parent dvcignore")
dvc.tree.__dict__.pop("dvcignore", None)
subrepo_dir = tmp_dir / "subdir"
assert not dvc.tree.exists(PathInfo(subrepo_dir / "foo"))
with subrepo_dir.chdir():
subrepo = Repo.init(subdir=True)
scm.add(str(subrepo_dir / "foo"))
scm.commit("subrepo init")
for _ in subrepo.brancher(all_commits=True):
assert subrepo.tree.exists(PathInfo(subrepo_dir / "foo"))
def test_ignore_blank_line(tmp_dir, dvc):
tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}})
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "foo\n\ndir/ignored")
dvc.tree.__dict__.pop("dvcignore", None)
path = PathInfo(tmp_dir)
assert set(dvc.tree.walk_files(path / "dir")) == {path / "dir" / "other"}
# It is not possible to re-include a file if a parent directory of
# that file is excluded.
# Git doesn’t list excluded directories for performance reasons,
# so any patterns on contained files have no effect,
# no matter where they are defined.
@pytest.mark.parametrize(
"data_struct, pattern_list, result_set",
[
(
{"dir": {"subdir": {"not_ignore": "121"}}},
["subdir/*", "!not_ignore"],
{os.path.join("dir", "subdir", "not_ignore")},
),
(
{"dir": {"subdir": {"should_ignore": "121"}}},
["subdir", "!should_ignore"],
set(),
),
(
{"dir": {"subdir": {"should_ignore": "121"}}},
["subdir/", "!should_ignore"],
set(),
),
],
)
def test_ignore_file_in_parent_path(
tmp_dir, dvc, data_struct, pattern_list, result_set
):
tmp_dir.gen(data_struct)
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "\n".join(pattern_list))
dvc.tree.__dict__.pop("dvcignore", None)
path = PathInfo(tmp_dir)
assert set(dvc.tree.walk_files(path / "dir")) == {
path / relpath for relpath in result_set
}
# If there is a separator at the end of the pattern then the pattern
# will only match directories,
# otherwise the pattern can match both files and directories.
# For example, a pattern doc/frotz/ matches doc/frotz directory,
# but not a/doc/frotz directory;
def test_ignore_sub_directory(tmp_dir, dvc):
tmp_dir.gen(
{
"dir": {
"doc": {"fortz": {"b": "b"}},
"a": {"doc": {"fortz": {"a": "a"}}},
}
}
)
tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "doc/fortz"}})
dvc.tree.__dict__.pop("dvcignore", None)
path = PathInfo(tmp_dir)
assert set(dvc.tree.walk_files(path / "dir")) == {
path / "dir" / "a" / "doc" / "fortz" / "a",
path / "dir" / DvcIgnore.DVCIGNORE_FILE,
}
# however frotz/ matches frotz and a/frotz that is a directory
def test_ignore_directory(tmp_dir, dvc):
tmp_dir.gen({"dir": {"fortz": {}, "a": {"fortz": {}}}})
tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "fortz"}})
dvc.tree.__dict__.pop("dvcignore", None)
path = PathInfo(tmp_dir)
assert set(dvc.tree.walk_files(path / "dir")) == {
path / "dir" / DvcIgnore.DVCIGNORE_FILE,
}
def test_multi_ignore_file(tmp_dir, dvc, monkeypatch):
tmp_dir.gen({"dir": {"subdir": {"should_ignore": "1", "not_ignore": "1"}}})
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/subdir/*_ignore")
tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "!subdir/not_ignore"}})
dvc.tree.__dict__.pop("dvcignore", None)
path = PathInfo(tmp_dir)
assert set(dvc.tree.walk_files(path / "dir")) == {
path / "dir" / "subdir" / "not_ignore",
path / "dir" / DvcIgnore.DVCIGNORE_FILE,
}
def test_pattern_trie_tree(tmp_dir, dvc):
tmp_dir.gen(
{
"top": {
"first": {
DvcIgnore.DVCIGNORE_FILE: "a\nb\nc",
"middle": {
"second": {
DvcIgnore.DVCIGNORE_FILE: "d\ne\nf",
"bottom": {},
}
},
},
},
"other": {DvcIgnore.DVCIGNORE_FILE: "1\n2\n3"},
}
)
dvc.tree.__dict__.pop("dvcignore", None)
ignore_pattern_trie = None
for ignore in dvc.tree.dvcignore.ignores:
if isinstance(ignore, DvcIgnorePatternsTrie):
ignore_pattern_trie = ignore
break
assert ignore_pattern_trie is not None
ignore_pattern_top = ignore_pattern_trie[os.fspath(tmp_dir / "top")]
ignore_pattern_other = ignore_pattern_trie[os.fspath(tmp_dir / "other")]
ignore_pattern_first = ignore_pattern_trie[
os.fspath(tmp_dir / "top" / "first")
]
ignore_pattern_middle = ignore_pattern_trie[
os.fspath(tmp_dir / "top" / "first" / "middle")
]
ignore_pattern_second = ignore_pattern_trie[
os.fspath(tmp_dir / "top" / "first" / "middle" / "second")
]
ignore_pattern_bottom = ignore_pattern_trie[
os.fspath(tmp_dir / "top" / "first" / "middle" / "second" / "bottom")
]
assert not ignore_pattern_top
assert (
DvcIgnorePatterns([], os.fspath(tmp_dir / "top")) == ignore_pattern_top
)
assert (
DvcIgnorePatterns(["1", "2", "3"], os.fspath(tmp_dir / "other"))
== ignore_pattern_other
)
assert (
DvcIgnorePatterns(
["a", "b", "c"], os.fspath(tmp_dir / "top" / "first")
)
== ignore_pattern_first
)
assert (
DvcIgnorePatterns(
["a", "b", "c"], os.fspath(tmp_dir / "top" / "first")
)
== ignore_pattern_middle
)
assert (
DvcIgnorePatterns(
[
"a",
"b",
"c",
"/middle/second/**/d",
"/middle/second/**/e",
"/middle/second/**/f",
],
os.fspath(tmp_dir / "top" / "first"),
)
== ignore_pattern_second
)
assert (
DvcIgnorePatterns(
[
"a",
"b",
"c",
"/middle/second/**/d",
"/middle/second/**/e",
"/middle/second/**/f",
],
os.fspath(tmp_dir / "top" / "first"),
)
== ignore_pattern_bottom
)