mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 23:11:44 +01:00
* dvcignore: move file/dir/parent checks into DvcIgnoreFilter * dvc: get rid of CleanTree CleanTree is a very awkward wrapper that spreads the context across the codebase and results in unexpected behaviour when using it. This PR starts moving dvcignore-related logic into the trees themselves (it makes a lot of sense, kinda like `state`) so they could deal with it however they like. There are at least two temporary ugly parts about this PR: 1) dvcignore is used by individual trees and not packed into tree/base.py yet; 2) `dvcignore_root` argument. This one is caused by the dvcignore trying to collect everything topdown starting from the certain root dir. What it should do instead is for a certain path that is being checked look up the tree through the parents until it finds repo root (.dvc dir) and then stop. That would handle subrepos as well. At the same time we need to leverage existing dvcignore trie structure to cache those results.
367 lines
11 KiB
Python
367 lines
11 KiB
Python
import os
|
||
import shutil
|
||
|
||
import pytest
|
||
|
||
from dvc.exceptions import DvcIgnoreInCollectedDirError
|
||
from dvc.ignore import (
|
||
DvcIgnore,
|
||
DvcIgnoreDirs,
|
||
DvcIgnorePatterns,
|
||
DvcIgnorePatternsTrie,
|
||
DvcIgnoreRepo,
|
||
)
|
||
from dvc.path_info import PathInfo
|
||
from dvc.repo import Repo
|
||
from dvc.tree.local import LocalRemoteTree
|
||
from dvc.utils import relpath
|
||
from dvc.utils.fs import get_mtime_and_size
|
||
from tests.dir_helpers import TmpDir
|
||
|
||
|
||
def test_ignore(tmp_dir, dvc, monkeypatch):
|
||
tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}})
|
||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored")
|
||
|
||
dvc.tree.__dict__.pop("dvcignore", None)
|
||
|
||
path = PathInfo(tmp_dir)
|
||
|
||
assert set(dvc.tree.walk_files(path / "dir")) == {path / "dir" / "other"}
|
||
|
||
|
||
def test_ignore_unicode(tmp_dir, dvc):
|
||
tmp_dir.gen({"dir": {"other": "text", "тест": "проверка"}})
|
||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/тест")
|
||
dvc.tree.__dict__.pop("dvcignore", None)
|
||
path = PathInfo(tmp_dir)
|
||
assert set(dvc.tree.walk_files(path / "dir")) == {path / "dir" / "other"}
|
||
|
||
|
||
def test_rename_ignored_file(tmp_dir, dvc):
|
||
tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}})
|
||
|
||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "ignored*")
|
||
dvc.tree.__dict__.pop("dvcignore", None)
|
||
|
||
mtime, size = get_mtime_and_size("dir", dvc.tree)
|
||
|
||
shutil.move("dir/ignored", "dir/ignored_new")
|
||
new_mtime, new_size = get_mtime_and_size("dir", dvc.tree)
|
||
|
||
assert new_mtime == mtime and new_size == size
|
||
|
||
|
||
def test_rename_file(tmp_dir, dvc):
|
||
tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
|
||
mtime, size = get_mtime_and_size("dir", dvc.tree)
|
||
|
||
shutil.move("dir/foo", "dir/foo_new")
|
||
new_mtime, new_size = get_mtime_and_size("dir", dvc.tree)
|
||
|
||
assert new_mtime != mtime and new_size == size
|
||
|
||
|
||
def test_remove_ignored_file(tmp_dir, dvc):
|
||
tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}})
|
||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored")
|
||
dvc.tree.__dict__.pop("dvcignore", None)
|
||
|
||
mtime, size = get_mtime_and_size("dir", dvc.tree)
|
||
|
||
os.remove("dir/ignored")
|
||
new_mtime, new_size = get_mtime_and_size("dir", dvc.tree)
|
||
|
||
assert new_mtime == mtime and new_size == size
|
||
|
||
|
||
def test_remove_file(tmp_dir, dvc):
|
||
tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
|
||
mtime, size = get_mtime_and_size("dir", dvc.tree)
|
||
|
||
os.remove("dir/foo")
|
||
new_mtime, new_size = get_mtime_and_size("dir", dvc.tree)
|
||
|
||
assert new_mtime != mtime and new_size != size
|
||
|
||
|
||
def test_dvcignore_in_out_dir(tmp_dir, dvc):
|
||
tmp_dir.gen({"dir": {"foo": "foo", DvcIgnore.DVCIGNORE_FILE: ""}})
|
||
|
||
with pytest.raises(DvcIgnoreInCollectedDirError):
|
||
dvc.add("dir")
|
||
|
||
|
||
@pytest.mark.parametrize("dname", ["dir", "dir/subdir"])
|
||
def test_ignore_collecting_dvcignores(tmp_dir, dvc, dname):
|
||
tmp_dir.gen({"dir": {"subdir": {}}})
|
||
|
||
top_ignore_file = (tmp_dir / dname).with_name(DvcIgnore.DVCIGNORE_FILE)
|
||
top_ignore_file.write_text(os.path.basename(dname))
|
||
dvc.tree.__dict__.pop("dvcignore", None)
|
||
|
||
ignore_file = tmp_dir / dname / DvcIgnore.DVCIGNORE_FILE
|
||
ignore_file.write_text("foo")
|
||
|
||
assert len(dvc.tree.dvcignore.ignores) == 3
|
||
assert DvcIgnoreDirs([".git", ".hg", ".dvc"]) in dvc.tree.dvcignore.ignores
|
||
ignore_pattern_trie = None
|
||
for ignore in dvc.tree.dvcignore.ignores:
|
||
if isinstance(ignore, DvcIgnorePatternsTrie):
|
||
ignore_pattern_trie = ignore
|
||
|
||
assert ignore_pattern_trie is not None
|
||
assert (
|
||
DvcIgnorePatterns.from_files(
|
||
os.fspath(top_ignore_file),
|
||
LocalRemoteTree(None, {"url": dvc.root_dir}),
|
||
)
|
||
== ignore_pattern_trie[os.fspath(ignore_file)]
|
||
)
|
||
|
||
assert any(
|
||
i for i in dvc.tree.dvcignore.ignores if isinstance(i, DvcIgnoreRepo)
|
||
)
|
||
|
||
|
||
def test_ignore_on_branch(tmp_dir, scm, dvc):
|
||
tmp_dir.scm_gen({"foo": "foo", "bar": "bar"}, commit="add files")
|
||
|
||
with tmp_dir.branch("branch", new=True):
|
||
tmp_dir.scm_gen(DvcIgnore.DVCIGNORE_FILE, "foo", commit="add ignore")
|
||
|
||
dvc.tree.__dict__.pop("dvcignore", None)
|
||
path = PathInfo(tmp_dir)
|
||
assert set(dvc.tree.walk_files(path)) == {
|
||
path / "foo",
|
||
path / "bar",
|
||
}
|
||
|
||
dvc.tree = scm.get_tree("branch", use_dvcignore=True)
|
||
assert set(dvc.tree.walk_files(path)) == {
|
||
os.fspath(path / DvcIgnore.DVCIGNORE_FILE),
|
||
os.fspath(path / "bar"),
|
||
}
|
||
|
||
|
||
def test_match_nested(tmp_dir, dvc):
|
||
tmp_dir.gen(
|
||
{
|
||
".dvcignore": "*.backup\ntmp",
|
||
"foo": "foo",
|
||
"tmp": "...",
|
||
"dir": {"x.backup": "x backup", "tmp": "content"},
|
||
}
|
||
)
|
||
dvc.tree.__dict__.pop("dvcignore", None)
|
||
result = {os.fspath(os.path.normpath(f)) for f in dvc.tree.walk_files(".")}
|
||
assert result == {".dvcignore", "foo"}
|
||
|
||
|
||
def test_ignore_external(tmp_dir, scm, dvc, tmp_path_factory):
|
||
tmp_dir.gen(".dvcignore", "*.backup\ntmp")
|
||
ext_dir = TmpDir(os.fspath(tmp_path_factory.mktemp("external_dir")))
|
||
ext_dir.gen({"y.backup": "y", "tmp": "ext tmp"})
|
||
|
||
result = {relpath(f, ext_dir) for f in dvc.tree.walk_files(ext_dir)}
|
||
assert result == {"y.backup", "tmp"}
|
||
|
||
|
||
def test_ignore_subrepo(tmp_dir, scm, dvc):
|
||
tmp_dir.gen({".dvcignore": "foo", "subdir": {"foo": "foo"}})
|
||
scm.add([".dvcignore"])
|
||
scm.commit("init parent dvcignore")
|
||
dvc.tree.__dict__.pop("dvcignore", None)
|
||
|
||
subrepo_dir = tmp_dir / "subdir"
|
||
assert not dvc.tree.exists(PathInfo(subrepo_dir / "foo"))
|
||
|
||
with subrepo_dir.chdir():
|
||
subrepo = Repo.init(subdir=True)
|
||
scm.add(str(subrepo_dir / "foo"))
|
||
scm.commit("subrepo init")
|
||
|
||
for _ in subrepo.brancher(all_commits=True):
|
||
assert subrepo.tree.exists(PathInfo(subrepo_dir / "foo"))
|
||
|
||
|
||
def test_ignore_blank_line(tmp_dir, dvc):
|
||
tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}})
|
||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "foo\n\ndir/ignored")
|
||
dvc.tree.__dict__.pop("dvcignore", None)
|
||
path = PathInfo(tmp_dir)
|
||
assert set(dvc.tree.walk_files(path / "dir")) == {path / "dir" / "other"}
|
||
|
||
|
||
# It is not possible to re-include a file if a parent directory of
|
||
# that file is excluded.
|
||
# Git doesn’t list excluded directories for performance reasons,
|
||
# so any patterns on contained files have no effect,
|
||
# no matter where they are defined.
|
||
@pytest.mark.parametrize(
|
||
"data_struct, pattern_list, result_set",
|
||
[
|
||
(
|
||
{"dir": {"subdir": {"not_ignore": "121"}}},
|
||
["subdir/*", "!not_ignore"],
|
||
{os.path.join("dir", "subdir", "not_ignore")},
|
||
),
|
||
(
|
||
{"dir": {"subdir": {"should_ignore": "121"}}},
|
||
["subdir", "!should_ignore"],
|
||
set(),
|
||
),
|
||
(
|
||
{"dir": {"subdir": {"should_ignore": "121"}}},
|
||
["subdir/", "!should_ignore"],
|
||
set(),
|
||
),
|
||
],
|
||
)
|
||
def test_ignore_file_in_parent_path(
|
||
tmp_dir, dvc, data_struct, pattern_list, result_set
|
||
):
|
||
tmp_dir.gen(data_struct)
|
||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "\n".join(pattern_list))
|
||
dvc.tree.__dict__.pop("dvcignore", None)
|
||
path = PathInfo(tmp_dir)
|
||
assert set(dvc.tree.walk_files(path / "dir")) == {
|
||
path / relpath for relpath in result_set
|
||
}
|
||
|
||
|
||
# If there is a separator at the end of the pattern then the pattern
|
||
# will only match directories,
|
||
# otherwise the pattern can match both files and directories.
|
||
# For example, a pattern doc/frotz/ matches doc/frotz directory,
|
||
# but not a/doc/frotz directory;
|
||
def test_ignore_sub_directory(tmp_dir, dvc):
|
||
tmp_dir.gen(
|
||
{
|
||
"dir": {
|
||
"doc": {"fortz": {"b": "b"}},
|
||
"a": {"doc": {"fortz": {"a": "a"}}},
|
||
}
|
||
}
|
||
)
|
||
tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "doc/fortz"}})
|
||
|
||
dvc.tree.__dict__.pop("dvcignore", None)
|
||
path = PathInfo(tmp_dir)
|
||
assert set(dvc.tree.walk_files(path / "dir")) == {
|
||
path / "dir" / "a" / "doc" / "fortz" / "a",
|
||
path / "dir" / DvcIgnore.DVCIGNORE_FILE,
|
||
}
|
||
|
||
|
||
# however frotz/ matches frotz and a/frotz that is a directory
|
||
def test_ignore_directory(tmp_dir, dvc):
|
||
tmp_dir.gen({"dir": {"fortz": {}, "a": {"fortz": {}}}})
|
||
tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "fortz"}})
|
||
dvc.tree.__dict__.pop("dvcignore", None)
|
||
path = PathInfo(tmp_dir)
|
||
assert set(dvc.tree.walk_files(path / "dir")) == {
|
||
path / "dir" / DvcIgnore.DVCIGNORE_FILE,
|
||
}
|
||
|
||
|
||
def test_multi_ignore_file(tmp_dir, dvc, monkeypatch):
|
||
tmp_dir.gen({"dir": {"subdir": {"should_ignore": "1", "not_ignore": "1"}}})
|
||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/subdir/*_ignore")
|
||
tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "!subdir/not_ignore"}})
|
||
dvc.tree.__dict__.pop("dvcignore", None)
|
||
path = PathInfo(tmp_dir)
|
||
assert set(dvc.tree.walk_files(path / "dir")) == {
|
||
path / "dir" / "subdir" / "not_ignore",
|
||
path / "dir" / DvcIgnore.DVCIGNORE_FILE,
|
||
}
|
||
|
||
|
||
def test_pattern_trie_tree(tmp_dir, dvc):
|
||
tmp_dir.gen(
|
||
{
|
||
"top": {
|
||
"first": {
|
||
DvcIgnore.DVCIGNORE_FILE: "a\nb\nc",
|
||
"middle": {
|
||
"second": {
|
||
DvcIgnore.DVCIGNORE_FILE: "d\ne\nf",
|
||
"bottom": {},
|
||
}
|
||
},
|
||
},
|
||
},
|
||
"other": {DvcIgnore.DVCIGNORE_FILE: "1\n2\n3"},
|
||
}
|
||
)
|
||
dvc.tree.__dict__.pop("dvcignore", None)
|
||
ignore_pattern_trie = None
|
||
for ignore in dvc.tree.dvcignore.ignores:
|
||
if isinstance(ignore, DvcIgnorePatternsTrie):
|
||
ignore_pattern_trie = ignore
|
||
break
|
||
|
||
assert ignore_pattern_trie is not None
|
||
ignore_pattern_top = ignore_pattern_trie[os.fspath(tmp_dir / "top")]
|
||
ignore_pattern_other = ignore_pattern_trie[os.fspath(tmp_dir / "other")]
|
||
ignore_pattern_first = ignore_pattern_trie[
|
||
os.fspath(tmp_dir / "top" / "first")
|
||
]
|
||
ignore_pattern_middle = ignore_pattern_trie[
|
||
os.fspath(tmp_dir / "top" / "first" / "middle")
|
||
]
|
||
ignore_pattern_second = ignore_pattern_trie[
|
||
os.fspath(tmp_dir / "top" / "first" / "middle" / "second")
|
||
]
|
||
ignore_pattern_bottom = ignore_pattern_trie[
|
||
os.fspath(tmp_dir / "top" / "first" / "middle" / "second" / "bottom")
|
||
]
|
||
assert not ignore_pattern_top
|
||
assert (
|
||
DvcIgnorePatterns([], os.fspath(tmp_dir / "top")) == ignore_pattern_top
|
||
)
|
||
assert (
|
||
DvcIgnorePatterns(["1", "2", "3"], os.fspath(tmp_dir / "other"))
|
||
== ignore_pattern_other
|
||
)
|
||
assert (
|
||
DvcIgnorePatterns(
|
||
["a", "b", "c"], os.fspath(tmp_dir / "top" / "first")
|
||
)
|
||
== ignore_pattern_first
|
||
)
|
||
assert (
|
||
DvcIgnorePatterns(
|
||
["a", "b", "c"], os.fspath(tmp_dir / "top" / "first")
|
||
)
|
||
== ignore_pattern_middle
|
||
)
|
||
assert (
|
||
DvcIgnorePatterns(
|
||
[
|
||
"a",
|
||
"b",
|
||
"c",
|
||
"/middle/second/**/d",
|
||
"/middle/second/**/e",
|
||
"/middle/second/**/f",
|
||
],
|
||
os.fspath(tmp_dir / "top" / "first"),
|
||
)
|
||
== ignore_pattern_second
|
||
)
|
||
assert (
|
||
DvcIgnorePatterns(
|
||
[
|
||
"a",
|
||
"b",
|
||
"c",
|
||
"/middle/second/**/d",
|
||
"/middle/second/**/e",
|
||
"/middle/second/**/f",
|
||
],
|
||
os.fspath(tmp_dir / "top" / "first"),
|
||
)
|
||
== ignore_pattern_bottom
|
||
)
|