mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 15:28:28 +01:00
This commit: * defines a `Ignore` protocol in `dvc.objects`, that expects a `find` method. This will be used in dvc-data and dvc-objects instead of depending on dvc.ignore. dvcignore is expected to respect this protocol, which it does. * renames `dvcignore` kwarg to `ignore` in `dvc-data` and `dvc-objects`. * teaches dvc.cli that `IgnoreInCollectedError` is a known exception and is logged correctly and adds a test for the behaviour. I am not happy that we need to hardcode `.dvcignore` in dvc-data but I don't see any easy way, since we always try to avoid staging if the tree contains .dvcignore file, even if .dvcignore is not passed.
422 lines
13 KiB
Python
422 lines
13 KiB
Python
import os
|
||
import shutil
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
|
||
from dvc.data.stage import IgnoreInCollectedDirError
|
||
from dvc.ignore import DvcIgnore, DvcIgnorePatterns
|
||
from dvc.objects.utils import get_mtime_and_size
|
||
from dvc.output import OutputIsIgnoredError
|
||
from dvc.pathspec_math import PatternInfo, merge_patterns
|
||
from dvc.repo import Repo
|
||
from dvc.testing.tmp_dir import TmpDir
|
||
from dvc.types import List
|
||
|
||
|
||
def _to_pattern_info_list(str_list: List):
|
||
return [PatternInfo(a, "") for a in str_list]
|
||
|
||
|
||
def walk_files(dvc, *args):
|
||
for fs_path in dvc.dvcignore.find(*args):
|
||
yield fs_path
|
||
|
||
|
||
@pytest.mark.parametrize("filename", ["ignored", "тест"])
|
||
def test_ignore(tmp_dir, dvc, filename):
|
||
tmp_dir.gen({"dir": {filename: filename, "other": "text2"}})
|
||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, f"dir/{filename}")
|
||
|
||
dvc._reset()
|
||
|
||
result = walk_files(dvc, dvc.fs, tmp_dir)
|
||
assert set(result) == {
|
||
(tmp_dir / DvcIgnore.DVCIGNORE_FILE).fs_path,
|
||
(tmp_dir / "dir" / "other").fs_path,
|
||
}
|
||
|
||
|
||
def test_rename_ignored_file(tmp_dir, dvc):
|
||
tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}})
|
||
|
||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "ignored*")
|
||
dvc._reset()
|
||
|
||
mtime, size = get_mtime_and_size("dir", dvc.fs, dvc.dvcignore)
|
||
|
||
shutil.move("dir/ignored", "dir/ignored_new")
|
||
new_mtime, new_size = get_mtime_and_size("dir", dvc.fs, dvc.dvcignore)
|
||
|
||
assert new_mtime == mtime and new_size == size
|
||
|
||
|
||
def test_rename_file(tmp_dir, dvc):
|
||
tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
|
||
mtime, size = get_mtime_and_size("dir", dvc.fs, dvc.dvcignore)
|
||
|
||
shutil.move("dir/foo", "dir/foo_new")
|
||
new_mtime, new_size = get_mtime_and_size("dir", dvc.fs, dvc.dvcignore)
|
||
|
||
assert new_mtime != mtime and new_size == size
|
||
|
||
|
||
def test_remove_ignored_file(tmp_dir, dvc):
|
||
tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}})
|
||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored")
|
||
dvc._reset()
|
||
|
||
mtime, size = get_mtime_and_size("dir", dvc.fs, dvc.dvcignore)
|
||
|
||
os.remove("dir/ignored")
|
||
new_mtime, new_size = get_mtime_and_size("dir", dvc.fs, dvc.dvcignore)
|
||
|
||
assert new_mtime == mtime and new_size == size
|
||
|
||
|
||
def test_remove_file(tmp_dir, dvc):
|
||
tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
|
||
mtime, size = get_mtime_and_size("dir", dvc.fs, dvc.dvcignore)
|
||
|
||
os.remove("dir/foo")
|
||
new_mtime, new_size = get_mtime_and_size("dir", dvc.fs, dvc.dvcignore)
|
||
|
||
assert new_mtime != mtime and new_size != size
|
||
|
||
|
||
def test_dvcignore_in_out_dir(tmp_dir, dvc):
|
||
tmp_dir.gen({"dir": {"foo": "foo", DvcIgnore.DVCIGNORE_FILE: ""}})
|
||
|
||
with pytest.raises(IgnoreInCollectedDirError):
|
||
dvc.add("dir")
|
||
|
||
|
||
@pytest.mark.parametrize("dname", ["dir", "dir/subdir"])
|
||
def test_ignore_collecting_dvcignores(tmp_dir, dvc, dname):
|
||
tmp_dir.gen({"dir": {"subdir": {}}})
|
||
|
||
top_ignore_file = (tmp_dir / dname).with_name(DvcIgnore.DVCIGNORE_FILE)
|
||
top_ignore_file.write_text(os.path.basename(dname))
|
||
dvc._reset()
|
||
|
||
ignore_file = tmp_dir / dname / DvcIgnore.DVCIGNORE_FILE
|
||
ignore_file.write_text("foo")
|
||
|
||
dvcignore = dvc.dvcignore
|
||
|
||
top_ignore_path = os.path.dirname(os.fspath(top_ignore_file))
|
||
|
||
sub_dir_path = os.path.dirname(os.fspath(ignore_file))
|
||
|
||
assert (
|
||
DvcIgnorePatterns(
|
||
*merge_patterns(
|
||
os.path,
|
||
_to_pattern_info_list([".hg/", ".git/", ".git", ".dvc/"]),
|
||
os.fspath(tmp_dir),
|
||
_to_pattern_info_list([os.path.basename(dname)]),
|
||
top_ignore_path,
|
||
),
|
||
os.sep,
|
||
)
|
||
== dvcignore._get_trie_pattern(top_ignore_path)
|
||
== dvcignore._get_trie_pattern(sub_dir_path)
|
||
)
|
||
|
||
|
||
def test_ignore_on_branch(tmp_dir, scm, dvc):
|
||
from dvc.fs import GitFileSystem
|
||
|
||
tmp_dir.scm_gen({"foo": "foo", "bar": "bar"}, commit="add files")
|
||
|
||
with tmp_dir.branch("branch", new=True):
|
||
tmp_dir.scm_gen(DvcIgnore.DVCIGNORE_FILE, "foo", commit="add ignore")
|
||
|
||
dvc._reset()
|
||
|
||
result = walk_files(dvc, dvc.fs, tmp_dir)
|
||
assert set(result) == {
|
||
(tmp_dir / "foo").fs_path,
|
||
(tmp_dir / "bar").fs_path,
|
||
(tmp_dir / DvcIgnore.DVCIGNORE_FILE).fs_path,
|
||
}
|
||
|
||
dvc.fs = GitFileSystem(scm=scm, rev="branch")
|
||
dvc.root_dir = "/"
|
||
assert dvc.dvcignore.is_ignored_file("/foo")
|
||
|
||
|
||
def test_match_nested(tmp_dir, dvc):
|
||
tmp_dir.gen(
|
||
{
|
||
".dvcignore": "*.backup\ntmp",
|
||
"foo": "foo",
|
||
"tmp": "...",
|
||
"dir": {"x.backup": "x backup", "tmp": "content"},
|
||
}
|
||
)
|
||
dvc._reset()
|
||
result = walk_files(dvc, dvc.fs, tmp_dir)
|
||
assert set(result) == {
|
||
(tmp_dir / DvcIgnore.DVCIGNORE_FILE).fs_path,
|
||
(tmp_dir / "foo").fs_path,
|
||
}
|
||
|
||
|
||
def test_ignore_external(tmp_dir, scm, dvc, tmp_path_factory):
|
||
tmp_dir.gen(".dvcignore", "*.backup\ntmp")
|
||
ext_dir = TmpDir(os.fspath(tmp_path_factory.mktemp("external_dir")))
|
||
ext_dir.gen({"y.backup": "y", "tmp": {"file": "ext tmp"}})
|
||
|
||
result = walk_files(dvc, dvc.fs, ext_dir)
|
||
assert set(result) == {
|
||
(ext_dir / "y.backup").fs_path,
|
||
(ext_dir / "tmp" / "file").fs_path,
|
||
}
|
||
assert dvc.dvcignore.is_ignored_dir(os.fspath(ext_dir / "tmp")) is False
|
||
assert (
|
||
dvc.dvcignore.is_ignored_file(os.fspath(ext_dir / "y.backup")) is False
|
||
)
|
||
|
||
|
||
def test_ignore_resurface_subrepo(tmp_dir, scm, dvc):
|
||
tmp_dir.dvc_gen({"foo": "foo"}, commit="add foo")
|
||
subrepo_dir = tmp_dir / "subdir"
|
||
subrepo_dir.mkdir()
|
||
with subrepo_dir.chdir():
|
||
Repo.init(subdir=True)
|
||
subrepo_dir.gen({"bar": {"bar": "bar"}})
|
||
|
||
dvc._reset()
|
||
|
||
files = ["foo"]
|
||
dirs = ["bar"]
|
||
root = os.fspath(subrepo_dir)
|
||
assert dvc.dvcignore(root, dirs, files, ignore_subrepos=False) == (
|
||
dirs,
|
||
files,
|
||
)
|
||
assert dvc.dvcignore(root, dirs, files) == ([], [])
|
||
|
||
assert dvc.dvcignore.is_ignored_dir(os.fspath(subrepo_dir / "bar"))
|
||
assert not dvc.dvcignore.is_ignored_dir(
|
||
os.fspath(subrepo_dir / "bar"), ignore_subrepos=False
|
||
)
|
||
|
||
|
||
def test_ignore_blank_line(tmp_dir, dvc):
|
||
tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}})
|
||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "foo\n\ndir/ignored")
|
||
dvc._reset()
|
||
result = walk_files(dvc, dvc.fs, tmp_dir / "dir")
|
||
assert set(result) == {(tmp_dir / "dir" / "other").fs_path}
|
||
|
||
|
||
# It is not possible to re-include a file if a parent directory of
|
||
# that file is excluded.
|
||
# Git doesn’t list excluded directories for performance reasons,
|
||
# so any patterns on contained files have no effect,
|
||
# no matter where they are defined.
|
||
@pytest.mark.parametrize(
|
||
"data_struct, pattern_list, result_set",
|
||
[
|
||
(
|
||
{"dir": {"subdir": {"not_ignore": "121"}}},
|
||
["subdir/*", "!not_ignore"],
|
||
{os.path.join("dir", "subdir", "not_ignore")},
|
||
),
|
||
(
|
||
{"dir": {"subdir": {"should_ignore": "121"}}},
|
||
["subdir", "!should_ignore"],
|
||
set(),
|
||
),
|
||
(
|
||
{"dir": {"subdir": {"should_ignore": "121"}}},
|
||
["subdir/", "!should_ignore"],
|
||
set(),
|
||
),
|
||
],
|
||
)
|
||
def test_ignore_file_in_parent_path(
|
||
tmp_dir, dvc, data_struct, pattern_list, result_set
|
||
):
|
||
tmp_dir.gen(data_struct)
|
||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "\n".join(pattern_list))
|
||
dvc._reset()
|
||
result = walk_files(dvc, dvc.fs, tmp_dir / "dir")
|
||
assert set(result) == {
|
||
(tmp_dir / relpath).fs_path for relpath in result_set
|
||
}
|
||
|
||
|
||
# If there is a separator at the end of the pattern then the pattern
|
||
# will only match directories,
|
||
# otherwise the pattern can match both files and directories.
|
||
# For example, a pattern doc/frotz/ matches doc/frotz directory,
|
||
# but not a/doc/frotz directory;
|
||
def test_ignore_sub_directory(tmp_dir, dvc):
|
||
tmp_dir.gen(
|
||
{
|
||
"dir": {
|
||
"doc": {"fortz": {"b": "b"}},
|
||
"a": {"doc": {"fortz": {"a": "a"}}},
|
||
}
|
||
}
|
||
)
|
||
tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "doc/fortz"}})
|
||
|
||
dvc._reset()
|
||
result = walk_files(dvc, dvc.fs, tmp_dir / "dir")
|
||
assert set(result) == {
|
||
(tmp_dir / "dir" / "a" / "doc" / "fortz" / "a").fs_path,
|
||
(tmp_dir / "dir" / DvcIgnore.DVCIGNORE_FILE).fs_path,
|
||
}
|
||
|
||
|
||
# however frotz/ matches frotz and a/frotz that is a directory
|
||
def test_ignore_directory(tmp_dir, dvc):
|
||
tmp_dir.gen({"dir": {"fortz": {}, "a": {"fortz": {}}}})
|
||
tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "fortz"}})
|
||
dvc._reset()
|
||
result = walk_files(dvc, dvc.fs, tmp_dir / "dir")
|
||
assert set(result) == {
|
||
(tmp_dir / "dir" / DvcIgnore.DVCIGNORE_FILE).fs_path
|
||
}
|
||
|
||
|
||
def test_multi_ignore_file(tmp_dir, dvc, monkeypatch):
|
||
tmp_dir.gen({"dir": {"subdir": {"should_ignore": "1", "not_ignore": "1"}}})
|
||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/subdir/*_ignore")
|
||
tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "!subdir/not_ignore"}})
|
||
dvc._reset()
|
||
result = walk_files(dvc, dvc.fs, tmp_dir / "dir")
|
||
assert set(result) == {
|
||
(tmp_dir / "dir" / "subdir" / "not_ignore").fs_path,
|
||
(tmp_dir / "dir" / DvcIgnore.DVCIGNORE_FILE).fs_path,
|
||
}
|
||
|
||
|
||
def test_pattern_trie_fs(tmp_dir, dvc):
|
||
tmp_dir.gen(
|
||
{
|
||
"top": {
|
||
"first": {
|
||
DvcIgnore.DVCIGNORE_FILE: "a\nb\nc",
|
||
"middle": {
|
||
"second": {
|
||
DvcIgnore.DVCIGNORE_FILE: "d\ne\nf",
|
||
"bottom": {},
|
||
}
|
||
},
|
||
}
|
||
},
|
||
"other": {DvcIgnore.DVCIGNORE_FILE: "1\n2\n3"},
|
||
}
|
||
)
|
||
dvc._reset()
|
||
dvcignore = dvc.dvcignore
|
||
|
||
ignore_pattern_top = dvcignore._get_trie_pattern(
|
||
os.fspath(tmp_dir / "top")
|
||
)
|
||
ignore_pattern_other = dvcignore._get_trie_pattern(
|
||
os.fspath(tmp_dir / "other")
|
||
)
|
||
ignore_pattern_first = dvcignore._get_trie_pattern(
|
||
os.fspath(tmp_dir / "top" / "first")
|
||
)
|
||
ignore_pattern_middle = dvcignore._get_trie_pattern(
|
||
os.fspath(tmp_dir / "top" / "first" / "middle")
|
||
)
|
||
ignore_pattern_second = dvcignore._get_trie_pattern(
|
||
os.fspath(tmp_dir / "top" / "first" / "middle" / "second")
|
||
)
|
||
ignore_pattern_bottom = dvcignore._get_trie_pattern(
|
||
os.fspath(tmp_dir / "top" / "first" / "middle" / "second" / "bottom")
|
||
)
|
||
|
||
base_pattern = (
|
||
_to_pattern_info_list([".hg/", ".git/", ".git", ".dvc/"]),
|
||
os.fspath(tmp_dir),
|
||
)
|
||
first_pattern = merge_patterns(
|
||
os.path,
|
||
*base_pattern,
|
||
_to_pattern_info_list(["a", "b", "c"]),
|
||
os.fspath(tmp_dir / "top" / "first"),
|
||
)
|
||
second_pattern = merge_patterns(
|
||
os.path,
|
||
*first_pattern,
|
||
_to_pattern_info_list(["d", "e", "f"]),
|
||
os.fspath(tmp_dir / "top" / "first" / "middle" / "second"),
|
||
)
|
||
other_pattern = merge_patterns(
|
||
os.path,
|
||
*base_pattern,
|
||
_to_pattern_info_list(["1", "2", "3"]),
|
||
os.fspath(tmp_dir / "other"),
|
||
)
|
||
|
||
assert DvcIgnorePatterns(*base_pattern, os.sep) == ignore_pattern_top
|
||
assert DvcIgnorePatterns(*other_pattern, os.sep) == ignore_pattern_other
|
||
assert (
|
||
DvcIgnorePatterns(*first_pattern, os.sep)
|
||
== ignore_pattern_first
|
||
== ignore_pattern_middle
|
||
)
|
||
assert (
|
||
DvcIgnorePatterns(*second_pattern, os.sep)
|
||
== ignore_pattern_second
|
||
== ignore_pattern_bottom
|
||
)
|
||
|
||
|
||
def test_ignore_in_added_dir(tmp_dir, dvc):
|
||
tmp_dir.gen(
|
||
{
|
||
"dir": {
|
||
"sub": {
|
||
"ignored": {"content": "ignored content"},
|
||
"not_ignored": "not ignored content",
|
||
}
|
||
},
|
||
".dvcignore": "**/ignored",
|
||
}
|
||
)
|
||
dvc._reset()
|
||
|
||
ignored_path = tmp_dir / "dir" / "sub" / "ignored"
|
||
result = walk_files(dvc, dvc.fs, ignored_path)
|
||
assert set(result) == set()
|
||
assert ignored_path.exists()
|
||
|
||
dvc.add("dir")
|
||
shutil.rmtree(ignored_path)
|
||
dvc.checkout()
|
||
|
||
assert not ignored_path.exists()
|
||
|
||
|
||
def test_ignored_output(tmp_dir, scm, dvc, run_copy):
|
||
tmp_dir.gen({".dvcignore": "*.log\n!foo.log", "foo": "foo content"})
|
||
|
||
with pytest.raises(OutputIsIgnoredError):
|
||
run_copy("foo", "abc.log", name="copy")
|
||
|
||
run_copy("foo", "foo.log", name="copy")
|
||
|
||
|
||
def test_ignored_output_nested(tmp_dir, scm, dvc, run_copy):
|
||
tmp_dir.gen({".dvcignore": "/*.log", "copy": {"foo": "foo content"}})
|
||
|
||
run_copy("foo", "foo.log", name="copy", wdir="copy")
|
||
|
||
assert Path("copy/foo.log").exists()
|
||
|
||
|
||
def test_run_dvcignored_dep(tmp_dir, dvc, run_copy):
|
||
tmp_dir.gen({".dvcignore": "dir\n", "dir": {"foo": "foo"}})
|
||
run_copy(os.path.join("dir", "foo"), "bar", name="copy-foo-to-bar")
|
||
assert (tmp_dir / "bar").read_text() == "foo"
|