diff --git a/dvc/ignore.py b/dvc/ignore.py index 54b04f507..8fdf8298a 100644 --- a/dvc/ignore.py +++ b/dvc/ignore.py @@ -77,10 +77,9 @@ class DvcIgnoreFilter(object): def __init__(self, tree): self.tree = tree self.ignores = {DvcIgnoreDirs([".git", ".hg", ".dvc"])} - self._update(self.tree.tree_root) - for root, dirs, _ in self.tree.walk(self.tree.tree_root): - for d in dirs: - self._update(os.path.join(root, d)) + for root, dirs, files in self.tree.walk(self.tree.tree_root): + self._update(root) + dirs[:], files[:] = self(root, dirs, files) def _update(self, dirname): ignore_file_path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE) diff --git a/dvc/repo/brancher.py b/dvc/repo/brancher.py index 99b6aab79..7213cb37a 100644 --- a/dvc/repo/brancher.py +++ b/dvc/repo/brancher.py @@ -59,7 +59,7 @@ def brancher( # noqa: E302 # code which might expect the tree on which exception was raised to # stay in place. This behavior is a subject to change. for sha, names in group_by(scm.resolve_rev, revs).items(): - self.tree = scm.get_tree(sha) + self.tree = CleanTree(scm.get_tree(sha)) yield ", ".join(names) self.tree = saved_tree diff --git a/dvc/scm/git/__init__.py b/dvc/scm/git/__init__.py index 194c714fd..96177503f 100644 --- a/dvc/scm/git/__init__.py +++ b/dvc/scm/git/__init__.py @@ -8,7 +8,6 @@ from funcy import cached_property from pathspec.patterns import GitWildMatchPattern from dvc.exceptions import GitHookAlreadyExistsError -from dvc.ignore import CleanTree from dvc.scm.base import Base from dvc.scm.base import CloneError from dvc.scm.base import FileNotInRepoError @@ -308,7 +307,7 @@ class Git(Base): return basename == self.ignore_file or Git.GIT_DIR in path_parts def get_tree(self, rev): - return CleanTree(GitTree(self.repo, rev)) + return GitTree(self.repo, rev) def _get_diff_trees(self, a_ref, b_ref): """Private method for getting the trees and commit hashes of 2 git @@ -322,6 +321,7 @@ class Git(Base): tuple: tuple with elements: (trees, commits) """ from gitdb.exc import BadObject, BadName + from dvc.ignore import CleanTree trees = {DIFF_A_TREE: None, DIFF_B_TREE: None} commits = [] @@ -334,8 +334,8 @@ class Git(Base): # /en/2.1.11/reference.html#git.objects.base.Object.__str__ commits.append(a_commit) commits.append(b_commit) - trees[DIFF_A_TREE] = self.get_tree(commits[0]) - trees[DIFF_B_TREE] = self.get_tree(commits[1]) + trees[DIFF_A_TREE] = CleanTree(self.get_tree(commits[0])) + trees[DIFF_B_TREE] = CleanTree(self.get_tree(commits[1])) except (BadName, BadObject) as e: raise SCMError("git problem", cause=e) return trees, commits diff --git a/dvc/scm/tree.py b/dvc/scm/tree.py index e95af7d57..5e28beae9 100644 --- a/dvc/scm/tree.py +++ b/dvc/scm/tree.py @@ -1,6 +1,6 @@ import os -from dvc.utils.compat import open +from dvc.utils.compat import open, fspath class BaseTree(object): @@ -65,6 +65,8 @@ class WorkingTree(BaseTree): - it could raise exceptions, there is no onerror argument """ + top = fspath(top) + def onerror(e): raise e diff --git a/dvc/utils/fs.py b/dvc/utils/fs.py index 533cdc01e..ebb4b6cbc 100644 --- a/dvc/utils/fs.py +++ b/dvc/utils/fs.py @@ -34,8 +34,6 @@ def get_inode(path): return inode -# FIXME: There is no use case of get_mtime_and_size without tree, we should -# move it to tree def get_mtime_and_size(path, tree): if os.path.isdir(fspath_py35(path)): size = 0 diff --git a/tests/func/test_ignore.py b/tests/func/test_ignore.py index 6adfb4484..22fef716e 100644 --- a/tests/func/test_ignore.py +++ b/tests/func/test_ignore.py @@ -5,7 +5,8 @@ import shutil import pytest from dvc.exceptions import DvcIgnoreInCollectedDirError -from dvc.ignore import DvcIgnore +from dvc.ignore import DvcIgnore, DvcIgnoreDirs, DvcIgnorePatterns, CleanTree +from dvc.scm.tree import WorkingTree from dvc.utils import relpath from dvc.utils.compat import fspath_py35 from dvc.utils.compat import fspath @@ -89,6 +90,22 @@ def test_dvcignore_in_out_dir(tmp_dir, dvc): dvc.add("dir") +@pytest.mark.parametrize("dname", ["dir", "dir/subdir"]) +def test_ignore_collecting_dvcignores(tmp_dir, dvc, dname): + tmp_dir.gen({"dir": {"subdir": {}}}) + + top_ignore_file = (tmp_dir / dname).with_name(DvcIgnore.DVCIGNORE_FILE) + top_ignore_file.write_text(os.path.basename(dname)) + + ignore_file = tmp_dir / dname / DvcIgnore.DVCIGNORE_FILE + ignore_file.write_text("foo") + + assert dvc.tree.dvcignore.ignores == { + DvcIgnoreDirs([".git", ".hg", ".dvc"]), + DvcIgnorePatterns(fspath(top_ignore_file), WorkingTree(dvc.root_dir)), + } + + def test_ignore_on_branch(tmp_dir, scm, dvc): tmp_dir.scm_gen({"foo": "foo", "bar": "bar"}, commit="add files") @@ -98,7 +115,7 @@ def test_ignore_on_branch(tmp_dir, scm, dvc): scm.checkout("master") assert _files_set(".", dvc.tree) == {"./foo", "./bar"} - tree = scm.get_tree("branch") + tree = CleanTree(scm.get_tree("branch")) files_on_branch = { f for r, ds, fs in tree.walk(tree.tree_root) for f in fs } diff --git a/tests/func/test_repo.py b/tests/func/test_repo.py index 8bb7abfed..51f3a8754 100644 --- a/tests/func/test_repo.py +++ b/tests/func/test_repo.py @@ -1,6 +1,5 @@ import os -from dvc.ignore import CleanTree from dvc.scm.git.tree import GitTree from dvc.cache import Cache from dvc.repo import Repo @@ -51,7 +50,7 @@ def test_collect(tmp_dir, scm, dvc, run_copy): assert collect_outs("bar.dvc", with_deps=True) == {"foo", "bar"} - dvc.tree = CleanTree(GitTree(scm.repo, "new-branch")) + dvc.tree = GitTree(scm.repo, "new-branch") assert collect_outs("buzz.dvc", with_deps=True) == {"foo", "bar", "buzz"} assert collect_outs("buzz.dvc", with_deps=False) == {"buzz"} diff --git a/tests/func/test_tree.py b/tests/func/test_tree.py index 29d8d5e21..ee38be6ae 100644 --- a/tests/func/test_tree.py +++ b/tests/func/test_tree.py @@ -82,14 +82,14 @@ class TestGitTree(TestGit, GitTreeTests): def setUp(self): super(TestGitTree, self).setUp() self.scm = SCM(self._root_dir) - self.tree = CleanTree(GitTree(self.git, "master")) + self.tree = GitTree(self.git, "master") class TestGitSubmoduleTree(TestGitSubmodule, GitTreeTests): def setUp(self): super(TestGitSubmoduleTree, self).setUp() self.scm = SCM(self._root_dir) - self.tree = CleanTree(GitTree(self.git, "master")) + self.tree = GitTree(self.git, "master") self._pushd(self._root_dir) @@ -108,7 +108,7 @@ class AssertWalkEqualMixin(object): class TestWalkInNoSCM(AssertWalkEqualMixin, TestDir): def test(self): - tree = CleanTree(WorkingTree(self._root_dir)) + tree = WorkingTree(self._root_dir) self.assertWalkEqual( tree.walk(self._root_dir), [ @@ -127,7 +127,7 @@ class TestWalkInNoSCM(AssertWalkEqualMixin, TestDir): ) def test_subdir(self): - tree = CleanTree(WorkingTree(self._root_dir)) + tree = WorkingTree(self._root_dir) self.assertWalkEqual( tree.walk(join("data_dir", "data_sub_dir")), [(join("data_dir", "data_sub_dir"), [], ["data_sub"])], @@ -154,7 +154,7 @@ class TestWalkInGit(AssertWalkEqualMixin, TestGit): scm = SCM(self._root_dir) scm.add([self.DATA_SUB_DIR]) scm.commit("add data_dir/data_sub_dir/data_sub") - tree = CleanTree(GitTree(self.git, "master")) + tree = GitTree(self.git, "master") self.assertWalkEqual( tree.walk("."), [