From f4ee01cffe36f3de392bd83408cf8ba3a77cd8da Mon Sep 17 00:00:00 2001 From: pawel Date: Fri, 20 Dec 2019 18:22:45 +0100 Subject: [PATCH] no need for dvcignore in walk args --- dvc/ignore.py | 10 +++++++--- dvc/repo/__init__.py | 8 +++----- dvc/repo/brancher.py | 4 ++-- dvc/scm/git/__init__.py | 6 ++---- dvc/scm/git/tree.py | 22 +++++++--------------- dvc/scm/tree.py | 27 ++++++++++++++++++--------- tests/func/test_repo.py | 4 ++-- tests/func/test_tree.py | 26 +++++++++++--------------- tests/unit/utils/test_fs.py | 6 ++---- 9 files changed, 54 insertions(+), 59 deletions(-) diff --git a/dvc/ignore.py b/dvc/ignore.py index fbfcff3ac..a09273554 100644 --- a/dvc/ignore.py +++ b/dvc/ignore.py @@ -72,11 +72,15 @@ class DvcIgnoreDirs(DvcIgnore): class DvcIgnoreFilter(object): - def __init__(self, root_dir, tree): + def __init__(self, tree): self.tree = tree self.ignores = {DvcIgnoreDirs([".git", ".hg", ".dvc"])} - self._update(root_dir) - for root, dirs, _ in self.tree.walk(root_dir, dvcignore=self): + + def reload(self): + self._update(self.tree.tree_root) + for root, dirs, _ in self.tree.walk( + self.tree.tree_root, dvcignore=self + ): for d in dirs: self._update(os.path.join(root, d)) diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index d81234782..ce3ce7117 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -9,7 +9,7 @@ from dvc.utils.compat import FileNotFoundError, fspath_py35, open as _open from funcy import cached_property -from dvc.scm.tree import IgnoreTree +from dvc.scm.tree import CleanTree from dvc.config import Config from dvc.exceptions import ( FileMissingError, @@ -84,7 +84,7 @@ class Repo(object): self.scm = SCM(self.root_dir) - self.tree = IgnoreTree(WorkingTree(self.root_dir)) + self.tree = CleanTree(WorkingTree(self.root_dir)) self.tmp_dir = os.path.join(self.dvc_dir, "tmp") makedirs(self.tmp_dir, exist_ok=True) @@ -391,9 +391,7 @@ class Repo(object): stages = [] outs = [] - for root, dirs, files in self.tree.walk( - self.root_dir, dvcignore=self.tree.dvcignore - ): + for root, dirs, files in self.tree.walk(self.root_dir): for fname in files: path = os.path.join(root, fname) if not Stage.is_valid_filename(path): diff --git a/dvc/repo/brancher.py b/dvc/repo/brancher.py index ff3e25d39..a24415a07 100644 --- a/dvc/repo/brancher.py +++ b/dvc/repo/brancher.py @@ -1,6 +1,6 @@ from funcy import group_by -from dvc.scm.tree import WorkingTree, IgnoreTree +from dvc.scm.tree import WorkingTree, CleanTree def brancher( # noqa: E302 @@ -34,7 +34,7 @@ def brancher( # noqa: E302 scm = self.scm - self.tree = IgnoreTree(WorkingTree(self.root_dir)) + self.tree = CleanTree(WorkingTree(self.root_dir)) yield "working tree" if all_commits: diff --git a/dvc/scm/git/__init__.py b/dvc/scm/git/__init__.py index 9a71a3afb..874c9e510 100644 --- a/dvc/scm/git/__init__.py +++ b/dvc/scm/git/__init__.py @@ -14,7 +14,7 @@ from dvc.scm.base import FileNotInRepoError from dvc.scm.base import RevError from dvc.scm.base import SCMError from dvc.scm.git.tree import GitTree -from dvc.scm.tree import IgnoreTree +from dvc.scm.tree import CleanTree from dvc.utils import fix_env from dvc.utils import is_binary from dvc.utils import relpath @@ -308,9 +308,7 @@ class Git(Base): return basename == self.ignore_file or Git.GIT_DIR in path_parts def get_tree(self, rev): - tree = GitTree(self.repo, rev) - - return IgnoreTree(tree) + return CleanTree(GitTree(self.repo, rev)) def _get_diff_trees(self, a_ref, b_ref): """Private method for getting the trees and commit hashes of 2 git diff --git a/dvc/scm/git/tree.py b/dvc/scm/git/tree.py index a7991d53e..ff0ce6e47 100644 --- a/dvc/scm/git/tree.py +++ b/dvc/scm/git/tree.py @@ -118,7 +118,7 @@ class GitTree(BaseTree): tree = tree[i] return tree - def _walk(self, tree, topdown=True, dvcignore=None): + def _walk(self, tree, topdown=True): dirs, nondirs = [], [] for i in _iter_tree(tree): if i.mode == GIT_MODE_DIR: @@ -127,24 +127,16 @@ class GitTree(BaseTree): nondirs.append(i.name) if topdown: - root = os.path.normpath(tree.abspath) - if dvcignore: - dirs[:], nondirs[:] = dvcignore(root, dirs, nondirs) - yield root, dirs, nondirs + yield os.path.normpath(tree.abspath), dirs, nondirs for i in dirs: - for root, dirs, nondirs in self._walk(tree[i], topdown=True): - if dvcignore: - dirs[:], nondirs[:] = dvcignore(root, dirs, nondirs) - yield root, dirs, nondirs + for x in self._walk(tree[i], topdown=True): + yield x if not topdown: - root = os.path.normpath(tree.abspath) - if dvcignore: - dirs[:], nondirs[:] = dvcignore(root, dirs, nondirs) - yield root, dirs, nondirs + yield os.path.normpath(tree.abspath), dirs, nondirs - def walk(self, top, topdown=True, dvcignore=None): + def walk(self, top, topdown=True, **kwargs): """Directory tree generator. See `os.walk` for the docs. Differences: @@ -156,5 +148,5 @@ class GitTree(BaseTree): if tree is None: raise IOError(errno.ENOENT, "No such file") - for x in self._walk(tree, topdown, dvcignore): + for x in self._walk(tree, topdown): yield x diff --git a/dvc/scm/tree.py b/dvc/scm/tree.py index 06062f996..4e8a84a0e 100644 --- a/dvc/scm/tree.py +++ b/dvc/scm/tree.py @@ -1,5 +1,7 @@ import os +from funcy import cached_property + from dvc.ignore import DvcIgnoreFilter from dvc.utils import dvc_walk from dvc.utils.compat import open @@ -12,6 +14,9 @@ class BaseTree(object): def tree_root(self): pass + def set_dvcignore(self, dvcignore): + self.dvcignore = dvcignore + def open(self, path, mode="r", encoding="utf-8"): """Open file and return a stream.""" @@ -24,7 +29,7 @@ class BaseTree(object): def isfile(self, path): """Test whether a path is a regular file""" - def walk(self, top, topdown=True, dvcignore=None): + def walk(self, top, topdown=True, **kwargs): """Directory tree generator. See `os.walk` for the docs. Differences: @@ -59,7 +64,7 @@ class WorkingTree(BaseTree): """Test whether a path is a regular file""" return os.path.isfile(path) - def walk(self, top, topdown=True, dvcignore=None): + def walk(self, top, topdown=True, dvcignore=None, **kwargs): """Directory tree generator. See `os.walk` for the docs. Differences: @@ -67,8 +72,6 @@ class WorkingTree(BaseTree): - it could raise exceptions, there is no onerror argument """ - assert dvcignore - def onerror(e): raise e @@ -78,13 +81,17 @@ class WorkingTree(BaseTree): yield os.path.normpath(root), dirs, files -class IgnoreTree(BaseTree): +class CleanTree(BaseTree): def __init__(self, tree): + super(CleanTree, self).__init__() self.tree = tree + self._dvcignore = DvcIgnoreFilter(self.tree) + self.tree.set_dvcignore(self._dvcignore) - @property + @cached_property def dvcignore(self): - return DvcIgnoreFilter(self.tree.tree_root, self.tree) + self._dvcignore.reload() + return self._dvcignore @property def tree_root(self): @@ -102,6 +109,8 @@ class IgnoreTree(BaseTree): def isfile(self, path): return self.tree.isfile(path) - def walk(self, top, topdown=True, dvcignore=None): - for r, d, fs in self.tree.walk(top, topdown, self.dvcignore): + def walk(self, top, topdown=True, **kwargs): + for r, d, fs in self.tree.walk(top, topdown, dvcignore=self.dvcignore): + d[:], fs[:] = self.dvcignore(r, d, fs) + yield r, d, fs diff --git a/tests/func/test_repo.py b/tests/func/test_repo.py index 28f881693..d78b55fa6 100644 --- a/tests/func/test_repo.py +++ b/tests/func/test_repo.py @@ -3,7 +3,7 @@ import os from dvc.scm.git.tree import GitTree from dvc.cache import Cache from dvc.repo import Repo -from dvc.scm.tree import IgnoreTree +from dvc.scm.tree import CleanTree from dvc.system import System from dvc.utils.compat import fspath @@ -51,7 +51,7 @@ def test_collect(tmp_dir, scm, dvc, run_copy): assert collect_outs("bar.dvc", with_deps=True) == {"foo", "bar"} - dvc.tree = IgnoreTree(GitTree(scm.repo, "new-branch")) + dvc.tree = CleanTree(GitTree(scm.repo, "new-branch")) assert collect_outs("buzz.dvc", with_deps=True) == {"foo", "bar", "buzz"} assert collect_outs("buzz.dvc", with_deps=False) == {"buzz"} diff --git a/tests/func/test_tree.py b/tests/func/test_tree.py index 2b2ad38b2..908449fe3 100644 --- a/tests/func/test_tree.py +++ b/tests/func/test_tree.py @@ -3,10 +3,9 @@ from __future__ import unicode_literals from os.path import join -from dvc.ignore import DvcIgnoreFilter from dvc.scm import SCM from dvc.scm.git import GitTree -from dvc.scm.tree import WorkingTree, IgnoreTree +from dvc.scm.tree import WorkingTree, CleanTree from tests.basic_env import TestDir from tests.basic_env import TestGit from tests.basic_env import TestGitSubmodule @@ -82,14 +81,14 @@ class TestGitTree(TestGit, GitTreeTests): def setUp(self): super(TestGitTree, self).setUp() self.scm = SCM(self._root_dir) - self.tree = IgnoreTree(GitTree(self.git, "master")) + self.tree = CleanTree(GitTree(self.git, "master")) class TestGitSubmoduleTree(TestGitSubmodule, GitTreeTests): def setUp(self): super(TestGitSubmoduleTree, self).setUp() self.scm = SCM(self._root_dir) - self.tree = IgnoreTree(GitTree(self.git, "master")) + self.tree = CleanTree(GitTree(self.git, "master")) self._pushd(self._root_dir) @@ -108,10 +107,9 @@ class AssertWalkEqualMixin(object): class TestWalkInNoSCM(AssertWalkEqualMixin, TestDir): def test(self): - tree = WorkingTree(self._root_dir) - dvcignore = DvcIgnoreFilter(self.root_dir, tree) + tree = CleanTree(WorkingTree(self._root_dir)) self.assertWalkEqual( - tree.walk(self._root_dir, dvcignore=dvcignore), + tree.walk(self._root_dir), [ ( self._root_dir, @@ -128,10 +126,9 @@ class TestWalkInNoSCM(AssertWalkEqualMixin, TestDir): ) def test_subdir(self): - tree = WorkingTree(self._root_dir) - dvcignore = DvcIgnoreFilter(self.root_dir, tree) + tree = CleanTree(WorkingTree(self._root_dir)) self.assertWalkEqual( - tree.walk(join("data_dir", "data_sub_dir"), dvcignore=dvcignore), + tree.walk(join("data_dir", "data_sub_dir")), [ ( join(self._root_dir, "data_dir", "data_sub_dir"), @@ -144,10 +141,9 @@ class TestWalkInNoSCM(AssertWalkEqualMixin, TestDir): class TestWalkInGit(AssertWalkEqualMixin, TestGit): def test_nobranch(self): - tree = WorkingTree(self._root_dir) - dvcignore = DvcIgnoreFilter(self._root_dir, tree) + tree = CleanTree(WorkingTree(self._root_dir)) self.assertWalkEqual( - tree.walk(".", dvcignore=dvcignore), + tree.walk("."), [ ( self._root_dir, @@ -163,7 +159,7 @@ class TestWalkInGit(AssertWalkEqualMixin, TestGit): ], ) self.assertWalkEqual( - tree.walk(join("data_dir", "data_sub_dir"), dvcignore=dvcignore), + tree.walk(join("data_dir", "data_sub_dir")), [ ( join(self._root_dir, "data_dir", "data_sub_dir"), @@ -177,7 +173,7 @@ class TestWalkInGit(AssertWalkEqualMixin, TestGit): scm = SCM(self._root_dir) scm.add([self.DATA_SUB_DIR]) scm.commit("add data_dir/data_sub_dir/data_sub") - tree = IgnoreTree(GitTree(self.git, "master")) + tree = CleanTree(GitTree(self.git, "master")) self.assertWalkEqual( tree.walk("."), [ diff --git a/tests/unit/utils/test_fs.py b/tests/unit/utils/test_fs.py index 809212907..7f4f580fc 100644 --- a/tests/unit/utils/test_fs.py +++ b/tests/unit/utils/test_fs.py @@ -23,7 +23,7 @@ from tests.utils import spy class TestMtimeAndSize(TestDir): def test(self): - dvcignore = DvcIgnoreFilter(self.root_dir, WorkingTree(self.root_dir)) + dvcignore = DvcIgnoreFilter(WorkingTree(self.root_dir)) file_time, file_size = get_mtime_and_size(self.DATA, dvcignore) dir_time, dir_size = get_mtime_and_size(self.DATA_DIR, dvcignore) @@ -127,9 +127,7 @@ def test_get_inode(repo_dir): def test_path_object_and_str_are_valid_types_get_mtime_and_size( path, repo_dir ): - dvcignore = DvcIgnoreFilter( - repo_dir.root_dir, WorkingTree(repo_dir.root_dir) - ) + dvcignore = DvcIgnoreFilter(WorkingTree(repo_dir.root_dir)) time, size = get_mtime_and_size(path, dvcignore) object_time, object_size = get_mtime_and_size(PathInfo(path), dvcignore) assert time == object_time