remove most occurences of dvc_walk

dvc/remote/local.py

@@ -83,7 +83,7 @@ class RemoteLOCAL(RemoteBASE):
 
     def list_cache_paths(self):
         assert self.path_info is not None
-        return walk_files(self.path_info, None)
+        return walk_files(self.path_info)
 
     def get(self, md5):
         if not md5:
@@ -138,7 +138,7 @@ class RemoteLOCAL(RemoteBASE):
         return os.path.getsize(fspath_py35(path_info))
 
     def walk_files(self, path_info):
-        for fname in walk_files(path_info, self.repo.tree.dvcignore):
+        for fname in self.repo.tree.walk_files(path_info):
             yield PathInfo(fname)
 
     def get_file_checksum(self, path_info):
@@ -427,7 +427,7 @@ class RemoteLOCAL(RemoteBASE):
         os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
 
     def _unprotect_dir(self, path):
-        for fname in walk_files(path, self.repo.tree.dvcignore):
+        for fname in self.repo.tree.walk_files(path):
             RemoteLOCAL._unprotect_file(fname)
 
     def unprotect(self, path_info):

dvc/repo/add.py

@@ -12,7 +12,6 @@ from dvc.progress import Tqdm
 from dvc.repo.scm_context import scm_context
 from dvc.stage import Stage
 from dvc.utils import LARGE_DIR_SIZE
-from dvc.utils import walk_files
 
 logger = logging.getLogger(__name__)
 
@@ -68,7 +67,7 @@ def _find_all_targets(repo, target, recursive):
     if os.path.isdir(target) and recursive:
         return [
             fname
-            for fname in walk_files(target, repo.tree.dvcignore)
+            for fname in repo.tree.walk_files(target)
             if not repo.is_dvc_internal(fname)
             if not Stage.is_stage_file(fname)
             if not repo.scm.belongs_to_scm(fname)

dvc/scm/tree.py

@@ -108,3 +108,8 @@ class CleanTree(BaseTree):
             d[:], fs[:] = self.dvcignore(r, d, fs)
 
             yield r, d, fs
+
+    def walk_files(self, top):
+        for root, _, files in self.walk(top):
+            for file in files:
+                yield os.path.join(root, file)

dvc/state.py

@@ -378,7 +378,7 @@ class State(object):  # pylint: disable=too-many-instance-attributes
         assert os.path.exists(fspath_py35(path_info))
 
         actual_mtime, actual_size = get_mtime_and_size(
-            path_info, self.repo.tree.dvcignore
+            path_info, self.repo.tree
         )
         actual_inode = get_inode(path_info)
 
@@ -410,9 +410,7 @@ class State(object):  # pylint: disable=too-many-instance-attributes
         if not os.path.exists(path):
             return None
 
-        actual_mtime, actual_size = get_mtime_and_size(
-            path, self.repo.tree.dvcignore
-        )
+        actual_mtime, actual_size = get_mtime_and_size(path, self.repo.tree)
         actual_inode = get_inode(path)
 
         existing_record = self.get_state_record_for_inode(actual_inode)
@@ -439,7 +437,7 @@ class State(object):  # pylint: disable=too-many-instance-attributes
         if not os.path.exists(path):
             return
 
-        mtime, _ = get_mtime_and_size(path, self.repo.tree.dvcignore)
+        mtime, _ = get_mtime_and_size(path, self.repo.tree)
         inode = get_inode(path)
         relative_path = relpath(path, self.root_dir)
 
@@ -469,9 +467,7 @@ class State(object):  # pylint: disable=too-many-instance-attributes
                 continue
 
             actual_inode = get_inode(path)
-            actual_mtime, _ = get_mtime_and_size(
-                path, self.repo.tree.dvcignore
-            )
+            actual_mtime, _ = get_mtime_and_size(path, self.repo.tree)
 
             if inode == actual_inode and mtime == actual_mtime:
                 logger.debug("Removing '{}' as unused link.".format(path))

dvc/utils/__init__.py

@@ -305,8 +305,8 @@ def dvc_walk(top, dvcignore, topdown=True, onerror=None, followlinks=False):
         yield root, dirs, files
 
 
-def walk_files(directory, dvcignore):
-    for root, _, files in dvc_walk(directory, dvcignore):
+def walk_files(directory):
+    for root, _, files in os.walk(fspath(directory)):
         for f in files:
             yield os.path.join(root, f)
 

dvc/utils/fs.py

@@ -15,7 +15,6 @@ from dvc.utils import dict_md5
 from dvc.utils import fspath
 from dvc.utils import fspath_py35
 from dvc.utils import relpath
-from dvc.utils import walk_files
 from dvc.utils.compat import str
 
 
@@ -35,11 +34,13 @@ def get_inode(path):
     return inode
 
 
-def get_mtime_and_size(path, dvcignore):
+# FIXME: There is no use case of get_mtime_and_size without tree, we should
+# move it to tree
+def get_mtime_and_size(path, tree):
     if os.path.isdir(fspath_py35(path)):
         size = 0
         files_mtimes = {}
-        for file_path in walk_files(path, dvcignore):
+        for file_path in tree.walk_files(path):
             try:
                 stat = os.stat(file_path)
             except OSError as exc:

tests/func/test_checkout.py

@@ -19,7 +19,6 @@ from dvc.stage import StageFileBadNameError
 from dvc.stage import StageFileDoesNotExistError
 from dvc.system import System
 from dvc.utils import relpath
-from dvc.utils import walk_files
 from dvc.utils.compat import is_py2
 from dvc.utils.stage import dump_stage_file
 from dvc.utils.stage import load_stage_file
@@ -137,7 +136,7 @@ class CheckoutBase(TestDvcGit):
         paths = [
             path
             for output in stage["outs"]
-            for path in walk_files(output["path"], self.dvc.tree.dvcignore)
+            for path in self.dvc.tree.walk_files(output["path"])
         ]
 
         return [

tests/func/test_ignore.py

@@ -7,8 +7,9 @@ import pytest
 from dvc.exceptions import DvcIgnoreInCollectedDirError
 from dvc.ignore import DvcIgnore, DvcIgnoreDirs, DvcIgnorePatterns
 from dvc.scm.tree import WorkingTree
-from dvc.utils import walk_files, relpath
-from dvc.utils.compat import fspath, fspath_py35
+from dvc.utils import relpath
+from dvc.utils.compat import fspath_py35
+from dvc.utils.compat import fspath
 from dvc.utils.fs import get_mtime_and_size
 from dvc.remote import RemoteLOCAL
 
@@ -20,10 +21,10 @@ def test_ignore(tmp_dir, dvc, monkeypatch):
     tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}})
     tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored")
 
-    assert _files_set("dir", dvc.tree.dvcignore) == {"dir/other"}
+    assert _files_set("dir", dvc.tree) == {"dir/other"}
 
     monkeypatch.chdir("dir")
-    assert _files_set(".", dvc.tree.dvcignore) == {"./other"}
+    assert _files_set(".", dvc.tree) == {"./other"}
 
 
 def test_ignore_unicode(tmp_dir, dvc):
@@ -35,27 +36,27 @@ def test_ignore_unicode(tmp_dir, dvc):
 
     tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/тест")
 
-    assert _files_set("dir", dvc.tree.dvcignore) == {"dir/other"}
+    assert _files_set("dir", dvc.tree) == {"dir/other"}
 
 
 def test_rename_ignored_file(tmp_dir, dvc):
     tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}})
 
     tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "ignored*")
-    mtime, size = get_mtime_and_size("dir", dvc.tree.dvcignore)
+    mtime, size = get_mtime_and_size("dir", dvc.tree)
 
     shutil.move("dir/ignored", "dir/ignored_new")
-    new_mtime, new_size = get_mtime_and_size("dir", dvc.tree.dvcignore)
+    new_mtime, new_size = get_mtime_and_size("dir", dvc.tree)
 
     assert new_mtime == mtime and new_size == size
 
 
 def test_rename_file(tmp_dir, dvc):
     tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
-    mtime, size = get_mtime_and_size("dir", dvc.tree.dvcignore)
+    mtime, size = get_mtime_and_size("dir", dvc.tree)
 
     shutil.move("dir/foo", "dir/foo_new")
-    new_mtime, new_size = get_mtime_and_size("dir", dvc.tree.dvcignore)
+    new_mtime, new_size = get_mtime_and_size("dir", dvc.tree)
 
     assert new_mtime != mtime and new_size == size
 
@@ -64,20 +65,20 @@ def test_remove_ignored_file(tmp_dir, dvc):
     tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}})
     tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored")
 
-    mtime, size = get_mtime_and_size("dir", dvc.tree.dvcignore)
+    mtime, size = get_mtime_and_size("dir", dvc.tree)
 
     os.remove("dir/ignored")
-    new_mtime, new_size = get_mtime_and_size("dir", dvc.tree.dvcignore)
+    new_mtime, new_size = get_mtime_and_size("dir", dvc.tree)
 
     assert new_mtime == mtime and new_size == size
 
 
 def test_remove_file(tmp_dir, dvc):
     tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
-    mtime, size = get_mtime_and_size("dir", dvc.tree.dvcignore)
+    mtime, size = get_mtime_and_size("dir", dvc.tree)
 
     os.remove("dir/foo")
-    new_mtime, new_size = get_mtime_and_size("dir", dvc.tree.dvcignore)
+    new_mtime, new_size = get_mtime_and_size("dir", dvc.tree)
 
     assert new_mtime != mtime and new_size != size
 
@@ -112,7 +113,7 @@ def test_ignore_on_branch(tmp_dir, scm, dvc):
     tmp_dir.scm_gen(DvcIgnore.DVCIGNORE_FILE, "foo", commit="add ignore")
 
     scm.checkout("master")
-    assert _files_set(".", dvc.tree.dvcignore) == {"./foo", "./bar"}
+    assert _files_set(".", dvc.tree) == {"./foo", "./bar"}
 
     tree = scm.get_tree("branch")
     files_on_branch = {
@@ -121,8 +122,8 @@ def test_ignore_on_branch(tmp_dir, scm, dvc):
     assert "foo" not in files_on_branch
 
 
-def _files_set(root, dvcignore):
-    return {to_posixpath(f) for f in walk_files(root, dvcignore)}
+def _files_set(root, tree):
+    return {to_posixpath(f) for f in tree.walk_files(root)}
 
 
 def test_match_nested(tmp_dir, dvc):

tests/unit/remote/test_remote_dir.py

@@ -142,7 +142,7 @@ def test_download_dir(remote, tmpdir):
     remote.download(remote.path_info / "data", to_info)
     assert os.path.isdir(path)
     data_dir = tmpdir / "data"
-    assert len(list(walk_files(path, None))) == 7
+    assert len(list(walk_files(path))) == 7
     assert (data_dir / "alice").read_text(encoding="utf-8") == "alice"
     assert (data_dir / "alpha").read_text(encoding="utf-8") == "alpha"
     assert (data_dir / "subdir-file.txt").read_text(

tests/unit/utils/test_fs.py

@@ -5,9 +5,8 @@ import pytest
 from mock import patch
 
 import dvc
-from dvc.ignore import DvcIgnoreFilter
 from dvc.path_info import PathInfo
-from dvc.scm.tree import WorkingTree
+from dvc.scm.tree import WorkingTree, CleanTree
 from dvc.system import System
 from dvc.utils import relpath
 from dvc.utils.compat import str
@@ -23,9 +22,9 @@ from tests.utils import spy
 
 class TestMtimeAndSize(TestDir):
     def test(self):
-        dvcignore = DvcIgnoreFilter(WorkingTree(self.root_dir))
-        file_time, file_size = get_mtime_and_size(self.DATA, dvcignore)
-        dir_time, dir_size = get_mtime_and_size(self.DATA_DIR, dvcignore)
+        tree = CleanTree(WorkingTree(self.root_dir))
+        file_time, file_size = get_mtime_and_size(self.DATA, tree)
+        dir_time, dir_size = get_mtime_and_size(self.DATA_DIR, tree)
 
         actual_file_size = os.path.getsize(self.DATA)
         actual_dir_size = os.path.getsize(self.DATA) + os.path.getsize(
@@ -127,9 +126,9 @@ def test_get_inode(repo_dir):
 def test_path_object_and_str_are_valid_types_get_mtime_and_size(
     path, repo_dir
 ):
-    dvcignore = DvcIgnoreFilter(WorkingTree(repo_dir.root_dir))
-    time, size = get_mtime_and_size(path, dvcignore)
-    object_time, object_size = get_mtime_and_size(PathInfo(path), dvcignore)
+    tree = CleanTree(WorkingTree(repo_dir.root_dir))
+    time, size = get_mtime_and_size(path, tree)
+    object_time, object_size = get_mtime_and_size(PathInfo(path), tree)
     assert time == object_time
     assert size == object_size