diff --git a/dvc/remote/local.py b/dvc/remote/local.py index 0da62ff25..81b63d061 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -83,7 +83,7 @@ class RemoteLOCAL(RemoteBASE): def list_cache_paths(self): assert self.path_info is not None - return walk_files(self.path_info, None) + return walk_files(self.path_info) def get(self, md5): if not md5: @@ -138,7 +138,7 @@ class RemoteLOCAL(RemoteBASE): return os.path.getsize(fspath_py35(path_info)) def walk_files(self, path_info): - for fname in walk_files(path_info, self.repo.tree.dvcignore): + for fname in self.repo.tree.walk_files(path_info): yield PathInfo(fname) def get_file_checksum(self, path_info): @@ -427,7 +427,7 @@ class RemoteLOCAL(RemoteBASE): os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE) def _unprotect_dir(self, path): - for fname in walk_files(path, self.repo.tree.dvcignore): + for fname in self.repo.tree.walk_files(path): RemoteLOCAL._unprotect_file(fname) def unprotect(self, path_info): diff --git a/dvc/repo/add.py b/dvc/repo/add.py index f0353482b..a94f0ef1a 100644 --- a/dvc/repo/add.py +++ b/dvc/repo/add.py @@ -12,7 +12,6 @@ from dvc.progress import Tqdm from dvc.repo.scm_context import scm_context from dvc.stage import Stage from dvc.utils import LARGE_DIR_SIZE -from dvc.utils import walk_files logger = logging.getLogger(__name__) @@ -68,7 +67,7 @@ def _find_all_targets(repo, target, recursive): if os.path.isdir(target) and recursive: return [ fname - for fname in walk_files(target, repo.tree.dvcignore) + for fname in repo.tree.walk_files(target) if not repo.is_dvc_internal(fname) if not Stage.is_stage_file(fname) if not repo.scm.belongs_to_scm(fname) diff --git a/dvc/scm/tree.py b/dvc/scm/tree.py index d830397b8..4df46cce7 100644 --- a/dvc/scm/tree.py +++ b/dvc/scm/tree.py @@ -108,3 +108,8 @@ class CleanTree(BaseTree): d[:], fs[:] = self.dvcignore(r, d, fs) yield r, d, fs + + def walk_files(self, top): + for root, _, files in self.walk(top): + for file in files: + yield os.path.join(root, file) diff --git a/dvc/state.py b/dvc/state.py index a3e27a8e9..5f10654d1 100644 --- a/dvc/state.py +++ b/dvc/state.py @@ -378,7 +378,7 @@ class State(object): # pylint: disable=too-many-instance-attributes assert os.path.exists(fspath_py35(path_info)) actual_mtime, actual_size = get_mtime_and_size( - path_info, self.repo.tree.dvcignore + path_info, self.repo.tree ) actual_inode = get_inode(path_info) @@ -410,9 +410,7 @@ class State(object): # pylint: disable=too-many-instance-attributes if not os.path.exists(path): return None - actual_mtime, actual_size = get_mtime_and_size( - path, self.repo.tree.dvcignore - ) + actual_mtime, actual_size = get_mtime_and_size(path, self.repo.tree) actual_inode = get_inode(path) existing_record = self.get_state_record_for_inode(actual_inode) @@ -439,7 +437,7 @@ class State(object): # pylint: disable=too-many-instance-attributes if not os.path.exists(path): return - mtime, _ = get_mtime_and_size(path, self.repo.tree.dvcignore) + mtime, _ = get_mtime_and_size(path, self.repo.tree) inode = get_inode(path) relative_path = relpath(path, self.root_dir) @@ -469,9 +467,7 @@ class State(object): # pylint: disable=too-many-instance-attributes continue actual_inode = get_inode(path) - actual_mtime, _ = get_mtime_and_size( - path, self.repo.tree.dvcignore - ) + actual_mtime, _ = get_mtime_and_size(path, self.repo.tree) if inode == actual_inode and mtime == actual_mtime: logger.debug("Removing '{}' as unused link.".format(path)) diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index 92fd34a8a..c9127be90 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -305,8 +305,8 @@ def dvc_walk(top, dvcignore, topdown=True, onerror=None, followlinks=False): yield root, dirs, files -def walk_files(directory, dvcignore): - for root, _, files in dvc_walk(directory, dvcignore): +def walk_files(directory): + for root, _, files in os.walk(fspath(directory)): for f in files: yield os.path.join(root, f) diff --git a/dvc/utils/fs.py b/dvc/utils/fs.py index 26ee05506..533cdc01e 100644 --- a/dvc/utils/fs.py +++ b/dvc/utils/fs.py @@ -15,7 +15,6 @@ from dvc.utils import dict_md5 from dvc.utils import fspath from dvc.utils import fspath_py35 from dvc.utils import relpath -from dvc.utils import walk_files from dvc.utils.compat import str @@ -35,11 +34,13 @@ def get_inode(path): return inode -def get_mtime_and_size(path, dvcignore): +# FIXME: There is no use case of get_mtime_and_size without tree, we should +# move it to tree +def get_mtime_and_size(path, tree): if os.path.isdir(fspath_py35(path)): size = 0 files_mtimes = {} - for file_path in walk_files(path, dvcignore): + for file_path in tree.walk_files(path): try: stat = os.stat(file_path) except OSError as exc: diff --git a/tests/func/test_checkout.py b/tests/func/test_checkout.py index 0760bd2d5..4f83f2cf8 100644 --- a/tests/func/test_checkout.py +++ b/tests/func/test_checkout.py @@ -19,7 +19,6 @@ from dvc.stage import StageFileBadNameError from dvc.stage import StageFileDoesNotExistError from dvc.system import System from dvc.utils import relpath -from dvc.utils import walk_files from dvc.utils.compat import is_py2 from dvc.utils.stage import dump_stage_file from dvc.utils.stage import load_stage_file @@ -137,7 +136,7 @@ class CheckoutBase(TestDvcGit): paths = [ path for output in stage["outs"] - for path in walk_files(output["path"], self.dvc.tree.dvcignore) + for path in self.dvc.tree.walk_files(output["path"]) ] return [ diff --git a/tests/func/test_ignore.py b/tests/func/test_ignore.py index a2a4aae2d..803315f6a 100644 --- a/tests/func/test_ignore.py +++ b/tests/func/test_ignore.py @@ -7,8 +7,9 @@ import pytest from dvc.exceptions import DvcIgnoreInCollectedDirError from dvc.ignore import DvcIgnore, DvcIgnoreDirs, DvcIgnorePatterns from dvc.scm.tree import WorkingTree -from dvc.utils import walk_files, relpath -from dvc.utils.compat import fspath, fspath_py35 +from dvc.utils import relpath +from dvc.utils.compat import fspath_py35 +from dvc.utils.compat import fspath from dvc.utils.fs import get_mtime_and_size from dvc.remote import RemoteLOCAL @@ -20,10 +21,10 @@ def test_ignore(tmp_dir, dvc, monkeypatch): tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}}) tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored") - assert _files_set("dir", dvc.tree.dvcignore) == {"dir/other"} + assert _files_set("dir", dvc.tree) == {"dir/other"} monkeypatch.chdir("dir") - assert _files_set(".", dvc.tree.dvcignore) == {"./other"} + assert _files_set(".", dvc.tree) == {"./other"} def test_ignore_unicode(tmp_dir, dvc): @@ -35,27 +36,27 @@ def test_ignore_unicode(tmp_dir, dvc): tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ั‚ะตัั‚") - assert _files_set("dir", dvc.tree.dvcignore) == {"dir/other"} + assert _files_set("dir", dvc.tree) == {"dir/other"} def test_rename_ignored_file(tmp_dir, dvc): tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}}) tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "ignored*") - mtime, size = get_mtime_and_size("dir", dvc.tree.dvcignore) + mtime, size = get_mtime_and_size("dir", dvc.tree) shutil.move("dir/ignored", "dir/ignored_new") - new_mtime, new_size = get_mtime_and_size("dir", dvc.tree.dvcignore) + new_mtime, new_size = get_mtime_and_size("dir", dvc.tree) assert new_mtime == mtime and new_size == size def test_rename_file(tmp_dir, dvc): tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}}) - mtime, size = get_mtime_and_size("dir", dvc.tree.dvcignore) + mtime, size = get_mtime_and_size("dir", dvc.tree) shutil.move("dir/foo", "dir/foo_new") - new_mtime, new_size = get_mtime_and_size("dir", dvc.tree.dvcignore) + new_mtime, new_size = get_mtime_and_size("dir", dvc.tree) assert new_mtime != mtime and new_size == size @@ -64,20 +65,20 @@ def test_remove_ignored_file(tmp_dir, dvc): tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}}) tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored") - mtime, size = get_mtime_and_size("dir", dvc.tree.dvcignore) + mtime, size = get_mtime_and_size("dir", dvc.tree) os.remove("dir/ignored") - new_mtime, new_size = get_mtime_and_size("dir", dvc.tree.dvcignore) + new_mtime, new_size = get_mtime_and_size("dir", dvc.tree) assert new_mtime == mtime and new_size == size def test_remove_file(tmp_dir, dvc): tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}}) - mtime, size = get_mtime_and_size("dir", dvc.tree.dvcignore) + mtime, size = get_mtime_and_size("dir", dvc.tree) os.remove("dir/foo") - new_mtime, new_size = get_mtime_and_size("dir", dvc.tree.dvcignore) + new_mtime, new_size = get_mtime_and_size("dir", dvc.tree) assert new_mtime != mtime and new_size != size @@ -112,7 +113,7 @@ def test_ignore_on_branch(tmp_dir, scm, dvc): tmp_dir.scm_gen(DvcIgnore.DVCIGNORE_FILE, "foo", commit="add ignore") scm.checkout("master") - assert _files_set(".", dvc.tree.dvcignore) == {"./foo", "./bar"} + assert _files_set(".", dvc.tree) == {"./foo", "./bar"} tree = scm.get_tree("branch") files_on_branch = { @@ -121,8 +122,8 @@ def test_ignore_on_branch(tmp_dir, scm, dvc): assert "foo" not in files_on_branch -def _files_set(root, dvcignore): - return {to_posixpath(f) for f in walk_files(root, dvcignore)} +def _files_set(root, tree): + return {to_posixpath(f) for f in tree.walk_files(root)} def test_match_nested(tmp_dir, dvc): diff --git a/tests/unit/remote/test_remote_dir.py b/tests/unit/remote/test_remote_dir.py index 6c868a1a6..b6d6c337b 100644 --- a/tests/unit/remote/test_remote_dir.py +++ b/tests/unit/remote/test_remote_dir.py @@ -142,7 +142,7 @@ def test_download_dir(remote, tmpdir): remote.download(remote.path_info / "data", to_info) assert os.path.isdir(path) data_dir = tmpdir / "data" - assert len(list(walk_files(path, None))) == 7 + assert len(list(walk_files(path))) == 7 assert (data_dir / "alice").read_text(encoding="utf-8") == "alice" assert (data_dir / "alpha").read_text(encoding="utf-8") == "alpha" assert (data_dir / "subdir-file.txt").read_text( diff --git a/tests/unit/utils/test_fs.py b/tests/unit/utils/test_fs.py index 7f4f580fc..0dca8c65b 100644 --- a/tests/unit/utils/test_fs.py +++ b/tests/unit/utils/test_fs.py @@ -5,9 +5,8 @@ import pytest from mock import patch import dvc -from dvc.ignore import DvcIgnoreFilter from dvc.path_info import PathInfo -from dvc.scm.tree import WorkingTree +from dvc.scm.tree import WorkingTree, CleanTree from dvc.system import System from dvc.utils import relpath from dvc.utils.compat import str @@ -23,9 +22,9 @@ from tests.utils import spy class TestMtimeAndSize(TestDir): def test(self): - dvcignore = DvcIgnoreFilter(WorkingTree(self.root_dir)) - file_time, file_size = get_mtime_and_size(self.DATA, dvcignore) - dir_time, dir_size = get_mtime_and_size(self.DATA_DIR, dvcignore) + tree = CleanTree(WorkingTree(self.root_dir)) + file_time, file_size = get_mtime_and_size(self.DATA, tree) + dir_time, dir_size = get_mtime_and_size(self.DATA_DIR, tree) actual_file_size = os.path.getsize(self.DATA) actual_dir_size = os.path.getsize(self.DATA) + os.path.getsize( @@ -127,9 +126,9 @@ def test_get_inode(repo_dir): def test_path_object_and_str_are_valid_types_get_mtime_and_size( path, repo_dir ): - dvcignore = DvcIgnoreFilter(WorkingTree(repo_dir.root_dir)) - time, size = get_mtime_and_size(path, dvcignore) - object_time, object_size = get_mtime_and_size(PathInfo(path), dvcignore) + tree = CleanTree(WorkingTree(repo_dir.root_dir)) + time, size = get_mtime_and_size(path, tree) + object_time, object_size = get_mtime_and_size(PathInfo(path), tree) assert time == object_time assert size == object_size