Merge pull request #2974 from pared/2914

repo: move dvcignore from repo to tree
This commit is contained in:
Ruslan Kuprieiev
2019-12-30 18:31:23 +02:00
committed by GitHub
16 changed files with 130 additions and 129 deletions
+42 -5
View File
@@ -3,9 +3,11 @@ from __future__ import unicode_literals
import logging
import os
from funcy import cached_property
from pathspec import PathSpec
from pathspec.patterns import GitWildMatchPattern
from dvc.scm.tree import BaseTree
from dvc.utils import relpath
logger = logging.getLogger(__name__)
@@ -72,13 +74,12 @@ class DvcIgnoreDirs(DvcIgnore):
class DvcIgnoreFilter(object):
def __init__(self, root_dir, tree):
def __init__(self, tree):
self.tree = tree
self.ignores = {DvcIgnoreDirs([".git", ".hg", ".dvc"])}
self._update(root_dir)
for root, dirs, _ in self.tree.walk(root_dir, dvcignore=self):
for d in dirs:
self._update(os.path.join(root, d))
for root, dirs, files in self.tree.walk(self.tree.tree_root):
self._update(root)
dirs[:], files[:] = self(root, dirs, files)
def _update(self, dirname):
ignore_file_path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
@@ -90,3 +91,39 @@ class DvcIgnoreFilter(object):
dirs, files = ignore(root, dirs, files)
return dirs, files
class CleanTree(BaseTree):
def __init__(self, tree):
self.tree = tree
@cached_property
def dvcignore(self):
return DvcIgnoreFilter(self.tree)
@property
def tree_root(self):
return self.tree.tree_root
def open(self, path, mode="r", encoding="utf-8"):
return self.tree.open(path, mode, encoding)
def exists(self, path):
return self.tree.exists(path)
def isdir(self, path):
return self.tree.isdir(path)
def isfile(self, path):
return self.tree.isfile(path)
def walk(self, top, topdown=True):
for root, dirs, files in self.tree.walk(top, topdown):
dirs[:], files[:] = self.dvcignore(root, dirs, files)
yield root, dirs, files
def walk_files(self, top):
for root, _, files in self.walk(top):
for file in files:
yield os.path.join(root, file)
+11 -3
View File
@@ -13,6 +13,7 @@ from dvc.config import Config
from dvc.exceptions import DownloadError
from dvc.exceptions import DvcException
from dvc.exceptions import UploadError
from dvc.ignore import CleanTree
from dvc.path_info import PathInfo
from dvc.progress import Tqdm
from dvc.remote.base import RemoteBASE
@@ -21,6 +22,7 @@ from dvc.remote.base import STATUS_MAP
from dvc.remote.base import STATUS_MISSING
from dvc.remote.base import STATUS_NEW
from dvc.scheme import Schemes
from dvc.scm.tree import WorkingTree
from dvc.system import System
from dvc.utils import copyfile
from dvc.utils import file_md5
@@ -83,7 +85,7 @@ class RemoteLOCAL(RemoteBASE):
def list_cache_paths(self):
assert self.path_info is not None
return walk_files(self.path_info, None)
return walk_files(self.path_info)
def get(self, md5):
if not md5:
@@ -138,7 +140,11 @@ class RemoteLOCAL(RemoteBASE):
return os.path.getsize(fspath_py35(path_info))
def walk_files(self, path_info):
for fname in walk_files(path_info, self.repo.dvcignore):
assert isinstance(self.repo.tree, CleanTree) and isinstance(
self.repo.tree.tree, WorkingTree
)
for fname in self.repo.tree.walk_files(path_info):
yield PathInfo(fname)
def get_file_checksum(self, path_info):
@@ -427,7 +433,9 @@ class RemoteLOCAL(RemoteBASE):
os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
def _unprotect_dir(self, path):
for fname in walk_files(path, self.repo.dvcignore):
assert isinstance(self.repo.tree, CleanTree)
for fname in self.repo.tree.walk_files(path):
RemoteLOCAL._unprotect_file(fname)
def unprotect(self, path_info):
+4 -9
View File
@@ -5,6 +5,8 @@ import os
from contextlib import contextmanager
from functools import wraps
from itertools import chain
from dvc.ignore import CleanTree
from dvc.utils.compat import FileNotFoundError, fspath_py35, open as _open
from funcy import cached_property
@@ -15,7 +17,6 @@ from dvc.exceptions import (
NotDvcRepoError,
OutputNotFoundError,
)
from dvc.ignore import DvcIgnoreFilter
from dvc.path_info import PathInfo
from dvc.remote.base import RemoteActionNotImplemented
from dvc.utils import relpath
@@ -84,7 +85,7 @@ class Repo(object):
self.scm = SCM(self.root_dir)
self.tree = WorkingTree(self.root_dir)
self.tree = CleanTree(WorkingTree(self.root_dir))
self.tmp_dir = os.path.join(self.dvc_dir, "tmp")
makedirs(self.tmp_dir, exist_ok=True)
@@ -391,9 +392,7 @@ class Repo(object):
stages = []
outs = []
for root, dirs, files in self.tree.walk(
self.root_dir, dvcignore=self.dvcignore
):
for root, dirs, files in self.tree.walk(self.root_dir):
for fname in files:
path = os.path.join(root, fname)
if not Stage.is_valid_filename(path):
@@ -487,10 +486,6 @@ class Repo(object):
return _open(cache_file, mode=mode, encoding=encoding)
@cached_property
def dvcignore(self):
return DvcIgnoreFilter(self.root_dir, self.tree)
def close(self):
self.scm.close()
+1 -2
View File
@@ -12,7 +12,6 @@ from dvc.progress import Tqdm
from dvc.repo.scm_context import scm_context
from dvc.stage import Stage
from dvc.utils import LARGE_DIR_SIZE
from dvc.utils import walk_files
logger = logging.getLogger(__name__)
@@ -68,7 +67,7 @@ def _find_all_targets(repo, target, recursive):
if os.path.isdir(target) and recursive:
return [
fname
for fname in walk_files(target, repo.dvcignore)
for fname in repo.tree.walk_files(target)
if not repo.is_dvc_internal(fname)
if not Stage.is_stage_file(fname)
if not repo.scm.belongs_to_scm(fname)
+3 -2
View File
@@ -1,5 +1,6 @@
from funcy import group_by
from dvc.ignore import CleanTree
from dvc.scm.tree import WorkingTree
@@ -34,7 +35,7 @@ def brancher( # noqa: E302
scm = self.scm
self.tree = WorkingTree(self.root_dir)
self.tree = CleanTree(WorkingTree(self.root_dir))
yield "working tree"
if all_commits:
@@ -58,7 +59,7 @@ def brancher( # noqa: E302
# code which might expect the tree on which exception was raised to
# stay in place. This behavior is a subject to change.
for sha, names in group_by(scm.resolve_rev, revs).items():
self.tree = scm.get_tree(sha)
self.tree = CleanTree(scm.get_tree(sha))
yield ", ".join(names)
self.tree = saved_tree
+3 -2
View File
@@ -4,6 +4,7 @@ import os
from errno import ENOENT
import dvc.logger as logger
from dvc.ignore import CleanTree
from . import locked
from dvc.scm.base import FileNotInCommitError
from dvc.scm.git import DIFF_A_REF
@@ -137,9 +138,9 @@ def _is_dir(path, a_outs, b_outs):
def _get_diff_outs(self, diff_dct):
self.tree = diff_dct[DIFF_A_TREE]
self.tree = CleanTree(diff_dct[DIFF_A_TREE])
a_outs = {str(out): out for st in self.stages for out in st.outs}
self.tree = diff_dct[DIFF_B_TREE]
self.tree = CleanTree(diff_dct[DIFF_B_TREE])
b_outs = {str(out): out for st in self.stages for out in st.outs}
outs_paths = set(a_outs.keys())
outs_paths.update(b_outs.keys())
+1 -1
View File
@@ -136,7 +136,7 @@ class GitTree(BaseTree):
if not topdown:
yield os.path.normpath(tree.abspath), dirs, nondirs
def walk(self, top, topdown=True, dvcignore=None):
def walk(self, top, topdown=True):
"""Directory tree generator.
See `os.walk` for the docs. Differences:
+6 -7
View File
@@ -1,7 +1,6 @@
import os
from dvc.utils import dvc_walk
from dvc.utils.compat import open
from dvc.utils.compat import open, fspath
class BaseTree(object):
@@ -23,7 +22,7 @@ class BaseTree(object):
def isfile(self, path):
"""Test whether a path is a regular file"""
def walk(self, top, topdown=True, dvcignore=None):
def walk(self, top, topdown=True):
"""Directory tree generator.
See `os.walk` for the docs. Differences:
@@ -58,7 +57,7 @@ class WorkingTree(BaseTree):
"""Test whether a path is a regular file"""
return os.path.isfile(path)
def walk(self, top, topdown=True, dvcignore=None):
def walk(self, top, topdown=True):
"""Directory tree generator.
See `os.walk` for the docs. Differences:
@@ -66,12 +65,12 @@ class WorkingTree(BaseTree):
- it could raise exceptions, there is no onerror argument
"""
assert dvcignore
top = fspath(top)
def onerror(e):
raise e
for root, dirs, files in dvc_walk(
os.path.abspath(top), dvcignore, topdown=topdown, onerror=onerror
for root, dirs, files in os.walk(
top, topdown=topdown, onerror=onerror
):
yield os.path.normpath(root), dirs, files
+4 -6
View File
@@ -378,7 +378,7 @@ class State(object): # pylint: disable=too-many-instance-attributes
assert os.path.exists(fspath_py35(path_info))
actual_mtime, actual_size = get_mtime_and_size(
path_info, self.repo.dvcignore
path_info, self.repo.tree
)
actual_inode = get_inode(path_info)
@@ -410,9 +410,7 @@ class State(object): # pylint: disable=too-many-instance-attributes
if not os.path.exists(path):
return None
actual_mtime, actual_size = get_mtime_and_size(
path, self.repo.dvcignore
)
actual_mtime, actual_size = get_mtime_and_size(path, self.repo.tree)
actual_inode = get_inode(path)
existing_record = self.get_state_record_for_inode(actual_inode)
@@ -439,7 +437,7 @@ class State(object): # pylint: disable=too-many-instance-attributes
if not os.path.exists(path):
return
mtime, _ = get_mtime_and_size(path, self.repo.dvcignore)
mtime, _ = get_mtime_and_size(path, self.repo.tree)
inode = get_inode(path)
relative_path = relpath(path, self.root_dir)
@@ -469,7 +467,7 @@ class State(object): # pylint: disable=too-many-instance-attributes
continue
actual_inode = get_inode(path)
actual_mtime, _ = get_mtime_and_size(path, self.repo.dvcignore)
actual_mtime, _ = get_mtime_and_size(path, self.repo.tree)
if inode == actual_inode and mtime == actual_mtime:
logger.debug("Removing '{}' as unused link.".format(path))
+2 -19
View File
@@ -288,25 +288,8 @@ def to_yaml_string(data):
return stream.getvalue()
def dvc_walk(top, dvcignore, topdown=True, onerror=None, followlinks=False):
"""
Proxy for `os.walk` directory tree generator.
Utilizes DvcIgnoreFilter functionality.
"""
top = fspath_py35(top)
for root, dirs, files in os.walk(
top, topdown=topdown, onerror=onerror, followlinks=followlinks
):
if dvcignore:
dirs[:], files[:] = dvcignore(root, dirs, files)
yield root, dirs, files
def walk_files(directory, dvcignore):
for root, _, files in dvc_walk(directory, dvcignore):
def walk_files(directory):
for root, _, files in os.walk(fspath(directory)):
for f in files:
yield os.path.join(root, f)
+6 -3
View File
@@ -15,7 +15,6 @@ from dvc.utils import dict_md5
from dvc.utils import fspath
from dvc.utils import fspath_py35
from dvc.utils import relpath
from dvc.utils import walk_files
from dvc.utils.compat import str
@@ -35,11 +34,15 @@ def get_inode(path):
return inode
def get_mtime_and_size(path, dvcignore):
def get_mtime_and_size(path, tree):
from dvc.ignore import CleanTree
assert isinstance(tree, CleanTree)
if os.path.isdir(fspath_py35(path)):
size = 0
files_mtimes = {}
for file_path in walk_files(path, dvcignore):
for file_path in tree.walk_files(path):
try:
stat = os.stat(file_path)
except OSError as exc:
+3 -6
View File
@@ -18,8 +18,7 @@ from dvc.stage import Stage
from dvc.stage import StageFileBadNameError
from dvc.stage import StageFileDoesNotExistError
from dvc.system import System
from dvc.utils import relpath
from dvc.utils import walk_files
from dvc.utils import relpath, walk_files
from dvc.utils.compat import is_py2
from dvc.utils.stage import dump_stage_file
from dvc.utils.stage import load_stage_file
@@ -137,7 +136,7 @@ class CheckoutBase(TestDvcGit):
paths = [
path
for output in stage["outs"]
for path in walk_files(output["path"], self.dvc.dvcignore)
for path in self.dvc.tree.walk_files(output["path"])
]
return [
@@ -520,6 +519,4 @@ def test_partial_checkout(tmp_dir, dvc, target):
tmp_dir.dvc_gen({"dir": {"subdir": {"file": "file"}, "other": "other"}})
shutil.rmtree("dir")
dvc.checkout([target])
assert list(walk_files("dir", None)) == [
os.path.join("dir", "subdir", "file")
]
assert list(walk_files("dir")) == [os.path.join("dir", "subdir", "file")]
+25 -20
View File
@@ -5,10 +5,11 @@ import shutil
import pytest
from dvc.exceptions import DvcIgnoreInCollectedDirError
from dvc.ignore import DvcIgnore, DvcIgnoreDirs, DvcIgnorePatterns
from dvc.ignore import DvcIgnore, DvcIgnoreDirs, DvcIgnorePatterns, CleanTree
from dvc.scm.tree import WorkingTree
from dvc.utils import walk_files, relpath
from dvc.utils.compat import fspath, fspath_py35
from dvc.utils import relpath
from dvc.utils.compat import fspath_py35
from dvc.utils.compat import fspath
from dvc.utils.fs import get_mtime_and_size
from dvc.remote import RemoteLOCAL
@@ -20,10 +21,10 @@ def test_ignore(tmp_dir, dvc, monkeypatch):
tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}})
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored")
assert _files_set("dir", dvc.dvcignore) == {"dir/other"}
assert _files_set("dir", dvc.tree) == {"dir/other"}
monkeypatch.chdir("dir")
assert _files_set(".", dvc.dvcignore) == {"./other"}
assert _files_set(".", dvc.tree) == {"./other"}
def test_ignore_unicode(tmp_dir, dvc):
@@ -35,27 +36,27 @@ def test_ignore_unicode(tmp_dir, dvc):
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/тест")
assert _files_set("dir", dvc.dvcignore) == {"dir/other"}
assert _files_set("dir", dvc.tree) == {"dir/other"}
def test_rename_ignored_file(tmp_dir, dvc):
tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}})
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "ignored*")
mtime, size = get_mtime_and_size("dir", dvc.dvcignore)
mtime, size = get_mtime_and_size("dir", dvc.tree)
shutil.move("dir/ignored", "dir/ignored_new")
new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore)
new_mtime, new_size = get_mtime_and_size("dir", dvc.tree)
assert new_mtime == mtime and new_size == size
def test_rename_file(tmp_dir, dvc):
tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
mtime, size = get_mtime_and_size("dir", dvc.dvcignore)
mtime, size = get_mtime_and_size("dir", dvc.tree)
shutil.move("dir/foo", "dir/foo_new")
new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore)
new_mtime, new_size = get_mtime_and_size("dir", dvc.tree)
assert new_mtime != mtime and new_size == size
@@ -64,20 +65,20 @@ def test_remove_ignored_file(tmp_dir, dvc):
tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}})
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored")
mtime, size = get_mtime_and_size("dir", dvc.dvcignore)
mtime, size = get_mtime_and_size("dir", dvc.tree)
os.remove("dir/ignored")
new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore)
new_mtime, new_size = get_mtime_and_size("dir", dvc.tree)
assert new_mtime == mtime and new_size == size
def test_remove_file(tmp_dir, dvc):
tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
mtime, size = get_mtime_and_size("dir", dvc.dvcignore)
mtime, size = get_mtime_and_size("dir", dvc.tree)
os.remove("dir/foo")
new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore)
new_mtime, new_size = get_mtime_and_size("dir", dvc.tree)
assert new_mtime != mtime and new_size != size
@@ -99,7 +100,7 @@ def test_ignore_collecting_dvcignores(tmp_dir, dvc, dname):
ignore_file = tmp_dir / dname / DvcIgnore.DVCIGNORE_FILE
ignore_file.write_text("foo")
assert dvc.dvcignore.ignores == {
assert dvc.tree.dvcignore.ignores == {
DvcIgnoreDirs([".git", ".hg", ".dvc"]),
DvcIgnorePatterns(fspath(top_ignore_file), WorkingTree(dvc.root_dir)),
}
@@ -112,14 +113,18 @@ def test_ignore_on_branch(tmp_dir, scm, dvc):
tmp_dir.scm_gen(DvcIgnore.DVCIGNORE_FILE, "foo", commit="add ignore")
scm.checkout("master")
assert _files_set(".", dvc.dvcignore) == {"./foo", "./bar"}
assert _files_set(".", dvc.tree) == {"./foo", "./bar"}
dvc.tree = scm.get_tree("branch")
assert _files_set(".", dvc.dvcignore) == {"./bar"}
tree = CleanTree(scm.get_tree("branch"))
assert _files_set(".", tree) == {
to_posixpath(os.path.join(tree.tree_root, DvcIgnore.DVCIGNORE_FILE)),
to_posixpath(os.path.join(tree.tree_root, "bar")),
}
def _files_set(root, dvcignore):
return {to_posixpath(f) for f in walk_files(root, dvcignore)}
def _files_set(root, tree):
return {to_posixpath(f) for f in tree.walk_files(root)}
def test_match_nested(tmp_dir, dvc):
+11 -34
View File
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
from os.path import join
from dvc.ignore import DvcIgnoreFilter
from dvc.ignore import CleanTree
from dvc.scm import SCM
from dvc.scm.git import GitTree
from dvc.scm.tree import WorkingTree
@@ -109,9 +109,8 @@ class AssertWalkEqualMixin(object):
class TestWalkInNoSCM(AssertWalkEqualMixin, TestDir):
def test(self):
tree = WorkingTree(self._root_dir)
dvcignore = DvcIgnoreFilter(self.root_dir, tree)
self.assertWalkEqual(
tree.walk(self._root_dir, dvcignore=dvcignore),
tree.walk(self._root_dir),
[
(
self._root_dir,
@@ -129,48 +128,26 @@ class TestWalkInNoSCM(AssertWalkEqualMixin, TestDir):
def test_subdir(self):
tree = WorkingTree(self._root_dir)
dvcignore = DvcIgnoreFilter(self.root_dir, tree)
self.assertWalkEqual(
tree.walk(join("data_dir", "data_sub_dir"), dvcignore=dvcignore),
[
(
join(self._root_dir, "data_dir", "data_sub_dir"),
[],
["data_sub"],
)
],
tree.walk(join("data_dir", "data_sub_dir")),
[(join("data_dir", "data_sub_dir"), [], ["data_sub"])],
)
class TestWalkInGit(AssertWalkEqualMixin, TestGit):
def test_nobranch(self):
tree = WorkingTree(self._root_dir)
dvcignore = DvcIgnoreFilter(self._root_dir, tree)
tree = CleanTree(WorkingTree(self._root_dir))
self.assertWalkEqual(
tree.walk(".", dvcignore=dvcignore),
tree.walk("."),
[
(
self._root_dir,
["data_dir"],
["bar", "тест", "code.py", "foo"],
),
(join(self._root_dir, "data_dir"), ["data_sub_dir"], ["data"]),
(
join(self._root_dir, "data_dir", "data_sub_dir"),
[],
["data_sub"],
),
(".", ["data_dir"], ["bar", "тест", "code.py", "foo"]),
(join("data_dir"), ["data_sub_dir"], ["data"]),
(join("data_dir", "data_sub_dir"), [], ["data_sub"]),
],
)
self.assertWalkEqual(
tree.walk(join("data_dir", "data_sub_dir"), dvcignore=dvcignore),
[
(
join(self._root_dir, "data_dir", "data_sub_dir"),
[],
["data_sub"],
)
],
tree.walk(join("data_dir", "data_sub_dir")),
[(join("data_dir", "data_sub_dir"), [], ["data_sub"])],
)
def test_branch(self):
+1 -1
View File
@@ -142,7 +142,7 @@ def test_download_dir(remote, tmpdir):
remote.download(remote.path_info / "data", to_info)
assert os.path.isdir(path)
data_dir = tmpdir / "data"
assert len(list(walk_files(path, None))) == 7
assert len(list(walk_files(path))) == 7
assert (data_dir / "alice").read_text(encoding="utf-8") == "alice"
assert (data_dir / "alpha").read_text(encoding="utf-8") == "alpha"
assert (data_dir / "subdir-file.txt").read_text(
+7 -9
View File
@@ -5,7 +5,7 @@ import pytest
from mock import patch
import dvc
from dvc.ignore import DvcIgnoreFilter
from dvc.ignore import CleanTree
from dvc.path_info import PathInfo
from dvc.scm.tree import WorkingTree
from dvc.system import System
@@ -23,9 +23,9 @@ from tests.utils import spy
class TestMtimeAndSize(TestDir):
def test(self):
dvcignore = DvcIgnoreFilter(self.root_dir, WorkingTree(self.root_dir))
file_time, file_size = get_mtime_and_size(self.DATA, dvcignore)
dir_time, dir_size = get_mtime_and_size(self.DATA_DIR, dvcignore)
tree = CleanTree(WorkingTree(self.root_dir))
file_time, file_size = get_mtime_and_size(self.DATA, tree)
dir_time, dir_size = get_mtime_and_size(self.DATA_DIR, tree)
actual_file_size = os.path.getsize(self.DATA)
actual_dir_size = os.path.getsize(self.DATA) + os.path.getsize(
@@ -127,11 +127,9 @@ def test_get_inode(repo_dir):
def test_path_object_and_str_are_valid_types_get_mtime_and_size(
path, repo_dir
):
dvcignore = DvcIgnoreFilter(
repo_dir.root_dir, WorkingTree(repo_dir.root_dir)
)
time, size = get_mtime_and_size(path, dvcignore)
object_time, object_size = get_mtime_and_size(PathInfo(path), dvcignore)
tree = CleanTree(WorkingTree(repo_dir.root_dir))
time, size = get_mtime_and_size(path, tree)
object_time, object_size = get_mtime_and_size(PathInfo(path), tree)
assert time == object_time
assert size == object_size