mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-08 07:18:27 +01:00
Merge pull request #2974 from pared/2914
repo: move dvcignore from repo to tree
This commit is contained in:
+42
-5
@@ -3,9 +3,11 @@ from __future__ import unicode_literals
|
||||
import logging
|
||||
import os
|
||||
|
||||
from funcy import cached_property
|
||||
from pathspec import PathSpec
|
||||
from pathspec.patterns import GitWildMatchPattern
|
||||
|
||||
from dvc.scm.tree import BaseTree
|
||||
from dvc.utils import relpath
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -72,13 +74,12 @@ class DvcIgnoreDirs(DvcIgnore):
|
||||
|
||||
|
||||
class DvcIgnoreFilter(object):
|
||||
def __init__(self, root_dir, tree):
|
||||
def __init__(self, tree):
|
||||
self.tree = tree
|
||||
self.ignores = {DvcIgnoreDirs([".git", ".hg", ".dvc"])}
|
||||
self._update(root_dir)
|
||||
for root, dirs, _ in self.tree.walk(root_dir, dvcignore=self):
|
||||
for d in dirs:
|
||||
self._update(os.path.join(root, d))
|
||||
for root, dirs, files in self.tree.walk(self.tree.tree_root):
|
||||
self._update(root)
|
||||
dirs[:], files[:] = self(root, dirs, files)
|
||||
|
||||
def _update(self, dirname):
|
||||
ignore_file_path = os.path.join(dirname, DvcIgnore.DVCIGNORE_FILE)
|
||||
@@ -90,3 +91,39 @@ class DvcIgnoreFilter(object):
|
||||
dirs, files = ignore(root, dirs, files)
|
||||
|
||||
return dirs, files
|
||||
|
||||
|
||||
class CleanTree(BaseTree):
|
||||
def __init__(self, tree):
|
||||
self.tree = tree
|
||||
|
||||
@cached_property
|
||||
def dvcignore(self):
|
||||
return DvcIgnoreFilter(self.tree)
|
||||
|
||||
@property
|
||||
def tree_root(self):
|
||||
return self.tree.tree_root
|
||||
|
||||
def open(self, path, mode="r", encoding="utf-8"):
|
||||
return self.tree.open(path, mode, encoding)
|
||||
|
||||
def exists(self, path):
|
||||
return self.tree.exists(path)
|
||||
|
||||
def isdir(self, path):
|
||||
return self.tree.isdir(path)
|
||||
|
||||
def isfile(self, path):
|
||||
return self.tree.isfile(path)
|
||||
|
||||
def walk(self, top, topdown=True):
|
||||
for root, dirs, files in self.tree.walk(top, topdown):
|
||||
dirs[:], files[:] = self.dvcignore(root, dirs, files)
|
||||
|
||||
yield root, dirs, files
|
||||
|
||||
def walk_files(self, top):
|
||||
for root, _, files in self.walk(top):
|
||||
for file in files:
|
||||
yield os.path.join(root, file)
|
||||
|
||||
+11
-3
@@ -13,6 +13,7 @@ from dvc.config import Config
|
||||
from dvc.exceptions import DownloadError
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.exceptions import UploadError
|
||||
from dvc.ignore import CleanTree
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.remote.base import RemoteBASE
|
||||
@@ -21,6 +22,7 @@ from dvc.remote.base import STATUS_MAP
|
||||
from dvc.remote.base import STATUS_MISSING
|
||||
from dvc.remote.base import STATUS_NEW
|
||||
from dvc.scheme import Schemes
|
||||
from dvc.scm.tree import WorkingTree
|
||||
from dvc.system import System
|
||||
from dvc.utils import copyfile
|
||||
from dvc.utils import file_md5
|
||||
@@ -83,7 +85,7 @@ class RemoteLOCAL(RemoteBASE):
|
||||
|
||||
def list_cache_paths(self):
|
||||
assert self.path_info is not None
|
||||
return walk_files(self.path_info, None)
|
||||
return walk_files(self.path_info)
|
||||
|
||||
def get(self, md5):
|
||||
if not md5:
|
||||
@@ -138,7 +140,11 @@ class RemoteLOCAL(RemoteBASE):
|
||||
return os.path.getsize(fspath_py35(path_info))
|
||||
|
||||
def walk_files(self, path_info):
|
||||
for fname in walk_files(path_info, self.repo.dvcignore):
|
||||
assert isinstance(self.repo.tree, CleanTree) and isinstance(
|
||||
self.repo.tree.tree, WorkingTree
|
||||
)
|
||||
|
||||
for fname in self.repo.tree.walk_files(path_info):
|
||||
yield PathInfo(fname)
|
||||
|
||||
def get_file_checksum(self, path_info):
|
||||
@@ -427,7 +433,9 @@ class RemoteLOCAL(RemoteBASE):
|
||||
os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
|
||||
|
||||
def _unprotect_dir(self, path):
|
||||
for fname in walk_files(path, self.repo.dvcignore):
|
||||
assert isinstance(self.repo.tree, CleanTree)
|
||||
|
||||
for fname in self.repo.tree.walk_files(path):
|
||||
RemoteLOCAL._unprotect_file(fname)
|
||||
|
||||
def unprotect(self, path_info):
|
||||
|
||||
@@ -5,6 +5,8 @@ import os
|
||||
from contextlib import contextmanager
|
||||
from functools import wraps
|
||||
from itertools import chain
|
||||
|
||||
from dvc.ignore import CleanTree
|
||||
from dvc.utils.compat import FileNotFoundError, fspath_py35, open as _open
|
||||
|
||||
from funcy import cached_property
|
||||
@@ -15,7 +17,6 @@ from dvc.exceptions import (
|
||||
NotDvcRepoError,
|
||||
OutputNotFoundError,
|
||||
)
|
||||
from dvc.ignore import DvcIgnoreFilter
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.remote.base import RemoteActionNotImplemented
|
||||
from dvc.utils import relpath
|
||||
@@ -84,7 +85,7 @@ class Repo(object):
|
||||
|
||||
self.scm = SCM(self.root_dir)
|
||||
|
||||
self.tree = WorkingTree(self.root_dir)
|
||||
self.tree = CleanTree(WorkingTree(self.root_dir))
|
||||
|
||||
self.tmp_dir = os.path.join(self.dvc_dir, "tmp")
|
||||
makedirs(self.tmp_dir, exist_ok=True)
|
||||
@@ -391,9 +392,7 @@ class Repo(object):
|
||||
stages = []
|
||||
outs = []
|
||||
|
||||
for root, dirs, files in self.tree.walk(
|
||||
self.root_dir, dvcignore=self.dvcignore
|
||||
):
|
||||
for root, dirs, files in self.tree.walk(self.root_dir):
|
||||
for fname in files:
|
||||
path = os.path.join(root, fname)
|
||||
if not Stage.is_valid_filename(path):
|
||||
@@ -487,10 +486,6 @@ class Repo(object):
|
||||
|
||||
return _open(cache_file, mode=mode, encoding=encoding)
|
||||
|
||||
@cached_property
|
||||
def dvcignore(self):
|
||||
return DvcIgnoreFilter(self.root_dir, self.tree)
|
||||
|
||||
def close(self):
|
||||
self.scm.close()
|
||||
|
||||
|
||||
+1
-2
@@ -12,7 +12,6 @@ from dvc.progress import Tqdm
|
||||
from dvc.repo.scm_context import scm_context
|
||||
from dvc.stage import Stage
|
||||
from dvc.utils import LARGE_DIR_SIZE
|
||||
from dvc.utils import walk_files
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -68,7 +67,7 @@ def _find_all_targets(repo, target, recursive):
|
||||
if os.path.isdir(target) and recursive:
|
||||
return [
|
||||
fname
|
||||
for fname in walk_files(target, repo.dvcignore)
|
||||
for fname in repo.tree.walk_files(target)
|
||||
if not repo.is_dvc_internal(fname)
|
||||
if not Stage.is_stage_file(fname)
|
||||
if not repo.scm.belongs_to_scm(fname)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from funcy import group_by
|
||||
|
||||
from dvc.ignore import CleanTree
|
||||
from dvc.scm.tree import WorkingTree
|
||||
|
||||
|
||||
@@ -34,7 +35,7 @@ def brancher( # noqa: E302
|
||||
|
||||
scm = self.scm
|
||||
|
||||
self.tree = WorkingTree(self.root_dir)
|
||||
self.tree = CleanTree(WorkingTree(self.root_dir))
|
||||
yield "working tree"
|
||||
|
||||
if all_commits:
|
||||
@@ -58,7 +59,7 @@ def brancher( # noqa: E302
|
||||
# code which might expect the tree on which exception was raised to
|
||||
# stay in place. This behavior is a subject to change.
|
||||
for sha, names in group_by(scm.resolve_rev, revs).items():
|
||||
self.tree = scm.get_tree(sha)
|
||||
self.tree = CleanTree(scm.get_tree(sha))
|
||||
yield ", ".join(names)
|
||||
|
||||
self.tree = saved_tree
|
||||
|
||||
+3
-2
@@ -4,6 +4,7 @@ import os
|
||||
from errno import ENOENT
|
||||
|
||||
import dvc.logger as logger
|
||||
from dvc.ignore import CleanTree
|
||||
from . import locked
|
||||
from dvc.scm.base import FileNotInCommitError
|
||||
from dvc.scm.git import DIFF_A_REF
|
||||
@@ -137,9 +138,9 @@ def _is_dir(path, a_outs, b_outs):
|
||||
|
||||
|
||||
def _get_diff_outs(self, diff_dct):
|
||||
self.tree = diff_dct[DIFF_A_TREE]
|
||||
self.tree = CleanTree(diff_dct[DIFF_A_TREE])
|
||||
a_outs = {str(out): out for st in self.stages for out in st.outs}
|
||||
self.tree = diff_dct[DIFF_B_TREE]
|
||||
self.tree = CleanTree(diff_dct[DIFF_B_TREE])
|
||||
b_outs = {str(out): out for st in self.stages for out in st.outs}
|
||||
outs_paths = set(a_outs.keys())
|
||||
outs_paths.update(b_outs.keys())
|
||||
|
||||
+1
-1
@@ -136,7 +136,7 @@ class GitTree(BaseTree):
|
||||
if not topdown:
|
||||
yield os.path.normpath(tree.abspath), dirs, nondirs
|
||||
|
||||
def walk(self, top, topdown=True, dvcignore=None):
|
||||
def walk(self, top, topdown=True):
|
||||
"""Directory tree generator.
|
||||
|
||||
See `os.walk` for the docs. Differences:
|
||||
|
||||
+6
-7
@@ -1,7 +1,6 @@
|
||||
import os
|
||||
|
||||
from dvc.utils import dvc_walk
|
||||
from dvc.utils.compat import open
|
||||
from dvc.utils.compat import open, fspath
|
||||
|
||||
|
||||
class BaseTree(object):
|
||||
@@ -23,7 +22,7 @@ class BaseTree(object):
|
||||
def isfile(self, path):
|
||||
"""Test whether a path is a regular file"""
|
||||
|
||||
def walk(self, top, topdown=True, dvcignore=None):
|
||||
def walk(self, top, topdown=True):
|
||||
"""Directory tree generator.
|
||||
|
||||
See `os.walk` for the docs. Differences:
|
||||
@@ -58,7 +57,7 @@ class WorkingTree(BaseTree):
|
||||
"""Test whether a path is a regular file"""
|
||||
return os.path.isfile(path)
|
||||
|
||||
def walk(self, top, topdown=True, dvcignore=None):
|
||||
def walk(self, top, topdown=True):
|
||||
"""Directory tree generator.
|
||||
|
||||
See `os.walk` for the docs. Differences:
|
||||
@@ -66,12 +65,12 @@ class WorkingTree(BaseTree):
|
||||
- it could raise exceptions, there is no onerror argument
|
||||
"""
|
||||
|
||||
assert dvcignore
|
||||
top = fspath(top)
|
||||
|
||||
def onerror(e):
|
||||
raise e
|
||||
|
||||
for root, dirs, files in dvc_walk(
|
||||
os.path.abspath(top), dvcignore, topdown=topdown, onerror=onerror
|
||||
for root, dirs, files in os.walk(
|
||||
top, topdown=topdown, onerror=onerror
|
||||
):
|
||||
yield os.path.normpath(root), dirs, files
|
||||
|
||||
+4
-6
@@ -378,7 +378,7 @@ class State(object): # pylint: disable=too-many-instance-attributes
|
||||
assert os.path.exists(fspath_py35(path_info))
|
||||
|
||||
actual_mtime, actual_size = get_mtime_and_size(
|
||||
path_info, self.repo.dvcignore
|
||||
path_info, self.repo.tree
|
||||
)
|
||||
actual_inode = get_inode(path_info)
|
||||
|
||||
@@ -410,9 +410,7 @@ class State(object): # pylint: disable=too-many-instance-attributes
|
||||
if not os.path.exists(path):
|
||||
return None
|
||||
|
||||
actual_mtime, actual_size = get_mtime_and_size(
|
||||
path, self.repo.dvcignore
|
||||
)
|
||||
actual_mtime, actual_size = get_mtime_and_size(path, self.repo.tree)
|
||||
actual_inode = get_inode(path)
|
||||
|
||||
existing_record = self.get_state_record_for_inode(actual_inode)
|
||||
@@ -439,7 +437,7 @@ class State(object): # pylint: disable=too-many-instance-attributes
|
||||
if not os.path.exists(path):
|
||||
return
|
||||
|
||||
mtime, _ = get_mtime_and_size(path, self.repo.dvcignore)
|
||||
mtime, _ = get_mtime_and_size(path, self.repo.tree)
|
||||
inode = get_inode(path)
|
||||
relative_path = relpath(path, self.root_dir)
|
||||
|
||||
@@ -469,7 +467,7 @@ class State(object): # pylint: disable=too-many-instance-attributes
|
||||
continue
|
||||
|
||||
actual_inode = get_inode(path)
|
||||
actual_mtime, _ = get_mtime_and_size(path, self.repo.dvcignore)
|
||||
actual_mtime, _ = get_mtime_and_size(path, self.repo.tree)
|
||||
|
||||
if inode == actual_inode and mtime == actual_mtime:
|
||||
logger.debug("Removing '{}' as unused link.".format(path))
|
||||
|
||||
+2
-19
@@ -288,25 +288,8 @@ def to_yaml_string(data):
|
||||
return stream.getvalue()
|
||||
|
||||
|
||||
def dvc_walk(top, dvcignore, topdown=True, onerror=None, followlinks=False):
|
||||
"""
|
||||
Proxy for `os.walk` directory tree generator.
|
||||
Utilizes DvcIgnoreFilter functionality.
|
||||
"""
|
||||
top = fspath_py35(top)
|
||||
|
||||
for root, dirs, files in os.walk(
|
||||
top, topdown=topdown, onerror=onerror, followlinks=followlinks
|
||||
):
|
||||
|
||||
if dvcignore:
|
||||
dirs[:], files[:] = dvcignore(root, dirs, files)
|
||||
|
||||
yield root, dirs, files
|
||||
|
||||
|
||||
def walk_files(directory, dvcignore):
|
||||
for root, _, files in dvc_walk(directory, dvcignore):
|
||||
def walk_files(directory):
|
||||
for root, _, files in os.walk(fspath(directory)):
|
||||
for f in files:
|
||||
yield os.path.join(root, f)
|
||||
|
||||
|
||||
+6
-3
@@ -15,7 +15,6 @@ from dvc.utils import dict_md5
|
||||
from dvc.utils import fspath
|
||||
from dvc.utils import fspath_py35
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils import walk_files
|
||||
from dvc.utils.compat import str
|
||||
|
||||
|
||||
@@ -35,11 +34,15 @@ def get_inode(path):
|
||||
return inode
|
||||
|
||||
|
||||
def get_mtime_and_size(path, dvcignore):
|
||||
def get_mtime_and_size(path, tree):
|
||||
from dvc.ignore import CleanTree
|
||||
|
||||
assert isinstance(tree, CleanTree)
|
||||
|
||||
if os.path.isdir(fspath_py35(path)):
|
||||
size = 0
|
||||
files_mtimes = {}
|
||||
for file_path in walk_files(path, dvcignore):
|
||||
for file_path in tree.walk_files(path):
|
||||
try:
|
||||
stat = os.stat(file_path)
|
||||
except OSError as exc:
|
||||
|
||||
@@ -18,8 +18,7 @@ from dvc.stage import Stage
|
||||
from dvc.stage import StageFileBadNameError
|
||||
from dvc.stage import StageFileDoesNotExistError
|
||||
from dvc.system import System
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils import walk_files
|
||||
from dvc.utils import relpath, walk_files
|
||||
from dvc.utils.compat import is_py2
|
||||
from dvc.utils.stage import dump_stage_file
|
||||
from dvc.utils.stage import load_stage_file
|
||||
@@ -137,7 +136,7 @@ class CheckoutBase(TestDvcGit):
|
||||
paths = [
|
||||
path
|
||||
for output in stage["outs"]
|
||||
for path in walk_files(output["path"], self.dvc.dvcignore)
|
||||
for path in self.dvc.tree.walk_files(output["path"])
|
||||
]
|
||||
|
||||
return [
|
||||
@@ -520,6 +519,4 @@ def test_partial_checkout(tmp_dir, dvc, target):
|
||||
tmp_dir.dvc_gen({"dir": {"subdir": {"file": "file"}, "other": "other"}})
|
||||
shutil.rmtree("dir")
|
||||
dvc.checkout([target])
|
||||
assert list(walk_files("dir", None)) == [
|
||||
os.path.join("dir", "subdir", "file")
|
||||
]
|
||||
assert list(walk_files("dir")) == [os.path.join("dir", "subdir", "file")]
|
||||
|
||||
+25
-20
@@ -5,10 +5,11 @@ import shutil
|
||||
import pytest
|
||||
|
||||
from dvc.exceptions import DvcIgnoreInCollectedDirError
|
||||
from dvc.ignore import DvcIgnore, DvcIgnoreDirs, DvcIgnorePatterns
|
||||
from dvc.ignore import DvcIgnore, DvcIgnoreDirs, DvcIgnorePatterns, CleanTree
|
||||
from dvc.scm.tree import WorkingTree
|
||||
from dvc.utils import walk_files, relpath
|
||||
from dvc.utils.compat import fspath, fspath_py35
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.compat import fspath_py35
|
||||
from dvc.utils.compat import fspath
|
||||
from dvc.utils.fs import get_mtime_and_size
|
||||
from dvc.remote import RemoteLOCAL
|
||||
|
||||
@@ -20,10 +21,10 @@ def test_ignore(tmp_dir, dvc, monkeypatch):
|
||||
tmp_dir.gen({"dir": {"ignored": "text", "other": "text2"}})
|
||||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored")
|
||||
|
||||
assert _files_set("dir", dvc.dvcignore) == {"dir/other"}
|
||||
assert _files_set("dir", dvc.tree) == {"dir/other"}
|
||||
|
||||
monkeypatch.chdir("dir")
|
||||
assert _files_set(".", dvc.dvcignore) == {"./other"}
|
||||
assert _files_set(".", dvc.tree) == {"./other"}
|
||||
|
||||
|
||||
def test_ignore_unicode(tmp_dir, dvc):
|
||||
@@ -35,27 +36,27 @@ def test_ignore_unicode(tmp_dir, dvc):
|
||||
|
||||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/тест")
|
||||
|
||||
assert _files_set("dir", dvc.dvcignore) == {"dir/other"}
|
||||
assert _files_set("dir", dvc.tree) == {"dir/other"}
|
||||
|
||||
|
||||
def test_rename_ignored_file(tmp_dir, dvc):
|
||||
tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}})
|
||||
|
||||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "ignored*")
|
||||
mtime, size = get_mtime_and_size("dir", dvc.dvcignore)
|
||||
mtime, size = get_mtime_and_size("dir", dvc.tree)
|
||||
|
||||
shutil.move("dir/ignored", "dir/ignored_new")
|
||||
new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore)
|
||||
new_mtime, new_size = get_mtime_and_size("dir", dvc.tree)
|
||||
|
||||
assert new_mtime == mtime and new_size == size
|
||||
|
||||
|
||||
def test_rename_file(tmp_dir, dvc):
|
||||
tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
|
||||
mtime, size = get_mtime_and_size("dir", dvc.dvcignore)
|
||||
mtime, size = get_mtime_and_size("dir", dvc.tree)
|
||||
|
||||
shutil.move("dir/foo", "dir/foo_new")
|
||||
new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore)
|
||||
new_mtime, new_size = get_mtime_and_size("dir", dvc.tree)
|
||||
|
||||
assert new_mtime != mtime and new_size == size
|
||||
|
||||
@@ -64,20 +65,20 @@ def test_remove_ignored_file(tmp_dir, dvc):
|
||||
tmp_dir.gen({"dir": {"ignored": "...", "other": "text"}})
|
||||
tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "dir/ignored")
|
||||
|
||||
mtime, size = get_mtime_and_size("dir", dvc.dvcignore)
|
||||
mtime, size = get_mtime_and_size("dir", dvc.tree)
|
||||
|
||||
os.remove("dir/ignored")
|
||||
new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore)
|
||||
new_mtime, new_size = get_mtime_and_size("dir", dvc.tree)
|
||||
|
||||
assert new_mtime == mtime and new_size == size
|
||||
|
||||
|
||||
def test_remove_file(tmp_dir, dvc):
|
||||
tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
|
||||
mtime, size = get_mtime_and_size("dir", dvc.dvcignore)
|
||||
mtime, size = get_mtime_and_size("dir", dvc.tree)
|
||||
|
||||
os.remove("dir/foo")
|
||||
new_mtime, new_size = get_mtime_and_size("dir", dvc.dvcignore)
|
||||
new_mtime, new_size = get_mtime_and_size("dir", dvc.tree)
|
||||
|
||||
assert new_mtime != mtime and new_size != size
|
||||
|
||||
@@ -99,7 +100,7 @@ def test_ignore_collecting_dvcignores(tmp_dir, dvc, dname):
|
||||
ignore_file = tmp_dir / dname / DvcIgnore.DVCIGNORE_FILE
|
||||
ignore_file.write_text("foo")
|
||||
|
||||
assert dvc.dvcignore.ignores == {
|
||||
assert dvc.tree.dvcignore.ignores == {
|
||||
DvcIgnoreDirs([".git", ".hg", ".dvc"]),
|
||||
DvcIgnorePatterns(fspath(top_ignore_file), WorkingTree(dvc.root_dir)),
|
||||
}
|
||||
@@ -112,14 +113,18 @@ def test_ignore_on_branch(tmp_dir, scm, dvc):
|
||||
tmp_dir.scm_gen(DvcIgnore.DVCIGNORE_FILE, "foo", commit="add ignore")
|
||||
|
||||
scm.checkout("master")
|
||||
assert _files_set(".", dvc.dvcignore) == {"./foo", "./bar"}
|
||||
assert _files_set(".", dvc.tree) == {"./foo", "./bar"}
|
||||
|
||||
dvc.tree = scm.get_tree("branch")
|
||||
assert _files_set(".", dvc.dvcignore) == {"./bar"}
|
||||
tree = CleanTree(scm.get_tree("branch"))
|
||||
|
||||
assert _files_set(".", tree) == {
|
||||
to_posixpath(os.path.join(tree.tree_root, DvcIgnore.DVCIGNORE_FILE)),
|
||||
to_posixpath(os.path.join(tree.tree_root, "bar")),
|
||||
}
|
||||
|
||||
|
||||
def _files_set(root, dvcignore):
|
||||
return {to_posixpath(f) for f in walk_files(root, dvcignore)}
|
||||
def _files_set(root, tree):
|
||||
return {to_posixpath(f) for f in tree.walk_files(root)}
|
||||
|
||||
|
||||
def test_match_nested(tmp_dir, dvc):
|
||||
|
||||
+11
-34
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
|
||||
|
||||
from os.path import join
|
||||
|
||||
from dvc.ignore import DvcIgnoreFilter
|
||||
from dvc.ignore import CleanTree
|
||||
from dvc.scm import SCM
|
||||
from dvc.scm.git import GitTree
|
||||
from dvc.scm.tree import WorkingTree
|
||||
@@ -109,9 +109,8 @@ class AssertWalkEqualMixin(object):
|
||||
class TestWalkInNoSCM(AssertWalkEqualMixin, TestDir):
|
||||
def test(self):
|
||||
tree = WorkingTree(self._root_dir)
|
||||
dvcignore = DvcIgnoreFilter(self.root_dir, tree)
|
||||
self.assertWalkEqual(
|
||||
tree.walk(self._root_dir, dvcignore=dvcignore),
|
||||
tree.walk(self._root_dir),
|
||||
[
|
||||
(
|
||||
self._root_dir,
|
||||
@@ -129,48 +128,26 @@ class TestWalkInNoSCM(AssertWalkEqualMixin, TestDir):
|
||||
|
||||
def test_subdir(self):
|
||||
tree = WorkingTree(self._root_dir)
|
||||
dvcignore = DvcIgnoreFilter(self.root_dir, tree)
|
||||
self.assertWalkEqual(
|
||||
tree.walk(join("data_dir", "data_sub_dir"), dvcignore=dvcignore),
|
||||
[
|
||||
(
|
||||
join(self._root_dir, "data_dir", "data_sub_dir"),
|
||||
[],
|
||||
["data_sub"],
|
||||
)
|
||||
],
|
||||
tree.walk(join("data_dir", "data_sub_dir")),
|
||||
[(join("data_dir", "data_sub_dir"), [], ["data_sub"])],
|
||||
)
|
||||
|
||||
|
||||
class TestWalkInGit(AssertWalkEqualMixin, TestGit):
|
||||
def test_nobranch(self):
|
||||
tree = WorkingTree(self._root_dir)
|
||||
dvcignore = DvcIgnoreFilter(self._root_dir, tree)
|
||||
tree = CleanTree(WorkingTree(self._root_dir))
|
||||
self.assertWalkEqual(
|
||||
tree.walk(".", dvcignore=dvcignore),
|
||||
tree.walk("."),
|
||||
[
|
||||
(
|
||||
self._root_dir,
|
||||
["data_dir"],
|
||||
["bar", "тест", "code.py", "foo"],
|
||||
),
|
||||
(join(self._root_dir, "data_dir"), ["data_sub_dir"], ["data"]),
|
||||
(
|
||||
join(self._root_dir, "data_dir", "data_sub_dir"),
|
||||
[],
|
||||
["data_sub"],
|
||||
),
|
||||
(".", ["data_dir"], ["bar", "тест", "code.py", "foo"]),
|
||||
(join("data_dir"), ["data_sub_dir"], ["data"]),
|
||||
(join("data_dir", "data_sub_dir"), [], ["data_sub"]),
|
||||
],
|
||||
)
|
||||
self.assertWalkEqual(
|
||||
tree.walk(join("data_dir", "data_sub_dir"), dvcignore=dvcignore),
|
||||
[
|
||||
(
|
||||
join(self._root_dir, "data_dir", "data_sub_dir"),
|
||||
[],
|
||||
["data_sub"],
|
||||
)
|
||||
],
|
||||
tree.walk(join("data_dir", "data_sub_dir")),
|
||||
[(join("data_dir", "data_sub_dir"), [], ["data_sub"])],
|
||||
)
|
||||
|
||||
def test_branch(self):
|
||||
|
||||
@@ -142,7 +142,7 @@ def test_download_dir(remote, tmpdir):
|
||||
remote.download(remote.path_info / "data", to_info)
|
||||
assert os.path.isdir(path)
|
||||
data_dir = tmpdir / "data"
|
||||
assert len(list(walk_files(path, None))) == 7
|
||||
assert len(list(walk_files(path))) == 7
|
||||
assert (data_dir / "alice").read_text(encoding="utf-8") == "alice"
|
||||
assert (data_dir / "alpha").read_text(encoding="utf-8") == "alpha"
|
||||
assert (data_dir / "subdir-file.txt").read_text(
|
||||
|
||||
@@ -5,7 +5,7 @@ import pytest
|
||||
from mock import patch
|
||||
|
||||
import dvc
|
||||
from dvc.ignore import DvcIgnoreFilter
|
||||
from dvc.ignore import CleanTree
|
||||
from dvc.path_info import PathInfo
|
||||
from dvc.scm.tree import WorkingTree
|
||||
from dvc.system import System
|
||||
@@ -23,9 +23,9 @@ from tests.utils import spy
|
||||
|
||||
class TestMtimeAndSize(TestDir):
|
||||
def test(self):
|
||||
dvcignore = DvcIgnoreFilter(self.root_dir, WorkingTree(self.root_dir))
|
||||
file_time, file_size = get_mtime_and_size(self.DATA, dvcignore)
|
||||
dir_time, dir_size = get_mtime_and_size(self.DATA_DIR, dvcignore)
|
||||
tree = CleanTree(WorkingTree(self.root_dir))
|
||||
file_time, file_size = get_mtime_and_size(self.DATA, tree)
|
||||
dir_time, dir_size = get_mtime_and_size(self.DATA_DIR, tree)
|
||||
|
||||
actual_file_size = os.path.getsize(self.DATA)
|
||||
actual_dir_size = os.path.getsize(self.DATA) + os.path.getsize(
|
||||
@@ -127,11 +127,9 @@ def test_get_inode(repo_dir):
|
||||
def test_path_object_and_str_are_valid_types_get_mtime_and_size(
|
||||
path, repo_dir
|
||||
):
|
||||
dvcignore = DvcIgnoreFilter(
|
||||
repo_dir.root_dir, WorkingTree(repo_dir.root_dir)
|
||||
)
|
||||
time, size = get_mtime_and_size(path, dvcignore)
|
||||
object_time, object_size = get_mtime_and_size(PathInfo(path), dvcignore)
|
||||
tree = CleanTree(WorkingTree(repo_dir.root_dir))
|
||||
time, size = get_mtime_and_size(path, tree)
|
||||
object_time, object_size = get_mtime_and_size(PathInfo(path), tree)
|
||||
assert time == object_time
|
||||
assert size == object_size
|
||||
|
||||
|
||||
Reference in New Issue
Block a user