Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

dvc: refactor objects (#5607)

Browse Source 
* dvc: get rid of DirInfo

DirInfo was created before we had proper objects, as a way to have a
structure that represents a staged directory. Right now we have a
dedicated Tree object that is specifically meant for that.

As a result, HashInfo no longer needs to have dir_info and we no longer
need `get_hash/get_dir_hash` helpers.

* objects: move Tree into tree.py
This commit is contained in:
Ruslan Kuprieiev
2021-03-12 01:10:34 +02:00
committed by GitHub
parent b691b15007
commit 4af3b3d20d
25 changed files with 476 additions and 475 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/api.py
+2 -3
View File
@@ -4,7 +4,6 @@ from contextlib import _GeneratorContextManager as GCM
from funcy import reraise
from dvc.exceptions import OutputNotFoundError, PathMissingError
from dvc.objects.stage import get_hash
from dvc.path_info import PathInfo
from dvc.repo import Repo
@@ -29,8 +28,8 @@ def get_url(path, repo=None, rev=None, remote=None):
raise OutputNotFoundError(path, repo)
cloud = metadata.repo.cloud
hash_info = get_hash(path_info, _repo.repo_fs, "md5", _repo.odb)
return cloud.get_url_for(remote, checksum=hash_info.value)
md5 = metadata.repo.dvcfs.info(path_info)["md5"]
return cloud.get_url_for(remote, checksum=md5)
def open( # noqa, pylint: disable=redefined-builtin
dvc/checkout.py
+8 -11
View File
@@ -10,7 +10,7 @@ from dvc.exceptions import (
DvcException,
)
from dvc.objects import ObjectFormatError, check, load
from dvc.objects.stage import get_hash
from dvc.objects.stage import stage
from dvc.remote.slow_link_detection import ( # type: ignore[attr-defined]
slow_link_guard,
)
@@ -30,7 +30,7 @@ def _changed(path_info, fs, obj, cache):
return True
try:
actual = get_hash(path_info, fs, obj.hash_info.name, cache)
actual = stage(cache, path_info, fs, obj.hash_info.name).hash_info
except FileNotFoundError:
logger.debug("'%s' doesn't exist.", path_info)
return True
@@ -56,7 +56,7 @@ def _remove(path_info, fs, cache, force=False):
fs.remove(path_info)
return
current = get_hash(path_info, fs, fs.PARAM_CHECKSUM, cache)
current = stage(cache, path_info, fs, fs.PARAM_CHECKSUM).hash_info
try:
obj = load(cache, current)
check(cache, obj)
@@ -174,10 +174,10 @@ def _checkout_file(
return modified
def _remove_redundant_files(path_info, fs, dir_info, cache, force):
def _remove_redundant_files(path_info, fs, obj, cache, force):
existing_files = set(fs.walk_files(path_info))
needed_files = {path_info.joinpath(*key) for key, _ in dir_info.items()}
needed_files = {path_info.joinpath(*key) for key, _ in obj}
redundant_files = existing_files - needed_files
for path in redundant_files:
_remove(path, fs, cache, force)
@@ -197,11 +197,11 @@ def _checkout_dir(
logger.debug("Linking directory '%s'.", path_info)
for entry_key, entry_hash_info in obj.hash_info.dir_info.items():
for entry_key, entry_obj in obj:
entry_modified = _checkout_file(
path_info.joinpath(*entry_key),
fs,
cache.get(entry_hash_info),
entry_obj,
cache,
force,
progress_callback,
@@ -211,10 +211,7 @@ def _checkout_dir(
modified = True
modified = (
_remove_redundant_files(
path_info, fs, obj.hash_info.dir_info, cache, force
)
or modified
_remove_redundant_files(path_info, fs, obj, cache, force) or modified
)
fs.repo.state.save(path_info, fs, obj.hash_info)
dvc/dependency/repo.py
+7 -1
View File
@@ -55,6 +55,7 @@ class RepoDependency(LocalDependency):
self.repo.odb.local,
path_info,
repo.repo_fs,
self.repo.odb.local.fs.PARAM_CHECKSUM,
follow_subrepos=False,
).hash_info
@@ -94,7 +95,12 @@ class RepoDependency(LocalDependency):
except (NoOutputOrStageError, NoRemoteError):
pass
obj = stage(
odb, path_info, repo.repo_fs, jobs=jobs, follow_subrepos=False,
odb,
path_info,
repo.repo_fs,
odb.fs.PARAM_CHECKSUM,
jobs=jobs,
follow_subrepos=False,
)
save(odb, obj, jobs=jobs)
dvc/dir_info.py
-117
View File
@@ -1,117 +0,0 @@
import posixpath
from operator import itemgetter
from funcy import cached_property
from pygtrie import Trie
from .hash_info import HashInfo
def _diff(ancestor, other, allow_removed=False):
from dictdiffer import diff
from dvc.exceptions import MergeError
allowed = ["add"]
if allow_removed:
allowed.append("remove")
result = list(diff(ancestor, other))
for typ, _, _ in result:
if typ not in allowed:
raise MergeError(
"unable to auto-merge directories with diff that contains "
f"'{typ}'ed files"
)
return result
def _merge(ancestor, our, their):
import copy
from dictdiffer import patch
our_diff = _diff(ancestor, our)
if not our_diff:
return copy.deepcopy(their)
their_diff = _diff(ancestor, their)
if not their_diff:
return copy.deepcopy(our)
# make sure there are no conflicting files
_diff(our, their, allow_removed=True)
return patch(our_diff + their_diff, ancestor)
class DirInfo:
PARAM_RELPATH = "relpath"
def __init__(self):
self._dict = {}
def as_dict(self):
return self._dict.copy()
def add(self, key, hash_info):
self.__dict__.pop("trie", None)
self._dict[key] = hash_info
def get(self, key, default=None):
return self._dict.get(key, default)
@cached_property
def trie(self):
return Trie(self._dict)
@property
def size(self):
try:
return sum(hash_info.size for _, hash_info in self._dict.items())
except TypeError:
return None
@property
def nfiles(self):
return len(self._dict)
def items(self):
for key, hash_info in self._dict.items():
yield key, hash_info
@classmethod
def from_list(cls, lst):
ret = DirInfo()
for _entry in lst:
entry = _entry.copy()
relpath = entry.pop(cls.PARAM_RELPATH)
parts = tuple(relpath.split(posixpath.sep))
hash_info = HashInfo.from_dict(entry)
ret.add(parts, hash_info)
return ret
def to_list(self):
# Sorting the list by path to ensure reproducibility
return sorted(
(
{
# NOTE: not using hash_info.to_dict() because we don't want
# size/nfiles fields at this point.
hash_info.name: hash_info.value,
self.PARAM_RELPATH: posixpath.sep.join(parts),
}
for parts, hash_info in self._dict.items() # noqa: B301
),
key=itemgetter(self.PARAM_RELPATH),
)
def merge(self, ancestor, their):
merged_dict = _merge(
ancestor.as_dict(), self.as_dict(), their.as_dict(),
)
merged = DirInfo()
for key, hi in merged_dict.items():
merged.add(key, hi)
return merged
dvc/fs/dvc.py
+9 -16
View File
@@ -48,9 +48,9 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
# NOTE: use string paths here for performance reasons
key = tuple(relpath(path_info, out.path_info).split(os.sep))
out.get_dir_cache(remote=remote)
file_hash = out.hash_info.dir_info.get(key)
if file_hash:
return file_hash
obj = out.obj.trie.get(key)
if obj:
return obj.hash_info
raise FileNotFoundError
def open( # type: ignore
@@ -131,21 +131,14 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
# pull dir cache if needed
out.get_dir_cache(**kwargs)
dir_cache = out.dir_cache
if not dir_cache:
raise FileNotFoundError
from dvc.objects import Tree
hash_info = Tree.save_dir_info(out.odb, dir_cache)
if hash_info != out.hash_info:
if not out.obj:
raise FileNotFoundError
def _add_dir(self, trie, out, **kwargs):
self._fetch_dir(out, **kwargs)
base = out.path_info.parts
for key, _ in out.dir_cache.items(): # noqa: B301
for key, _ in out.obj: # noqa: B301
trie[base + key] = None
def _walk(self, root, trie, topdown=True, **kwargs):
@@ -241,9 +234,9 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
elif meta.part_of_output:
(out,) = meta.outs
key = path_info.relative_to(out.path_info).parts
hash_info = out.hash_info.dir_info.trie.get(key)
if hash_info:
ret["size"] = hash_info.size
ret[hash_info.name] = hash_info.value
obj = out.obj.trie.get(key)
if obj:
ret["size"] = obj.size
ret[obj.hash_info.name] = obj.hash_info.value
return ret
dvc/fs/memory.py
+4 -1
View File
@@ -13,13 +13,16 @@ class MemoryFileSystem(BaseFileSystem):
self.fs = MemFS()
def exists(self, path_info, use_dvcignore=True):
return self.fs.exists(path_info.path)
return self.fs.exists(path_info.fspath)
def open(self, path_info, mode="r", encoding=None, **kwargs):
return self.fs.open(
path_info.fspath, mode=mode, encoding=encoding, **kwargs
)
def info(self, path_info):
return self.fs.info(path_info.fspath)
def stat(self, path_info):
import os
dvc/hash_info.py
+1 -4
View File
@@ -1,11 +1,9 @@
from collections import OrderedDict
from dataclasses import dataclass, field
from typing import Dict, Optional
from typing import Optional
HASH_DIR_SUFFIX = ".dir"
DirInfo = Dict[str, str]
@dataclass
class HashInfo:
@@ -14,7 +12,6 @@ class HashInfo:
name: Optional[str]
value: Optional[str]
dir_info: Optional[DirInfo] = field(default=None, compare=False)
size: Optional[int] = field(default=None, compare=False)
nfiles: Optional[int] = field(default=None, compare=False)
dvc/objects/__init__.py
+40 -176
View File
@@ -1,13 +1,11 @@
import itertools
import json
import logging
from concurrent import futures
from dvc.dir_info import DirInfo
from dvc.exceptions import DvcException
from dvc.progress import Tqdm
from .stage import get_hash
from .stage import get_file_hash
logger = logging.getLogger(__name__)
@@ -26,6 +24,12 @@ class HashFile:
self.fs = fs
self.hash_info = hash_info
@property
def size(self):
if not (self.path_info and self.fs):
return None
return self.fs.getsize(self.path_info)
def __len__(self):
return 1
@@ -35,8 +39,19 @@ class HashFile:
def __bool__(self):
return bool(self.hash_info)
def __eq__(self, other):
if not isinstance(other, HashFile):
return False
return (
self.path_info == other.path_info
and self.fs == other.fs
and self.hash_info == other.hash_info
)
def check(self, odb):
actual = get_hash(self.path_info, self.fs, odb.fs.PARAM_CHECKSUM, odb)
actual = get_file_hash(
self.path_info, self.fs, self.hash_info.name, odb.repo.state
)
logger.trace(
"cache '%s' expected '%s' actual '%s'",
@@ -53,187 +68,32 @@ class HashFile:
def load(cls, odb, hash_info):
return odb.get(hash_info)
def save(self, odb, **kwargs):
odb.add(self.path_info, self.fs, self.hash_info, **kwargs)
class File(HashFile):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.src = None
@classmethod
def stage(cls, odb, path_info, fs, **kwargs):
hash_info = get_hash(
path_info, fs, odb.fs.PARAM_CHECKSUM, odb, **kwargs
)
raw = odb.get(hash_info)
obj = cls(raw.path_info, raw.fs, hash_info)
obj.src = HashFile(path_info, fs, hash_info)
return obj
def save(self, odb, **kwargs):
self.src.save(odb, **kwargs)
class Tree(HashFile):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.src_hash_info = None
self.src_path_info = None
self.src_fs = None
def __len__(self):
return self.hash_info.dir_info.nfiles
def __iter__(self):
yield from self.hash_info.dir_info.items()
@classmethod
def save_dir_info(cls, odb, dir_info, hash_info=None):
if hash_info and hash_info.name == odb.fs.PARAM_CHECKSUM:
try:
odb.check(hash_info)
assert hash_info.dir_info == dir_info
return hash_info
except (FileNotFoundError, ObjectFormatError):
pass
from dvc.fs.memory import MemoryFileSystem
from dvc.path_info import PathInfo
from dvc.utils import tmp_fname
fs = MemoryFileSystem(None, {})
path_info = PathInfo(tmp_fname(""))
with fs.open(path_info, "w") as fobj:
json.dump(dir_info.to_list(), fobj, sort_keys=True)
tmp_info = odb.fs.path_info / tmp_fname("")
with fs.open(path_info, "rb") as fobj:
odb.fs.upload_fobj(fobj, tmp_info)
hash_info = get_hash(tmp_info, odb.fs, odb.fs.PARAM_CHECKSUM, odb)
hash_info.value += odb.fs.CHECKSUM_DIR_SUFFIX
hash_info.dir_info = dir_info
hash_info.nfiles = dir_info.nfiles
odb.add(tmp_info, odb.fs, hash_info)
return hash_info
@classmethod
def stage(cls, odb, path_info, fs, **kwargs):
hash_info = get_hash(
path_info, fs, odb.fs.PARAM_CHECKSUM, odb, **kwargs
)
hi = cls.save_dir_info(odb, hash_info.dir_info, hash_info)
hi.size = hash_info.size
raw = odb.get(hi)
obj = cls(raw.path_info, raw.fs, hi)
obj.src_hash_info = hash_info
obj.src_path_info = path_info
obj.src_fs = fs
return obj
@classmethod
def load(cls, odb, hash_info):
obj = odb.get(hash_info)
try:
with obj.fs.open(obj.path_info, "r") as fobj:
raw = json.load(fobj)
except ValueError as exc:
raise ObjectFormatError(f"{obj} is corrupted") from exc
if not isinstance(raw, list):
logger.error(
"dir cache file format error '%s' [skipping the file]",
obj.path_info,
)
raise ObjectFormatError(f"{obj} is corrupted")
dir_info = DirInfo.from_list(raw)
hash_info.dir_info = dir_info
hash_info.nfiles = dir_info.nfiles
return cls(obj.path_info, obj.fs, hash_info)
def save(self, odb, **kwargs):
assert self.src_hash_info.dir_info
hi = self.save_dir_info(
odb, self.src_hash_info.dir_info, self.hash_info
)
for entry_key, entry_hash in Tqdm(
hi.dir_info.items(),
desc="Saving " + self.src_path_info.name,
unit="file",
):
entry_info = self.src_path_info.joinpath(*entry_key)
entry_obj = HashFile(entry_info, self.src_fs, entry_hash)
entry_obj.save(odb, **kwargs)
self.src_fs.repo.state.save(self.src_path_info, self.src_fs, hi)
def filter(self, odb, prefix):
hash_info = self.hash_info.dir_info.get(prefix)
if hash_info:
return load(odb, hash_info)
depth = len(prefix)
dir_info = DirInfo()
try:
for key, value in self.hash_info.dir_info.trie.items(prefix):
dir_info.add(key[depth:], value)
except KeyError:
return None
return load(odb, self.save_dir_info(odb, dir_info))
def save(odb, obj, **kwargs):
obj.save(odb, **kwargs)
from .tree import Tree
if isinstance(obj, Tree):
for _, entry in Tqdm(obj):
odb.add(entry.path_info, entry.fs, entry.hash_info, **kwargs)
odb.add(obj.path_info, obj.fs, obj.hash_info, **kwargs)
def check(odb, obj):
from .tree import Tree
odb.check(obj.hash_info)
if isinstance(obj, Tree):
for _, hash_info in obj:
odb.check(hash_info)
for _, entry in obj:
odb.check(entry.hash_info)
def load(odb, hash_info):
from .tree import Tree
if hash_info.isdir:
return Tree.load(odb, hash_info)
return File.load(odb, hash_info)
def _get_dir_size(odb, dir_info):
try:
return sum(
odb.fs.getsize(odb.hash_to_path_info(hi.value))
for _, hi in dir_info.items()
)
except FileNotFoundError:
return None
def merge(odb, ancestor_info, our_info, their_info):
assert our_info
assert their_info
if ancestor_info:
ancestor = load(odb, ancestor_info).hash_info.dir_info
else:
ancestor = DirInfo()
our = load(odb, our_info).hash_info.dir_info
their = load(odb, their_info).hash_info.dir_info
merged = our.merge(ancestor, their)
hash_info = Tree.save_dir_info(odb, merged)
hash_info.size = _get_dir_size(odb, merged)
return hash_info
return odb.get(hash_info)
def _transfer_file(odb, from_fs, from_info):
@@ -286,17 +146,21 @@ def _transfer_directory_contents(odb, from_fs, from_info, jobs, pbar):
def _transfer_directory(odb, from_fs, from_info, jobs, no_progress_bar=False):
dir_info = DirInfo()
from .tree import Tree
tree = Tree(None, None, None)
with Tqdm(total=1, unit="Files", disable=no_progress_bar) as pbar:
for (
entry_info,
(entry_tmp_info, entry_hash),
) in _transfer_directory_contents(odb, from_fs, from_info, jobs, pbar):
odb.add(entry_tmp_info, odb.fs, entry_hash)
dir_info.add(entry_info.parts, entry_hash)
obj = HashFile(entry_tmp_info, odb.fs, entry_hash)
tree.add(entry_info.parts, obj)
return Tree.save_dir_info(odb, dir_info)
tree.digest()
save(odb, tree)
return tree.hash_info
def transfer(odb, from_fs, from_info, jobs=None, no_progress_bar=False):
dvc/objects/stage.py
+77 -42
View File
@@ -9,7 +9,7 @@ from dvc.progress import Tqdm
from dvc.utils import file_md5
def get_file_hash(path_info, fs, name):
def _get_file_hash(path_info, fs, name):
info = fs.info(path_info)
if name in info:
assert not info[name].endswith(".dir")
@@ -27,20 +27,36 @@ def get_file_hash(path_info, fs, name):
raise NotImplementedError
def _calculate_hashes(path_info, fs, name, state, **kwargs):
def _get_file_hash(file_info):
def get_file_hash(path_info, fs, name, state=None):
if state:
hash_info = state.get( # pylint: disable=assignment-from-none
file_info, fs,
path_info, fs,
)
if not hash_info:
hash_info = get_file_hash(file_info, fs, name)
state.save(file_info, fs, hash_info)
return file_info, hash_info
if hash_info:
return hash_info
if not fs.exists(path_info):
raise FileNotFoundError(
errno.ENOENT, os.strerror(errno.ENOENT), path_info
)
hash_info = _get_file_hash(path_info, fs, name)
if state:
assert ".dir" not in hash_info.value
state.save(path_info, fs, hash_info)
return hash_info
def _calculate_hashes(path_info, fs, name, state, **kwargs):
def _hash(file_info):
return file_info, get_file_hash(file_info, fs, name, state)
with Tqdm(
unit="md5", desc="Computing file/dir hashes (only done once)",
) as pbar:
worker = pbar.wrap_fn(_get_file_hash)
worker = pbar.wrap_fn(_hash)
with ThreadPoolExecutor(max_workers=fs.hash_jobs) as executor:
pairs = executor.map(worker, fs.walk_files(path_info, **kwargs))
return dict(pairs)
@@ -60,14 +76,17 @@ def _iter_hashes(path_info, fs, name, state, **kwargs):
yield from _calculate_hashes(path_info, fs, name, state, **kwargs).items()
def _collect_dir(path_info, fs, name, state, **kwargs):
from dvc.dir_info import DirInfo
def _build_tree(path_info, fs, name, state, **kwargs):
from . import HashFile
from .tree import Tree
dir_info = DirInfo()
tree = Tree(None, None, None)
for fi, hi in _iter_hashes(path_info, fs, name, state, **kwargs):
if DvcIgnore.DVCIGNORE_FILE == fi.name:
raise DvcIgnoreInCollectedDirError(fi.parent)
obj = HashFile(fi, fs, hi)
# NOTE: this is lossy transformation:
# "hey\there" -> "hey/there"
# "hey/there" -> "hey/there"
@@ -76,31 +95,44 @@ def _collect_dir(path_info, fs, name, state, **kwargs):
#
# Yes, this is a BUG, as long as we permit "/" in
# filenames on Windows and "\" on Unix
dir_info.add(fi.relative_to(path_info).parts, hi)
tree.add(fi.relative_to(path_info).parts, obj)
return dir_info
tree.digest()
return tree
def get_dir_hash(path_info, fs, name, odb, state, **kwargs):
from . import Tree
def _get_tree_obj(path_info, fs, name, odb, state, **kwargs):
from .tree import Tree
value = fs.info(path_info).get(name)
if value:
hash_info = HashInfo(name, value)
try:
Tree.load(odb, hash_info)
return hash_info
return Tree.load(odb, hash_info)
except FileNotFoundError:
pass
dir_info = _collect_dir(path_info, fs, name, state, **kwargs)
hash_info = Tree.save_dir_info(fs.repo.odb.local, dir_info)
hash_info.size = dir_info.size
hash_info.dir_info = dir_info
return hash_info
tree = _build_tree(path_info, fs, name, state, **kwargs)
odb.add(tree.path_info, tree.fs, tree.hash_info)
if name != "md5":
# NOTE: used only for external outputs. Initial reasoning was to be
# able to validate .dir files right in the workspace (e.g. check s3
# etag), but could be dropped for manual validation with regular md5,
# that would be universal for all clouds.
raw = odb.get(tree.hash_info)
hash_info = get_file_hash(raw.path_info, raw.fs, name, state)
tree.hash_info.name = hash_info.name
tree.hash_info.value = hash_info.value
if not tree.hash_info.value.endswith(".dir"):
tree.hash_info.value += ".dir"
odb.add(tree.path_info, tree.fs, tree.hash_info)
return tree
def get_hash(path_info, fs, name, odb, **kwargs):
def stage(odb, path_info, fs, name, **kwargs):
assert path_info and (
isinstance(path_info, str) or path_info.scheme == fs.scheme
)
@@ -125,28 +157,31 @@ def get_hash(path_info, fs, name, odb, **kwargs):
hash_info = None
if hash_info:
if hash_info.isdir:
from . import Tree
from . import load
from .tree import Tree
# NOTE: loading the fs will restore hash_info.dir_info
Tree.load(odb, hash_info)
assert hash_info.name == name
return hash_info
obj = load(odb, hash_info)
if isinstance(obj, Tree):
obj.hash_info.nfiles = len(obj)
for key, entry in obj:
entry.fs = fs
entry.path_info = path_info.joinpath(*key)
else:
obj.fs = fs
obj.path_info = path_info
assert obj.hash_info.name == name
obj.hash_info.size = hash_info.size
return obj
if fs.isdir(path_info):
hash_info = get_dir_hash(path_info, fs, name, odb, state, **kwargs)
obj = _get_tree_obj(path_info, fs, name, odb, state, **kwargs)
else:
hash_info = get_file_hash(path_info, fs, name)
from . import HashFile
if hash_info and fs.exists(path_info):
state.save(path_info, fs, hash_info)
hash_info = get_file_hash(path_info, fs, name, state)
obj = HashFile(path_info, fs, hash_info)
return hash_info
if obj.hash_info and fs.exists(path_info):
state.save(path_info, fs, obj.hash_info)
def stage(odb, path_info, fs, **kwargs):
from . import File, Tree
if fs.isdir(path_info):
return Tree.stage(odb, path_info, fs, **kwargs)
return File.stage(odb, path_info, fs, **kwargs)
return obj
dvc/objects/tree.py
+210
View File
@@ -0,0 +1,210 @@
import json
import logging
import posixpath
from funcy import cached_property
from . import HashFile, ObjectFormatError
from .stage import get_file_hash
logger = logging.getLogger(__name__)
class Tree(HashFile):
PARAM_RELPATH = "relpath"
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._dict = {}
@cached_property
def trie(self):
from pygtrie import Trie
return Trie(self._dict)
@property
def size(self):
try:
return sum(obj.size for _, obj in self)
except TypeError:
return None
def add(self, key, obj):
self.__dict__.pop("trie", None)
self._dict[key] = obj
def digest(self):
from dvc.fs.memory import MemoryFileSystem
from dvc.path_info import PathInfo
from dvc.utils import tmp_fname
memfs = MemoryFileSystem(None, {})
path_info = PathInfo(tmp_fname(""))
with memfs.open(path_info, "wb") as fobj:
fobj.write(self.as_bytes())
self.fs = memfs
self.path_info = path_info
self.hash_info = get_file_hash(path_info, memfs, "md5")
self.hash_info.value += ".dir"
self.hash_info.size = self.size
self.hash_info.nfiles = len(self)
def __len__(self):
return len(self._dict)
def __iter__(self):
yield from self._dict.items()
def as_dict(self):
return self._dict.copy()
def as_list(self):
from operator import itemgetter
# Sorting the list by path to ensure reproducibility
return sorted(
(
{
# NOTE: not using hash_info.to_dict() because we don't want
# size/nfiles fields at this point.
obj.hash_info.name: obj.hash_info.value,
self.PARAM_RELPATH: posixpath.sep.join(parts),
}
for parts, obj in self._dict.items() # noqa: B301
),
key=itemgetter(self.PARAM_RELPATH),
)
def as_bytes(self):
return json.dumps(self.as_list(), sort_keys=True).encode("utf-8")
@classmethod
def from_list(cls, lst):
from dvc.hash_info import HashInfo
tree = cls(None, None, None)
for _entry in lst:
entry = _entry.copy()
relpath = entry.pop(cls.PARAM_RELPATH)
parts = tuple(relpath.split(posixpath.sep))
hash_info = HashInfo.from_dict(entry)
obj = HashFile(None, None, hash_info)
tree.add(parts, obj)
return tree
@classmethod
def load(cls, odb, hash_info):
obj = odb.get(hash_info)
try:
with obj.fs.open(obj.path_info, "r") as fobj:
raw = json.load(fobj)
except ValueError as exc:
raise ObjectFormatError(f"{obj} is corrupted") from exc
if not isinstance(raw, list):
logger.error(
"dir cache file format error '%s' [skipping the file]",
obj.path_info,
)
raise ObjectFormatError(f"{obj} is corrupted")
tree = cls.from_list(raw)
tree.path_info = obj.path_info
tree.fs = obj.fs
tree.hash_info = hash_info
return tree
def filter(self, odb, prefix):
obj = self._dict.get(prefix)
if obj:
return obj
depth = len(prefix)
tree = Tree(None, None, None)
try:
for key, obj in self.trie.items(prefix):
tree.add(key[depth:], obj)
except KeyError:
return None
tree.digest()
odb.add(tree.path_info, tree.fs, tree.hash_info)
return tree
def _get_dir_size(odb, tree):
try:
return sum(
odb.fs.getsize(odb.hash_to_path_info(obj.hash_info.value))
for _, obj in tree
)
except FileNotFoundError:
return None
def _diff(ancestor, other, allow_removed=False):
from dictdiffer import diff
from dvc.exceptions import MergeError
allowed = ["add"]
if allow_removed:
allowed.append("remove")
result = list(diff(ancestor, other))
for typ, _, _ in result:
if typ not in allowed:
raise MergeError(
"unable to auto-merge directories with diff that contains "
f"'{typ}'ed files"
)
return result
def _merge(ancestor, our, their):
import copy
from dictdiffer import patch
our_diff = _diff(ancestor, our)
if not our_diff:
return copy.deepcopy(their)
their_diff = _diff(ancestor, their)
if not their_diff:
return copy.deepcopy(our)
# make sure there are no conflicting files
_diff(our, their, allow_removed=True)
return patch(our_diff + their_diff, ancestor)
def merge(odb, ancestor_info, our_info, their_info):
from . import load
assert our_info
assert their_info
if ancestor_info:
ancestor = load(odb, ancestor_info)
else:
ancestor = Tree(None, None, None)
our = load(odb, our_info)
their = load(odb, their_info)
merged_dict = _merge(ancestor.as_dict(), our.as_dict(), their.as_dict(),)
merged = Tree(None, None, None)
for key, hi in merged_dict.items():
merged.add(key, hi)
merged.digest()
odb.add(merged.path_info, merged.fs, merged.hash_info)
hash_info = merged.hash_info
hash_info.size = _get_dir_size(odb, merged)
return hash_info
dvc/output/base.py
+25 -27
View File
@@ -18,7 +18,6 @@ from dvc.exceptions import (
)
from dvc.hash_info import HashInfo
from dvc.objects.db import NamedCache
from dvc.objects.stage import get_hash
from dvc.objects.stage import stage as ostage
from ..fs.base import BaseFileSystem
@@ -183,23 +182,21 @@ class BaseOutput:
def odb(self):
return getattr(self.repo.odb, self.scheme)
@property
def dir_cache(self):
return self.hash_info.dir_info
@property
def cache_path(self):
return self.odb.hash_to_path_info(self.hash_info.value).url
def get_hash(self):
if not self.use_cache:
return get_hash(
return ostage(
self.repo.odb.local,
self.path_info,
self.fs,
self.fs.PARAM_CHECKSUM,
self.repo.odb.local,
)
return ostage(self.odb, self.path_info, self.fs).hash_info
).hash_info
return ostage(
self.odb, self.path_info, self.fs, self.odb.fs.PARAM_CHECKSUM
).hash_info
@property
def is_dir_checksum(self):
@@ -307,7 +304,9 @@ class BaseOutput:
logger.debug("Output '%s' didn't change. Skipping saving.", self)
return
self.obj = ostage(self.odb, self.path_info, self.fs)
self.obj = ostage(
self.odb, self.path_info, self.fs, self.odb.fs.PARAM_CHECKSUM
)
self.hash_info = self.obj.hash_info
self.isexec = self.isfile() and self.fs.isexec(self.path_info)
@@ -322,7 +321,12 @@ class BaseOutput:
assert self.hash_info
if self.use_cache:
obj = ostage(self.odb, filter_info or self.path_info, self.fs)
obj = ostage(
self.odb,
filter_info or self.path_info,
self.fs,
self.odb.fs.PARAM_CHECKSUM,
)
objects.save(self.odb, obj)
checkout(
filter_info or self.path_info,
@@ -460,8 +464,6 @@ class BaseOutput:
self.repo.scm.ignore(self.fspath)
def get_files_number(self, filter_info=None):
from funcy import ilen
if not self.use_cache or not self.hash_info:
return 0
@@ -471,13 +473,8 @@ class BaseOutput:
if not filter_info or filter_info == self.path_info:
return self.hash_info.nfiles or 0
if not self.hash_info.dir_info:
return 0
return ilen(
filter_info.isin_or_eq(self.path_info.joinpath(*relpath))
for relpath, _ in self.hash_info.dir_info.items()
)
obj = self.get_obj(filter_info=filter_info)
return len(obj) if obj else 0
def unprotect(self):
if self.exists:
@@ -498,12 +495,11 @@ class BaseOutput:
)
try:
objects.load(self.odb, self.hash_info)
assert self.hash_info.dir_info
self.obj = objects.load(self.odb, self.hash_info)
except (objects.ObjectFormatError, FileNotFoundError):
self.hash_info.dir_info = None
self.obj = None
return self.dir_cache
return self.obj
def collect_used_dir_cache(
self, remote=None, force=False, jobs=None, filter_info=None
@@ -551,14 +547,14 @@ class BaseOutput:
path = str(self.path_info)
filter_path = str(filter_info) if filter_info else None
for entry_key, entry_hash_info in self.dir_cache.items():
for entry_key, entry_obj in self.obj:
entry_path = os.path.join(path, *entry_key)
if (
not filter_path
or entry_path == filter_path
or entry_path.startswith(filter_path + os.sep)
):
cache.add(self.scheme, entry_hash_info.value, entry_path)
cache.add(self.scheme, entry_obj.hash_info.value, entry_path)
return cache
@@ -651,6 +647,8 @@ class BaseOutput:
)
def merge(self, ancestor, other):
from dvc.objects.tree import merge
assert other
if ancestor:
@@ -662,6 +660,6 @@ class BaseOutput:
self._check_can_merge(self)
self._check_can_merge(other)
self.hash_info = objects.merge(
self.hash_info = merge(
self.odb, ancestor_info, self.hash_info, other.hash_info
)
dvc/repo/__init__.py
+6
View File
@@ -463,6 +463,12 @@ class Repo:
path_parts = os.path.normpath(path).split(os.path.sep)
return self.DVC_DIR in path_parts
@cached_property
def dvcfs(self):
from dvc.fs.dvc import DvcFileSystem
return DvcFileSystem(self)
@cached_property
def repo_fs(self):
from dvc.fs.repo import RepoFileSystem
dvc/repo/diff.py
+1 -1
View File
@@ -130,7 +130,7 @@ def _output_paths(repo, repo_fs, targets):
def _to_checksum(output):
if on_working_fs:
return ostage(
repo.odb.local, output.path_info, repo.odb.local.fs,
repo.odb.local, output.path_info, repo.odb.local.fs, "md5",
).hash_info.value
return output.hash_info.value
dvc/repo/fetch.py
+1
View File
@@ -115,6 +115,7 @@ def _fetch_external(self, repo_url, repo_rev, files, jobs):
odb,
path_info,
repo.repo_fs,
"md5",
jobs=jobs,
follow_subrepos=False,
)
tests/func/remote/test_index.py
+2 -2
View File
@@ -32,7 +32,7 @@ def remote(tmp_dir, dvc, tmp_path_factory, mocker):
def test_indexed_on_status(tmp_dir, dvc, tmp_path_factory, remote):
foo = tmp_dir.dvc_gen({"foo": "foo content"})[0].outs[0]
bar = tmp_dir.dvc_gen({"bar": {"baz": "baz content"}})[0].outs[0]
baz_hash = bar.dir_cache.trie.get(("baz",))
baz_hash = bar.obj.trie.get(("baz",)).hash_info
dvc.push()
with remote.index:
remote.index.clear()
@@ -49,7 +49,7 @@ def test_indexed_on_status(tmp_dir, dvc, tmp_path_factory, remote):
def test_indexed_on_push(tmp_dir, dvc, tmp_path_factory, remote):
foo = tmp_dir.dvc_gen({"foo": "foo content"})[0].outs[0]
bar = tmp_dir.dvc_gen({"bar": {"baz": "baz content"}})[0].outs[0]
baz_hash = bar.dir_cache.trie.get(("baz",))
baz_hash = bar.obj.trie.get(("baz",)).hash_info
dvc.push()
with remote.index:
tests/func/test_add.py
+6 -5
View File
@@ -109,9 +109,10 @@ def test_add_directory(tmp_dir, dvc):
hash_info = stage.outs[0].hash_info
dir_info = load(dvc.odb.local, hash_info).hash_info.dir_info
for path, _ in dir_info.trie.items():
assert "\\" not in path
obj = load(dvc.odb.local, hash_info)
for key, _ in obj:
for part in key:
assert "\\" not in part
class TestAddDirectoryRecursive(TestDvc):
@@ -516,7 +517,7 @@ class TestAddCommit(TestDvc):
def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc):
tmp_dir.gen({"data/data": "foo"})
get_dir_hash_counter = mocker.spy(dvc_module.objects.stage, "get_dir_hash")
counter = mocker.spy(dvc_module.objects.stage, "_get_tree_obj")
ret = main(["add", "data"])
assert ret == 0
@@ -525,7 +526,7 @@ def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc):
ret = main(["status"])
assert ret == 0
assert get_dir_hash_counter.mock.call_count == 1
assert counter.mock.call_count == 1
class TestShouldPlaceStageInDataDirIfRepositoryBelowSymlink(TestDvc):
tests/func/test_checkout.py
+3 -4
View File
@@ -100,11 +100,10 @@ class TestCheckoutCorruptedCacheDir(TestDvc):
# NOTE: modifying cache file for one of the files inside the directory
# to check if dvc will detect that the cache is corrupted.
_, entry_hash = next(
load(self.dvc.odb.local, out.hash_info).hash_info.dir_info.items()
)
obj = load(self.dvc.odb.local, out.hash_info)
_, entry_obj = list(obj)[0]
cache = os.fspath(
self.dvc.odb.local.hash_to_path_info(entry_hash.value)
self.dvc.odb.local.hash_to_path_info(entry_obj.hash_info.value)
)
os.chmod(cache, 0o644)
tests/func/test_data_cloud.py
+1 -1
View File
@@ -287,7 +287,7 @@ def test_verify_hashes(
remove("dir")
remove(dvc.odb.local.cache_dir)
hash_spy = mocker.spy(dvc_module.objects.stage, "get_file_hash")
hash_spy = mocker.spy(dvc_module.objects.stage, "_get_file_hash")
dvc.pull()
assert hash_spy.call_count == 0
tests/func/test_external_repo.py
+1
View File
@@ -205,6 +205,7 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
repo.odb.local,
PathInfo(repo.root_dir) / "dir",
repo.repo_fs,
"md5",
follow_subrepos=False,
)
save(repo.odb.local, obj)
tests/func/test_remote.py
+12 -8
View File
@@ -7,7 +7,6 @@ import pytest
from mock import patch
from dvc.config import Config
from dvc.dir_info import DirInfo
from dvc.exceptions import DownloadError, RemoteCacheRequiredError, UploadError
from dvc.fs.local import LocalFileSystem
from dvc.main import main
@@ -147,26 +146,31 @@ class TestRemoteShouldHandleUppercaseRemoteName(TestDvc):
def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
from dvc.objects.stage import stage
from dvc.objects.tree import Tree
tmp_dir.gen({"data": {"1": "1 content", "2": "2 content"}})
path_info = PathInfo("data")
dir_info = DirInfo.from_list(
tree = Tree.from_list(
[{"relpath": "1", "md5": "1"}, {"relpath": "2", "md5": "2"}]
)
with patch(
"dvc.objects.stage._collect_dir", return_value=dir_info,
"dvc.objects.stage._get_tree_obj", return_value=tree,
):
hash1 = stage(dvc.odb.local, path_info, dvc.odb.local.fs).hash_info
hash1 = stage(
dvc.odb.local, path_info, dvc.odb.local.fs, "md5"
).hash_info
dir_info = DirInfo.from_list(
tree = Tree.from_list(
[{"md5": "1", "relpath": "1"}, {"md5": "2", "relpath": "2"}]
)
with patch(
"dvc.objects.stage._collect_dir", return_value=dir_info,
"dvc.objects.stage._get_tree_obj", return_value=tree,
):
hash2 = stage(dvc.odb.local, path_info, dvc.odb.local.fs).hash_info
hash2 = stage(
dvc.odb.local, path_info, dvc.odb.local.fs, "md5"
).hash_info
assert hash1 == hash2
@@ -278,7 +282,7 @@ def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker, local_remote):
remote = dvc.cloud.get_remote("upstream")
foo_path = remote.odb.hash_to_path_info(foo.hash_info.value)
bar_path = remote.odb.hash_to_path_info(
foo.hash_info.dir_info.trie[("bar",)].value
foo.obj.trie[("bar",)].hash_info.value
)
paths = [args[1] for args, _ in mocked_upload.call_args_list]
assert paths.index(foo_path) > paths.index(bar_path)
tests/unit/fs/test_dvc.py
+6 -6
View File
@@ -229,15 +229,15 @@ def test_get_hash_granular(tmp_dir, dvc):
fs = DvcFileSystem(dvc)
subdir = PathInfo(tmp_dir) / "dir" / "subdir"
assert fs.info(subdir).get("md5") is None
assert stage(dvc.odb.local, subdir, fs).hash_info == HashInfo(
assert stage(dvc.odb.local, subdir, fs, "md5").hash_info == HashInfo(
"md5", "af314506f1622d107e0ed3f14ec1a3b5.dir",
)
assert (
fs.info(subdir / "data")["md5"] == "8d777f385d3dfec8815d20f7496026dc"
)
assert stage(dvc.odb.local, subdir / "data", fs).hash_info == HashInfo(
"md5", "8d777f385d3dfec8815d20f7496026dc",
)
assert stage(
dvc.odb.local, subdir / "data", fs, "md5"
).hash_info == HashInfo("md5", "8d777f385d3dfec8815d20f7496026dc",)
def test_get_hash_dirty_file(tmp_dir, dvc):
@@ -248,7 +248,7 @@ def test_get_hash_dirty_file(tmp_dir, dvc):
expected = "8c7dd922ad47494fc02c388e12c00eac"
assert fs.info(PathInfo(tmp_dir) / "file").get("md5") == expected
assert stage(
dvc.odb.local, PathInfo(tmp_dir) / "file", fs
dvc.odb.local, PathInfo(tmp_dir) / "file", fs, "md5"
).hash_info == HashInfo("md5", expected)
@@ -260,5 +260,5 @@ def test_get_hash_dirty_dir(tmp_dir, dvc):
expected = "5ea40360f5b4ec688df672a4db9c17d1.dir"
assert fs.info(PathInfo(tmp_dir) / "dir").get("md5") == expected
assert stage(
dvc.odb.local, PathInfo(tmp_dir) / "dir", fs
dvc.odb.local, PathInfo(tmp_dir) / "dir", fs, "md5"
).hash_info == HashInfo("md5", expected)
tests/unit/fs/test_repo.py
+20 -12
View File
@@ -483,7 +483,7 @@ def test_get_hash_cached_file(tmp_dir, dvc, mocker):
expected = "acbd18db4cc2f85cedef654fccc4a4d8"
assert fs.info(PathInfo(tmp_dir) / "foo").get("md5") is None
assert stage(
dvc.odb.local, PathInfo(tmp_dir) / "foo", fs
dvc.odb.local, PathInfo(tmp_dir) / "foo", fs, "md5",
).hash_info == HashInfo("md5", expected,)
(tmp_dir / "foo").unlink()
assert fs.info(PathInfo(tmp_dir) / "foo")["md5"] == expected
@@ -497,13 +497,13 @@ def test_get_hash_cached_dir(tmp_dir, dvc, mocker):
expected = "8761c4e9acad696bee718615e23e22db.dir"
assert fs.info(PathInfo(tmp_dir) / "dir").get("md5") is None
assert stage(
dvc.odb.local, PathInfo(tmp_dir) / "dir", fs
dvc.odb.local, PathInfo(tmp_dir) / "dir", fs, "md5",
).hash_info == HashInfo("md5", "8761c4e9acad696bee718615e23e22db.dir",)
shutil.rmtree(tmp_dir / "dir")
assert fs.info(PathInfo(tmp_dir) / "dir")["md5"] == expected
assert stage(
dvc.odb.local, PathInfo(tmp_dir) / "dir", fs
dvc.odb.local, PathInfo(tmp_dir) / "dir", fs, "md5",
).hash_info == HashInfo("md5", "8761c4e9acad696bee718615e23e22db.dir",)
@@ -514,13 +514,13 @@ def test_get_hash_cached_granular(tmp_dir, dvc, mocker):
fs = RepoFileSystem(dvc)
subdir = PathInfo(tmp_dir) / "dir" / "subdir"
assert fs.info(subdir).get("md5") is None
assert stage(dvc.odb.local, subdir, fs).hash_info == HashInfo(
assert stage(dvc.odb.local, subdir, fs, "md5").hash_info == HashInfo(
"md5", "af314506f1622d107e0ed3f14ec1a3b5.dir",
)
assert fs.info(subdir / "data").get("md5") is None
assert stage(dvc.odb.local, subdir / "data", fs).hash_info == HashInfo(
"md5", "8d777f385d3dfec8815d20f7496026dc",
)
assert stage(
dvc.odb.local, subdir / "data", fs, "md5"
).hash_info == HashInfo("md5", "8d777f385d3dfec8815d20f7496026dc",)
(tmp_dir / "dir" / "subdir" / "data").unlink()
assert (
fs.info(subdir / "data")["md5"] == "8d777f385d3dfec8815d20f7496026dc"
@@ -540,7 +540,9 @@ def test_get_hash_mixed_dir(tmp_dir, scm, dvc):
tmp_dir.scm.commit("add dir")
fs = RepoFileSystem(dvc)
actual = stage(dvc.odb.local, PathInfo(tmp_dir) / "dir", fs).hash_info
actual = stage(
dvc.odb.local, PathInfo(tmp_dir) / "dir", fs, "md5"
).hash_info
expected = HashInfo("md5", "e1d9e8eae5374860ae025ec84cfd85c7.dir")
assert actual == expected
@@ -551,7 +553,9 @@ def test_get_hash_dirty_file(tmp_dir, dvc):
fs = RepoFileSystem(dvc)
assert fs.info(PathInfo(tmp_dir) / "file").get("md5") is None
actual = stage(dvc.odb.local, PathInfo(tmp_dir) / "file", fs).hash_info
actual = stage(
dvc.odb.local, PathInfo(tmp_dir) / "file", fs, "md5"
).hash_info
expected = HashInfo("md5", "437b930db84b8079c2dd804a71936b5f")
assert actual == expected
@@ -560,7 +564,9 @@ def test_get_hash_dirty_file(tmp_dir, dvc):
fs.info(PathInfo(tmp_dir) / "file")["md5"]
== "8c7dd922ad47494fc02c388e12c00eac"
)
actual = stage(dvc.odb.local, PathInfo(tmp_dir) / "file", fs).hash_info
actual = stage(
dvc.odb.local, PathInfo(tmp_dir) / "file", fs, "md5"
).hash_info
expected = HashInfo("md5", "8c7dd922ad47494fc02c388e12c00eac")
assert actual == expected
@@ -570,10 +576,12 @@ def test_get_hash_dirty_dir(tmp_dir, dvc):
(tmp_dir / "dir" / "baz").write_text("baz")
fs = RepoFileSystem(dvc)
actual = stage(dvc.odb.local, PathInfo(tmp_dir) / "dir", fs).hash_info
actual = stage(
dvc.odb.local, PathInfo(tmp_dir) / "dir", fs, "md5"
).hash_info
expected = HashInfo("md5", "ba75a2162ca9c29acecb7957105a0bc2.dir")
assert actual == expected
assert actual.dir_info.nfiles == 3
assert actual.nfiles == 3
@pytest.mark.parametrize("traverse_subrepos", [True, False])
tests/unit/objects/__init__.py
View File
tests/unit/test_dir_info.py → tests/unit/objects/test_tree.py
+33 -26
View File
@@ -2,8 +2,9 @@ from operator import itemgetter
import pytest
from dvc.dir_info import DirInfo, _merge
from dvc.hash_info import HashInfo
from dvc.objects import HashFile
from dvc.objects.tree import Tree, _merge
@pytest.mark.parametrize(
@@ -18,10 +19,10 @@ from dvc.hash_info import HashInfo
{"md5": "456", "relpath": "bar"},
],
{
("zzz",): HashInfo("md5", "def"),
("foo",): HashInfo("md5", "123"),
("bar",): HashInfo("md5", "456"),
("aaa",): HashInfo("md5", "abc"),
("zzz",): HashFile(None, None, HashInfo("md5", "def")),
("foo",): HashFile(None, None, HashInfo("md5", "123")),
("bar",): HashFile(None, None, HashInfo("md5", "456")),
("aaa",): HashFile(None, None, HashInfo("md5", "abc")),
},
),
(
@@ -37,23 +38,29 @@ from dvc.hash_info import HashInfo
{"md5": "pqr", "relpath": "dir/subdir/a"},
],
{
("dir", "b"): HashInfo("md5", "123"),
("dir", "z"): HashInfo("md5", "456"),
("dir", "a"): HashInfo("md5", "789"),
("b",): HashInfo("md5", "abc"),
("a",): HashInfo("md5", "def"),
("z",): HashInfo("md5", "ghi"),
("dir", "subdir", "b"): HashInfo("md5", "jkl"),
("dir", "subdir", "z"): HashInfo("md5", "mno"),
("dir", "subdir", "a"): HashInfo("md5", "pqr"),
("dir", "b"): HashFile(None, None, HashInfo("md5", "123")),
("dir", "z"): HashFile(None, None, HashInfo("md5", "456")),
("dir", "a"): HashFile(None, None, HashInfo("md5", "789")),
("b",): HashFile(None, None, HashInfo("md5", "abc")),
("a",): HashFile(None, None, HashInfo("md5", "def")),
("z",): HashFile(None, None, HashInfo("md5", "ghi")),
("dir", "subdir", "b"): HashFile(
None, None, HashInfo("md5", "jkl")
),
("dir", "subdir", "z"): HashFile(
None, None, HashInfo("md5", "mno")
),
("dir", "subdir", "a"): HashFile(
None, None, HashInfo("md5", "pqr")
),
},
),
],
)
def test_list(lst, trie_dict):
dir_info = DirInfo.from_list(lst)
assert dir_info._dict == trie_dict
assert dir_info.to_list() == sorted(lst, key=itemgetter("relpath"))
tree = Tree.from_list(lst)
assert tree._dict == trie_dict
assert tree.as_list() == sorted(lst, key=itemgetter("relpath"))
@pytest.mark.parametrize(
@@ -81,9 +88,9 @@ def test_list(lst, trie_dict):
],
)
def test_size(trie_dict, size):
dir_info = DirInfo()
dir_info._dict = trie_dict
assert dir_info.size == size
tree = Tree(None, None, None)
tree._dict = trie_dict
assert tree.size == size
@pytest.mark.parametrize(
@@ -111,9 +118,9 @@ def test_size(trie_dict, size):
],
)
def test_nfiles(trie_dict, nfiles):
dir_info = DirInfo()
dir_info._dict = trie_dict
assert dir_info.nfiles == nfiles
tree = Tree(None, None, None)
tree._dict = trie_dict
assert len(tree) == nfiles
@pytest.mark.parametrize(
@@ -134,9 +141,9 @@ def test_nfiles(trie_dict, nfiles):
],
)
def test_items(trie_dict):
dir_info = DirInfo()
dir_info._dict = trie_dict
assert list(dir_info.items()) == list(trie_dict.items())
tree = Tree(None, None, None)
tree._dict = trie_dict
assert list(tree) == list(trie_dict.items())
@pytest.mark.parametrize(
tests/unit/output/test_local.py
+1 -12
View File
@@ -2,7 +2,6 @@ import os
from mock import patch
from dvc.dir_info import DirInfo
from dvc.hash_info import HashInfo
from dvc.output import LocalOutput
from dvc.stage import Stage
@@ -80,17 +79,7 @@ class TestGetFilesNumber(TestDvc):
def test_return_multiple_for_dir(self):
o = self._get_output()
o.hash_info = HashInfo(
"md5",
"12345678.dir",
nfiles=2,
dir_info=DirInfo.from_list(
[
{"relpath": "foo", "md5": "asdf"},
{"relpath": "bar", "md5": "qwe"},
]
),
)
o.hash_info = HashInfo("md5", "12345678.dir", nfiles=2,)
self.assertEqual(2, o.get_files_number())
@patch.object(LocalOutput, "is_dir_checksum", False)