Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

objects: extract size/nfiles to Meta (#6784)

Browse Source 
* stage: capture metadata

* tree: capture metadata

* objects: extract size/nfiles from HashInfo/HashFile/Tree to Meta
This commit is contained in:
Ruslan Kuprieiev
2021-10-12 03:06:18 +03:00
committed by GitHub
parent 5cbaa8d98e
commit 32467ada32
37 changed files with 284 additions and 251 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/dependency/param.py
+20
View File
@@ -1,3 +1,4 @@
import logging
import os
from collections import defaultdict
@@ -10,6 +11,8 @@ from dvc.utils.serialize import LOADERS, ParseError
from .base import Dependency
logger = logging.getLogger(__name__)
class MissingParamsError(DvcException):
pass
@@ -133,3 +136,20 @@ class ParamsDependency(Dependency):
)
return HashInfo(self.PARAM_PARAMS, info)
def save(self):
if not self.exists:
raise self.DoesNotExistError(self)
if not self.isfile and not self.isdir:
raise self.IsNotFileOrDirError(self)
if self.is_empty:
logger.warning(f"'{self}' is empty.")
self.ignore()
if self.metric or self.plot:
self.verify_metric()
self.hash_info = self.get_hash()
dvc/dependency/repo.py
+5 -3
View File
@@ -132,7 +132,7 @@ class RepoDependency(Dependency):
pass
try:
staging, staged_obj = stage(
staging, _, staged_obj = stage(
local_odb,
path_info,
repo.repo_fs,
@@ -149,7 +149,7 @@ class RepoDependency(Dependency):
used_obj_ids[staging].add(staged_obj.hash_info)
if isinstance(staged_obj, Tree):
used_obj_ids[staging].update(
entry.hash_info for _, entry in staged_obj
entry.hash_info for _, _, entry in staged_obj
)
return used_obj_ids, staged_obj
@@ -166,7 +166,9 @@ class RepoDependency(Dependency):
for hash_info in obj_ids:
if hash_info.isdir:
tree = Tree.load(odb, hash_info)
yield from (odb.get(entry.hash_info) for _, entry in tree)
yield from (
odb.get(entry.hash_info) for _, _, entry in tree
)
else:
yield odb.get(hash_info)
dvc/fs/dvc.py
+7 -7
View File
@@ -58,7 +58,7 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
out.get_dir_cache(remote=remote)
if out.obj is None:
raise FileNotFoundError
obj = out.obj.trie.get(key)
(_, obj) = out.obj.trie.get(key) or (None, None)
if obj:
return obj.hash_info
raise FileNotFoundError
@@ -154,7 +154,7 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
self._fetch_dir(out, **kwargs)
base = out.path_info.parts
for key, _ in out.obj: # noqa: B301
for key, _, _ in out.obj: # noqa: B301
trie[base + key] = None
def _walk(self, root, trie, topdown=True, **kwargs):
@@ -244,15 +244,15 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method
meta = self.metadata(path_info)
ret = {"type": "directory" if meta.isdir else "file"}
if meta.is_output and len(meta.outs) == 1 and meta.outs[0].hash_info:
hash_info = meta.outs[0].hash_info
ret["size"] = hash_info.size
ret[hash_info.name] = hash_info.value
out = meta.outs[0]
ret["size"] = out.meta.size
ret[out.hash_info.name] = out.hash_info.value
elif meta.part_of_output:
(out,) = meta.outs
key = path_info.relative_to(out.path_info).parts
obj = out.obj.trie.get(key)
(obj_meta, obj) = out.obj.trie.get(key) or (None, None)
if obj:
ret["size"] = obj.size
ret["size"] = obj_meta.size if obj_meta else 0
ret[obj.hash_info.name] = obj.hash_info.value
return ret
dvc/hash_info.py
+3 -16
View File
@@ -7,13 +7,8 @@ HASH_DIR_SUFFIX = ".dir"
@dataclass
class HashInfo:
PARAM_SIZE = "size"
PARAM_NFILES = "nfiles"
name: Optional[str]
value: Optional[str]
size: Optional[int] = field(default=None, compare=False)
nfiles: Optional[int] = field(default=None, compare=False)
obj_name: Optional[str] = field(default=None, compare=False)
def __bool__(self):
@@ -27,15 +22,11 @@ class HashInfo:
@classmethod
def from_dict(cls, d):
_d = d.copy() if d else {}
size = _d.pop(cls.PARAM_SIZE, None)
nfiles = _d.pop(cls.PARAM_NFILES, None)
if not _d:
if not d:
return cls(None, None)
((name, value),) = _d.items()
return cls(name, value, size=size, nfiles=nfiles)
((name, value),) = d.items()
return cls(name, value)
def to_dict(self):
ret = OrderedDict()
@@ -43,10 +34,6 @@ class HashInfo:
return ret
ret[self.name] = self.value
if self.size is not None:
ret[self.PARAM_SIZE] = self.size
if self.nfiles is not None:
ret[self.PARAM_NFILES] = self.nfiles
return ret
@property
dvc/objects/__init__.py
+1 -1
View File
@@ -14,7 +14,7 @@ logger = logging.getLogger(__name__)
def check(odb: "ObjectDB", obj: "HashFile", **kwargs):
if isinstance(obj, Tree):
for _, entry in obj:
for _, _, entry in obj:
odb.check(entry.hash_info, **kwargs)
odb.check(obj.hash_info, **kwargs)
dvc/objects/checkout.py
+1 -1
View File
@@ -166,7 +166,7 @@ def _diff(
):
old = None
try:
_, old = stage(
_, _, old = stage(
cache,
path_info,
fs,
dvc/objects/diff.py
+1 -1
View File
@@ -83,7 +83,7 @@ def diff(
if not obj:
return []
return [ROOT] + (
[key for key, _ in obj] if isinstance(obj, Tree) else []
[key for key, _, _ in obj] if isinstance(obj, Tree) else []
)
old_keys = set(_get_keys(old))
dvc/objects/file.py
+1 -5
View File
@@ -28,10 +28,6 @@ class HashFile:
self.hash_info = hash_info
self.name = name
@property
def size(self):
return self.hash_info.size
def __len__(self):
return 1
@@ -74,7 +70,7 @@ class HashFile:
def _check_hash(self, odb):
from .stage import get_file_hash
actual = get_file_hash(
_, actual = get_file_hash(
self.path_info, self.fs, self.hash_info.name, odb.state
)
dvc/objects/meta.py
+32
View File
@@ -0,0 +1,32 @@
from collections import OrderedDict
from dataclasses import dataclass, field
from typing import Optional
@dataclass
class Meta:
PARAM_SIZE = "size"
PARAM_NFILES = "nfiles"
size: Optional[int] = field(default=None)
nfiles: Optional[int] = field(default=None)
@classmethod
def from_dict(cls, d):
if not d:
return cls()
size = d.pop(cls.PARAM_SIZE, None)
nfiles = d.pop(cls.PARAM_NFILES, None)
return cls(size=size, nfiles=nfiles)
def to_dict(self):
ret = OrderedDict()
if self.size is not None:
ret[self.PARAM_SIZE] = self.size
if self.nfiles is not None:
ret[self.PARAM_NFILES] = self.nfiles
return ret
dvc/objects/stage.py
+37 -31
View File
@@ -14,6 +14,7 @@ from dvc.utils import file_md5
from .db.reference import ReferenceObjectDB
from .file import HashFile
from .meta import Meta
if TYPE_CHECKING:
from dvc.fs.base import BaseFileSystem
@@ -38,7 +39,8 @@ def _upload_file(path_info, fs, odb, upload_odb):
upload_odb.fs.upload(stream, tmp_info, desc=path_info.name, total=size)
odb.add(tmp_info, upload_odb.fs, stream.hash_info)
return path_info, odb.get(stream.hash_info)
meta = Meta(size=size)
return path_info, meta, odb.get(stream.hash_info)
def _get_file_hash(path_info, fs, name):
@@ -54,31 +56,31 @@ def _get_file_hash(path_info, fs, name):
else:
raise NotImplementedError
return HashInfo(name, hash_value, size=info["size"])
meta = Meta(size=info["size"])
hash_info = HashInfo(name, hash_value)
return meta, hash_info
def get_file_hash(path_info, fs, name, state=None):
if state:
hash_info = state.get( # pylint: disable=assignment-from-none
meta, hash_info = state.get( # pylint: disable=assignment-from-none
path_info, fs
)
if hash_info:
return hash_info
return meta, hash_info
hash_info = _get_file_hash(path_info, fs, name)
meta, hash_info = _get_file_hash(path_info, fs, name)
if state:
assert ".dir" not in hash_info.value
state.save(path_info, fs, hash_info)
return hash_info
return meta, hash_info
def _get_file_obj(
path_info, fs, name, odb=None, upload_odb=None, dry_run=False
):
def _stage_file(path_info, fs, name, odb=None, upload_odb=None, dry_run=False):
state = odb.state if odb else None
hash_info = get_file_hash(path_info, fs, name, state=state)
meta, hash_info = get_file_hash(path_info, fs, name, state=state)
if upload_odb and not dry_run:
assert odb and name == "md5"
return _upload_file(path_info, fs, odb, upload_odb)
@@ -88,7 +90,7 @@ def _get_file_obj(
else:
odb.add(path_info, fs, hash_info, move=False)
obj = odb.get(hash_info)
return path_info, obj
return path_info, meta, obj
def _build_objects(
@@ -111,7 +113,7 @@ def _build_objects(
) as pbar:
worker = pbar.wrap_fn(
partial(
_get_file_obj,
_stage_file,
fs=fs,
name=name,
**kwargs,
@@ -130,8 +132,9 @@ def _iter_objects(path_info, fs, name, **kwargs):
def _build_tree(path_info, fs, name, **kwargs):
from .tree import Tree
tree_meta = Meta(size=0, nfiles=0)
tree = Tree(None, None, None)
for file_info, obj in _iter_objects(path_info, fs, name, **kwargs):
for file_info, meta, obj in _iter_objects(path_info, fs, name, **kwargs):
if DvcIgnore.DVCIGNORE_FILE == file_info.name:
raise DvcIgnoreInCollectedDirError(file_info.parent)
@@ -143,11 +146,15 @@ def _build_tree(path_info, fs, name, **kwargs):
#
# Yes, this is a BUG, as long as we permit "/" in
# filenames on Windows and "\" on Unix
tree.add(file_info.relative_to(path_info).parts, obj)
return tree
tree.add(file_info.relative_to(path_info).parts, meta, obj)
tree_meta.size += meta.size
tree_meta.nfiles += 1
return tree_meta, tree
def _get_tree_obj(path_info, fs, fs_info, name, odb=None, **kwargs):
def _stage_tree(path_info, fs, fs_info, name, odb=None, **kwargs):
from .tree import Tree
value = fs_info.get(name)
@@ -158,18 +165,18 @@ def _get_tree_obj(path_info, fs, fs_info, name, odb=None, **kwargs):
# NOTE: loaded entries are naive objects with hash_infos but no
# path_info. For staging trees, obj.path_info should be relative
# to the staging src `path_info` and src fs
for key, entry in tree:
for key, _, entry in tree:
entry.fs = fs
entry.path_info = path_info.joinpath(*key)
return tree
return Meta(nfiles=len(tree)), tree
except FileNotFoundError:
pass
tree = _build_tree(path_info, fs, name, odb=odb, **kwargs)
meta, tree = _build_tree(path_info, fs, name, odb=odb, **kwargs)
state = odb.state if odb and odb.state else None
hash_info = None
if state:
hash_info = state.get( # pylint: disable=assignment-from-none
_, hash_info = state.get( # pylint: disable=assignment-from-none
path_info, fs
)
tree.digest(hash_info=hash_info)
@@ -181,7 +188,7 @@ def _get_tree_obj(path_info, fs, fs_info, name, odb=None, **kwargs):
tree.fs.remove(tree.path_info)
tree.fs = raw.fs
tree.path_info = raw.path_info
return tree
return meta, tree
_url_cache: Dict[str, str] = {}
@@ -223,7 +230,7 @@ def _load_from_state(odb, staging, path_info, fs, name):
from .tree import Tree
state = odb.state
hash_info = state.get(path_info, fs)
meta, hash_info = state.get(path_info, fs)
if hash_info:
for odb_ in (odb, staging):
if odb_.exists(hash_info):
@@ -231,16 +238,15 @@ def _load_from_state(odb, staging, path_info, fs, name):
obj = load(odb_, hash_info)
check(odb_, obj, check_hash=False)
if isinstance(obj, Tree):
obj.hash_info.nfiles = len(obj)
for key, entry in obj:
meta.nfiles = len(obj)
for key, _, entry in obj:
entry.fs = fs
entry.path_info = path_info.joinpath(*key)
else:
obj.fs = fs
obj.path_info = path_info
assert obj.hash_info.name == name
obj.hash_info.size = hash_info.size
return odb_, obj
return odb_, meta, obj
except (ObjectFormatError, FileNotFoundError):
pass
raise FileNotFoundError
@@ -255,7 +261,7 @@ def _stage_external_tree_info(odb, tree, name):
odb.add(tree.path_info, tree.fs, tree.hash_info)
raw = odb.get(tree.hash_info)
hash_info = get_file_hash(raw.path_info, raw.fs, name, state=odb.state)
_, hash_info = get_file_hash(raw.path_info, raw.fs, name, state=odb.state)
tree.path_info = raw.path_info
tree.fs = raw.fs
tree.hash_info.name = hash_info.name
@@ -273,7 +279,7 @@ def stage(
upload: bool = False,
dry_run: bool = False,
**kwargs,
) -> Tuple["ObjectDB", "HashFile"]:
) -> Tuple["ObjectDB", "Meta", "HashFile"]:
"""Stage (prepare) objects from the given path for addition to an ODB.
Returns at tuple of (staging_odb, object) where addition to the ODB can
@@ -298,7 +304,7 @@ def stage(
pass
if details["type"] == "directory":
obj = _get_tree_obj(
meta, obj = _stage_tree(
path_info,
fs,
details,
@@ -312,7 +318,7 @@ def stage(
if name != "md5":
obj = _stage_external_tree_info(odb, obj, name)
else:
_, obj = _get_file_obj(
_, meta, obj = _stage_file(
path_info,
fs,
name,
@@ -324,4 +330,4 @@ def stage(
if odb and odb.state and obj.hash_info:
odb.state.save(path_info, fs, obj.hash_info)
return staging, obj
return staging, meta, obj
dvc/objects/status.py
+2 -2
View File
@@ -56,7 +56,7 @@ def _indexed_dir_hashes(odb, index, dir_objs, name, cache_odb):
tree = Tree.load(cache_odb, HashInfo(name, dir_hash))
except FileNotFoundError:
continue
file_hashes = [entry.hash_info.value for _, entry in tree]
file_hashes = [entry.hash_info.value for _, _, entry in tree]
if dir_hash not in index:
logger.debug(
"Indexing new .dir '%s' with '%s' nested files",
@@ -102,7 +102,7 @@ def status(
tree = None
else:
tree = Tree.load(cache_odb, hash_info)
for _, entry in tree:
for _, _, entry in tree:
assert entry.hash_info and entry.hash_info.value
hash_infos[entry.hash_info.value] = entry.hash_info
if index:
dvc/objects/transfer.py
+2 -2
View File
@@ -74,7 +74,7 @@ def _do_transfer(
dir_obj = find_tree_by_obj_id([cache_odb, src], dir_hash)
assert dir_obj
entry_ids = {entry.hash_info for _, entry in dir_obj}
entry_ids = {entry.hash_info for _, _, entry in dir_obj}
bound_file_ids = all_file_ids & entry_ids
all_file_ids -= entry_ids
@@ -117,7 +117,7 @@ def _do_transfer(
# index successfully pushed dirs
if dest_index:
for dir_obj in succeeded_dir_objs:
file_hashes = {entry.hash_info.value for _, entry in dir_obj}
file_hashes = {entry.hash_info.value for _, _, entry in dir_obj}
logger.debug(
"Indexing pushed dir '%s' with '%s' nested files",
dir_obj.hash_info,
dvc/objects/tree.py
+22 -26
View File
@@ -12,6 +12,8 @@ from .stage import get_file_hash
if TYPE_CHECKING:
from dvc.hash_info import HashInfo
from .meta import Meta
logger = logging.getLogger(__name__)
@@ -20,7 +22,7 @@ class Tree(HashFile):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._dict: Dict[Tuple[str], "HashFile"] = {}
self._dict: Dict[Tuple[str], Tuple["Meta", "HashFile"]] = {}
@cached_property
def trie(self):
@@ -28,9 +30,9 @@ class Tree(HashFile):
return Trie(self._dict)
def add(self, key, obj):
def add(self, key, meta, obj):
self.__dict__.pop("trie", None)
self._dict[key] = obj
self._dict[key] = (meta, obj)
def digest(self, hash_info: Optional["HashInfo"] = None):
from dvc.fs.memory import MemoryFileSystem
@@ -46,20 +48,17 @@ class Tree(HashFile):
if hash_info:
self.hash_info = hash_info
else:
self.hash_info = get_file_hash(path_info, memfs, "md5")
_, self.hash_info = get_file_hash(path_info, memfs, "md5")
assert self.hash_info.value
self.hash_info.value += ".dir"
try:
self.hash_info.size = sum(obj.size for _, obj in self)
except TypeError:
self.hash_info.size = None
self.hash_info.nfiles = len(self)
def __len__(self):
return len(self._dict)
def __iter__(self):
yield from self._dict.items()
yield from (
(key, value[0], value[1]) for key, value in self._dict.items()
)
def as_dict(self):
return self._dict.copy()
@@ -76,7 +75,7 @@ class Tree(HashFile):
obj.hash_info.name: obj.hash_info.value,
self.PARAM_RELPATH: posixpath.sep.join(parts),
}
for parts, obj in self._dict.items() # noqa: B301
for parts, _, obj in self # noqa: B301
),
key=itemgetter(self.PARAM_RELPATH),
)
@@ -95,7 +94,7 @@ class Tree(HashFile):
parts = tuple(relpath.split(posixpath.sep))
hash_info = HashInfo.from_dict(entry)
obj = HashFile(None, None, hash_info)
tree.add(parts, obj)
tree.add(parts, None, obj)
return tree
@classmethod
@@ -118,7 +117,7 @@ class Tree(HashFile):
tree = cls.from_list(raw)
tree.path_info = obj.path_info
tree.fs = obj.fs
for _, entry_obj in tree:
for _, _, entry_obj in tree:
entry_obj.fs = obj.fs
tree.hash_info = hash_info
@@ -135,8 +134,8 @@ class Tree(HashFile):
"""
tree = Tree(self.path_info, self.fs, self.hash_info)
try:
for key, obj in self.trie.items(prefix):
tree.add(key, obj)
for key, (meta, obj) in self.trie.items(prefix):
tree.add(key, meta, obj)
except KeyError:
pass
return tree
@@ -146,26 +145,26 @@ class Tree(HashFile):
Returns None if no object exists at the specified prefix.
"""
obj = self._dict.get(prefix)
_, obj = self._dict.get(prefix) or (None, None)
if obj:
return obj
tree = Tree(None, None, None)
depth = len(prefix)
try:
for key, obj in self.trie.items(prefix):
tree.add(key[depth:], obj)
for key, (meta, obj) in self.trie.items(prefix):
tree.add(key[depth:], meta, obj)
except KeyError:
return None
tree.digest()
return tree
def _get_dir_size(odb, tree):
def du(odb, tree):
try:
return sum(
odb.fs.getsize(odb.hash_to_path_info(obj.hash_info.value))
for _, obj in tree
for _, _, obj in tree
)
except FileNotFoundError:
return None
@@ -226,11 +225,8 @@ def merge(odb, ancestor_info, our_info, their_info):
merged_dict = _merge(ancestor.as_dict(), our.as_dict(), their.as_dict())
merged = Tree(None, None, None)
for key, hi in merged_dict.items():
merged.add(key, hi)
for key, (meta, obj) in merged_dict.items():
merged.add(key, meta, obj)
merged.digest()
odb.add(merged.path_info, merged.fs, merged.hash_info)
hash_info = merged.hash_info
hash_info.size = _get_dir_size(odb, merged)
return hash_info
return merged
dvc/output.py
+34 -17
View File
@@ -1,7 +1,6 @@
import logging
import os
from collections import defaultdict
from copy import copy
from typing import TYPE_CHECKING, Dict, Optional, Set, Type
from urllib.parse import urlparse
@@ -26,6 +25,7 @@ from .hash_info import HashInfo
from .istextfile import istextfile
from .objects import Tree
from .objects.errors import ObjectFormatError
from .objects.meta import Meta
from .objects.stage import stage as ostage
from .objects.transfer import transfer as otransfer
from .scheme import Schemes
@@ -316,6 +316,7 @@ class Output:
# By resolved path, which contains actual location,
# should be absolute and don't contain remote:// refs.
self.stage = stage
self.meta = Meta.from_dict(info)
self.hash_info = HashInfo.from_dict(info)
self.use_cache = False if self.IS_DEPENDENCY else cache
self.metric = False if self.IS_DEPENDENCY else metric
@@ -415,7 +416,7 @@ class Output:
else:
odb = self.repo.odb.local
name = self.fs.PARAM_CHECKSUM
_, obj = ostage(
_, _, obj = ostage(
odb,
self.path_info,
self.fs,
@@ -537,7 +538,15 @@ class Output:
self.verify_metric()
if not self.use_cache:
self.hash_info = self.get_hash()
_, self.meta, obj = ostage(
self.repo.odb.local,
self.path_info,
self.fs,
self.fs.PARAM_CHECKSUM,
dvcignore=self.dvcignore,
dry_run=True,
)
self.hash_info = obj.hash_info
if not self.IS_DEPENDENCY:
logger.debug(
"Output '%s' doesn't use cache. Skipping saving.", self
@@ -550,7 +559,7 @@ class Output:
logger.debug("Output '%s' didn't change. Skipping saving.", self)
return
_, self.obj = ostage(
_, self.meta, self.obj = ostage(
self.odb,
self.path_info,
self.fs,
@@ -579,7 +588,7 @@ class Output:
if granular:
obj = self._commit_granular_dir(filter_info)
else:
staging, obj = ostage(
staging, _, obj = ostage(
self.odb,
filter_info or self.path_info,
self.fs,
@@ -606,7 +615,7 @@ class Output:
def _commit_granular_dir(self, filter_info):
prefix = filter_info.relative_to(self.path_info).parts
staging, save_obj = ostage(
staging, _, save_obj = ostage(
self.odb,
self.path_info,
self.fs,
@@ -618,14 +627,15 @@ class Output:
otransfer(
staging,
self.odb,
{save_obj.hash_info} | {entry.hash_info for _, entry in save_obj},
{save_obj.hash_info}
| {entry.hash_info for _, _, entry in save_obj},
shallow=True,
move=True,
)
return checkout_obj
def dumpd(self):
ret = copy(self.hash_info.to_dict())
ret = {**self.hash_info.to_dict(), **self.meta.to_dict()}
if self.is_in_repo:
path = self.path_info.relpath(self.stage.wdir).as_posix()
@@ -803,7 +813,7 @@ class Output:
upload = not (update and from_fs.isdir(from_info))
jobs = jobs or min((from_fs.jobs, odb.fs.jobs))
staging, obj = ostage(
staging, self.meta, obj = ostage(
odb,
from_info,
from_fs,
@@ -832,7 +842,7 @@ class Output:
return 1
if not filter_info or filter_info == self.path_info:
return self.hash_info.nfiles or 0
return self.meta.nfiles or 0
obj = self.get_obj(filter_info=filter_info)
return len(obj) if obj else 0
@@ -946,7 +956,7 @@ class Output:
obj.hash_info.obj_name = name
oids = {obj.hash_info}
if isinstance(obj, Tree):
for key, entry_obj in obj:
for key, _, entry_obj in obj:
entry_obj.hash_info.obj_name = self.fs.sep.join([name, *key])
oids.add(entry_obj.hash_info)
return oids
@@ -981,8 +991,8 @@ class Output:
ignored = [
self.fs.PARAM_CHECKSUM,
HashInfo.PARAM_SIZE,
HashInfo.PARAM_NFILES,
Meta.PARAM_SIZE,
Meta.PARAM_NFILES,
]
for opt in ignored:
@@ -1000,7 +1010,7 @@ class Output:
)
def merge(self, ancestor, other):
from dvc.objects.tree import merge
from dvc.objects.tree import du, merge
assert other
@@ -1013,9 +1023,16 @@ class Output:
self._check_can_merge(self)
self._check_can_merge(other)
self.hash_info = merge(
merged = merge(
self.odb, ancestor_info, self.hash_info, other.hash_info
)
self.odb.add(merged.path_info, merged.fs, merged.hash_info)
self.hash_info = merged.hash_info
self.meta = Meta(
size=du(self.odb, merged),
nfiles=len(merged),
)
@property
def fspath(self):
@@ -1040,8 +1057,8 @@ ARTIFACT_SCHEMA = {
Output.PARAM_PLOT: bool,
Output.PARAM_PERSIST: bool,
Output.PARAM_CHECKPOINT: bool,
HashInfo.PARAM_SIZE: int,
HashInfo.PARAM_NFILES: int,
Meta.PARAM_SIZE: int,
Meta.PARAM_NFILES: int,
Output.PARAM_ISEXEC: bool,
}
dvc/repo/diff.py
+2 -2
View File
@@ -137,7 +137,7 @@ def _output_paths(repo, targets):
)
if on_working_fs:
_, obj = ostage(
_, _, obj = ostage(
repo.odb.local,
output.path_info,
repo.odb.local.fs,
@@ -164,7 +164,7 @@ def _output_paths(repo, targets):
def _dir_output_paths(path_info, obj, targets=None):
for key, entry_obj in obj:
for key, _, entry_obj in obj:
fname = path_info.joinpath(*key)
if targets is None or any(
fname.isin_or_eq(target) for target in targets
dvc/schema.py
+3 -3
View File
@@ -3,7 +3,7 @@ from collections.abc import Mapping
from voluptuous import Any, Optional, Required, Schema
from dvc import dependency, output
from dvc.hash_info import HashInfo
from dvc.objects.meta import Meta
from dvc.output import CHECKSUMS_SCHEMA, Output
from dvc.parsing import DO_KWD, FOREACH_KWD, VARS_KWD
from dvc.parsing.versions import SCHEMA_KWD, lockfile_version_schema
@@ -26,8 +26,8 @@ SINGLE_STAGE_SCHEMA = {
DATA_SCHEMA = {
**CHECKSUMS_SCHEMA,
Required("path"): str,
HashInfo.PARAM_SIZE: int,
HashInfo.PARAM_NFILES: int,
Meta.PARAM_SIZE: int,
Meta.PARAM_NFILES: int,
Output.PARAM_ISEXEC: bool,
}
LOCK_FILE_STAGE_SCHEMA = {
dvc/stage/cache.py
+2 -4
View File
@@ -19,13 +19,11 @@ class RunCacheNotFoundError(DvcException):
def _get_cache_hash(cache, key=False):
from dvc.hash_info import HashInfo
from dvc.objects.meta import Meta
if key:
cache["outs"] = [out["path"] for out in cache.get("outs", [])]
return dict_sha256(
cache, exclude=[HashInfo.PARAM_SIZE, HashInfo.PARAM_NFILES]
)
return dict_sha256(cache, exclude=[Meta.PARAM_SIZE, Meta.PARAM_NFILES])
def _can_hash(stage):
dvc/stage/loader.py
+2
View File
@@ -7,6 +7,7 @@ from funcy import cached_property, get_in, lcat, once, project
from dvc import dependency, output
from dvc.hash_info import HashInfo
from dvc.objects.meta import Meta
from dvc.parsing import FOREACH_KWD, JOIN, DataResolver, EntryNotFound
from dvc.parsing.versions import LOCKFILE_VERSION
from dvc.path_info import PathInfo
@@ -69,6 +70,7 @@ class StageLoader(Mapping):
info = info.copy()
info.pop("path", None)
item.isexec = info.pop("isexec", None)
item.meta = Meta.from_dict(info)
item.hash_info = HashInfo.from_dict(info)
@classmethod
dvc/stage/serialize.py
+1
View File
@@ -164,6 +164,7 @@ def to_single_stage_lockfile(stage: "Stage") -> dict:
ret = [
(item.PARAM_PATH, item.def_path),
*item.hash_info.to_dict().items(),
*item.meta.to_dict().items(),
]
if item.isexec:
dvc/stage/utils.py
+3 -3
View File
@@ -5,8 +5,8 @@ from typing import TYPE_CHECKING, Union
from funcy import concat, first, lsplit, rpartial, without
from dvc.exceptions import InvalidArgumentError
from dvc.objects.meta import Meta
from ..hash_info import HashInfo
from .exceptions import (
MissingDataSource,
StageExternalOutputsError,
@@ -197,8 +197,8 @@ def compute_md5(stage):
Output.PARAM_PERSIST,
Output.PARAM_CHECKPOINT,
Output.PARAM_ISEXEC,
HashInfo.PARAM_SIZE,
HashInfo.PARAM_NFILES,
Meta.PARAM_SIZE,
Meta.PARAM_NFILES,
],
)
dvc/state.py
+7 -5
View File
@@ -38,7 +38,7 @@ class StateNoop(StateBase):
pass
def get(self, path_info, fs): # pylint: disable=unused-argument
return None
return None, None
def save_link(self, path_info, fs):
pass
@@ -103,22 +103,24 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes
HashInfo or None: hash for the specified path info or None if it
doesn't exist in the state database.
"""
from .objects.meta import Meta
if not isinstance(fs, LocalFileSystem):
return None
return None, None
try:
mtime, size = get_mtime_and_size(path_info, fs, self.dvcignore)
except FileNotFoundError:
return None
return None, None
inode = get_inode(path_info)
value = self.md5s.get(inode)
if not value or value[0] != mtime or value[1] != str(size):
return None
return None, None
return HashInfo("md5", value[2], size=size)
return Meta(size=size), HashInfo("md5", value[2])
def save_link(self, path_info, fs):
"""Adds the specified path to the list of links created by dvc. This
dvc/utils/stream.py
+1 -3
View File
@@ -47,6 +47,4 @@ class HashedStreamReader(io.IOBase):
@property
def hash_info(self):
return HashInfo(
self.PARAM_CHECKSUM, self.md5.hexdigest(), size=self.total_read
)
return HashInfo(self.PARAM_CHECKSUM, self.md5.hexdigest())
tests/func/objects/db/test_index.py
+3 -3
View File
@@ -16,7 +16,7 @@ def index(dvc, local_remote, mocker):
def test_indexed_on_status(tmp_dir, dvc, index):
foo = tmp_dir.dvc_gen({"foo": "foo content"})[0].outs[0]
bar = tmp_dir.dvc_gen({"bar": {"baz": "baz content"}})[0].outs[0]
baz_hash = bar.obj.trie.get(("baz",)).hash_info
baz_hash = bar.obj.trie.get(("baz",))[1].hash_info
clean_staging()
dvc.push()
index.clear()
@@ -30,7 +30,7 @@ def test_indexed_on_status(tmp_dir, dvc, index):
def test_indexed_on_push(tmp_dir, dvc, index):
foo = tmp_dir.dvc_gen({"foo": "foo content"})[0].outs[0]
bar = tmp_dir.dvc_gen({"bar": {"baz": "baz content"}})[0].outs[0]
baz_hash = bar.obj.trie.get(("baz",)).hash_info
baz_hash = bar.obj.trie.get(("baz",))[1].hash_info
clean_staging()
dvc.push()
@@ -60,7 +60,7 @@ def test_clear_on_download_err(tmp_dir, dvc, index, mocker):
out = tmp_dir.dvc_gen({"dir": {"foo": "foo content"}})[0].outs[0]
dvc.push()
for _, entry in out.obj:
for _, _, entry in out.obj:
remove(dvc.odb.local.get(entry.hash_info).path_info)
remove(out.path_info)
tests/func/test_add.py
+6 -5
View File
@@ -112,7 +112,7 @@ def test_add_directory(tmp_dir, dvc):
hash_info = stage.outs[0].hash_info
obj = load(dvc.odb.local, hash_info)
for key, _ in obj:
for key, _, _ in obj:
for part in key:
assert "\\" not in part
@@ -534,7 +534,7 @@ class TestAddCommit(TestDvc):
def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc):
tmp_dir.gen({"data/data": "foo"})
counter = mocker.spy(dvc_module.objects.stage, "_get_tree_obj")
counter = mocker.spy(dvc_module.objects.stage, "_stage_tree")
ret = main(["add", "data"])
assert ret == 0
assert counter.mock.call_count == 2
@@ -1026,8 +1026,9 @@ def test_add_to_remote(tmp_dir, dvc, local_cloud, local_remote):
assert len(stage.outs) == 1
hash_info = stage.outs[0].hash_info
meta = stage.outs[0].meta
assert local_remote.hash_to_path_info(hash_info.value).read_text() == "foo"
assert hash_info.size == len("foo")
assert meta.size == len("foo")
def test_add_to_remote_absolute(tmp_dir, make_tmp_dir, dvc, local_remote):
@@ -1071,8 +1072,8 @@ def test_add_to_cache_dir(tmp_dir, dvc, local_cloud):
(stage,) = dvc.add(str(local_cloud / "data"), out="data")
assert len(stage.deps) == 0
assert len(stage.outs) == 1
assert stage.outs[0].hash_info.size == len("foo") + len("bar")
assert stage.outs[0].hash_info.nfiles == 2
assert stage.outs[0].meta.size == len("foo") + len("bar")
assert stage.outs[0].meta.nfiles == 2
data = tmp_dir / "data"
assert data.read_text() == {"foo": "foo", "bar": "bar"}
tests/func/test_checkout.py
+1 -1
View File
@@ -101,7 +101,7 @@ class TestCheckoutCorruptedCacheDir(TestDvc):
# NOTE: modifying cache file for one of the files inside the directory
# to check if dvc will detect that the cache is corrupted.
obj = load(self.dvc.odb.local, out.hash_info)
_, entry_obj = list(obj)[0]
_, _, entry_obj = list(obj)[0]
cache = os.fspath(
self.dvc.odb.local.hash_to_path_info(entry_obj.hash_info.value)
)
tests/func/test_data_cloud.py
+1 -1
View File
@@ -58,7 +58,7 @@ def test_cloud(tmp_dir, dvc, remote): # pylint:disable=unused-argument
cache_dir = out_dir.cache_path
dir_hash = out_dir.hash_info
dir_hashes = {dir_hash} | {
entry_obj.hash_info for _, entry_obj in out_dir.obj
entry_obj.hash_info for _, _, entry_obj in out_dir.obj
}
def _check_status(status, **kwargs):
tests/func/test_external_repo.py
+1 -1
View File
@@ -203,7 +203,7 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir):
clean_staging()
makedirs(cache_dir)
staging, obj = stage(
staging, _, obj = stage(
repo.odb.local,
PathInfo(repo.root_dir) / "dir",
repo.repo_fs,
tests/func/test_import_url.py
+1 -1
View File
@@ -291,7 +291,7 @@ def test_import_url_to_remote_single_file(
hash_info = stage.outs[0].hash_info
assert local_remote.hash_to_path_info(hash_info.value).read_text() == "foo"
assert hash_info.size == len("foo")
assert stage.outs[0].meta.size == len("foo")
@pytest.mark.parametrize(
tests/func/test_lockfile.py
+2 -4
View File
@@ -167,14 +167,12 @@ def test_params_dump(tmp_dir, dvc, run_head):
def v1_repo_lock(tmp_dir, dvc):
"""Generates a repo having v1 format lockfile"""
size = 5 if os.name == "nt" else 4
hi = HashInfo(
name="md5", size=size, value="c157a79031e1c40f85931829bc5fc552"
)
hi = HashInfo(name="md5", value="c157a79031e1c40f85931829bc5fc552")
v1_lockdata = {
"foo": {"cmd": "echo foo"},
"bar": {
"cmd": "echo bar>bar.txt",
"outs": [{"path": "bar.txt", **hi.to_dict()}],
"outs": [{"path": "bar.txt", **hi.to_dict(), "size": size}],
},
}
dvc.run(cmd="echo foo", name="foo", no_exec=True)
tests/func/test_remote.py
+6 -6
View File
@@ -156,16 +156,16 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc):
[{"relpath": "1", "md5": "1"}, {"relpath": "2", "md5": "2"}]
)
tree.digest()
with patch("dvc.objects.stage._get_tree_obj", return_value=tree):
_, obj = stage(dvc.odb.local, path_info, dvc.odb.local.fs, "md5")
with patch("dvc.objects.stage._stage_tree", return_value=(None, tree)):
_, _, obj = stage(dvc.odb.local, path_info, dvc.odb.local.fs, "md5")
hash1 = obj.hash_info
tree = Tree.from_list(
[{"md5": "1", "relpath": "1"}, {"md5": "2", "relpath": "2"}]
)
tree.digest()
with patch("dvc.objects.stage._get_tree_obj", return_value=tree):
_, obj = stage(dvc.odb.local, path_info, dvc.odb.local.fs, "md5")
with patch("dvc.objects.stage._stage_tree", return_value=(None, tree)):
_, _, obj = stage(dvc.odb.local, path_info, dvc.odb.local.fs, "md5")
hash2 = obj.hash_info
assert hash1 == hash2
@@ -274,7 +274,7 @@ def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker, local_remote):
# foo .dir file should be uploaded after bar
odb = dvc.cloud.get_remote_odb("upstream")
foo_path = odb.hash_to_path_info(foo.hash_info.value)
bar_path = odb.hash_to_path_info(foo.obj.trie[("bar",)].hash_info.value)
bar_path = odb.hash_to_path_info(foo.obj.trie[("bar",)][1].hash_info.value)
paths = [args[1] for args, _ in mocked_upload.call_args_list]
assert paths.index(foo_path) > paths.index(bar_path)
@@ -424,7 +424,7 @@ def test_push_incomplete_dir(tmp_dir, dvc, mocker, local_remote):
odb = dvc.odb.local
out = stage.outs[0]
file_objs = [entry_obj for _, entry_obj in out.obj]
file_objs = [entry_obj for _, _, entry_obj in out.obj]
# remove one of the cache files for directory
remove(odb.hash_to_path_info(file_objs[0].hash_info.value))
tests/func/test_repo_index.py
-3
View File
@@ -272,7 +272,6 @@ def test_used_objs(tmp_dir, scm, dvc, run_copy, rev):
HashInfo(
name="md5",
value="acbd18db4cc2f85cedef654fccc4a4d8",
size=3,
obj_name="bar",
),
HashInfo(
@@ -283,8 +282,6 @@ def test_used_objs(tmp_dir, scm, dvc, run_copy, rev):
HashInfo(
name="md5",
value="d28c9e28591aeb7e303dc6772ffa6f6b.dir",
size=4,
nfiles=1,
obj_name="dir",
),
]
tests/func/test_state.py
+3 -3
View File
@@ -17,17 +17,17 @@ def test_state(tmp_dir, dvc):
state = State(dvc.root_dir, dvc.tmp_dir, dvc.dvcignore)
state.save(path_info, dvc.fs, hash_info)
assert state.get(path_info, dvc.fs) == hash_info
assert state.get(path_info, dvc.fs)[1] == hash_info
path.unlink()
path.write_text("1")
assert state.get(path_info, dvc.fs) is None
assert state.get(path_info, dvc.fs) == (None, None)
hash_info = HashInfo("md5", file_md5(path, dvc.fs))
state.save(path_info, dvc.fs, hash_info)
assert state.get(path_info, dvc.fs) == hash_info
assert state.get(path_info, dvc.fs)[1] == hash_info
def test_state_overflow(tmp_dir, dvc):
tests/unit/fs/test_dvc.py
+4 -4
View File
@@ -243,14 +243,14 @@ def test_get_hash_granular(tmp_dir, dvc):
fs = DvcFileSystem(repo=dvc)
subdir = PathInfo(tmp_dir) / "dir" / "subdir"
assert fs.info(subdir).get("md5") is None
_, obj = stage(dvc.odb.local, subdir, fs, "md5", dry_run=True)
_, _, obj = stage(dvc.odb.local, subdir, fs, "md5", dry_run=True)
assert obj.hash_info == HashInfo(
"md5", "af314506f1622d107e0ed3f14ec1a3b5.dir"
)
assert (
fs.info(subdir / "data")["md5"] == "8d777f385d3dfec8815d20f7496026dc"
)
_, obj = stage(dvc.odb.local, subdir / "data", fs, "md5", dry_run=True)
_, _, obj = stage(dvc.odb.local, subdir / "data", fs, "md5", dry_run=True)
assert obj.hash_info == HashInfo("md5", "8d777f385d3dfec8815d20f7496026dc")
@@ -261,7 +261,7 @@ def test_get_hash_dirty_file(tmp_dir, dvc):
fs = DvcFileSystem(repo=dvc)
expected = "8c7dd922ad47494fc02c388e12c00eac"
assert fs.info(PathInfo(tmp_dir) / "file").get("md5") == expected
_, obj = stage(
_, _, obj = stage(
dvc.odb.local, PathInfo(tmp_dir) / "file", fs, "md5", dry_run=True
)
assert obj.hash_info == HashInfo("md5", expected)
@@ -274,7 +274,7 @@ def test_get_hash_dirty_dir(tmp_dir, dvc):
fs = DvcFileSystem(repo=dvc)
expected = "5ea40360f5b4ec688df672a4db9c17d1.dir"
assert fs.info(PathInfo(tmp_dir) / "dir").get("md5") == expected
_, obj = stage(
_, _, obj = stage(
dvc.odb.local, PathInfo(tmp_dir) / "dir", fs, "md5", dry_run=True
)
assert obj.hash_info == HashInfo("md5", expected)
tests/unit/fs/test_repo.py
+13 -11
View File
@@ -513,7 +513,7 @@ def test_get_hash_cached_file(tmp_dir, dvc, mocker):
fs = RepoFileSystem(repo=dvc)
expected = "acbd18db4cc2f85cedef654fccc4a4d8"
assert fs.info(PathInfo(tmp_dir) / "foo").get("md5") is None
_, obj = stage(dvc.odb.local, PathInfo(tmp_dir) / "foo", fs, "md5")
_, _, obj = stage(dvc.odb.local, PathInfo(tmp_dir) / "foo", fs, "md5")
assert obj.hash_info == HashInfo("md5", expected)
(tmp_dir / "foo").unlink()
assert fs.info(PathInfo(tmp_dir) / "foo")["md5"] == expected
@@ -526,14 +526,14 @@ def test_get_hash_cached_dir(tmp_dir, dvc, mocker):
fs = RepoFileSystem(repo=dvc)
expected = "8761c4e9acad696bee718615e23e22db.dir"
assert fs.info(PathInfo(tmp_dir) / "dir").get("md5") is None
_, obj = stage(dvc.odb.local, PathInfo(tmp_dir) / "dir", fs, "md5")
_, _, obj = stage(dvc.odb.local, PathInfo(tmp_dir) / "dir", fs, "md5")
assert obj.hash_info == HashInfo(
"md5", "8761c4e9acad696bee718615e23e22db.dir"
)
shutil.rmtree(tmp_dir / "dir")
assert fs.info(PathInfo(tmp_dir) / "dir")["md5"] == expected
_, obj = stage(dvc.odb.local, PathInfo(tmp_dir) / "dir", fs, "md5")
_, _, obj = stage(dvc.odb.local, PathInfo(tmp_dir) / "dir", fs, "md5")
assert obj.hash_info == HashInfo(
"md5", "8761c4e9acad696bee718615e23e22db.dir"
)
@@ -546,12 +546,12 @@ def test_get_hash_cached_granular(tmp_dir, dvc, mocker):
fs = RepoFileSystem(repo=dvc)
subdir = PathInfo(tmp_dir) / "dir" / "subdir"
assert fs.info(subdir).get("md5") is None
_, obj = stage(dvc.odb.local, subdir, fs, "md5")
_, _, obj = stage(dvc.odb.local, subdir, fs, "md5")
assert obj.hash_info == HashInfo(
"md5", "af314506f1622d107e0ed3f14ec1a3b5.dir"
)
assert fs.info(subdir / "data").get("md5") is None
_, obj = stage(dvc.odb.local, subdir / "data", fs, "md5")
_, _, obj = stage(dvc.odb.local, subdir / "data", fs, "md5")
assert obj.hash_info == HashInfo("md5", "8d777f385d3dfec8815d20f7496026dc")
(tmp_dir / "dir" / "subdir" / "data").unlink()
assert (
@@ -573,7 +573,7 @@ def test_get_hash_mixed_dir(tmp_dir, scm, dvc):
clean_staging()
fs = RepoFileSystem(repo=dvc)
_, obj = stage(dvc.odb.local, PathInfo(tmp_dir) / "dir", fs, "md5")
_, _, obj = stage(dvc.odb.local, PathInfo(tmp_dir) / "dir", fs, "md5")
assert obj.hash_info == HashInfo(
"md5", "e1d9e8eae5374860ae025ec84cfd85c7.dir"
)
@@ -596,7 +596,9 @@ def test_get_hash_dirty_file(tmp_dir, dvc):
# get_file_hash(file) should return workspace hash, not DVC cached hash
fs = RepoFileSystem(repo=dvc)
assert fs.info(PathInfo(tmp_dir) / "file").get("md5") is None
staging, obj = stage(dvc.odb.local, PathInfo(tmp_dir) / "file", fs, "md5")
staging, _, obj = stage(
dvc.odb.local, PathInfo(tmp_dir) / "file", fs, "md5"
)
assert obj.hash_info == something_hash_info
check(staging, obj)
@@ -608,14 +610,14 @@ def test_get_hash_dirty_file(tmp_dir, dvc):
# get_file_hash(file) should return DVC cached hash
assert fs.info(PathInfo(tmp_dir) / "file")["md5"] == file_hash_info.value
hash_info = get_file_hash(
_, hash_info = get_file_hash(
PathInfo(tmp_dir) / "file", fs, "md5", state=dvc.state
)
assert hash_info == file_hash_info
# tmp_dir/file can be staged even though it is missing in workspace since
# repofs will use the DVC cached hash (and refer to the local cache object)
_, obj = stage(dvc.odb.local, PathInfo(tmp_dir) / "file", fs, "md5")
_, _, obj = stage(dvc.odb.local, PathInfo(tmp_dir) / "file", fs, "md5")
assert obj.hash_info == file_hash_info
@@ -625,11 +627,11 @@ def test_get_hash_dirty_dir(tmp_dir, dvc):
clean_staging()
fs = RepoFileSystem(repo=dvc)
_, obj = stage(dvc.odb.local, PathInfo(tmp_dir) / "dir", fs, "md5")
_, meta, obj = stage(dvc.odb.local, PathInfo(tmp_dir) / "dir", fs, "md5")
assert obj.hash_info == HashInfo(
"md5", "ba75a2162ca9c29acecb7957105a0bc2.dir"
)
assert obj.hash_info.nfiles == 3
assert meta.nfiles == 3
@pytest.mark.parametrize("traverse_subrepos", [True, False])
tests/unit/objects/db/test_local.py
+2 -2
View File
@@ -94,7 +94,7 @@ def test_staging_file(tmp_dir, dvc):
fs = LocalFileSystem()
local_odb = dvc.odb.local
staging_odb, obj = stage(local_odb, tmp_dir / "foo", fs, "md5")
staging_odb, _, obj = stage(local_odb, tmp_dir / "foo", fs, "md5")
assert not local_odb.exists(obj.hash_info)
assert staging_odb.exists(obj.hash_info)
@@ -121,7 +121,7 @@ def test_staging_dir(tmp_dir, dvc):
fs = LocalFileSystem()
local_odb = dvc.odb.local
staging_odb, obj = stage(local_odb, tmp_dir / "dir", fs, "md5")
staging_odb, _, obj = stage(local_odb, tmp_dir / "dir", fs, "md5")
assert not local_odb.exists(obj.hash_info)
assert staging_odb.exists(obj.hash_info)
tests/unit/objects/test_tree.py
+49 -73
View File
@@ -4,6 +4,7 @@ import pytest
from dvc.hash_info import HashInfo
from dvc.objects.file import HashFile
from dvc.objects.meta import Meta
from dvc.objects.tree import Tree, _merge
@@ -19,10 +20,10 @@ from dvc.objects.tree import Tree, _merge
{"md5": "456", "relpath": "bar"},
],
{
("zzz",): HashFile(None, None, HashInfo("md5", "def")),
("foo",): HashFile(None, None, HashInfo("md5", "123")),
("bar",): HashFile(None, None, HashInfo("md5", "456")),
("aaa",): HashFile(None, None, HashInfo("md5", "abc")),
("zzz",): (None, HashFile(None, None, HashInfo("md5", "def"))),
("foo",): (None, HashFile(None, None, HashInfo("md5", "123"))),
("bar",): (None, HashFile(None, None, HashInfo("md5", "456"))),
("aaa",): (None, HashFile(None, None, HashInfo("md5", "abc"))),
},
),
(
@@ -38,20 +39,32 @@ from dvc.objects.tree import Tree, _merge
{"md5": "pqr", "relpath": "dir/subdir/a"},
],
{
("dir", "b"): HashFile(None, None, HashInfo("md5", "123")),
("dir", "z"): HashFile(None, None, HashInfo("md5", "456")),
("dir", "a"): HashFile(None, None, HashInfo("md5", "789")),
("b",): HashFile(None, None, HashInfo("md5", "abc")),
("a",): HashFile(None, None, HashInfo("md5", "def")),
("z",): HashFile(None, None, HashInfo("md5", "ghi")),
("dir", "subdir", "b"): HashFile(
None, None, HashInfo("md5", "jkl")
("dir", "b"): (
None,
HashFile(None, None, HashInfo("md5", "123")),
),
("dir", "subdir", "z"): HashFile(
None, None, HashInfo("md5", "mno")
("dir", "z"): (
None,
HashFile(None, None, HashInfo("md5", "456")),
),
("dir", "subdir", "a"): HashFile(
None, None, HashInfo("md5", "pqr")
("dir", "a"): (
None,
HashFile(None, None, HashInfo("md5", "789")),
),
("b",): (None, HashFile(None, None, HashInfo("md5", "abc"))),
("a",): (None, HashFile(None, None, HashInfo("md5", "def"))),
("z",): (None, HashFile(None, None, HashInfo("md5", "ghi"))),
("dir", "subdir", "b"): (
None,
HashFile(None, None, HashInfo("md5", "jkl")),
),
("dir", "subdir", "z"): (
None,
HashFile(None, None, HashInfo("md5", "mno")),
),
("dir", "subdir", "a"): (
None,
HashFile(None, None, HashInfo("md5", "pqr")),
),
},
),
@@ -63,64 +76,25 @@ def test_list(lst, trie_dict):
assert tree.as_list() == sorted(lst, key=itemgetter("relpath"))
@pytest.mark.parametrize(
"trie_dict, size",
[
({}, 0),
(
{
("a",): HashFile(None, None, HashInfo("md5", "abc", size=1)),
("b",): HashFile(None, None, HashInfo("md5", "def", size=2)),
("c",): HashFile(None, None, HashInfo("md5", "ghi", size=3)),
("dir", "foo"): HashFile(
None, None, HashInfo("md5", "jkl", size=4)
),
("dir", "bar"): HashFile(
None, None, HashInfo("md5", "mno", size=5)
),
("dir", "baz"): HashFile(
None, None, HashInfo("md5", "pqr", size=6)
),
},
21,
),
(
{
("a",): HashFile(None, None, HashInfo("md5", "abc", size=1)),
("b",): HashFile(
None, None, HashInfo("md5", "def", size=None)
),
},
None,
),
],
)
def test_size(trie_dict, size):
tree = Tree(None, None, None)
tree._dict = trie_dict
tree.digest()
assert tree.size == size
@pytest.mark.parametrize(
"trie_dict, nfiles",
[
({}, 0),
(
{
("a",): HashInfo("md5", "abc", size=1),
("b",): HashInfo("md5", "def", size=2),
("c",): HashInfo("md5", "ghi", size=3),
("dir", "foo"): HashInfo("md5", "jkl", size=4),
("dir", "bar"): HashInfo("md5", "mno", size=5),
("dir", "baz"): HashInfo("md5", "pqr", size=6),
("a",): (Meta(size=1), HashInfo("md5", "abc")),
("b",): (Meta(size=2), HashInfo("md5", "def")),
("c",): (Meta(size=3), HashInfo("md5", "ghi")),
("dir", "foo"): (Meta(size=4), HashInfo("md5", "jkl")),
("dir", "bar"): (Meta(size=5), HashInfo("md5", "mno")),
("dir", "baz"): (Meta(size=6), HashInfo("md5", "pqr")),
},
6,
),
(
{
("a",): HashInfo("md5", "abc", size=1),
("b",): HashInfo("md5", "def", size=None),
("a",): (Meta(size=1), HashInfo("md5", "abc")),
("b",): (Meta(), HashInfo("md5", "def")),
},
2,
),
@@ -137,22 +111,24 @@ def test_nfiles(trie_dict, nfiles):
[
{},
{
("a",): HashInfo("md5", "abc"),
("b",): HashInfo("md5", "def"),
("c",): HashInfo("md5", "ghi"),
("dir", "foo"): HashInfo("md5", "jkl"),
("dir", "bar"): HashInfo("md5", "mno"),
("dir", "baz"): HashInfo("md5", "pqr"),
("dir", "subdir", "1"): HashInfo("md5", "stu"),
("dir", "subdir", "2"): HashInfo("md5", "vwx"),
("dir", "subdir", "3"): HashInfo("md5", "yz"),
("a",): (None, HashInfo("md5", "abc")),
("b",): (None, HashInfo("md5", "def")),
("c",): (None, HashInfo("md5", "ghi")),
("dir", "foo"): (None, HashInfo("md5", "jkl")),
("dir", "bar"): (None, HashInfo("md5", "mno")),
("dir", "baz"): (None, HashInfo("md5", "pqr")),
("dir", "subdir", "1"): (None, HashInfo("md5", "stu")),
("dir", "subdir", "2"): (None, HashInfo("md5", "vwx")),
("dir", "subdir", "3"): (None, HashInfo("md5", "yz")),
},
],
)
def test_items(trie_dict):
tree = Tree(None, None, None)
tree._dict = trie_dict
assert list(tree) == list(trie_dict.items())
assert list(tree) == [
(key, value[0], value[1]) for key, value in trie_dict.items()
]
@pytest.mark.parametrize(
tests/unit/output/test_local.py
+3 -1
View File
@@ -2,6 +2,7 @@ import os
from unittest.mock import patch
from dvc.hash_info import HashInfo
from dvc.objects.meta import Meta
from dvc.output import Output
from dvc.stage import Stage
from dvc.utils import relpath
@@ -63,7 +64,8 @@ class TestGetFilesNumber(TestDvc):
def test_return_multiple_for_dir(self):
o = self._get_output()
o.hash_info = HashInfo("md5", "12345678.dir", nfiles=2)
o.hash_info = HashInfo("md5", "12345678.dir")
o.meta = Meta(nfiles=2)
self.assertEqual(2, o.get_files_number())
@patch.object(Output, "is_dir_checksum", False)