diff --git a/dvc/dependency/param.py b/dvc/dependency/param.py index 717196de8..cf8f1f428 100644 --- a/dvc/dependency/param.py +++ b/dvc/dependency/param.py @@ -1,3 +1,4 @@ +import logging import os from collections import defaultdict @@ -10,6 +11,8 @@ from dvc.utils.serialize import LOADERS, ParseError from .base import Dependency +logger = logging.getLogger(__name__) + class MissingParamsError(DvcException): pass @@ -133,3 +136,20 @@ class ParamsDependency(Dependency): ) return HashInfo(self.PARAM_PARAMS, info) + + def save(self): + if not self.exists: + raise self.DoesNotExistError(self) + + if not self.isfile and not self.isdir: + raise self.IsNotFileOrDirError(self) + + if self.is_empty: + logger.warning(f"'{self}' is empty.") + + self.ignore() + + if self.metric or self.plot: + self.verify_metric() + + self.hash_info = self.get_hash() diff --git a/dvc/dependency/repo.py b/dvc/dependency/repo.py index 8a2a6509e..21d96a889 100644 --- a/dvc/dependency/repo.py +++ b/dvc/dependency/repo.py @@ -132,7 +132,7 @@ class RepoDependency(Dependency): pass try: - staging, staged_obj = stage( + staging, _, staged_obj = stage( local_odb, path_info, repo.repo_fs, @@ -149,7 +149,7 @@ class RepoDependency(Dependency): used_obj_ids[staging].add(staged_obj.hash_info) if isinstance(staged_obj, Tree): used_obj_ids[staging].update( - entry.hash_info for _, entry in staged_obj + entry.hash_info for _, _, entry in staged_obj ) return used_obj_ids, staged_obj @@ -166,7 +166,9 @@ class RepoDependency(Dependency): for hash_info in obj_ids: if hash_info.isdir: tree = Tree.load(odb, hash_info) - yield from (odb.get(entry.hash_info) for _, entry in tree) + yield from ( + odb.get(entry.hash_info) for _, _, entry in tree + ) else: yield odb.get(hash_info) diff --git a/dvc/fs/dvc.py b/dvc/fs/dvc.py index 7623d53dc..fe8b5f15d 100644 --- a/dvc/fs/dvc.py +++ b/dvc/fs/dvc.py @@ -58,7 +58,7 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method out.get_dir_cache(remote=remote) if out.obj is None: raise FileNotFoundError - obj = out.obj.trie.get(key) + (_, obj) = out.obj.trie.get(key) or (None, None) if obj: return obj.hash_info raise FileNotFoundError @@ -154,7 +154,7 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method self._fetch_dir(out, **kwargs) base = out.path_info.parts - for key, _ in out.obj: # noqa: B301 + for key, _, _ in out.obj: # noqa: B301 trie[base + key] = None def _walk(self, root, trie, topdown=True, **kwargs): @@ -244,15 +244,15 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method meta = self.metadata(path_info) ret = {"type": "directory" if meta.isdir else "file"} if meta.is_output and len(meta.outs) == 1 and meta.outs[0].hash_info: - hash_info = meta.outs[0].hash_info - ret["size"] = hash_info.size - ret[hash_info.name] = hash_info.value + out = meta.outs[0] + ret["size"] = out.meta.size + ret[out.hash_info.name] = out.hash_info.value elif meta.part_of_output: (out,) = meta.outs key = path_info.relative_to(out.path_info).parts - obj = out.obj.trie.get(key) + (obj_meta, obj) = out.obj.trie.get(key) or (None, None) if obj: - ret["size"] = obj.size + ret["size"] = obj_meta.size if obj_meta else 0 ret[obj.hash_info.name] = obj.hash_info.value return ret diff --git a/dvc/hash_info.py b/dvc/hash_info.py index d12dd52a4..3c028543c 100644 --- a/dvc/hash_info.py +++ b/dvc/hash_info.py @@ -7,13 +7,8 @@ HASH_DIR_SUFFIX = ".dir" @dataclass class HashInfo: - PARAM_SIZE = "size" - PARAM_NFILES = "nfiles" - name: Optional[str] value: Optional[str] - size: Optional[int] = field(default=None, compare=False) - nfiles: Optional[int] = field(default=None, compare=False) obj_name: Optional[str] = field(default=None, compare=False) def __bool__(self): @@ -27,15 +22,11 @@ class HashInfo: @classmethod def from_dict(cls, d): - _d = d.copy() if d else {} - size = _d.pop(cls.PARAM_SIZE, None) - nfiles = _d.pop(cls.PARAM_NFILES, None) - - if not _d: + if not d: return cls(None, None) - ((name, value),) = _d.items() - return cls(name, value, size=size, nfiles=nfiles) + ((name, value),) = d.items() + return cls(name, value) def to_dict(self): ret = OrderedDict() @@ -43,10 +34,6 @@ class HashInfo: return ret ret[self.name] = self.value - if self.size is not None: - ret[self.PARAM_SIZE] = self.size - if self.nfiles is not None: - ret[self.PARAM_NFILES] = self.nfiles return ret @property diff --git a/dvc/objects/__init__.py b/dvc/objects/__init__.py index 159b8fb6a..a57b8f3c3 100644 --- a/dvc/objects/__init__.py +++ b/dvc/objects/__init__.py @@ -14,7 +14,7 @@ logger = logging.getLogger(__name__) def check(odb: "ObjectDB", obj: "HashFile", **kwargs): if isinstance(obj, Tree): - for _, entry in obj: + for _, _, entry in obj: odb.check(entry.hash_info, **kwargs) odb.check(obj.hash_info, **kwargs) diff --git a/dvc/objects/checkout.py b/dvc/objects/checkout.py index b22df2d24..834959459 100644 --- a/dvc/objects/checkout.py +++ b/dvc/objects/checkout.py @@ -166,7 +166,7 @@ def _diff( ): old = None try: - _, old = stage( + _, _, old = stage( cache, path_info, fs, diff --git a/dvc/objects/diff.py b/dvc/objects/diff.py index 458e19636..62f60047f 100644 --- a/dvc/objects/diff.py +++ b/dvc/objects/diff.py @@ -83,7 +83,7 @@ def diff( if not obj: return [] return [ROOT] + ( - [key for key, _ in obj] if isinstance(obj, Tree) else [] + [key for key, _, _ in obj] if isinstance(obj, Tree) else [] ) old_keys = set(_get_keys(old)) diff --git a/dvc/objects/file.py b/dvc/objects/file.py index 2e4fc4300..65319f021 100644 --- a/dvc/objects/file.py +++ b/dvc/objects/file.py @@ -28,10 +28,6 @@ class HashFile: self.hash_info = hash_info self.name = name - @property - def size(self): - return self.hash_info.size - def __len__(self): return 1 @@ -74,7 +70,7 @@ class HashFile: def _check_hash(self, odb): from .stage import get_file_hash - actual = get_file_hash( + _, actual = get_file_hash( self.path_info, self.fs, self.hash_info.name, odb.state ) diff --git a/dvc/objects/meta.py b/dvc/objects/meta.py new file mode 100644 index 000000000..ef3930e08 --- /dev/null +++ b/dvc/objects/meta.py @@ -0,0 +1,32 @@ +from collections import OrderedDict +from dataclasses import dataclass, field +from typing import Optional + + +@dataclass +class Meta: + PARAM_SIZE = "size" + PARAM_NFILES = "nfiles" + + size: Optional[int] = field(default=None) + nfiles: Optional[int] = field(default=None) + + @classmethod + def from_dict(cls, d): + if not d: + return cls() + + size = d.pop(cls.PARAM_SIZE, None) + nfiles = d.pop(cls.PARAM_NFILES, None) + return cls(size=size, nfiles=nfiles) + + def to_dict(self): + ret = OrderedDict() + + if self.size is not None: + ret[self.PARAM_SIZE] = self.size + + if self.nfiles is not None: + ret[self.PARAM_NFILES] = self.nfiles + + return ret diff --git a/dvc/objects/stage.py b/dvc/objects/stage.py index 99d99675c..8ed4f0f3e 100644 --- a/dvc/objects/stage.py +++ b/dvc/objects/stage.py @@ -14,6 +14,7 @@ from dvc.utils import file_md5 from .db.reference import ReferenceObjectDB from .file import HashFile +from .meta import Meta if TYPE_CHECKING: from dvc.fs.base import BaseFileSystem @@ -38,7 +39,8 @@ def _upload_file(path_info, fs, odb, upload_odb): upload_odb.fs.upload(stream, tmp_info, desc=path_info.name, total=size) odb.add(tmp_info, upload_odb.fs, stream.hash_info) - return path_info, odb.get(stream.hash_info) + meta = Meta(size=size) + return path_info, meta, odb.get(stream.hash_info) def _get_file_hash(path_info, fs, name): @@ -54,31 +56,31 @@ def _get_file_hash(path_info, fs, name): else: raise NotImplementedError - return HashInfo(name, hash_value, size=info["size"]) + meta = Meta(size=info["size"]) + hash_info = HashInfo(name, hash_value) + return meta, hash_info def get_file_hash(path_info, fs, name, state=None): if state: - hash_info = state.get( # pylint: disable=assignment-from-none + meta, hash_info = state.get( # pylint: disable=assignment-from-none path_info, fs ) if hash_info: - return hash_info + return meta, hash_info - hash_info = _get_file_hash(path_info, fs, name) + meta, hash_info = _get_file_hash(path_info, fs, name) if state: assert ".dir" not in hash_info.value state.save(path_info, fs, hash_info) - return hash_info + return meta, hash_info -def _get_file_obj( - path_info, fs, name, odb=None, upload_odb=None, dry_run=False -): +def _stage_file(path_info, fs, name, odb=None, upload_odb=None, dry_run=False): state = odb.state if odb else None - hash_info = get_file_hash(path_info, fs, name, state=state) + meta, hash_info = get_file_hash(path_info, fs, name, state=state) if upload_odb and not dry_run: assert odb and name == "md5" return _upload_file(path_info, fs, odb, upload_odb) @@ -88,7 +90,7 @@ def _get_file_obj( else: odb.add(path_info, fs, hash_info, move=False) obj = odb.get(hash_info) - return path_info, obj + return path_info, meta, obj def _build_objects( @@ -111,7 +113,7 @@ def _build_objects( ) as pbar: worker = pbar.wrap_fn( partial( - _get_file_obj, + _stage_file, fs=fs, name=name, **kwargs, @@ -130,8 +132,9 @@ def _iter_objects(path_info, fs, name, **kwargs): def _build_tree(path_info, fs, name, **kwargs): from .tree import Tree + tree_meta = Meta(size=0, nfiles=0) tree = Tree(None, None, None) - for file_info, obj in _iter_objects(path_info, fs, name, **kwargs): + for file_info, meta, obj in _iter_objects(path_info, fs, name, **kwargs): if DvcIgnore.DVCIGNORE_FILE == file_info.name: raise DvcIgnoreInCollectedDirError(file_info.parent) @@ -143,11 +146,15 @@ def _build_tree(path_info, fs, name, **kwargs): # # Yes, this is a BUG, as long as we permit "/" in # filenames on Windows and "\" on Unix - tree.add(file_info.relative_to(path_info).parts, obj) - return tree + tree.add(file_info.relative_to(path_info).parts, meta, obj) + + tree_meta.size += meta.size + tree_meta.nfiles += 1 + + return tree_meta, tree -def _get_tree_obj(path_info, fs, fs_info, name, odb=None, **kwargs): +def _stage_tree(path_info, fs, fs_info, name, odb=None, **kwargs): from .tree import Tree value = fs_info.get(name) @@ -158,18 +165,18 @@ def _get_tree_obj(path_info, fs, fs_info, name, odb=None, **kwargs): # NOTE: loaded entries are naive objects with hash_infos but no # path_info. For staging trees, obj.path_info should be relative # to the staging src `path_info` and src fs - for key, entry in tree: + for key, _, entry in tree: entry.fs = fs entry.path_info = path_info.joinpath(*key) - return tree + return Meta(nfiles=len(tree)), tree except FileNotFoundError: pass - tree = _build_tree(path_info, fs, name, odb=odb, **kwargs) + meta, tree = _build_tree(path_info, fs, name, odb=odb, **kwargs) state = odb.state if odb and odb.state else None hash_info = None if state: - hash_info = state.get( # pylint: disable=assignment-from-none + _, hash_info = state.get( # pylint: disable=assignment-from-none path_info, fs ) tree.digest(hash_info=hash_info) @@ -181,7 +188,7 @@ def _get_tree_obj(path_info, fs, fs_info, name, odb=None, **kwargs): tree.fs.remove(tree.path_info) tree.fs = raw.fs tree.path_info = raw.path_info - return tree + return meta, tree _url_cache: Dict[str, str] = {} @@ -223,7 +230,7 @@ def _load_from_state(odb, staging, path_info, fs, name): from .tree import Tree state = odb.state - hash_info = state.get(path_info, fs) + meta, hash_info = state.get(path_info, fs) if hash_info: for odb_ in (odb, staging): if odb_.exists(hash_info): @@ -231,16 +238,15 @@ def _load_from_state(odb, staging, path_info, fs, name): obj = load(odb_, hash_info) check(odb_, obj, check_hash=False) if isinstance(obj, Tree): - obj.hash_info.nfiles = len(obj) - for key, entry in obj: + meta.nfiles = len(obj) + for key, _, entry in obj: entry.fs = fs entry.path_info = path_info.joinpath(*key) else: obj.fs = fs obj.path_info = path_info assert obj.hash_info.name == name - obj.hash_info.size = hash_info.size - return odb_, obj + return odb_, meta, obj except (ObjectFormatError, FileNotFoundError): pass raise FileNotFoundError @@ -255,7 +261,7 @@ def _stage_external_tree_info(odb, tree, name): odb.add(tree.path_info, tree.fs, tree.hash_info) raw = odb.get(tree.hash_info) - hash_info = get_file_hash(raw.path_info, raw.fs, name, state=odb.state) + _, hash_info = get_file_hash(raw.path_info, raw.fs, name, state=odb.state) tree.path_info = raw.path_info tree.fs = raw.fs tree.hash_info.name = hash_info.name @@ -273,7 +279,7 @@ def stage( upload: bool = False, dry_run: bool = False, **kwargs, -) -> Tuple["ObjectDB", "HashFile"]: +) -> Tuple["ObjectDB", "Meta", "HashFile"]: """Stage (prepare) objects from the given path for addition to an ODB. Returns at tuple of (staging_odb, object) where addition to the ODB can @@ -298,7 +304,7 @@ def stage( pass if details["type"] == "directory": - obj = _get_tree_obj( + meta, obj = _stage_tree( path_info, fs, details, @@ -312,7 +318,7 @@ def stage( if name != "md5": obj = _stage_external_tree_info(odb, obj, name) else: - _, obj = _get_file_obj( + _, meta, obj = _stage_file( path_info, fs, name, @@ -324,4 +330,4 @@ def stage( if odb and odb.state and obj.hash_info: odb.state.save(path_info, fs, obj.hash_info) - return staging, obj + return staging, meta, obj diff --git a/dvc/objects/status.py b/dvc/objects/status.py index 0dfaabea1..f79908b6f 100644 --- a/dvc/objects/status.py +++ b/dvc/objects/status.py @@ -56,7 +56,7 @@ def _indexed_dir_hashes(odb, index, dir_objs, name, cache_odb): tree = Tree.load(cache_odb, HashInfo(name, dir_hash)) except FileNotFoundError: continue - file_hashes = [entry.hash_info.value for _, entry in tree] + file_hashes = [entry.hash_info.value for _, _, entry in tree] if dir_hash not in index: logger.debug( "Indexing new .dir '%s' with '%s' nested files", @@ -102,7 +102,7 @@ def status( tree = None else: tree = Tree.load(cache_odb, hash_info) - for _, entry in tree: + for _, _, entry in tree: assert entry.hash_info and entry.hash_info.value hash_infos[entry.hash_info.value] = entry.hash_info if index: diff --git a/dvc/objects/transfer.py b/dvc/objects/transfer.py index c6ee2c004..95aad3b70 100644 --- a/dvc/objects/transfer.py +++ b/dvc/objects/transfer.py @@ -74,7 +74,7 @@ def _do_transfer( dir_obj = find_tree_by_obj_id([cache_odb, src], dir_hash) assert dir_obj - entry_ids = {entry.hash_info for _, entry in dir_obj} + entry_ids = {entry.hash_info for _, _, entry in dir_obj} bound_file_ids = all_file_ids & entry_ids all_file_ids -= entry_ids @@ -117,7 +117,7 @@ def _do_transfer( # index successfully pushed dirs if dest_index: for dir_obj in succeeded_dir_objs: - file_hashes = {entry.hash_info.value for _, entry in dir_obj} + file_hashes = {entry.hash_info.value for _, _, entry in dir_obj} logger.debug( "Indexing pushed dir '%s' with '%s' nested files", dir_obj.hash_info, diff --git a/dvc/objects/tree.py b/dvc/objects/tree.py index fad5c0439..e7be2e191 100644 --- a/dvc/objects/tree.py +++ b/dvc/objects/tree.py @@ -12,6 +12,8 @@ from .stage import get_file_hash if TYPE_CHECKING: from dvc.hash_info import HashInfo + from .meta import Meta + logger = logging.getLogger(__name__) @@ -20,7 +22,7 @@ class Tree(HashFile): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._dict: Dict[Tuple[str], "HashFile"] = {} + self._dict: Dict[Tuple[str], Tuple["Meta", "HashFile"]] = {} @cached_property def trie(self): @@ -28,9 +30,9 @@ class Tree(HashFile): return Trie(self._dict) - def add(self, key, obj): + def add(self, key, meta, obj): self.__dict__.pop("trie", None) - self._dict[key] = obj + self._dict[key] = (meta, obj) def digest(self, hash_info: Optional["HashInfo"] = None): from dvc.fs.memory import MemoryFileSystem @@ -46,20 +48,17 @@ class Tree(HashFile): if hash_info: self.hash_info = hash_info else: - self.hash_info = get_file_hash(path_info, memfs, "md5") + _, self.hash_info = get_file_hash(path_info, memfs, "md5") assert self.hash_info.value self.hash_info.value += ".dir" - try: - self.hash_info.size = sum(obj.size for _, obj in self) - except TypeError: - self.hash_info.size = None - self.hash_info.nfiles = len(self) def __len__(self): return len(self._dict) def __iter__(self): - yield from self._dict.items() + yield from ( + (key, value[0], value[1]) for key, value in self._dict.items() + ) def as_dict(self): return self._dict.copy() @@ -76,7 +75,7 @@ class Tree(HashFile): obj.hash_info.name: obj.hash_info.value, self.PARAM_RELPATH: posixpath.sep.join(parts), } - for parts, obj in self._dict.items() # noqa: B301 + for parts, _, obj in self # noqa: B301 ), key=itemgetter(self.PARAM_RELPATH), ) @@ -95,7 +94,7 @@ class Tree(HashFile): parts = tuple(relpath.split(posixpath.sep)) hash_info = HashInfo.from_dict(entry) obj = HashFile(None, None, hash_info) - tree.add(parts, obj) + tree.add(parts, None, obj) return tree @classmethod @@ -118,7 +117,7 @@ class Tree(HashFile): tree = cls.from_list(raw) tree.path_info = obj.path_info tree.fs = obj.fs - for _, entry_obj in tree: + for _, _, entry_obj in tree: entry_obj.fs = obj.fs tree.hash_info = hash_info @@ -135,8 +134,8 @@ class Tree(HashFile): """ tree = Tree(self.path_info, self.fs, self.hash_info) try: - for key, obj in self.trie.items(prefix): - tree.add(key, obj) + for key, (meta, obj) in self.trie.items(prefix): + tree.add(key, meta, obj) except KeyError: pass return tree @@ -146,26 +145,26 @@ class Tree(HashFile): Returns None if no object exists at the specified prefix. """ - obj = self._dict.get(prefix) + _, obj = self._dict.get(prefix) or (None, None) if obj: return obj tree = Tree(None, None, None) depth = len(prefix) try: - for key, obj in self.trie.items(prefix): - tree.add(key[depth:], obj) + for key, (meta, obj) in self.trie.items(prefix): + tree.add(key[depth:], meta, obj) except KeyError: return None tree.digest() return tree -def _get_dir_size(odb, tree): +def du(odb, tree): try: return sum( odb.fs.getsize(odb.hash_to_path_info(obj.hash_info.value)) - for _, obj in tree + for _, _, obj in tree ) except FileNotFoundError: return None @@ -226,11 +225,8 @@ def merge(odb, ancestor_info, our_info, their_info): merged_dict = _merge(ancestor.as_dict(), our.as_dict(), their.as_dict()) merged = Tree(None, None, None) - for key, hi in merged_dict.items(): - merged.add(key, hi) + for key, (meta, obj) in merged_dict.items(): + merged.add(key, meta, obj) merged.digest() - odb.add(merged.path_info, merged.fs, merged.hash_info) - hash_info = merged.hash_info - hash_info.size = _get_dir_size(odb, merged) - return hash_info + return merged diff --git a/dvc/output.py b/dvc/output.py index 530684335..b8f734945 100644 --- a/dvc/output.py +++ b/dvc/output.py @@ -1,7 +1,6 @@ import logging import os from collections import defaultdict -from copy import copy from typing import TYPE_CHECKING, Dict, Optional, Set, Type from urllib.parse import urlparse @@ -26,6 +25,7 @@ from .hash_info import HashInfo from .istextfile import istextfile from .objects import Tree from .objects.errors import ObjectFormatError +from .objects.meta import Meta from .objects.stage import stage as ostage from .objects.transfer import transfer as otransfer from .scheme import Schemes @@ -316,6 +316,7 @@ class Output: # By resolved path, which contains actual location, # should be absolute and don't contain remote:// refs. self.stage = stage + self.meta = Meta.from_dict(info) self.hash_info = HashInfo.from_dict(info) self.use_cache = False if self.IS_DEPENDENCY else cache self.metric = False if self.IS_DEPENDENCY else metric @@ -415,7 +416,7 @@ class Output: else: odb = self.repo.odb.local name = self.fs.PARAM_CHECKSUM - _, obj = ostage( + _, _, obj = ostage( odb, self.path_info, self.fs, @@ -537,7 +538,15 @@ class Output: self.verify_metric() if not self.use_cache: - self.hash_info = self.get_hash() + _, self.meta, obj = ostage( + self.repo.odb.local, + self.path_info, + self.fs, + self.fs.PARAM_CHECKSUM, + dvcignore=self.dvcignore, + dry_run=True, + ) + self.hash_info = obj.hash_info if not self.IS_DEPENDENCY: logger.debug( "Output '%s' doesn't use cache. Skipping saving.", self @@ -550,7 +559,7 @@ class Output: logger.debug("Output '%s' didn't change. Skipping saving.", self) return - _, self.obj = ostage( + _, self.meta, self.obj = ostage( self.odb, self.path_info, self.fs, @@ -579,7 +588,7 @@ class Output: if granular: obj = self._commit_granular_dir(filter_info) else: - staging, obj = ostage( + staging, _, obj = ostage( self.odb, filter_info or self.path_info, self.fs, @@ -606,7 +615,7 @@ class Output: def _commit_granular_dir(self, filter_info): prefix = filter_info.relative_to(self.path_info).parts - staging, save_obj = ostage( + staging, _, save_obj = ostage( self.odb, self.path_info, self.fs, @@ -618,14 +627,15 @@ class Output: otransfer( staging, self.odb, - {save_obj.hash_info} | {entry.hash_info for _, entry in save_obj}, + {save_obj.hash_info} + | {entry.hash_info for _, _, entry in save_obj}, shallow=True, move=True, ) return checkout_obj def dumpd(self): - ret = copy(self.hash_info.to_dict()) + ret = {**self.hash_info.to_dict(), **self.meta.to_dict()} if self.is_in_repo: path = self.path_info.relpath(self.stage.wdir).as_posix() @@ -803,7 +813,7 @@ class Output: upload = not (update and from_fs.isdir(from_info)) jobs = jobs or min((from_fs.jobs, odb.fs.jobs)) - staging, obj = ostage( + staging, self.meta, obj = ostage( odb, from_info, from_fs, @@ -832,7 +842,7 @@ class Output: return 1 if not filter_info or filter_info == self.path_info: - return self.hash_info.nfiles or 0 + return self.meta.nfiles or 0 obj = self.get_obj(filter_info=filter_info) return len(obj) if obj else 0 @@ -946,7 +956,7 @@ class Output: obj.hash_info.obj_name = name oids = {obj.hash_info} if isinstance(obj, Tree): - for key, entry_obj in obj: + for key, _, entry_obj in obj: entry_obj.hash_info.obj_name = self.fs.sep.join([name, *key]) oids.add(entry_obj.hash_info) return oids @@ -981,8 +991,8 @@ class Output: ignored = [ self.fs.PARAM_CHECKSUM, - HashInfo.PARAM_SIZE, - HashInfo.PARAM_NFILES, + Meta.PARAM_SIZE, + Meta.PARAM_NFILES, ] for opt in ignored: @@ -1000,7 +1010,7 @@ class Output: ) def merge(self, ancestor, other): - from dvc.objects.tree import merge + from dvc.objects.tree import du, merge assert other @@ -1013,9 +1023,16 @@ class Output: self._check_can_merge(self) self._check_can_merge(other) - self.hash_info = merge( + merged = merge( self.odb, ancestor_info, self.hash_info, other.hash_info ) + self.odb.add(merged.path_info, merged.fs, merged.hash_info) + + self.hash_info = merged.hash_info + self.meta = Meta( + size=du(self.odb, merged), + nfiles=len(merged), + ) @property def fspath(self): @@ -1040,8 +1057,8 @@ ARTIFACT_SCHEMA = { Output.PARAM_PLOT: bool, Output.PARAM_PERSIST: bool, Output.PARAM_CHECKPOINT: bool, - HashInfo.PARAM_SIZE: int, - HashInfo.PARAM_NFILES: int, + Meta.PARAM_SIZE: int, + Meta.PARAM_NFILES: int, Output.PARAM_ISEXEC: bool, } diff --git a/dvc/repo/diff.py b/dvc/repo/diff.py index 4ee08317c..7a947a86c 100644 --- a/dvc/repo/diff.py +++ b/dvc/repo/diff.py @@ -137,7 +137,7 @@ def _output_paths(repo, targets): ) if on_working_fs: - _, obj = ostage( + _, _, obj = ostage( repo.odb.local, output.path_info, repo.odb.local.fs, @@ -164,7 +164,7 @@ def _output_paths(repo, targets): def _dir_output_paths(path_info, obj, targets=None): - for key, entry_obj in obj: + for key, _, entry_obj in obj: fname = path_info.joinpath(*key) if targets is None or any( fname.isin_or_eq(target) for target in targets diff --git a/dvc/schema.py b/dvc/schema.py index 1e9e2a85e..0f5fcb1d7 100644 --- a/dvc/schema.py +++ b/dvc/schema.py @@ -3,7 +3,7 @@ from collections.abc import Mapping from voluptuous import Any, Optional, Required, Schema from dvc import dependency, output -from dvc.hash_info import HashInfo +from dvc.objects.meta import Meta from dvc.output import CHECKSUMS_SCHEMA, Output from dvc.parsing import DO_KWD, FOREACH_KWD, VARS_KWD from dvc.parsing.versions import SCHEMA_KWD, lockfile_version_schema @@ -26,8 +26,8 @@ SINGLE_STAGE_SCHEMA = { DATA_SCHEMA = { **CHECKSUMS_SCHEMA, Required("path"): str, - HashInfo.PARAM_SIZE: int, - HashInfo.PARAM_NFILES: int, + Meta.PARAM_SIZE: int, + Meta.PARAM_NFILES: int, Output.PARAM_ISEXEC: bool, } LOCK_FILE_STAGE_SCHEMA = { diff --git a/dvc/stage/cache.py b/dvc/stage/cache.py index 9177a7b98..1c96c30f4 100644 --- a/dvc/stage/cache.py +++ b/dvc/stage/cache.py @@ -19,13 +19,11 @@ class RunCacheNotFoundError(DvcException): def _get_cache_hash(cache, key=False): - from dvc.hash_info import HashInfo + from dvc.objects.meta import Meta if key: cache["outs"] = [out["path"] for out in cache.get("outs", [])] - return dict_sha256( - cache, exclude=[HashInfo.PARAM_SIZE, HashInfo.PARAM_NFILES] - ) + return dict_sha256(cache, exclude=[Meta.PARAM_SIZE, Meta.PARAM_NFILES]) def _can_hash(stage): diff --git a/dvc/stage/loader.py b/dvc/stage/loader.py index e2368f727..0216a97ef 100644 --- a/dvc/stage/loader.py +++ b/dvc/stage/loader.py @@ -7,6 +7,7 @@ from funcy import cached_property, get_in, lcat, once, project from dvc import dependency, output from dvc.hash_info import HashInfo +from dvc.objects.meta import Meta from dvc.parsing import FOREACH_KWD, JOIN, DataResolver, EntryNotFound from dvc.parsing.versions import LOCKFILE_VERSION from dvc.path_info import PathInfo @@ -69,6 +70,7 @@ class StageLoader(Mapping): info = info.copy() info.pop("path", None) item.isexec = info.pop("isexec", None) + item.meta = Meta.from_dict(info) item.hash_info = HashInfo.from_dict(info) @classmethod diff --git a/dvc/stage/serialize.py b/dvc/stage/serialize.py index ec145dbe7..606d5943c 100644 --- a/dvc/stage/serialize.py +++ b/dvc/stage/serialize.py @@ -164,6 +164,7 @@ def to_single_stage_lockfile(stage: "Stage") -> dict: ret = [ (item.PARAM_PATH, item.def_path), *item.hash_info.to_dict().items(), + *item.meta.to_dict().items(), ] if item.isexec: diff --git a/dvc/stage/utils.py b/dvc/stage/utils.py index 462f76125..4e6fc7651 100644 --- a/dvc/stage/utils.py +++ b/dvc/stage/utils.py @@ -5,8 +5,8 @@ from typing import TYPE_CHECKING, Union from funcy import concat, first, lsplit, rpartial, without from dvc.exceptions import InvalidArgumentError +from dvc.objects.meta import Meta -from ..hash_info import HashInfo from .exceptions import ( MissingDataSource, StageExternalOutputsError, @@ -197,8 +197,8 @@ def compute_md5(stage): Output.PARAM_PERSIST, Output.PARAM_CHECKPOINT, Output.PARAM_ISEXEC, - HashInfo.PARAM_SIZE, - HashInfo.PARAM_NFILES, + Meta.PARAM_SIZE, + Meta.PARAM_NFILES, ], ) diff --git a/dvc/state.py b/dvc/state.py index ffdb051e3..50cf3efb9 100644 --- a/dvc/state.py +++ b/dvc/state.py @@ -38,7 +38,7 @@ class StateNoop(StateBase): pass def get(self, path_info, fs): # pylint: disable=unused-argument - return None + return None, None def save_link(self, path_info, fs): pass @@ -103,22 +103,24 @@ class State(StateBase): # pylint: disable=too-many-instance-attributes HashInfo or None: hash for the specified path info or None if it doesn't exist in the state database. """ + from .objects.meta import Meta + if not isinstance(fs, LocalFileSystem): - return None + return None, None try: mtime, size = get_mtime_and_size(path_info, fs, self.dvcignore) except FileNotFoundError: - return None + return None, None inode = get_inode(path_info) value = self.md5s.get(inode) if not value or value[0] != mtime or value[1] != str(size): - return None + return None, None - return HashInfo("md5", value[2], size=size) + return Meta(size=size), HashInfo("md5", value[2]) def save_link(self, path_info, fs): """Adds the specified path to the list of links created by dvc. This diff --git a/dvc/utils/stream.py b/dvc/utils/stream.py index 053e5178a..a0a7ac8f9 100644 --- a/dvc/utils/stream.py +++ b/dvc/utils/stream.py @@ -47,6 +47,4 @@ class HashedStreamReader(io.IOBase): @property def hash_info(self): - return HashInfo( - self.PARAM_CHECKSUM, self.md5.hexdigest(), size=self.total_read - ) + return HashInfo(self.PARAM_CHECKSUM, self.md5.hexdigest()) diff --git a/tests/func/objects/db/test_index.py b/tests/func/objects/db/test_index.py index 7b85aea6a..7e79c90c7 100644 --- a/tests/func/objects/db/test_index.py +++ b/tests/func/objects/db/test_index.py @@ -16,7 +16,7 @@ def index(dvc, local_remote, mocker): def test_indexed_on_status(tmp_dir, dvc, index): foo = tmp_dir.dvc_gen({"foo": "foo content"})[0].outs[0] bar = tmp_dir.dvc_gen({"bar": {"baz": "baz content"}})[0].outs[0] - baz_hash = bar.obj.trie.get(("baz",)).hash_info + baz_hash = bar.obj.trie.get(("baz",))[1].hash_info clean_staging() dvc.push() index.clear() @@ -30,7 +30,7 @@ def test_indexed_on_status(tmp_dir, dvc, index): def test_indexed_on_push(tmp_dir, dvc, index): foo = tmp_dir.dvc_gen({"foo": "foo content"})[0].outs[0] bar = tmp_dir.dvc_gen({"bar": {"baz": "baz content"}})[0].outs[0] - baz_hash = bar.obj.trie.get(("baz",)).hash_info + baz_hash = bar.obj.trie.get(("baz",))[1].hash_info clean_staging() dvc.push() @@ -60,7 +60,7 @@ def test_clear_on_download_err(tmp_dir, dvc, index, mocker): out = tmp_dir.dvc_gen({"dir": {"foo": "foo content"}})[0].outs[0] dvc.push() - for _, entry in out.obj: + for _, _, entry in out.obj: remove(dvc.odb.local.get(entry.hash_info).path_info) remove(out.path_info) diff --git a/tests/func/test_add.py b/tests/func/test_add.py index 6b8768056..d1970f413 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -112,7 +112,7 @@ def test_add_directory(tmp_dir, dvc): hash_info = stage.outs[0].hash_info obj = load(dvc.odb.local, hash_info) - for key, _ in obj: + for key, _, _ in obj: for part in key: assert "\\" not in part @@ -534,7 +534,7 @@ class TestAddCommit(TestDvc): def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc): tmp_dir.gen({"data/data": "foo"}) - counter = mocker.spy(dvc_module.objects.stage, "_get_tree_obj") + counter = mocker.spy(dvc_module.objects.stage, "_stage_tree") ret = main(["add", "data"]) assert ret == 0 assert counter.mock.call_count == 2 @@ -1026,8 +1026,9 @@ def test_add_to_remote(tmp_dir, dvc, local_cloud, local_remote): assert len(stage.outs) == 1 hash_info = stage.outs[0].hash_info + meta = stage.outs[0].meta assert local_remote.hash_to_path_info(hash_info.value).read_text() == "foo" - assert hash_info.size == len("foo") + assert meta.size == len("foo") def test_add_to_remote_absolute(tmp_dir, make_tmp_dir, dvc, local_remote): @@ -1071,8 +1072,8 @@ def test_add_to_cache_dir(tmp_dir, dvc, local_cloud): (stage,) = dvc.add(str(local_cloud / "data"), out="data") assert len(stage.deps) == 0 assert len(stage.outs) == 1 - assert stage.outs[0].hash_info.size == len("foo") + len("bar") - assert stage.outs[0].hash_info.nfiles == 2 + assert stage.outs[0].meta.size == len("foo") + len("bar") + assert stage.outs[0].meta.nfiles == 2 data = tmp_dir / "data" assert data.read_text() == {"foo": "foo", "bar": "bar"} diff --git a/tests/func/test_checkout.py b/tests/func/test_checkout.py index 1fa4fce82..e4f374f3d 100644 --- a/tests/func/test_checkout.py +++ b/tests/func/test_checkout.py @@ -101,7 +101,7 @@ class TestCheckoutCorruptedCacheDir(TestDvc): # NOTE: modifying cache file for one of the files inside the directory # to check if dvc will detect that the cache is corrupted. obj = load(self.dvc.odb.local, out.hash_info) - _, entry_obj = list(obj)[0] + _, _, entry_obj = list(obj)[0] cache = os.fspath( self.dvc.odb.local.hash_to_path_info(entry_obj.hash_info.value) ) diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index 82a0f44d8..d08aa7ca3 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -58,7 +58,7 @@ def test_cloud(tmp_dir, dvc, remote): # pylint:disable=unused-argument cache_dir = out_dir.cache_path dir_hash = out_dir.hash_info dir_hashes = {dir_hash} | { - entry_obj.hash_info for _, entry_obj in out_dir.obj + entry_obj.hash_info for _, _, entry_obj in out_dir.obj } def _check_status(status, **kwargs): diff --git a/tests/func/test_external_repo.py b/tests/func/test_external_repo.py index 536894e63..996783735 100644 --- a/tests/func/test_external_repo.py +++ b/tests/func/test_external_repo.py @@ -203,7 +203,7 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir): clean_staging() makedirs(cache_dir) - staging, obj = stage( + staging, _, obj = stage( repo.odb.local, PathInfo(repo.root_dir) / "dir", repo.repo_fs, diff --git a/tests/func/test_import_url.py b/tests/func/test_import_url.py index b005e85da..b61e4529f 100644 --- a/tests/func/test_import_url.py +++ b/tests/func/test_import_url.py @@ -291,7 +291,7 @@ def test_import_url_to_remote_single_file( hash_info = stage.outs[0].hash_info assert local_remote.hash_to_path_info(hash_info.value).read_text() == "foo" - assert hash_info.size == len("foo") + assert stage.outs[0].meta.size == len("foo") @pytest.mark.parametrize( diff --git a/tests/func/test_lockfile.py b/tests/func/test_lockfile.py index 549ae75b7..dfd7d4e15 100644 --- a/tests/func/test_lockfile.py +++ b/tests/func/test_lockfile.py @@ -167,14 +167,12 @@ def test_params_dump(tmp_dir, dvc, run_head): def v1_repo_lock(tmp_dir, dvc): """Generates a repo having v1 format lockfile""" size = 5 if os.name == "nt" else 4 - hi = HashInfo( - name="md5", size=size, value="c157a79031e1c40f85931829bc5fc552" - ) + hi = HashInfo(name="md5", value="c157a79031e1c40f85931829bc5fc552") v1_lockdata = { "foo": {"cmd": "echo foo"}, "bar": { "cmd": "echo bar>bar.txt", - "outs": [{"path": "bar.txt", **hi.to_dict()}], + "outs": [{"path": "bar.txt", **hi.to_dict(), "size": size}], }, } dvc.run(cmd="echo foo", name="foo", no_exec=True) diff --git a/tests/func/test_remote.py b/tests/func/test_remote.py index 11729c59f..e0072600e 100644 --- a/tests/func/test_remote.py +++ b/tests/func/test_remote.py @@ -156,16 +156,16 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc): [{"relpath": "1", "md5": "1"}, {"relpath": "2", "md5": "2"}] ) tree.digest() - with patch("dvc.objects.stage._get_tree_obj", return_value=tree): - _, obj = stage(dvc.odb.local, path_info, dvc.odb.local.fs, "md5") + with patch("dvc.objects.stage._stage_tree", return_value=(None, tree)): + _, _, obj = stage(dvc.odb.local, path_info, dvc.odb.local.fs, "md5") hash1 = obj.hash_info tree = Tree.from_list( [{"md5": "1", "relpath": "1"}, {"md5": "2", "relpath": "2"}] ) tree.digest() - with patch("dvc.objects.stage._get_tree_obj", return_value=tree): - _, obj = stage(dvc.odb.local, path_info, dvc.odb.local.fs, "md5") + with patch("dvc.objects.stage._stage_tree", return_value=(None, tree)): + _, _, obj = stage(dvc.odb.local, path_info, dvc.odb.local.fs, "md5") hash2 = obj.hash_info assert hash1 == hash2 @@ -274,7 +274,7 @@ def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker, local_remote): # foo .dir file should be uploaded after bar odb = dvc.cloud.get_remote_odb("upstream") foo_path = odb.hash_to_path_info(foo.hash_info.value) - bar_path = odb.hash_to_path_info(foo.obj.trie[("bar",)].hash_info.value) + bar_path = odb.hash_to_path_info(foo.obj.trie[("bar",)][1].hash_info.value) paths = [args[1] for args, _ in mocked_upload.call_args_list] assert paths.index(foo_path) > paths.index(bar_path) @@ -424,7 +424,7 @@ def test_push_incomplete_dir(tmp_dir, dvc, mocker, local_remote): odb = dvc.odb.local out = stage.outs[0] - file_objs = [entry_obj for _, entry_obj in out.obj] + file_objs = [entry_obj for _, _, entry_obj in out.obj] # remove one of the cache files for directory remove(odb.hash_to_path_info(file_objs[0].hash_info.value)) diff --git a/tests/func/test_repo_index.py b/tests/func/test_repo_index.py index 31c981a6c..3cfda15ac 100644 --- a/tests/func/test_repo_index.py +++ b/tests/func/test_repo_index.py @@ -272,7 +272,6 @@ def test_used_objs(tmp_dir, scm, dvc, run_copy, rev): HashInfo( name="md5", value="acbd18db4cc2f85cedef654fccc4a4d8", - size=3, obj_name="bar", ), HashInfo( @@ -283,8 +282,6 @@ def test_used_objs(tmp_dir, scm, dvc, run_copy, rev): HashInfo( name="md5", value="d28c9e28591aeb7e303dc6772ffa6f6b.dir", - size=4, - nfiles=1, obj_name="dir", ), ] diff --git a/tests/func/test_state.py b/tests/func/test_state.py index f69afbad6..c0c615371 100644 --- a/tests/func/test_state.py +++ b/tests/func/test_state.py @@ -17,17 +17,17 @@ def test_state(tmp_dir, dvc): state = State(dvc.root_dir, dvc.tmp_dir, dvc.dvcignore) state.save(path_info, dvc.fs, hash_info) - assert state.get(path_info, dvc.fs) == hash_info + assert state.get(path_info, dvc.fs)[1] == hash_info path.unlink() path.write_text("1") - assert state.get(path_info, dvc.fs) is None + assert state.get(path_info, dvc.fs) == (None, None) hash_info = HashInfo("md5", file_md5(path, dvc.fs)) state.save(path_info, dvc.fs, hash_info) - assert state.get(path_info, dvc.fs) == hash_info + assert state.get(path_info, dvc.fs)[1] == hash_info def test_state_overflow(tmp_dir, dvc): diff --git a/tests/unit/fs/test_dvc.py b/tests/unit/fs/test_dvc.py index 9c0b789e3..b7595789c 100644 --- a/tests/unit/fs/test_dvc.py +++ b/tests/unit/fs/test_dvc.py @@ -243,14 +243,14 @@ def test_get_hash_granular(tmp_dir, dvc): fs = DvcFileSystem(repo=dvc) subdir = PathInfo(tmp_dir) / "dir" / "subdir" assert fs.info(subdir).get("md5") is None - _, obj = stage(dvc.odb.local, subdir, fs, "md5", dry_run=True) + _, _, obj = stage(dvc.odb.local, subdir, fs, "md5", dry_run=True) assert obj.hash_info == HashInfo( "md5", "af314506f1622d107e0ed3f14ec1a3b5.dir" ) assert ( fs.info(subdir / "data")["md5"] == "8d777f385d3dfec8815d20f7496026dc" ) - _, obj = stage(dvc.odb.local, subdir / "data", fs, "md5", dry_run=True) + _, _, obj = stage(dvc.odb.local, subdir / "data", fs, "md5", dry_run=True) assert obj.hash_info == HashInfo("md5", "8d777f385d3dfec8815d20f7496026dc") @@ -261,7 +261,7 @@ def test_get_hash_dirty_file(tmp_dir, dvc): fs = DvcFileSystem(repo=dvc) expected = "8c7dd922ad47494fc02c388e12c00eac" assert fs.info(PathInfo(tmp_dir) / "file").get("md5") == expected - _, obj = stage( + _, _, obj = stage( dvc.odb.local, PathInfo(tmp_dir) / "file", fs, "md5", dry_run=True ) assert obj.hash_info == HashInfo("md5", expected) @@ -274,7 +274,7 @@ def test_get_hash_dirty_dir(tmp_dir, dvc): fs = DvcFileSystem(repo=dvc) expected = "5ea40360f5b4ec688df672a4db9c17d1.dir" assert fs.info(PathInfo(tmp_dir) / "dir").get("md5") == expected - _, obj = stage( + _, _, obj = stage( dvc.odb.local, PathInfo(tmp_dir) / "dir", fs, "md5", dry_run=True ) assert obj.hash_info == HashInfo("md5", expected) diff --git a/tests/unit/fs/test_repo.py b/tests/unit/fs/test_repo.py index e927d32c6..baff82200 100644 --- a/tests/unit/fs/test_repo.py +++ b/tests/unit/fs/test_repo.py @@ -513,7 +513,7 @@ def test_get_hash_cached_file(tmp_dir, dvc, mocker): fs = RepoFileSystem(repo=dvc) expected = "acbd18db4cc2f85cedef654fccc4a4d8" assert fs.info(PathInfo(tmp_dir) / "foo").get("md5") is None - _, obj = stage(dvc.odb.local, PathInfo(tmp_dir) / "foo", fs, "md5") + _, _, obj = stage(dvc.odb.local, PathInfo(tmp_dir) / "foo", fs, "md5") assert obj.hash_info == HashInfo("md5", expected) (tmp_dir / "foo").unlink() assert fs.info(PathInfo(tmp_dir) / "foo")["md5"] == expected @@ -526,14 +526,14 @@ def test_get_hash_cached_dir(tmp_dir, dvc, mocker): fs = RepoFileSystem(repo=dvc) expected = "8761c4e9acad696bee718615e23e22db.dir" assert fs.info(PathInfo(tmp_dir) / "dir").get("md5") is None - _, obj = stage(dvc.odb.local, PathInfo(tmp_dir) / "dir", fs, "md5") + _, _, obj = stage(dvc.odb.local, PathInfo(tmp_dir) / "dir", fs, "md5") assert obj.hash_info == HashInfo( "md5", "8761c4e9acad696bee718615e23e22db.dir" ) shutil.rmtree(tmp_dir / "dir") assert fs.info(PathInfo(tmp_dir) / "dir")["md5"] == expected - _, obj = stage(dvc.odb.local, PathInfo(tmp_dir) / "dir", fs, "md5") + _, _, obj = stage(dvc.odb.local, PathInfo(tmp_dir) / "dir", fs, "md5") assert obj.hash_info == HashInfo( "md5", "8761c4e9acad696bee718615e23e22db.dir" ) @@ -546,12 +546,12 @@ def test_get_hash_cached_granular(tmp_dir, dvc, mocker): fs = RepoFileSystem(repo=dvc) subdir = PathInfo(tmp_dir) / "dir" / "subdir" assert fs.info(subdir).get("md5") is None - _, obj = stage(dvc.odb.local, subdir, fs, "md5") + _, _, obj = stage(dvc.odb.local, subdir, fs, "md5") assert obj.hash_info == HashInfo( "md5", "af314506f1622d107e0ed3f14ec1a3b5.dir" ) assert fs.info(subdir / "data").get("md5") is None - _, obj = stage(dvc.odb.local, subdir / "data", fs, "md5") + _, _, obj = stage(dvc.odb.local, subdir / "data", fs, "md5") assert obj.hash_info == HashInfo("md5", "8d777f385d3dfec8815d20f7496026dc") (tmp_dir / "dir" / "subdir" / "data").unlink() assert ( @@ -573,7 +573,7 @@ def test_get_hash_mixed_dir(tmp_dir, scm, dvc): clean_staging() fs = RepoFileSystem(repo=dvc) - _, obj = stage(dvc.odb.local, PathInfo(tmp_dir) / "dir", fs, "md5") + _, _, obj = stage(dvc.odb.local, PathInfo(tmp_dir) / "dir", fs, "md5") assert obj.hash_info == HashInfo( "md5", "e1d9e8eae5374860ae025ec84cfd85c7.dir" ) @@ -596,7 +596,9 @@ def test_get_hash_dirty_file(tmp_dir, dvc): # get_file_hash(file) should return workspace hash, not DVC cached hash fs = RepoFileSystem(repo=dvc) assert fs.info(PathInfo(tmp_dir) / "file").get("md5") is None - staging, obj = stage(dvc.odb.local, PathInfo(tmp_dir) / "file", fs, "md5") + staging, _, obj = stage( + dvc.odb.local, PathInfo(tmp_dir) / "file", fs, "md5" + ) assert obj.hash_info == something_hash_info check(staging, obj) @@ -608,14 +610,14 @@ def test_get_hash_dirty_file(tmp_dir, dvc): # get_file_hash(file) should return DVC cached hash assert fs.info(PathInfo(tmp_dir) / "file")["md5"] == file_hash_info.value - hash_info = get_file_hash( + _, hash_info = get_file_hash( PathInfo(tmp_dir) / "file", fs, "md5", state=dvc.state ) assert hash_info == file_hash_info # tmp_dir/file can be staged even though it is missing in workspace since # repofs will use the DVC cached hash (and refer to the local cache object) - _, obj = stage(dvc.odb.local, PathInfo(tmp_dir) / "file", fs, "md5") + _, _, obj = stage(dvc.odb.local, PathInfo(tmp_dir) / "file", fs, "md5") assert obj.hash_info == file_hash_info @@ -625,11 +627,11 @@ def test_get_hash_dirty_dir(tmp_dir, dvc): clean_staging() fs = RepoFileSystem(repo=dvc) - _, obj = stage(dvc.odb.local, PathInfo(tmp_dir) / "dir", fs, "md5") + _, meta, obj = stage(dvc.odb.local, PathInfo(tmp_dir) / "dir", fs, "md5") assert obj.hash_info == HashInfo( "md5", "ba75a2162ca9c29acecb7957105a0bc2.dir" ) - assert obj.hash_info.nfiles == 3 + assert meta.nfiles == 3 @pytest.mark.parametrize("traverse_subrepos", [True, False]) diff --git a/tests/unit/objects/db/test_local.py b/tests/unit/objects/db/test_local.py index da305b9a1..2c71d3960 100644 --- a/tests/unit/objects/db/test_local.py +++ b/tests/unit/objects/db/test_local.py @@ -94,7 +94,7 @@ def test_staging_file(tmp_dir, dvc): fs = LocalFileSystem() local_odb = dvc.odb.local - staging_odb, obj = stage(local_odb, tmp_dir / "foo", fs, "md5") + staging_odb, _, obj = stage(local_odb, tmp_dir / "foo", fs, "md5") assert not local_odb.exists(obj.hash_info) assert staging_odb.exists(obj.hash_info) @@ -121,7 +121,7 @@ def test_staging_dir(tmp_dir, dvc): fs = LocalFileSystem() local_odb = dvc.odb.local - staging_odb, obj = stage(local_odb, tmp_dir / "dir", fs, "md5") + staging_odb, _, obj = stage(local_odb, tmp_dir / "dir", fs, "md5") assert not local_odb.exists(obj.hash_info) assert staging_odb.exists(obj.hash_info) diff --git a/tests/unit/objects/test_tree.py b/tests/unit/objects/test_tree.py index fc6953451..cc6d1783d 100644 --- a/tests/unit/objects/test_tree.py +++ b/tests/unit/objects/test_tree.py @@ -4,6 +4,7 @@ import pytest from dvc.hash_info import HashInfo from dvc.objects.file import HashFile +from dvc.objects.meta import Meta from dvc.objects.tree import Tree, _merge @@ -19,10 +20,10 @@ from dvc.objects.tree import Tree, _merge {"md5": "456", "relpath": "bar"}, ], { - ("zzz",): HashFile(None, None, HashInfo("md5", "def")), - ("foo",): HashFile(None, None, HashInfo("md5", "123")), - ("bar",): HashFile(None, None, HashInfo("md5", "456")), - ("aaa",): HashFile(None, None, HashInfo("md5", "abc")), + ("zzz",): (None, HashFile(None, None, HashInfo("md5", "def"))), + ("foo",): (None, HashFile(None, None, HashInfo("md5", "123"))), + ("bar",): (None, HashFile(None, None, HashInfo("md5", "456"))), + ("aaa",): (None, HashFile(None, None, HashInfo("md5", "abc"))), }, ), ( @@ -38,20 +39,32 @@ from dvc.objects.tree import Tree, _merge {"md5": "pqr", "relpath": "dir/subdir/a"}, ], { - ("dir", "b"): HashFile(None, None, HashInfo("md5", "123")), - ("dir", "z"): HashFile(None, None, HashInfo("md5", "456")), - ("dir", "a"): HashFile(None, None, HashInfo("md5", "789")), - ("b",): HashFile(None, None, HashInfo("md5", "abc")), - ("a",): HashFile(None, None, HashInfo("md5", "def")), - ("z",): HashFile(None, None, HashInfo("md5", "ghi")), - ("dir", "subdir", "b"): HashFile( - None, None, HashInfo("md5", "jkl") + ("dir", "b"): ( + None, + HashFile(None, None, HashInfo("md5", "123")), ), - ("dir", "subdir", "z"): HashFile( - None, None, HashInfo("md5", "mno") + ("dir", "z"): ( + None, + HashFile(None, None, HashInfo("md5", "456")), ), - ("dir", "subdir", "a"): HashFile( - None, None, HashInfo("md5", "pqr") + ("dir", "a"): ( + None, + HashFile(None, None, HashInfo("md5", "789")), + ), + ("b",): (None, HashFile(None, None, HashInfo("md5", "abc"))), + ("a",): (None, HashFile(None, None, HashInfo("md5", "def"))), + ("z",): (None, HashFile(None, None, HashInfo("md5", "ghi"))), + ("dir", "subdir", "b"): ( + None, + HashFile(None, None, HashInfo("md5", "jkl")), + ), + ("dir", "subdir", "z"): ( + None, + HashFile(None, None, HashInfo("md5", "mno")), + ), + ("dir", "subdir", "a"): ( + None, + HashFile(None, None, HashInfo("md5", "pqr")), ), }, ), @@ -63,64 +76,25 @@ def test_list(lst, trie_dict): assert tree.as_list() == sorted(lst, key=itemgetter("relpath")) -@pytest.mark.parametrize( - "trie_dict, size", - [ - ({}, 0), - ( - { - ("a",): HashFile(None, None, HashInfo("md5", "abc", size=1)), - ("b",): HashFile(None, None, HashInfo("md5", "def", size=2)), - ("c",): HashFile(None, None, HashInfo("md5", "ghi", size=3)), - ("dir", "foo"): HashFile( - None, None, HashInfo("md5", "jkl", size=4) - ), - ("dir", "bar"): HashFile( - None, None, HashInfo("md5", "mno", size=5) - ), - ("dir", "baz"): HashFile( - None, None, HashInfo("md5", "pqr", size=6) - ), - }, - 21, - ), - ( - { - ("a",): HashFile(None, None, HashInfo("md5", "abc", size=1)), - ("b",): HashFile( - None, None, HashInfo("md5", "def", size=None) - ), - }, - None, - ), - ], -) -def test_size(trie_dict, size): - tree = Tree(None, None, None) - tree._dict = trie_dict - tree.digest() - assert tree.size == size - - @pytest.mark.parametrize( "trie_dict, nfiles", [ ({}, 0), ( { - ("a",): HashInfo("md5", "abc", size=1), - ("b",): HashInfo("md5", "def", size=2), - ("c",): HashInfo("md5", "ghi", size=3), - ("dir", "foo"): HashInfo("md5", "jkl", size=4), - ("dir", "bar"): HashInfo("md5", "mno", size=5), - ("dir", "baz"): HashInfo("md5", "pqr", size=6), + ("a",): (Meta(size=1), HashInfo("md5", "abc")), + ("b",): (Meta(size=2), HashInfo("md5", "def")), + ("c",): (Meta(size=3), HashInfo("md5", "ghi")), + ("dir", "foo"): (Meta(size=4), HashInfo("md5", "jkl")), + ("dir", "bar"): (Meta(size=5), HashInfo("md5", "mno")), + ("dir", "baz"): (Meta(size=6), HashInfo("md5", "pqr")), }, 6, ), ( { - ("a",): HashInfo("md5", "abc", size=1), - ("b",): HashInfo("md5", "def", size=None), + ("a",): (Meta(size=1), HashInfo("md5", "abc")), + ("b",): (Meta(), HashInfo("md5", "def")), }, 2, ), @@ -137,22 +111,24 @@ def test_nfiles(trie_dict, nfiles): [ {}, { - ("a",): HashInfo("md5", "abc"), - ("b",): HashInfo("md5", "def"), - ("c",): HashInfo("md5", "ghi"), - ("dir", "foo"): HashInfo("md5", "jkl"), - ("dir", "bar"): HashInfo("md5", "mno"), - ("dir", "baz"): HashInfo("md5", "pqr"), - ("dir", "subdir", "1"): HashInfo("md5", "stu"), - ("dir", "subdir", "2"): HashInfo("md5", "vwx"), - ("dir", "subdir", "3"): HashInfo("md5", "yz"), + ("a",): (None, HashInfo("md5", "abc")), + ("b",): (None, HashInfo("md5", "def")), + ("c",): (None, HashInfo("md5", "ghi")), + ("dir", "foo"): (None, HashInfo("md5", "jkl")), + ("dir", "bar"): (None, HashInfo("md5", "mno")), + ("dir", "baz"): (None, HashInfo("md5", "pqr")), + ("dir", "subdir", "1"): (None, HashInfo("md5", "stu")), + ("dir", "subdir", "2"): (None, HashInfo("md5", "vwx")), + ("dir", "subdir", "3"): (None, HashInfo("md5", "yz")), }, ], ) def test_items(trie_dict): tree = Tree(None, None, None) tree._dict = trie_dict - assert list(tree) == list(trie_dict.items()) + assert list(tree) == [ + (key, value[0], value[1]) for key, value in trie_dict.items() + ] @pytest.mark.parametrize( diff --git a/tests/unit/output/test_local.py b/tests/unit/output/test_local.py index a071eaf01..c78c1bd40 100644 --- a/tests/unit/output/test_local.py +++ b/tests/unit/output/test_local.py @@ -2,6 +2,7 @@ import os from unittest.mock import patch from dvc.hash_info import HashInfo +from dvc.objects.meta import Meta from dvc.output import Output from dvc.stage import Stage from dvc.utils import relpath @@ -63,7 +64,8 @@ class TestGetFilesNumber(TestDvc): def test_return_multiple_for_dir(self): o = self._get_output() - o.hash_info = HashInfo("md5", "12345678.dir", nfiles=2) + o.hash_info = HashInfo("md5", "12345678.dir") + o.meta = Meta(nfiles=2) self.assertEqual(2, o.get_files_number()) @patch.object(Output, "is_dir_checksum", False)