diff --git a/dvc/api.py b/dvc/api.py index c6420b840..56ec03b70 100644 --- a/dvc/api.py +++ b/dvc/api.py @@ -4,7 +4,6 @@ from contextlib import _GeneratorContextManager as GCM from funcy import reraise from dvc.exceptions import OutputNotFoundError, PathMissingError -from dvc.objects.stage import get_hash from dvc.path_info import PathInfo from dvc.repo import Repo @@ -29,8 +28,8 @@ def get_url(path, repo=None, rev=None, remote=None): raise OutputNotFoundError(path, repo) cloud = metadata.repo.cloud - hash_info = get_hash(path_info, _repo.repo_fs, "md5", _repo.odb) - return cloud.get_url_for(remote, checksum=hash_info.value) + md5 = metadata.repo.dvcfs.info(path_info)["md5"] + return cloud.get_url_for(remote, checksum=md5) def open( # noqa, pylint: disable=redefined-builtin diff --git a/dvc/checkout.py b/dvc/checkout.py index 6a0034833..606d60755 100644 --- a/dvc/checkout.py +++ b/dvc/checkout.py @@ -10,7 +10,7 @@ from dvc.exceptions import ( DvcException, ) from dvc.objects import ObjectFormatError, check, load -from dvc.objects.stage import get_hash +from dvc.objects.stage import stage from dvc.remote.slow_link_detection import ( # type: ignore[attr-defined] slow_link_guard, ) @@ -30,7 +30,7 @@ def _changed(path_info, fs, obj, cache): return True try: - actual = get_hash(path_info, fs, obj.hash_info.name, cache) + actual = stage(cache, path_info, fs, obj.hash_info.name).hash_info except FileNotFoundError: logger.debug("'%s' doesn't exist.", path_info) return True @@ -56,7 +56,7 @@ def _remove(path_info, fs, cache, force=False): fs.remove(path_info) return - current = get_hash(path_info, fs, fs.PARAM_CHECKSUM, cache) + current = stage(cache, path_info, fs, fs.PARAM_CHECKSUM).hash_info try: obj = load(cache, current) check(cache, obj) @@ -174,10 +174,10 @@ def _checkout_file( return modified -def _remove_redundant_files(path_info, fs, dir_info, cache, force): +def _remove_redundant_files(path_info, fs, obj, cache, force): existing_files = set(fs.walk_files(path_info)) - needed_files = {path_info.joinpath(*key) for key, _ in dir_info.items()} + needed_files = {path_info.joinpath(*key) for key, _ in obj} redundant_files = existing_files - needed_files for path in redundant_files: _remove(path, fs, cache, force) @@ -197,11 +197,11 @@ def _checkout_dir( logger.debug("Linking directory '%s'.", path_info) - for entry_key, entry_hash_info in obj.hash_info.dir_info.items(): + for entry_key, entry_obj in obj: entry_modified = _checkout_file( path_info.joinpath(*entry_key), fs, - cache.get(entry_hash_info), + entry_obj, cache, force, progress_callback, @@ -211,10 +211,7 @@ def _checkout_dir( modified = True modified = ( - _remove_redundant_files( - path_info, fs, obj.hash_info.dir_info, cache, force - ) - or modified + _remove_redundant_files(path_info, fs, obj, cache, force) or modified ) fs.repo.state.save(path_info, fs, obj.hash_info) diff --git a/dvc/dependency/repo.py b/dvc/dependency/repo.py index 0014b4655..909080a8e 100644 --- a/dvc/dependency/repo.py +++ b/dvc/dependency/repo.py @@ -55,6 +55,7 @@ class RepoDependency(LocalDependency): self.repo.odb.local, path_info, repo.repo_fs, + self.repo.odb.local.fs.PARAM_CHECKSUM, follow_subrepos=False, ).hash_info @@ -94,7 +95,12 @@ class RepoDependency(LocalDependency): except (NoOutputOrStageError, NoRemoteError): pass obj = stage( - odb, path_info, repo.repo_fs, jobs=jobs, follow_subrepos=False, + odb, + path_info, + repo.repo_fs, + odb.fs.PARAM_CHECKSUM, + jobs=jobs, + follow_subrepos=False, ) save(odb, obj, jobs=jobs) diff --git a/dvc/dir_info.py b/dvc/dir_info.py deleted file mode 100644 index beea89a20..000000000 --- a/dvc/dir_info.py +++ /dev/null @@ -1,117 +0,0 @@ -import posixpath -from operator import itemgetter - -from funcy import cached_property -from pygtrie import Trie - -from .hash_info import HashInfo - - -def _diff(ancestor, other, allow_removed=False): - from dictdiffer import diff - - from dvc.exceptions import MergeError - - allowed = ["add"] - if allow_removed: - allowed.append("remove") - - result = list(diff(ancestor, other)) - for typ, _, _ in result: - if typ not in allowed: - raise MergeError( - "unable to auto-merge directories with diff that contains " - f"'{typ}'ed files" - ) - return result - - -def _merge(ancestor, our, their): - import copy - - from dictdiffer import patch - - our_diff = _diff(ancestor, our) - if not our_diff: - return copy.deepcopy(their) - - their_diff = _diff(ancestor, their) - if not their_diff: - return copy.deepcopy(our) - - # make sure there are no conflicting files - _diff(our, their, allow_removed=True) - - return patch(our_diff + their_diff, ancestor) - - -class DirInfo: - PARAM_RELPATH = "relpath" - - def __init__(self): - self._dict = {} - - def as_dict(self): - return self._dict.copy() - - def add(self, key, hash_info): - self.__dict__.pop("trie", None) - self._dict[key] = hash_info - - def get(self, key, default=None): - return self._dict.get(key, default) - - @cached_property - def trie(self): - return Trie(self._dict) - - @property - def size(self): - try: - return sum(hash_info.size for _, hash_info in self._dict.items()) - except TypeError: - return None - - @property - def nfiles(self): - return len(self._dict) - - def items(self): - for key, hash_info in self._dict.items(): - yield key, hash_info - - @classmethod - def from_list(cls, lst): - ret = DirInfo() - for _entry in lst: - entry = _entry.copy() - relpath = entry.pop(cls.PARAM_RELPATH) - parts = tuple(relpath.split(posixpath.sep)) - hash_info = HashInfo.from_dict(entry) - ret.add(parts, hash_info) - return ret - - def to_list(self): - # Sorting the list by path to ensure reproducibility - return sorted( - ( - { - # NOTE: not using hash_info.to_dict() because we don't want - # size/nfiles fields at this point. - hash_info.name: hash_info.value, - self.PARAM_RELPATH: posixpath.sep.join(parts), - } - for parts, hash_info in self._dict.items() # noqa: B301 - ), - key=itemgetter(self.PARAM_RELPATH), - ) - - def merge(self, ancestor, their): - merged_dict = _merge( - ancestor.as_dict(), self.as_dict(), their.as_dict(), - ) - - merged = DirInfo() - for key, hi in merged_dict.items(): - merged.add(key, hi) - return merged diff --git a/dvc/fs/dvc.py b/dvc/fs/dvc.py index ea07d7b53..a49701cc7 100644 --- a/dvc/fs/dvc.py +++ b/dvc/fs/dvc.py @@ -48,9 +48,9 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method # NOTE: use string paths here for performance reasons key = tuple(relpath(path_info, out.path_info).split(os.sep)) out.get_dir_cache(remote=remote) - file_hash = out.hash_info.dir_info.get(key) - if file_hash: - return file_hash + obj = out.obj.trie.get(key) + if obj: + return obj.hash_info raise FileNotFoundError def open( # type: ignore @@ -131,21 +131,14 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method # pull dir cache if needed out.get_dir_cache(**kwargs) - dir_cache = out.dir_cache - if not dir_cache: - raise FileNotFoundError - - from dvc.objects import Tree - - hash_info = Tree.save_dir_info(out.odb, dir_cache) - if hash_info != out.hash_info: + if not out.obj: raise FileNotFoundError def _add_dir(self, trie, out, **kwargs): self._fetch_dir(out, **kwargs) base = out.path_info.parts - for key, _ in out.dir_cache.items(): # noqa: B301 + for key, _ in out.obj: # noqa: B301 trie[base + key] = None def _walk(self, root, trie, topdown=True, **kwargs): @@ -241,9 +234,9 @@ class DvcFileSystem(BaseFileSystem): # pylint:disable=abstract-method elif meta.part_of_output: (out,) = meta.outs key = path_info.relative_to(out.path_info).parts - hash_info = out.hash_info.dir_info.trie.get(key) - if hash_info: - ret["size"] = hash_info.size - ret[hash_info.name] = hash_info.value + obj = out.obj.trie.get(key) + if obj: + ret["size"] = obj.size + ret[obj.hash_info.name] = obj.hash_info.value return ret diff --git a/dvc/fs/memory.py b/dvc/fs/memory.py index 20f8e7e34..c9b2ef628 100644 --- a/dvc/fs/memory.py +++ b/dvc/fs/memory.py @@ -13,13 +13,16 @@ class MemoryFileSystem(BaseFileSystem): self.fs = MemFS() def exists(self, path_info, use_dvcignore=True): - return self.fs.exists(path_info.path) + return self.fs.exists(path_info.fspath) def open(self, path_info, mode="r", encoding=None, **kwargs): return self.fs.open( path_info.fspath, mode=mode, encoding=encoding, **kwargs ) + def info(self, path_info): + return self.fs.info(path_info.fspath) + def stat(self, path_info): import os diff --git a/dvc/hash_info.py b/dvc/hash_info.py index 077faf1f1..6fba0c21f 100644 --- a/dvc/hash_info.py +++ b/dvc/hash_info.py @@ -1,11 +1,9 @@ from collections import OrderedDict from dataclasses import dataclass, field -from typing import Dict, Optional +from typing import Optional HASH_DIR_SUFFIX = ".dir" -DirInfo = Dict[str, str] - @dataclass class HashInfo: @@ -14,7 +12,6 @@ class HashInfo: name: Optional[str] value: Optional[str] - dir_info: Optional[DirInfo] = field(default=None, compare=False) size: Optional[int] = field(default=None, compare=False) nfiles: Optional[int] = field(default=None, compare=False) diff --git a/dvc/objects/__init__.py b/dvc/objects/__init__.py index eb2656b93..8b8c9c611 100644 --- a/dvc/objects/__init__.py +++ b/dvc/objects/__init__.py @@ -1,13 +1,11 @@ import itertools -import json import logging from concurrent import futures -from dvc.dir_info import DirInfo from dvc.exceptions import DvcException from dvc.progress import Tqdm -from .stage import get_hash +from .stage import get_file_hash logger = logging.getLogger(__name__) @@ -26,6 +24,12 @@ class HashFile: self.fs = fs self.hash_info = hash_info + @property + def size(self): + if not (self.path_info and self.fs): + return None + return self.fs.getsize(self.path_info) + def __len__(self): return 1 @@ -35,8 +39,19 @@ class HashFile: def __bool__(self): return bool(self.hash_info) + def __eq__(self, other): + if not isinstance(other, HashFile): + return False + return ( + self.path_info == other.path_info + and self.fs == other.fs + and self.hash_info == other.hash_info + ) + def check(self, odb): - actual = get_hash(self.path_info, self.fs, odb.fs.PARAM_CHECKSUM, odb) + actual = get_file_hash( + self.path_info, self.fs, self.hash_info.name, odb.repo.state + ) logger.trace( "cache '%s' expected '%s' actual '%s'", @@ -53,187 +68,32 @@ class HashFile: def load(cls, odb, hash_info): return odb.get(hash_info) - def save(self, odb, **kwargs): - odb.add(self.path_info, self.fs, self.hash_info, **kwargs) - - -class File(HashFile): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.src = None - - @classmethod - def stage(cls, odb, path_info, fs, **kwargs): - hash_info = get_hash( - path_info, fs, odb.fs.PARAM_CHECKSUM, odb, **kwargs - ) - raw = odb.get(hash_info) - obj = cls(raw.path_info, raw.fs, hash_info) - obj.src = HashFile(path_info, fs, hash_info) - return obj - - def save(self, odb, **kwargs): - self.src.save(odb, **kwargs) - - -class Tree(HashFile): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.src_hash_info = None - self.src_path_info = None - self.src_fs = None - - def __len__(self): - return self.hash_info.dir_info.nfiles - - def __iter__(self): - yield from self.hash_info.dir_info.items() - - @classmethod - def save_dir_info(cls, odb, dir_info, hash_info=None): - if hash_info and hash_info.name == odb.fs.PARAM_CHECKSUM: - try: - odb.check(hash_info) - assert hash_info.dir_info == dir_info - return hash_info - except (FileNotFoundError, ObjectFormatError): - pass - - from dvc.fs.memory import MemoryFileSystem - from dvc.path_info import PathInfo - from dvc.utils import tmp_fname - - fs = MemoryFileSystem(None, {}) - path_info = PathInfo(tmp_fname("")) - with fs.open(path_info, "w") as fobj: - json.dump(dir_info.to_list(), fobj, sort_keys=True) - - tmp_info = odb.fs.path_info / tmp_fname("") - with fs.open(path_info, "rb") as fobj: - odb.fs.upload_fobj(fobj, tmp_info) - - hash_info = get_hash(tmp_info, odb.fs, odb.fs.PARAM_CHECKSUM, odb) - hash_info.value += odb.fs.CHECKSUM_DIR_SUFFIX - hash_info.dir_info = dir_info - hash_info.nfiles = dir_info.nfiles - - odb.add(tmp_info, odb.fs, hash_info) - - return hash_info - - @classmethod - def stage(cls, odb, path_info, fs, **kwargs): - hash_info = get_hash( - path_info, fs, odb.fs.PARAM_CHECKSUM, odb, **kwargs - ) - hi = cls.save_dir_info(odb, hash_info.dir_info, hash_info) - hi.size = hash_info.size - raw = odb.get(hi) - obj = cls(raw.path_info, raw.fs, hi) - obj.src_hash_info = hash_info - obj.src_path_info = path_info - obj.src_fs = fs - return obj - - @classmethod - def load(cls, odb, hash_info): - - obj = odb.get(hash_info) - - try: - with obj.fs.open(obj.path_info, "r") as fobj: - raw = json.load(fobj) - except ValueError as exc: - raise ObjectFormatError(f"{obj} is corrupted") from exc - - if not isinstance(raw, list): - logger.error( - "dir cache file format error '%s' [skipping the file]", - obj.path_info, - ) - raise ObjectFormatError(f"{obj} is corrupted") - - dir_info = DirInfo.from_list(raw) - hash_info.dir_info = dir_info - hash_info.nfiles = dir_info.nfiles - - return cls(obj.path_info, obj.fs, hash_info) - - def save(self, odb, **kwargs): - assert self.src_hash_info.dir_info - hi = self.save_dir_info( - odb, self.src_hash_info.dir_info, self.hash_info - ) - for entry_key, entry_hash in Tqdm( - hi.dir_info.items(), - desc="Saving " + self.src_path_info.name, - unit="file", - ): - entry_info = self.src_path_info.joinpath(*entry_key) - entry_obj = HashFile(entry_info, self.src_fs, entry_hash) - entry_obj.save(odb, **kwargs) - self.src_fs.repo.state.save(self.src_path_info, self.src_fs, hi) - - def filter(self, odb, prefix): - hash_info = self.hash_info.dir_info.get(prefix) - if hash_info: - return load(odb, hash_info) - - depth = len(prefix) - dir_info = DirInfo() - try: - for key, value in self.hash_info.dir_info.trie.items(prefix): - dir_info.add(key[depth:], value) - except KeyError: - return None - - return load(odb, self.save_dir_info(odb, dir_info)) - def save(odb, obj, **kwargs): - obj.save(odb, **kwargs) + from .tree import Tree + + if isinstance(obj, Tree): + for _, entry in Tqdm(obj): + odb.add(entry.path_info, entry.fs, entry.hash_info, **kwargs) + odb.add(obj.path_info, obj.fs, obj.hash_info, **kwargs) def check(odb, obj): + from .tree import Tree + odb.check(obj.hash_info) if isinstance(obj, Tree): - for _, hash_info in obj: - odb.check(hash_info) + for _, entry in obj: + odb.check(entry.hash_info) def load(odb, hash_info): + from .tree import Tree + if hash_info.isdir: return Tree.load(odb, hash_info) - return File.load(odb, hash_info) - - -def _get_dir_size(odb, dir_info): - try: - return sum( - odb.fs.getsize(odb.hash_to_path_info(hi.value)) - for _, hi in dir_info.items() - ) - except FileNotFoundError: - return None - - -def merge(odb, ancestor_info, our_info, their_info): - assert our_info - assert their_info - - if ancestor_info: - ancestor = load(odb, ancestor_info).hash_info.dir_info - else: - ancestor = DirInfo() - - our = load(odb, our_info).hash_info.dir_info - their = load(odb, their_info).hash_info.dir_info - - merged = our.merge(ancestor, their) - hash_info = Tree.save_dir_info(odb, merged) - hash_info.size = _get_dir_size(odb, merged) - return hash_info + return odb.get(hash_info) def _transfer_file(odb, from_fs, from_info): @@ -286,17 +146,21 @@ def _transfer_directory_contents(odb, from_fs, from_info, jobs, pbar): def _transfer_directory(odb, from_fs, from_info, jobs, no_progress_bar=False): - dir_info = DirInfo() + from .tree import Tree + + tree = Tree(None, None, None) with Tqdm(total=1, unit="Files", disable=no_progress_bar) as pbar: for ( entry_info, (entry_tmp_info, entry_hash), ) in _transfer_directory_contents(odb, from_fs, from_info, jobs, pbar): - odb.add(entry_tmp_info, odb.fs, entry_hash) - dir_info.add(entry_info.parts, entry_hash) + obj = HashFile(entry_tmp_info, odb.fs, entry_hash) + tree.add(entry_info.parts, obj) - return Tree.save_dir_info(odb, dir_info) + tree.digest() + save(odb, tree) + return tree.hash_info def transfer(odb, from_fs, from_info, jobs=None, no_progress_bar=False): diff --git a/dvc/objects/stage.py b/dvc/objects/stage.py index f1dadf564..71a8a2d3b 100644 --- a/dvc/objects/stage.py +++ b/dvc/objects/stage.py @@ -9,7 +9,7 @@ from dvc.progress import Tqdm from dvc.utils import file_md5 -def get_file_hash(path_info, fs, name): +def _get_file_hash(path_info, fs, name): info = fs.info(path_info) if name in info: assert not info[name].endswith(".dir") @@ -27,20 +27,36 @@ def get_file_hash(path_info, fs, name): raise NotImplementedError -def _calculate_hashes(path_info, fs, name, state, **kwargs): - def _get_file_hash(file_info): +def get_file_hash(path_info, fs, name, state=None): + if state: hash_info = state.get( # pylint: disable=assignment-from-none - file_info, fs, + path_info, fs, ) - if not hash_info: - hash_info = get_file_hash(file_info, fs, name) - state.save(file_info, fs, hash_info) - return file_info, hash_info + if hash_info: + return hash_info + + if not fs.exists(path_info): + raise FileNotFoundError( + errno.ENOENT, os.strerror(errno.ENOENT), path_info + ) + + hash_info = _get_file_hash(path_info, fs, name) + + if state: + assert ".dir" not in hash_info.value + state.save(path_info, fs, hash_info) + + return hash_info + + +def _calculate_hashes(path_info, fs, name, state, **kwargs): + def _hash(file_info): + return file_info, get_file_hash(file_info, fs, name, state) with Tqdm( unit="md5", desc="Computing file/dir hashes (only done once)", ) as pbar: - worker = pbar.wrap_fn(_get_file_hash) + worker = pbar.wrap_fn(_hash) with ThreadPoolExecutor(max_workers=fs.hash_jobs) as executor: pairs = executor.map(worker, fs.walk_files(path_info, **kwargs)) return dict(pairs) @@ -60,14 +76,17 @@ def _iter_hashes(path_info, fs, name, state, **kwargs): yield from _calculate_hashes(path_info, fs, name, state, **kwargs).items() -def _collect_dir(path_info, fs, name, state, **kwargs): - from dvc.dir_info import DirInfo +def _build_tree(path_info, fs, name, state, **kwargs): + from . import HashFile + from .tree import Tree - dir_info = DirInfo() + tree = Tree(None, None, None) for fi, hi in _iter_hashes(path_info, fs, name, state, **kwargs): if DvcIgnore.DVCIGNORE_FILE == fi.name: raise DvcIgnoreInCollectedDirError(fi.parent) + obj = HashFile(fi, fs, hi) + # NOTE: this is lossy transformation: # "hey\there" -> "hey/there" # "hey/there" -> "hey/there" @@ -76,31 +95,44 @@ def _collect_dir(path_info, fs, name, state, **kwargs): # # Yes, this is a BUG, as long as we permit "/" in # filenames on Windows and "\" on Unix - dir_info.add(fi.relative_to(path_info).parts, hi) + tree.add(fi.relative_to(path_info).parts, obj) - return dir_info + tree.digest() + + return tree -def get_dir_hash(path_info, fs, name, odb, state, **kwargs): - from . import Tree +def _get_tree_obj(path_info, fs, name, odb, state, **kwargs): + from .tree import Tree value = fs.info(path_info).get(name) if value: hash_info = HashInfo(name, value) try: - Tree.load(odb, hash_info) - return hash_info + return Tree.load(odb, hash_info) except FileNotFoundError: pass - dir_info = _collect_dir(path_info, fs, name, state, **kwargs) - hash_info = Tree.save_dir_info(fs.repo.odb.local, dir_info) - hash_info.size = dir_info.size - hash_info.dir_info = dir_info - return hash_info + tree = _build_tree(path_info, fs, name, state, **kwargs) + + odb.add(tree.path_info, tree.fs, tree.hash_info) + if name != "md5": + # NOTE: used only for external outputs. Initial reasoning was to be + # able to validate .dir files right in the workspace (e.g. check s3 + # etag), but could be dropped for manual validation with regular md5, + # that would be universal for all clouds. + raw = odb.get(tree.hash_info) + hash_info = get_file_hash(raw.path_info, raw.fs, name, state) + tree.hash_info.name = hash_info.name + tree.hash_info.value = hash_info.value + if not tree.hash_info.value.endswith(".dir"): + tree.hash_info.value += ".dir" + odb.add(tree.path_info, tree.fs, tree.hash_info) + + return tree -def get_hash(path_info, fs, name, odb, **kwargs): +def stage(odb, path_info, fs, name, **kwargs): assert path_info and ( isinstance(path_info, str) or path_info.scheme == fs.scheme ) @@ -125,28 +157,31 @@ def get_hash(path_info, fs, name, odb, **kwargs): hash_info = None if hash_info: - if hash_info.isdir: - from . import Tree + from . import load + from .tree import Tree - # NOTE: loading the fs will restore hash_info.dir_info - Tree.load(odb, hash_info) - assert hash_info.name == name - return hash_info + obj = load(odb, hash_info) + if isinstance(obj, Tree): + obj.hash_info.nfiles = len(obj) + for key, entry in obj: + entry.fs = fs + entry.path_info = path_info.joinpath(*key) + else: + obj.fs = fs + obj.path_info = path_info + assert obj.hash_info.name == name + obj.hash_info.size = hash_info.size + return obj if fs.isdir(path_info): - hash_info = get_dir_hash(path_info, fs, name, odb, state, **kwargs) + obj = _get_tree_obj(path_info, fs, name, odb, state, **kwargs) else: - hash_info = get_file_hash(path_info, fs, name) + from . import HashFile - if hash_info and fs.exists(path_info): - state.save(path_info, fs, hash_info) + hash_info = get_file_hash(path_info, fs, name, state) + obj = HashFile(path_info, fs, hash_info) - return hash_info + if obj.hash_info and fs.exists(path_info): + state.save(path_info, fs, obj.hash_info) - -def stage(odb, path_info, fs, **kwargs): - from . import File, Tree - - if fs.isdir(path_info): - return Tree.stage(odb, path_info, fs, **kwargs) - return File.stage(odb, path_info, fs, **kwargs) + return obj diff --git a/dvc/objects/tree.py b/dvc/objects/tree.py new file mode 100644 index 000000000..b885515ac --- /dev/null +++ b/dvc/objects/tree.py @@ -0,0 +1,210 @@ +import json +import logging +import posixpath + +from funcy import cached_property + +from . import HashFile, ObjectFormatError +from .stage import get_file_hash + +logger = logging.getLogger(__name__) + + +class Tree(HashFile): + PARAM_RELPATH = "relpath" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._dict = {} + + @cached_property + def trie(self): + from pygtrie import Trie + + return Trie(self._dict) + + @property + def size(self): + try: + return sum(obj.size for _, obj in self) + except TypeError: + return None + + def add(self, key, obj): + self.__dict__.pop("trie", None) + self._dict[key] = obj + + def digest(self): + from dvc.fs.memory import MemoryFileSystem + from dvc.path_info import PathInfo + from dvc.utils import tmp_fname + + memfs = MemoryFileSystem(None, {}) + path_info = PathInfo(tmp_fname("")) + with memfs.open(path_info, "wb") as fobj: + fobj.write(self.as_bytes()) + self.fs = memfs + self.path_info = path_info + self.hash_info = get_file_hash(path_info, memfs, "md5") + self.hash_info.value += ".dir" + self.hash_info.size = self.size + self.hash_info.nfiles = len(self) + + def __len__(self): + return len(self._dict) + + def __iter__(self): + yield from self._dict.items() + + def as_dict(self): + return self._dict.copy() + + def as_list(self): + from operator import itemgetter + + # Sorting the list by path to ensure reproducibility + return sorted( + ( + { + # NOTE: not using hash_info.to_dict() because we don't want + # size/nfiles fields at this point. + obj.hash_info.name: obj.hash_info.value, + self.PARAM_RELPATH: posixpath.sep.join(parts), + } + for parts, obj in self._dict.items() # noqa: B301 + ), + key=itemgetter(self.PARAM_RELPATH), + ) + + def as_bytes(self): + return json.dumps(self.as_list(), sort_keys=True).encode("utf-8") + + @classmethod + def from_list(cls, lst): + from dvc.hash_info import HashInfo + + tree = cls(None, None, None) + for _entry in lst: + entry = _entry.copy() + relpath = entry.pop(cls.PARAM_RELPATH) + parts = tuple(relpath.split(posixpath.sep)) + hash_info = HashInfo.from_dict(entry) + obj = HashFile(None, None, hash_info) + tree.add(parts, obj) + return tree + + @classmethod + def load(cls, odb, hash_info): + + obj = odb.get(hash_info) + + try: + with obj.fs.open(obj.path_info, "r") as fobj: + raw = json.load(fobj) + except ValueError as exc: + raise ObjectFormatError(f"{obj} is corrupted") from exc + + if not isinstance(raw, list): + logger.error( + "dir cache file format error '%s' [skipping the file]", + obj.path_info, + ) + raise ObjectFormatError(f"{obj} is corrupted") + + tree = cls.from_list(raw) + tree.path_info = obj.path_info + tree.fs = obj.fs + tree.hash_info = hash_info + + return tree + + def filter(self, odb, prefix): + obj = self._dict.get(prefix) + if obj: + return obj + + depth = len(prefix) + tree = Tree(None, None, None) + try: + for key, obj in self.trie.items(prefix): + tree.add(key[depth:], obj) + except KeyError: + return None + tree.digest() + odb.add(tree.path_info, tree.fs, tree.hash_info) + return tree + + +def _get_dir_size(odb, tree): + try: + return sum( + odb.fs.getsize(odb.hash_to_path_info(obj.hash_info.value)) + for _, obj in tree + ) + except FileNotFoundError: + return None + + +def _diff(ancestor, other, allow_removed=False): + from dictdiffer import diff + + from dvc.exceptions import MergeError + + allowed = ["add"] + if allow_removed: + allowed.append("remove") + + result = list(diff(ancestor, other)) + for typ, _, _ in result: + if typ not in allowed: + raise MergeError( + "unable to auto-merge directories with diff that contains " + f"'{typ}'ed files" + ) + return result + + +def _merge(ancestor, our, their): + import copy + + from dictdiffer import patch + + our_diff = _diff(ancestor, our) + if not our_diff: + return copy.deepcopy(their) + + their_diff = _diff(ancestor, their) + if not their_diff: + return copy.deepcopy(our) + + # make sure there are no conflicting files + _diff(our, their, allow_removed=True) + + return patch(our_diff + their_diff, ancestor) + + +def merge(odb, ancestor_info, our_info, their_info): + from . import load + + assert our_info + assert their_info + + if ancestor_info: + ancestor = load(odb, ancestor_info) + else: + ancestor = Tree(None, None, None) + + our = load(odb, our_info) + their = load(odb, their_info) + + merged_dict = _merge(ancestor.as_dict(), our.as_dict(), their.as_dict(),) + + merged = Tree(None, None, None) + for key, hi in merged_dict.items(): + merged.add(key, hi) + merged.digest() + + odb.add(merged.path_info, merged.fs, merged.hash_info) + hash_info = merged.hash_info + hash_info.size = _get_dir_size(odb, merged) + return hash_info diff --git a/dvc/output/base.py b/dvc/output/base.py index 30565d218..5011e3e0f 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -18,7 +18,6 @@ from dvc.exceptions import ( ) from dvc.hash_info import HashInfo from dvc.objects.db import NamedCache -from dvc.objects.stage import get_hash from dvc.objects.stage import stage as ostage from ..fs.base import BaseFileSystem @@ -183,23 +182,21 @@ class BaseOutput: def odb(self): return getattr(self.repo.odb, self.scheme) - @property - def dir_cache(self): - return self.hash_info.dir_info - @property def cache_path(self): return self.odb.hash_to_path_info(self.hash_info.value).url def get_hash(self): if not self.use_cache: - return get_hash( + return ostage( + self.repo.odb.local, self.path_info, self.fs, self.fs.PARAM_CHECKSUM, - self.repo.odb.local, - ) - return ostage(self.odb, self.path_info, self.fs).hash_info + ).hash_info + return ostage( + self.odb, self.path_info, self.fs, self.odb.fs.PARAM_CHECKSUM + ).hash_info @property def is_dir_checksum(self): @@ -307,7 +304,9 @@ class BaseOutput: logger.debug("Output '%s' didn't change. Skipping saving.", self) return - self.obj = ostage(self.odb, self.path_info, self.fs) + self.obj = ostage( + self.odb, self.path_info, self.fs, self.odb.fs.PARAM_CHECKSUM + ) self.hash_info = self.obj.hash_info self.isexec = self.isfile() and self.fs.isexec(self.path_info) @@ -322,7 +321,12 @@ class BaseOutput: assert self.hash_info if self.use_cache: - obj = ostage(self.odb, filter_info or self.path_info, self.fs) + obj = ostage( + self.odb, + filter_info or self.path_info, + self.fs, + self.odb.fs.PARAM_CHECKSUM, + ) objects.save(self.odb, obj) checkout( filter_info or self.path_info, @@ -460,8 +464,6 @@ class BaseOutput: self.repo.scm.ignore(self.fspath) def get_files_number(self, filter_info=None): - from funcy import ilen - if not self.use_cache or not self.hash_info: return 0 @@ -471,13 +473,8 @@ class BaseOutput: if not filter_info or filter_info == self.path_info: return self.hash_info.nfiles or 0 - if not self.hash_info.dir_info: - return 0 - - return ilen( - filter_info.isin_or_eq(self.path_info.joinpath(*relpath)) - for relpath, _ in self.hash_info.dir_info.items() - ) + obj = self.get_obj(filter_info=filter_info) + return len(obj) if obj else 0 def unprotect(self): if self.exists: @@ -498,12 +495,11 @@ class BaseOutput: ) try: - objects.load(self.odb, self.hash_info) - assert self.hash_info.dir_info + self.obj = objects.load(self.odb, self.hash_info) except (objects.ObjectFormatError, FileNotFoundError): - self.hash_info.dir_info = None + self.obj = None - return self.dir_cache + return self.obj def collect_used_dir_cache( self, remote=None, force=False, jobs=None, filter_info=None @@ -551,14 +547,14 @@ class BaseOutput: path = str(self.path_info) filter_path = str(filter_info) if filter_info else None - for entry_key, entry_hash_info in self.dir_cache.items(): + for entry_key, entry_obj in self.obj: entry_path = os.path.join(path, *entry_key) if ( not filter_path or entry_path == filter_path or entry_path.startswith(filter_path + os.sep) ): - cache.add(self.scheme, entry_hash_info.value, entry_path) + cache.add(self.scheme, entry_obj.hash_info.value, entry_path) return cache @@ -651,6 +647,8 @@ class BaseOutput: ) def merge(self, ancestor, other): + from dvc.objects.tree import merge + assert other if ancestor: @@ -662,6 +660,6 @@ class BaseOutput: self._check_can_merge(self) self._check_can_merge(other) - self.hash_info = objects.merge( + self.hash_info = merge( self.odb, ancestor_info, self.hash_info, other.hash_info ) diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 92818a23f..863979c69 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -463,6 +463,12 @@ class Repo: path_parts = os.path.normpath(path).split(os.path.sep) return self.DVC_DIR in path_parts + @cached_property + def dvcfs(self): + from dvc.fs.dvc import DvcFileSystem + + return DvcFileSystem(self) + @cached_property def repo_fs(self): from dvc.fs.repo import RepoFileSystem diff --git a/dvc/repo/diff.py b/dvc/repo/diff.py index d75520ce1..b971294a9 100644 --- a/dvc/repo/diff.py +++ b/dvc/repo/diff.py @@ -130,7 +130,7 @@ def _output_paths(repo, repo_fs, targets): def _to_checksum(output): if on_working_fs: return ostage( - repo.odb.local, output.path_info, repo.odb.local.fs, + repo.odb.local, output.path_info, repo.odb.local.fs, "md5", ).hash_info.value return output.hash_info.value diff --git a/dvc/repo/fetch.py b/dvc/repo/fetch.py index 6bfc73db6..b25627f7b 100644 --- a/dvc/repo/fetch.py +++ b/dvc/repo/fetch.py @@ -115,6 +115,7 @@ def _fetch_external(self, repo_url, repo_rev, files, jobs): odb, path_info, repo.repo_fs, + "md5", jobs=jobs, follow_subrepos=False, ) diff --git a/tests/func/remote/test_index.py b/tests/func/remote/test_index.py index 150452342..104930090 100644 --- a/tests/func/remote/test_index.py +++ b/tests/func/remote/test_index.py @@ -32,7 +32,7 @@ def remote(tmp_dir, dvc, tmp_path_factory, mocker): def test_indexed_on_status(tmp_dir, dvc, tmp_path_factory, remote): foo = tmp_dir.dvc_gen({"foo": "foo content"})[0].outs[0] bar = tmp_dir.dvc_gen({"bar": {"baz": "baz content"}})[0].outs[0] - baz_hash = bar.dir_cache.trie.get(("baz",)) + baz_hash = bar.obj.trie.get(("baz",)).hash_info dvc.push() with remote.index: remote.index.clear() @@ -49,7 +49,7 @@ def test_indexed_on_status(tmp_dir, dvc, tmp_path_factory, remote): def test_indexed_on_push(tmp_dir, dvc, tmp_path_factory, remote): foo = tmp_dir.dvc_gen({"foo": "foo content"})[0].outs[0] bar = tmp_dir.dvc_gen({"bar": {"baz": "baz content"}})[0].outs[0] - baz_hash = bar.dir_cache.trie.get(("baz",)) + baz_hash = bar.obj.trie.get(("baz",)).hash_info dvc.push() with remote.index: diff --git a/tests/func/test_add.py b/tests/func/test_add.py index a37ca0770..3c688ef09 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -109,9 +109,10 @@ def test_add_directory(tmp_dir, dvc): hash_info = stage.outs[0].hash_info - dir_info = load(dvc.odb.local, hash_info).hash_info.dir_info - for path, _ in dir_info.trie.items(): - assert "\\" not in path + obj = load(dvc.odb.local, hash_info) + for key, _ in obj: + for part in key: + assert "\\" not in part class TestAddDirectoryRecursive(TestDvc): @@ -516,7 +517,7 @@ class TestAddCommit(TestDvc): def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc): tmp_dir.gen({"data/data": "foo"}) - get_dir_hash_counter = mocker.spy(dvc_module.objects.stage, "get_dir_hash") + counter = mocker.spy(dvc_module.objects.stage, "_get_tree_obj") ret = main(["add", "data"]) assert ret == 0 @@ -525,7 +526,7 @@ def test_should_collect_dir_cache_only_once(mocker, tmp_dir, dvc): ret = main(["status"]) assert ret == 0 - assert get_dir_hash_counter.mock.call_count == 1 + assert counter.mock.call_count == 1 class TestShouldPlaceStageInDataDirIfRepositoryBelowSymlink(TestDvc): diff --git a/tests/func/test_checkout.py b/tests/func/test_checkout.py index 98d604684..bfd111229 100644 --- a/tests/func/test_checkout.py +++ b/tests/func/test_checkout.py @@ -100,11 +100,10 @@ class TestCheckoutCorruptedCacheDir(TestDvc): # NOTE: modifying cache file for one of the files inside the directory # to check if dvc will detect that the cache is corrupted. - _, entry_hash = next( - load(self.dvc.odb.local, out.hash_info).hash_info.dir_info.items() - ) + obj = load(self.dvc.odb.local, out.hash_info) + _, entry_obj = list(obj)[0] cache = os.fspath( - self.dvc.odb.local.hash_to_path_info(entry_hash.value) + self.dvc.odb.local.hash_to_path_info(entry_obj.hash_info.value) ) os.chmod(cache, 0o644) diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index a0ab15032..958c1c161 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -287,7 +287,7 @@ def test_verify_hashes( remove("dir") remove(dvc.odb.local.cache_dir) - hash_spy = mocker.spy(dvc_module.objects.stage, "get_file_hash") + hash_spy = mocker.spy(dvc_module.objects.stage, "_get_file_hash") dvc.pull() assert hash_spy.call_count == 0 diff --git a/tests/func/test_external_repo.py b/tests/func/test_external_repo.py index 84b3e62a1..973ebdaa7 100644 --- a/tests/func/test_external_repo.py +++ b/tests/func/test_external_repo.py @@ -205,6 +205,7 @@ def test_subrepos_are_ignored(tmp_dir, erepo_dir): repo.odb.local, PathInfo(repo.root_dir) / "dir", repo.repo_fs, + "md5", follow_subrepos=False, ) save(repo.odb.local, obj) diff --git a/tests/func/test_remote.py b/tests/func/test_remote.py index 90093d677..1f057b49e 100644 --- a/tests/func/test_remote.py +++ b/tests/func/test_remote.py @@ -7,7 +7,6 @@ import pytest from mock import patch from dvc.config import Config -from dvc.dir_info import DirInfo from dvc.exceptions import DownloadError, RemoteCacheRequiredError, UploadError from dvc.fs.local import LocalFileSystem from dvc.main import main @@ -147,26 +146,31 @@ class TestRemoteShouldHandleUppercaseRemoteName(TestDvc): def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc): from dvc.objects.stage import stage + from dvc.objects.tree import Tree tmp_dir.gen({"data": {"1": "1 content", "2": "2 content"}}) path_info = PathInfo("data") - dir_info = DirInfo.from_list( + tree = Tree.from_list( [{"relpath": "1", "md5": "1"}, {"relpath": "2", "md5": "2"}] ) with patch( - "dvc.objects.stage._collect_dir", return_value=dir_info, + "dvc.objects.stage._get_tree_obj", return_value=tree, ): - hash1 = stage(dvc.odb.local, path_info, dvc.odb.local.fs).hash_info + hash1 = stage( + dvc.odb.local, path_info, dvc.odb.local.fs, "md5" + ).hash_info - dir_info = DirInfo.from_list( + tree = Tree.from_list( [{"md5": "1", "relpath": "1"}, {"md5": "2", "relpath": "2"}] ) with patch( - "dvc.objects.stage._collect_dir", return_value=dir_info, + "dvc.objects.stage._get_tree_obj", return_value=tree, ): - hash2 = stage(dvc.odb.local, path_info, dvc.odb.local.fs).hash_info + hash2 = stage( + dvc.odb.local, path_info, dvc.odb.local.fs, "md5" + ).hash_info assert hash1 == hash2 @@ -278,7 +282,7 @@ def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker, local_remote): remote = dvc.cloud.get_remote("upstream") foo_path = remote.odb.hash_to_path_info(foo.hash_info.value) bar_path = remote.odb.hash_to_path_info( - foo.hash_info.dir_info.trie[("bar",)].value + foo.obj.trie[("bar",)].hash_info.value ) paths = [args[1] for args, _ in mocked_upload.call_args_list] assert paths.index(foo_path) > paths.index(bar_path) diff --git a/tests/unit/fs/test_dvc.py b/tests/unit/fs/test_dvc.py index 36b8bf29c..d19210268 100644 --- a/tests/unit/fs/test_dvc.py +++ b/tests/unit/fs/test_dvc.py @@ -229,15 +229,15 @@ def test_get_hash_granular(tmp_dir, dvc): fs = DvcFileSystem(dvc) subdir = PathInfo(tmp_dir) / "dir" / "subdir" assert fs.info(subdir).get("md5") is None - assert stage(dvc.odb.local, subdir, fs).hash_info == HashInfo( + assert stage(dvc.odb.local, subdir, fs, "md5").hash_info == HashInfo( "md5", "af314506f1622d107e0ed3f14ec1a3b5.dir", ) assert ( fs.info(subdir / "data")["md5"] == "8d777f385d3dfec8815d20f7496026dc" ) - assert stage(dvc.odb.local, subdir / "data", fs).hash_info == HashInfo( - "md5", "8d777f385d3dfec8815d20f7496026dc", - ) + assert stage( + dvc.odb.local, subdir / "data", fs, "md5" + ).hash_info == HashInfo("md5", "8d777f385d3dfec8815d20f7496026dc",) def test_get_hash_dirty_file(tmp_dir, dvc): @@ -248,7 +248,7 @@ def test_get_hash_dirty_file(tmp_dir, dvc): expected = "8c7dd922ad47494fc02c388e12c00eac" assert fs.info(PathInfo(tmp_dir) / "file").get("md5") == expected assert stage( - dvc.odb.local, PathInfo(tmp_dir) / "file", fs + dvc.odb.local, PathInfo(tmp_dir) / "file", fs, "md5" ).hash_info == HashInfo("md5", expected) @@ -260,5 +260,5 @@ def test_get_hash_dirty_dir(tmp_dir, dvc): expected = "5ea40360f5b4ec688df672a4db9c17d1.dir" assert fs.info(PathInfo(tmp_dir) / "dir").get("md5") == expected assert stage( - dvc.odb.local, PathInfo(tmp_dir) / "dir", fs + dvc.odb.local, PathInfo(tmp_dir) / "dir", fs, "md5" ).hash_info == HashInfo("md5", expected) diff --git a/tests/unit/fs/test_repo.py b/tests/unit/fs/test_repo.py index 64808b4fa..8f0b33ee2 100644 --- a/tests/unit/fs/test_repo.py +++ b/tests/unit/fs/test_repo.py @@ -483,7 +483,7 @@ def test_get_hash_cached_file(tmp_dir, dvc, mocker): expected = "acbd18db4cc2f85cedef654fccc4a4d8" assert fs.info(PathInfo(tmp_dir) / "foo").get("md5") is None assert stage( - dvc.odb.local, PathInfo(tmp_dir) / "foo", fs + dvc.odb.local, PathInfo(tmp_dir) / "foo", fs, "md5", ).hash_info == HashInfo("md5", expected,) (tmp_dir / "foo").unlink() assert fs.info(PathInfo(tmp_dir) / "foo")["md5"] == expected @@ -497,13 +497,13 @@ def test_get_hash_cached_dir(tmp_dir, dvc, mocker): expected = "8761c4e9acad696bee718615e23e22db.dir" assert fs.info(PathInfo(tmp_dir) / "dir").get("md5") is None assert stage( - dvc.odb.local, PathInfo(tmp_dir) / "dir", fs + dvc.odb.local, PathInfo(tmp_dir) / "dir", fs, "md5", ).hash_info == HashInfo("md5", "8761c4e9acad696bee718615e23e22db.dir",) shutil.rmtree(tmp_dir / "dir") assert fs.info(PathInfo(tmp_dir) / "dir")["md5"] == expected assert stage( - dvc.odb.local, PathInfo(tmp_dir) / "dir", fs + dvc.odb.local, PathInfo(tmp_dir) / "dir", fs, "md5", ).hash_info == HashInfo("md5", "8761c4e9acad696bee718615e23e22db.dir",) @@ -514,13 +514,13 @@ def test_get_hash_cached_granular(tmp_dir, dvc, mocker): fs = RepoFileSystem(dvc) subdir = PathInfo(tmp_dir) / "dir" / "subdir" assert fs.info(subdir).get("md5") is None - assert stage(dvc.odb.local, subdir, fs).hash_info == HashInfo( + assert stage(dvc.odb.local, subdir, fs, "md5").hash_info == HashInfo( "md5", "af314506f1622d107e0ed3f14ec1a3b5.dir", ) assert fs.info(subdir / "data").get("md5") is None - assert stage(dvc.odb.local, subdir / "data", fs).hash_info == HashInfo( - "md5", "8d777f385d3dfec8815d20f7496026dc", - ) + assert stage( + dvc.odb.local, subdir / "data", fs, "md5" + ).hash_info == HashInfo("md5", "8d777f385d3dfec8815d20f7496026dc",) (tmp_dir / "dir" / "subdir" / "data").unlink() assert ( fs.info(subdir / "data")["md5"] == "8d777f385d3dfec8815d20f7496026dc" @@ -540,7 +540,9 @@ def test_get_hash_mixed_dir(tmp_dir, scm, dvc): tmp_dir.scm.commit("add dir") fs = RepoFileSystem(dvc) - actual = stage(dvc.odb.local, PathInfo(tmp_dir) / "dir", fs).hash_info + actual = stage( + dvc.odb.local, PathInfo(tmp_dir) / "dir", fs, "md5" + ).hash_info expected = HashInfo("md5", "e1d9e8eae5374860ae025ec84cfd85c7.dir") assert actual == expected @@ -551,7 +553,9 @@ def test_get_hash_dirty_file(tmp_dir, dvc): fs = RepoFileSystem(dvc) assert fs.info(PathInfo(tmp_dir) / "file").get("md5") is None - actual = stage(dvc.odb.local, PathInfo(tmp_dir) / "file", fs).hash_info + actual = stage( + dvc.odb.local, PathInfo(tmp_dir) / "file", fs, "md5" + ).hash_info expected = HashInfo("md5", "437b930db84b8079c2dd804a71936b5f") assert actual == expected @@ -560,7 +564,9 @@ def test_get_hash_dirty_file(tmp_dir, dvc): fs.info(PathInfo(tmp_dir) / "file")["md5"] == "8c7dd922ad47494fc02c388e12c00eac" ) - actual = stage(dvc.odb.local, PathInfo(tmp_dir) / "file", fs).hash_info + actual = stage( + dvc.odb.local, PathInfo(tmp_dir) / "file", fs, "md5" + ).hash_info expected = HashInfo("md5", "8c7dd922ad47494fc02c388e12c00eac") assert actual == expected @@ -570,10 +576,12 @@ def test_get_hash_dirty_dir(tmp_dir, dvc): (tmp_dir / "dir" / "baz").write_text("baz") fs = RepoFileSystem(dvc) - actual = stage(dvc.odb.local, PathInfo(tmp_dir) / "dir", fs).hash_info + actual = stage( + dvc.odb.local, PathInfo(tmp_dir) / "dir", fs, "md5" + ).hash_info expected = HashInfo("md5", "ba75a2162ca9c29acecb7957105a0bc2.dir") assert actual == expected - assert actual.dir_info.nfiles == 3 + assert actual.nfiles == 3 @pytest.mark.parametrize("traverse_subrepos", [True, False]) diff --git a/tests/unit/objects/__init__.py b/tests/unit/objects/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/test_dir_info.py b/tests/unit/objects/test_tree.py similarity index 76% rename from tests/unit/test_dir_info.py rename to tests/unit/objects/test_tree.py index 53a0f920c..7e57f0fd0 100644 --- a/tests/unit/test_dir_info.py +++ b/tests/unit/objects/test_tree.py @@ -2,8 +2,9 @@ from operator import itemgetter import pytest -from dvc.dir_info import DirInfo, _merge from dvc.hash_info import HashInfo +from dvc.objects import HashFile +from dvc.objects.tree import Tree, _merge @pytest.mark.parametrize( @@ -18,10 +19,10 @@ from dvc.hash_info import HashInfo {"md5": "456", "relpath": "bar"}, ], { - ("zzz",): HashInfo("md5", "def"), - ("foo",): HashInfo("md5", "123"), - ("bar",): HashInfo("md5", "456"), - ("aaa",): HashInfo("md5", "abc"), + ("zzz",): HashFile(None, None, HashInfo("md5", "def")), + ("foo",): HashFile(None, None, HashInfo("md5", "123")), + ("bar",): HashFile(None, None, HashInfo("md5", "456")), + ("aaa",): HashFile(None, None, HashInfo("md5", "abc")), }, ), ( @@ -37,23 +38,29 @@ from dvc.hash_info import HashInfo {"md5": "pqr", "relpath": "dir/subdir/a"}, ], { - ("dir", "b"): HashInfo("md5", "123"), - ("dir", "z"): HashInfo("md5", "456"), - ("dir", "a"): HashInfo("md5", "789"), - ("b",): HashInfo("md5", "abc"), - ("a",): HashInfo("md5", "def"), - ("z",): HashInfo("md5", "ghi"), - ("dir", "subdir", "b"): HashInfo("md5", "jkl"), - ("dir", "subdir", "z"): HashInfo("md5", "mno"), - ("dir", "subdir", "a"): HashInfo("md5", "pqr"), + ("dir", "b"): HashFile(None, None, HashInfo("md5", "123")), + ("dir", "z"): HashFile(None, None, HashInfo("md5", "456")), + ("dir", "a"): HashFile(None, None, HashInfo("md5", "789")), + ("b",): HashFile(None, None, HashInfo("md5", "abc")), + ("a",): HashFile(None, None, HashInfo("md5", "def")), + ("z",): HashFile(None, None, HashInfo("md5", "ghi")), + ("dir", "subdir", "b"): HashFile( + None, None, HashInfo("md5", "jkl") + ), + ("dir", "subdir", "z"): HashFile( + None, None, HashInfo("md5", "mno") + ), + ("dir", "subdir", "a"): HashFile( + None, None, HashInfo("md5", "pqr") + ), }, ), ], ) def test_list(lst, trie_dict): - dir_info = DirInfo.from_list(lst) - assert dir_info._dict == trie_dict - assert dir_info.to_list() == sorted(lst, key=itemgetter("relpath")) + tree = Tree.from_list(lst) + assert tree._dict == trie_dict + assert tree.as_list() == sorted(lst, key=itemgetter("relpath")) @pytest.mark.parametrize( @@ -81,9 +88,9 @@ def test_list(lst, trie_dict): ], ) def test_size(trie_dict, size): - dir_info = DirInfo() - dir_info._dict = trie_dict - assert dir_info.size == size + tree = Tree(None, None, None) + tree._dict = trie_dict + assert tree.size == size @pytest.mark.parametrize( @@ -111,9 +118,9 @@ def test_size(trie_dict, size): ], ) def test_nfiles(trie_dict, nfiles): - dir_info = DirInfo() - dir_info._dict = trie_dict - assert dir_info.nfiles == nfiles + tree = Tree(None, None, None) + tree._dict = trie_dict + assert len(tree) == nfiles @pytest.mark.parametrize( @@ -134,9 +141,9 @@ def test_nfiles(trie_dict, nfiles): ], ) def test_items(trie_dict): - dir_info = DirInfo() - dir_info._dict = trie_dict - assert list(dir_info.items()) == list(trie_dict.items()) + tree = Tree(None, None, None) + tree._dict = trie_dict + assert list(tree) == list(trie_dict.items()) @pytest.mark.parametrize( diff --git a/tests/unit/output/test_local.py b/tests/unit/output/test_local.py index 49b61de52..59c89c25b 100644 --- a/tests/unit/output/test_local.py +++ b/tests/unit/output/test_local.py @@ -2,7 +2,6 @@ import os from mock import patch -from dvc.dir_info import DirInfo from dvc.hash_info import HashInfo from dvc.output import LocalOutput from dvc.stage import Stage @@ -80,17 +79,7 @@ class TestGetFilesNumber(TestDvc): def test_return_multiple_for_dir(self): o = self._get_output() - o.hash_info = HashInfo( - "md5", - "12345678.dir", - nfiles=2, - dir_info=DirInfo.from_list( - [ - {"relpath": "foo", "md5": "asdf"}, - {"relpath": "bar", "md5": "qwe"}, - ] - ), - ) + o.hash_info = HashInfo("md5", "12345678.dir", nfiles=2,) self.assertEqual(2, o.get_files_number()) @patch.object(LocalOutput, "is_dir_checksum", False)