diff --git a/dvc/cache/base.py b/dvc/cache/base.py index 6a7aabf3c..c8a46a501 100644 --- a/dvc/cache/base.py +++ b/dvc/cache/base.py @@ -263,6 +263,8 @@ class CloudCache: from dvc.utils import tmp_fname # Sorting the list by path to ensure reproducibility + if isinstance(dir_info, dict): + dir_info = self._from_dict(dir_info) dir_info = sorted(dir_info, key=itemgetter(self.tree.PARAM_RELPATH)) tmp = tempfile.NamedTemporaryFile(delete=False).name @@ -275,7 +277,7 @@ class CloudCache: hash_info = self.tree.get_file_hash(to_info) hash_info.value += self.tree.CHECKSUM_DIR_SUFFIX - hash_info.dir_info = dir_info + hash_info.dir_info = self._to_dict(dir_info) return hash_info, to_info @@ -299,15 +301,14 @@ class CloudCache: def _save_dir(self, path_info, tree, hash_info, save_link=True, **kwargs): if not hash_info.dir_info: - hash_info.dir_info = tree.cache.get_dir_cache(hash_info) - hi = self.save_dir_info(hash_info.dir_info, hash_info) - for entry in Tqdm( - hi.dir_info, desc="Saving " + path_info.name, unit="file" - ): - entry_info = path_info / entry[self.tree.PARAM_RELPATH] - entry_hash = HashInfo( - self.tree.PARAM_CHECKSUM, entry[self.tree.PARAM_CHECKSUM] + hash_info.dir_info = self._to_dict( + tree.cache.get_dir_cache(hash_info) ) + hi = self.save_dir_info(hash_info.dir_info, hash_info) + for relpath, entry_hash in Tqdm( + hi.dir_info.items(), desc="Saving " + path_info.name, unit="file" + ): + entry_info = path_info / relpath self._save_file( entry_info, tree, entry_hash, save_link=False, **kwargs ) @@ -614,18 +615,25 @@ class CloudCache: ) def _to_dict(self, dir_info): - return { - entry[self.tree.PARAM_RELPATH]: entry[self.tree.PARAM_CHECKSUM] - for entry in dir_info - } + info = {} + for entry in dir_info: + relpath = None + hash_info = None + for key, value in entry.items(): + if key == self.tree.PARAM_RELPATH: + relpath = value + else: + hash_info = HashInfo(key, value) + info[relpath] = hash_info + return info def _from_dict(self, dir_dict): return [ { self.tree.PARAM_RELPATH: relpath, - self.tree.PARAM_CHECKSUM: checksum, + hash_info.name: hash_info.value, } - for relpath, checksum in dir_dict.items() + for relpath, hash_info in dir_dict.items() ] @staticmethod @@ -692,3 +700,8 @@ class CloudCache: return hash_info return self.save_dir_info(hash_info.dir_info, hash_info) + + def set_dir_info(self, hash_info): + assert hash_info.isdir + + hash_info.dir_info = self._to_dict(self.get_dir_cache(hash_info)) diff --git a/dvc/hash_info.py b/dvc/hash_info.py index 715e76da3..826b4e1df 100644 --- a/dvc/hash_info.py +++ b/dvc/hash_info.py @@ -7,7 +7,7 @@ HASH_DIR_SUFFIX = ".dir" class HashInfo: name: str value: str - dir_info: list = field(default=None, compare=False) + dir_info: dict = field(default=None, compare=False) def __bool__(self): return bool(self.value) diff --git a/dvc/output/base.py b/dvc/output/base.py index 0a7b04454..f575685b9 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -385,6 +385,9 @@ class BaseOutput: show_checksums=False, **kwargs, ) + self.hash_info.dir_info = None + if not self.hash_info.dir_info: + self.cache.set_dir_info(self.hash_info) return self.dir_cache def collect_used_dir_cache( diff --git a/dvc/repo/diff.py b/dvc/repo/diff.py index 17ef236c8..cb5857a60 100644 --- a/dvc/repo/diff.py +++ b/dvc/repo/diff.py @@ -111,7 +111,7 @@ def _dir_output_paths(repo_tree, output): try: for fname in repo_tree.walk_files(output.path_info): - yield str(fname), repo_tree.get_hash(fname).value + yield str(fname), repo_tree.get_file_hash(fname).value except NoRemoteError: logger.warning("dir cache entry for '%s' is missing", output) diff --git a/dvc/tree/base.py b/dvc/tree/base.py index 466bb0ecf..718142670 100644 --- a/dvc/tree/base.py +++ b/dvc/tree/base.py @@ -256,7 +256,7 @@ class BaseTree: if hash_: hash_info = HashInfo(self.PARAM_CHECKSUM, hash_) if hash_info.isdir: - hash_info.dir_info = self.cache.get_dir_cache(hash_info) + self.cache.set_dir_info(hash_info) return hash_info if self.isdir(path_info): diff --git a/dvc/tree/dvc.py b/dvc/tree/dvc.py index e497a7879..2290d28b1 100644 --- a/dvc/tree/dvc.py +++ b/dvc/tree/dvc.py @@ -3,8 +3,8 @@ import os import typing from dvc.exceptions import OutputNotFoundError -from dvc.hash_info import HashInfo from dvc.path_info import PathInfo +from dvc.utils import relpath from ._metadata import Metadata from .base import BaseTree, RemoteActionNotImplemented @@ -32,6 +32,8 @@ class DvcTree(BaseTree): # pylint:disable=abstract-method scheme = "local" PARAM_CHECKSUM = "md5" + _cached_dir_cache = None + _dir_entry_hashes = {} def __init__(self, repo, fetch=False, stream=False): super().__init__(repo, {"url": repo.root_dir}) @@ -50,19 +52,21 @@ class DvcTree(BaseTree): # pylint:disable=abstract-method return outs - def _get_granular_checksum( + def _get_granular_hash( self, path_info: PathInfo, out: "BaseOutput", remote=None ): assert isinstance(path_info, PathInfo) if not self.fetch and not self.stream: raise FileNotFoundError - dir_cache = out.get_dir_cache(remote=remote) - for entry in dir_cache: - entry_relpath = entry[out.tree.PARAM_RELPATH] - if os.name == "nt": - entry_relpath = entry_relpath.replace("/", os.sep) - if path_info == out.path_info / entry_relpath: - return entry[out.tree.PARAM_CHECKSUM] + + # NOTE: use string paths here for performance reasons + path_str = relpath(path_info, out.path_info) + if os.name == "nt": + path_str = path_str.replace(os.sep, "/") + out.get_dir_cache(remote=remote) + file_hash = out.hash_info.dir_info.get(path_str) + if file_hash: + return file_hash raise FileNotFoundError def open( @@ -86,7 +90,7 @@ class DvcTree(BaseTree): # pylint:disable=abstract-method remote_obj = self.repo.cloud.get_remote(remote) if self.stream: if out.is_dir_checksum: - checksum = self._get_granular_checksum(path, out) + checksum = self._get_granular_hash(path, out).value else: checksum = out.hash_info.value try: @@ -100,7 +104,7 @@ class DvcTree(BaseTree): # pylint:disable=abstract-method self.repo.cloud.pull(cache_info, remote=remote) if out.is_dir_checksum: - checksum = self._get_granular_checksum(path, out) + checksum = self._get_granular_hash(path, out).value cache_path = out.cache.tree.hash_to_path_info(checksum).url else: cache_path = out.cache_path @@ -131,7 +135,7 @@ class DvcTree(BaseTree): # pylint:disable=abstract-method return True try: - self._get_granular_checksum(path_info, out) + self._get_granular_hash(path_info, out) return False except FileNotFoundError: return True @@ -259,10 +263,7 @@ class DvcTree(BaseTree): # pylint:disable=abstract-method raise OutputNotFoundError out = outs[0] if out.is_dir_checksum: - return HashInfo( - out.tree.PARAM_CHECKSUM, - self._get_granular_checksum(path_info, out), - ) + return self._get_granular_hash(path_info, out) return out.hash_info def metadata(self, path_info):