diff/dvctree: optimize dir cache access (#4626)

* dvctree: use string paths for performance reasons in
_get_granular_checksum()

* dvctree: cache most recently accessed metadata

* dvctree: cache output dir cache during walk() calls

* diff: use get_file_hash directly instead of get_hash when appropriate

* dvctree: cache most recen dir_cache hashes in _get_granular_checksum

* revert metadata cache changes

- mostly unnecessary after granular checksum changes, will be better
  addressed by the future exception handling updates

* fix windows path issue

* HashInfo: store HashInfo.dir_info as a dict instead of list

* DvcTree: use hash_info.dir_info for getting granular file hashes

* use str relpath

* use correct dir_info hash type

* fix windows paths

dvc/cache/base.py

@@ -263,6 +263,8 @@ class CloudCache:
         from dvc.utils import tmp_fname
 
         # Sorting the list by path to ensure reproducibility
+        if isinstance(dir_info, dict):
+            dir_info = self._from_dict(dir_info)
         dir_info = sorted(dir_info, key=itemgetter(self.tree.PARAM_RELPATH))
 
         tmp = tempfile.NamedTemporaryFile(delete=False).name
@@ -275,7 +277,7 @@ class CloudCache:
 
         hash_info = self.tree.get_file_hash(to_info)
         hash_info.value += self.tree.CHECKSUM_DIR_SUFFIX
-        hash_info.dir_info = dir_info
+        hash_info.dir_info = self._to_dict(dir_info)
 
         return hash_info, to_info
 
@@ -299,15 +301,14 @@ class CloudCache:
 
     def _save_dir(self, path_info, tree, hash_info, save_link=True, **kwargs):
         if not hash_info.dir_info:
-            hash_info.dir_info = tree.cache.get_dir_cache(hash_info)
-        hi = self.save_dir_info(hash_info.dir_info, hash_info)
-        for entry in Tqdm(
-            hi.dir_info, desc="Saving " + path_info.name, unit="file"
-        ):
-            entry_info = path_info / entry[self.tree.PARAM_RELPATH]
-            entry_hash = HashInfo(
-                self.tree.PARAM_CHECKSUM, entry[self.tree.PARAM_CHECKSUM]
+            hash_info.dir_info = self._to_dict(
+                tree.cache.get_dir_cache(hash_info)
             )
+        hi = self.save_dir_info(hash_info.dir_info, hash_info)
+        for relpath, entry_hash in Tqdm(
+            hi.dir_info.items(), desc="Saving " + path_info.name, unit="file"
+        ):
+            entry_info = path_info / relpath
             self._save_file(
                 entry_info, tree, entry_hash, save_link=False, **kwargs
             )
@@ -614,18 +615,25 @@ class CloudCache:
         )
 
     def _to_dict(self, dir_info):
-        return {
-            entry[self.tree.PARAM_RELPATH]: entry[self.tree.PARAM_CHECKSUM]
-            for entry in dir_info
-        }
+        info = {}
+        for entry in dir_info:
+            relpath = None
+            hash_info = None
+            for key, value in entry.items():
+                if key == self.tree.PARAM_RELPATH:
+                    relpath = value
+                else:
+                    hash_info = HashInfo(key, value)
+            info[relpath] = hash_info
+        return info
 
     def _from_dict(self, dir_dict):
         return [
             {
                 self.tree.PARAM_RELPATH: relpath,
-                self.tree.PARAM_CHECKSUM: checksum,
+                hash_info.name: hash_info.value,
             }
-            for relpath, checksum in dir_dict.items()
+            for relpath, hash_info in dir_dict.items()
         ]
 
     @staticmethod
@@ -692,3 +700,8 @@ class CloudCache:
             return hash_info
 
         return self.save_dir_info(hash_info.dir_info, hash_info)
+
+    def set_dir_info(self, hash_info):
+        assert hash_info.isdir
+
+        hash_info.dir_info = self._to_dict(self.get_dir_cache(hash_info))

dvc/hash_info.py

@@ -7,7 +7,7 @@ HASH_DIR_SUFFIX = ".dir"
 class HashInfo:
     name: str
     value: str
-    dir_info: list = field(default=None, compare=False)
+    dir_info: dict = field(default=None, compare=False)
 
     def __bool__(self):
         return bool(self.value)

dvc/output/base.py

@@ -385,6 +385,9 @@ class BaseOutput:
                 show_checksums=False,
                 **kwargs,
             )
+            self.hash_info.dir_info = None
+        if not self.hash_info.dir_info:
+            self.cache.set_dir_info(self.hash_info)
         return self.dir_cache
 
     def collect_used_dir_cache(

dvc/repo/diff.py

@@ -111,7 +111,7 @@ def _dir_output_paths(repo_tree, output):
 
     try:
         for fname in repo_tree.walk_files(output.path_info):
-            yield str(fname), repo_tree.get_hash(fname).value
+            yield str(fname), repo_tree.get_file_hash(fname).value
     except NoRemoteError:
         logger.warning("dir cache entry for '%s' is missing", output)
 

dvc/tree/base.py

@@ -256,7 +256,7 @@ class BaseTree:
         if hash_:
             hash_info = HashInfo(self.PARAM_CHECKSUM, hash_)
             if hash_info.isdir:
-                hash_info.dir_info = self.cache.get_dir_cache(hash_info)
+                self.cache.set_dir_info(hash_info)
             return hash_info
 
         if self.isdir(path_info):

dvc/tree/dvc.py

@@ -3,8 +3,8 @@ import os
 import typing
 
 from dvc.exceptions import OutputNotFoundError
-from dvc.hash_info import HashInfo
 from dvc.path_info import PathInfo
+from dvc.utils import relpath
 
 from ._metadata import Metadata
 from .base import BaseTree, RemoteActionNotImplemented
@@ -32,6 +32,8 @@ class DvcTree(BaseTree):  # pylint:disable=abstract-method
 
     scheme = "local"
     PARAM_CHECKSUM = "md5"
+    _cached_dir_cache = None
+    _dir_entry_hashes = {}
 
     def __init__(self, repo, fetch=False, stream=False):
         super().__init__(repo, {"url": repo.root_dir})
@@ -50,19 +52,21 @@ class DvcTree(BaseTree):  # pylint:disable=abstract-method
 
         return outs
 
-    def _get_granular_checksum(
+    def _get_granular_hash(
         self, path_info: PathInfo, out: "BaseOutput", remote=None
     ):
         assert isinstance(path_info, PathInfo)
         if not self.fetch and not self.stream:
             raise FileNotFoundError
-        dir_cache = out.get_dir_cache(remote=remote)
-        for entry in dir_cache:
-            entry_relpath = entry[out.tree.PARAM_RELPATH]
-            if os.name == "nt":
-                entry_relpath = entry_relpath.replace("/", os.sep)
-            if path_info == out.path_info / entry_relpath:
-                return entry[out.tree.PARAM_CHECKSUM]
+
+        # NOTE: use string paths here for performance reasons
+        path_str = relpath(path_info, out.path_info)
+        if os.name == "nt":
+            path_str = path_str.replace(os.sep, "/")
+        out.get_dir_cache(remote=remote)
+        file_hash = out.hash_info.dir_info.get(path_str)
+        if file_hash:
+            return file_hash
         raise FileNotFoundError
 
     def open(
@@ -86,7 +90,7 @@ class DvcTree(BaseTree):  # pylint:disable=abstract-method
             remote_obj = self.repo.cloud.get_remote(remote)
             if self.stream:
                 if out.is_dir_checksum:
-                    checksum = self._get_granular_checksum(path, out)
+                    checksum = self._get_granular_hash(path, out).value
                 else:
                     checksum = out.hash_info.value
                 try:
@@ -100,7 +104,7 @@ class DvcTree(BaseTree):  # pylint:disable=abstract-method
             self.repo.cloud.pull(cache_info, remote=remote)
 
         if out.is_dir_checksum:
-            checksum = self._get_granular_checksum(path, out)
+            checksum = self._get_granular_hash(path, out).value
             cache_path = out.cache.tree.hash_to_path_info(checksum).url
         else:
             cache_path = out.cache_path
@@ -131,7 +135,7 @@ class DvcTree(BaseTree):  # pylint:disable=abstract-method
             return True
 
         try:
-            self._get_granular_checksum(path_info, out)
+            self._get_granular_hash(path_info, out)
             return False
         except FileNotFoundError:
             return True
@@ -259,10 +263,7 @@ class DvcTree(BaseTree):  # pylint:disable=abstract-method
             raise OutputNotFoundError
         out = outs[0]
         if out.is_dir_checksum:
-            return HashInfo(
-                out.tree.PARAM_CHECKSUM,
-                self._get_granular_checksum(path_info, out),
-            )
+            return self._get_granular_hash(path_info, out)
         return out.hash_info
 
     def metadata(self, path_info):