Make repotree lazy (#5220)

Also removed dvcfile related imports on dvc.tree.repo.
This was making imports slower on dvc.main

dvc/external_repo.py

@@ -3,28 +3,22 @@ import os
 import tempfile
 import threading
 from contextlib import contextmanager
-from distutils.dir_util import copy_tree
 from typing import Dict, Iterable
 
 from funcy import cached_property, reraise, retry, wrap_with
 
-from dvc.config import NoRemoteError, NotDvcRepoError
 from dvc.exceptions import (
     DvcException,
     FileMissingError,
     NoOutputInExternalRepoError,
     NoRemoteInExternalRepoError,
+    NotDvcRepoError,
     OutputNotFoundError,
     PathMissingError,
 )
 from dvc.path_info import PathInfo
 from dvc.repo import Repo
-from dvc.scm.base import CloneError
-from dvc.scm.git import Git
-from dvc.tree.local import LocalTree
-from dvc.tree.repo import RepoTree
 from dvc.utils import relpath
-from dvc.utils.fs import remove
 
 logger = logging.getLogger(__name__)
 
@@ -35,6 +29,9 @@ class IsADVCRepoError(DvcException):
 
 @contextmanager
 def external_repo(url, rev=None, for_write=False, **kwargs):
+    from dvc.config import NoRemoteError
+    from dvc.scm.git import Git
+
     logger.debug("Creating external repo %s@%s", url, rev)
     path = _cached_clone(url, rev, for_write=for_write)
     # Local HEAD points to the tip of whatever branch we first cloned from
@@ -158,6 +155,8 @@ class ExternalRepo(Repo):
         )
 
     def get_rev(self):
+        from dvc.tree.local import LocalTree
+
         assert self.scm
         if isinstance(self.tree, LocalTree):
             return self.scm.get_rev()
@@ -246,6 +245,8 @@ class ExternalRepo(Repo):
         """
         Provides a combined tree of a single repo with dvc + git/local tree.
         """
+        from dvc.tree.repo import RepoTree
+
         kw = {**self.tree_confs, **kwargs}
         if "fetch" not in kw:
             kw["fetch"] = True
@@ -275,6 +276,8 @@ def _cached_clone(url, rev, for_write=False):
     revision checked out. If for_write is set prevents reusing this dir via
     cache.
     """
+    from distutils.dir_util import copy_tree
+
     # even if we have already cloned this repo, we may need to
     # fetch/fast-forward to get specified rev
     clone_path, shallow = _clone_default_branch(url, rev, for_write=for_write)
@@ -301,6 +304,8 @@ def _clone_default_branch(url, rev, for_write=False):
 
     The cloned is reactualized with git pull unless rev is a known sha.
     """
+    from dvc.scm.git import Git
+
     clone_path, shallow = CLONES.get(url, (None, False))
 
     git = None
@@ -328,6 +333,8 @@ def _clone_default_branch(url, rev, for_write=False):
             clone_path = tempfile.mkdtemp("dvc-clone")
             if not for_write and rev and not Git.is_sha(rev):
                 # If rev is a tag or branch name try shallow clone first
+                from dvc.scm.base import CloneError
+
                 try:
                     git = Git.clone(url, clone_path, shallow_branch=rev)
                     shallow = True
@@ -361,6 +368,8 @@ def _unshallow(git):
 
 
 def _git_checkout(repo_path, rev):
+    from dvc.scm.git import Git
+
     logger.debug("erepo: git checkout %s@%s", repo_path, rev)
     git = Git(repo_path)
     try:
@@ -370,6 +379,8 @@ def _git_checkout(repo_path, rev):
 
 
 def _remove(path):
+    from dvc.utils.fs import remove
+
     if os.name == "nt":
         # git.exe may hang for a while not permitting to remove temp dir
         os_retry = retry(5, errors=OSError, timeout=0.1)

dvc/repo/__init__.py

@@ -11,15 +11,14 @@ from dvc.exceptions import FileMissingError
 from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError
 from dvc.exceptions import NotDvcRepoError, OutputNotFoundError
 from dvc.path_info import PathInfo
-from dvc.scm import Base
 from dvc.scm.base import SCMError
-from dvc.tree.repo import RepoTree
 from dvc.utils.fs import path_isin
 
 from .graph import build_graph, build_outs_graph, get_pipelines
 from .trie import build_outs_trie
 
 if TYPE_CHECKING:
+    from dvc.scm import Base
     from dvc.tree.base import BaseTree
 
 
@@ -88,7 +87,7 @@ class Repo:
     def _get_repo_dirs(
         self,
         root_dir: str = None,
-        scm: Base = None,
+        scm: "Base" = None,
         rev: str = None,
         uninitialized: bool = False,
     ):
@@ -115,6 +114,8 @@ class Repo:
             except SCMError:
                 scm = SCM(os.curdir, no_scm=True)
 
+            from dvc.scm import Base
+
             assert isinstance(scm, Base)
             root_dir = scm.root_dir
 
@@ -415,11 +416,14 @@ class Repo:
 
     @cached_property
     def repo_tree(self):
+        from dvc.tree.repo import RepoTree
+
         return RepoTree(self, subrepos=self.subrepos, fetch=True)
 
     @contextmanager
     def open_by_relpath(self, path, remote=None, mode="r", encoding=None):
         """Opens a specified resource as a file descriptor"""
+        from dvc.tree.repo import RepoTree
 
         tree = RepoTree(self, stream=True, subrepos=True)
         path = PathInfo(self.root_dir) / path

dvc/repo/diff.py

@@ -3,8 +3,6 @@ import os
 
 from dvc.exceptions import PathMissingError
 from dvc.repo import locked
-from dvc.tree.local import LocalTree
-from dvc.tree.repo import RepoTree
 
 logger = logging.getLogger(__name__)
 
@@ -22,6 +20,10 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
     if self.scm.no_commits:
         return {}
 
+    from dvc.tree.repo import RepoTree
+
+    repo_tree = RepoTree(self, stream=True)
+
     b_rev = b_rev if b_rev else "workspace"
     results = {}
     missing_targets = {}
@@ -35,10 +37,10 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
         if targets is not None:
             # convert targets to path_infos, and capture any missing targets
             targets_path_infos, missing_targets[rev] = _targets_to_path_infos(
-                self, targets
+                repo_tree, targets
             )
 
-        results[rev] = _paths_checksums(self, targets_path_infos)
+        results[rev] = _paths_checksums(self, repo_tree, targets_path_infos)
 
     if targets is not None:
         # check for overlapping missing targets between a_rev and b_rev
@@ -57,7 +59,7 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
     if b_rev == "workspace":
         # missing status is only applicable when diffing local workspace
         # against a commit
-        missing = sorted(_filter_missing(self, deleted_or_missing))
+        missing = sorted(_filter_missing(repo_tree, deleted_or_missing))
     else:
         missing = []
     deleted = sorted(deleted_or_missing - set(missing))
@@ -79,7 +81,7 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None):
     return ret if any(ret.values()) else {}
 
 
-def _paths_checksums(repo, targets):
+def _paths_checksums(repo, repo_tree, targets):
     """
     A dictionary of checksums addressed by relpaths collected from
     the current tree outputs.
@@ -91,11 +93,12 @@ def _paths_checksums(repo, targets):
         file:      "data"
     """
 
-    return dict(_output_paths(repo, targets))
+    return dict(_output_paths(repo, repo_tree, targets))
 
 
-def _output_paths(repo, targets):
-    repo_tree = RepoTree(repo, stream=True)
+def _output_paths(repo, repo_tree, targets):
+    from dvc.tree.local import LocalTree
+
     on_working_tree = isinstance(repo.tree, LocalTree)
 
     def _exists(output):
@@ -145,8 +148,7 @@ def _dir_output_paths(repo_tree, output, targets=None):
         logger.warning("dir cache entry for '%s' is missing", output)
 
 
-def _filter_missing(repo, paths):
-    repo_tree = RepoTree(repo, stream=True)
+def _filter_missing(repo_tree, paths):
     for path in paths:
         try:
             metadata = repo_tree.metadata(path)
@@ -158,12 +160,10 @@ def _filter_missing(repo, paths):
             pass
 
 
-def _targets_to_path_infos(repo, targets):
+def _targets_to_path_infos(repo_tree, targets):
     path_infos = []
     missing = []
 
-    repo_tree = RepoTree(repo, stream=True)
-
     for target in targets:
         if repo_tree.exists(target):
             path_infos.append(repo_tree.metadata(target).path_info)

dvc/tree/repo.py

@@ -8,14 +8,9 @@ from typing import TYPE_CHECKING, Callable, Optional, Tuple, Type, Union
 
 from funcy import lfilter, wrap_with
 
-from dvc.dvcfile import is_valid_filename
 from dvc.hash_info import HashInfo
 from dvc.path_info import PathInfo
-from dvc.utils import file_md5, is_exec
-from dvc.utils.collections import PathStringTrie
-from dvc.utils.fs import copy_fobj_to_file, makedirs
 
-from ._metadata import Metadata
 from .base import BaseTree
 from .dvc import DvcTree
 
@@ -45,6 +40,8 @@ class RepoTree(BaseTree):  # pylint:disable=abstract-method
     ):
         super().__init__(repo, {"url": repo.root_dir})
 
+        from dvc.utils.collections import PathStringTrie
+
         if not repo_factory:
             from dvc.repo import Repo
 
@@ -261,6 +258,8 @@ class RepoTree(BaseTree):  # pylint:disable=abstract-method
         yield from self._walk(tree_walk, dvc_walk, **kwargs)
 
     def _walk(self, repo_walk, dvc_walk=None, dvcfiles=False):
+        from dvc.dvcfile import is_valid_filename
+
         assert repo_walk
         try:
             _, dvc_dirs, dvc_fnames = (
@@ -392,6 +391,8 @@ class RepoTree(BaseTree):  # pylint:disable=abstract-method
         will be used. If path_info is a git file, MD5 will be computed for
         the git object.
         """
+        from dvc.utils import file_md5
+
         if not self.exists(path_info):
             raise FileNotFoundError
         _, dvc_tree = self._get_tree_pair(path_info)
@@ -403,6 +404,8 @@ class RepoTree(BaseTree):  # pylint:disable=abstract-method
         return HashInfo(self.PARAM_CHECKSUM, file_md5(path_info, self)[0])
 
     def copytree(self, top, dest):
+        from dvc.utils.fs import copy_fobj_to_file, makedirs
+
         top = PathInfo(top)
         dest = PathInfo(dest)
 
@@ -445,6 +448,8 @@ class RepoTree(BaseTree):  # pylint:disable=abstract-method
         if not stat_result and not dvc_meta:
             raise FileNotFoundError
 
+        from ._metadata import Metadata
+
         meta = dvc_meta or Metadata(
             path_info=path_info,
             repo=self._get_repo(abspath) or self._main_repo,
@@ -454,5 +459,7 @@ class RepoTree(BaseTree):  # pylint:disable=abstract-method
         meta.isdir = meta.isdir or isdir
 
         if not dvc_meta:
+            from dvc.utils import is_exec
+
             meta.is_exec = bool(stat_result) and is_exec(stat_result.st_mode)
         return meta