diff --git a/dvc/external_repo.py b/dvc/external_repo.py index fd53c1f87..9a7ee45d4 100644 --- a/dvc/external_repo.py +++ b/dvc/external_repo.py @@ -3,28 +3,22 @@ import os import tempfile import threading from contextlib import contextmanager -from distutils.dir_util import copy_tree from typing import Dict, Iterable from funcy import cached_property, reraise, retry, wrap_with -from dvc.config import NoRemoteError, NotDvcRepoError from dvc.exceptions import ( DvcException, FileMissingError, NoOutputInExternalRepoError, NoRemoteInExternalRepoError, + NotDvcRepoError, OutputNotFoundError, PathMissingError, ) from dvc.path_info import PathInfo from dvc.repo import Repo -from dvc.scm.base import CloneError -from dvc.scm.git import Git -from dvc.tree.local import LocalTree -from dvc.tree.repo import RepoTree from dvc.utils import relpath -from dvc.utils.fs import remove logger = logging.getLogger(__name__) @@ -35,6 +29,9 @@ class IsADVCRepoError(DvcException): @contextmanager def external_repo(url, rev=None, for_write=False, **kwargs): + from dvc.config import NoRemoteError + from dvc.scm.git import Git + logger.debug("Creating external repo %s@%s", url, rev) path = _cached_clone(url, rev, for_write=for_write) # Local HEAD points to the tip of whatever branch we first cloned from @@ -158,6 +155,8 @@ class ExternalRepo(Repo): ) def get_rev(self): + from dvc.tree.local import LocalTree + assert self.scm if isinstance(self.tree, LocalTree): return self.scm.get_rev() @@ -246,6 +245,8 @@ class ExternalRepo(Repo): """ Provides a combined tree of a single repo with dvc + git/local tree. """ + from dvc.tree.repo import RepoTree + kw = {**self.tree_confs, **kwargs} if "fetch" not in kw: kw["fetch"] = True @@ -275,6 +276,8 @@ def _cached_clone(url, rev, for_write=False): revision checked out. If for_write is set prevents reusing this dir via cache. """ + from distutils.dir_util import copy_tree + # even if we have already cloned this repo, we may need to # fetch/fast-forward to get specified rev clone_path, shallow = _clone_default_branch(url, rev, for_write=for_write) @@ -301,6 +304,8 @@ def _clone_default_branch(url, rev, for_write=False): The cloned is reactualized with git pull unless rev is a known sha. """ + from dvc.scm.git import Git + clone_path, shallow = CLONES.get(url, (None, False)) git = None @@ -328,6 +333,8 @@ def _clone_default_branch(url, rev, for_write=False): clone_path = tempfile.mkdtemp("dvc-clone") if not for_write and rev and not Git.is_sha(rev): # If rev is a tag or branch name try shallow clone first + from dvc.scm.base import CloneError + try: git = Git.clone(url, clone_path, shallow_branch=rev) shallow = True @@ -361,6 +368,8 @@ def _unshallow(git): def _git_checkout(repo_path, rev): + from dvc.scm.git import Git + logger.debug("erepo: git checkout %s@%s", repo_path, rev) git = Git(repo_path) try: @@ -370,6 +379,8 @@ def _git_checkout(repo_path, rev): def _remove(path): + from dvc.utils.fs import remove + if os.name == "nt": # git.exe may hang for a while not permitting to remove temp dir os_retry = retry(5, errors=OSError, timeout=0.1) diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 7ee9fc60c..9e8149aca 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -11,15 +11,14 @@ from dvc.exceptions import FileMissingError from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError from dvc.exceptions import NotDvcRepoError, OutputNotFoundError from dvc.path_info import PathInfo -from dvc.scm import Base from dvc.scm.base import SCMError -from dvc.tree.repo import RepoTree from dvc.utils.fs import path_isin from .graph import build_graph, build_outs_graph, get_pipelines from .trie import build_outs_trie if TYPE_CHECKING: + from dvc.scm import Base from dvc.tree.base import BaseTree @@ -88,7 +87,7 @@ class Repo: def _get_repo_dirs( self, root_dir: str = None, - scm: Base = None, + scm: "Base" = None, rev: str = None, uninitialized: bool = False, ): @@ -115,6 +114,8 @@ class Repo: except SCMError: scm = SCM(os.curdir, no_scm=True) + from dvc.scm import Base + assert isinstance(scm, Base) root_dir = scm.root_dir @@ -415,11 +416,14 @@ class Repo: @cached_property def repo_tree(self): + from dvc.tree.repo import RepoTree + return RepoTree(self, subrepos=self.subrepos, fetch=True) @contextmanager def open_by_relpath(self, path, remote=None, mode="r", encoding=None): """Opens a specified resource as a file descriptor""" + from dvc.tree.repo import RepoTree tree = RepoTree(self, stream=True, subrepos=True) path = PathInfo(self.root_dir) / path diff --git a/dvc/repo/diff.py b/dvc/repo/diff.py index 8d9958acf..afd872a09 100644 --- a/dvc/repo/diff.py +++ b/dvc/repo/diff.py @@ -3,8 +3,6 @@ import os from dvc.exceptions import PathMissingError from dvc.repo import locked -from dvc.tree.local import LocalTree -from dvc.tree.repo import RepoTree logger = logging.getLogger(__name__) @@ -22,6 +20,10 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None): if self.scm.no_commits: return {} + from dvc.tree.repo import RepoTree + + repo_tree = RepoTree(self, stream=True) + b_rev = b_rev if b_rev else "workspace" results = {} missing_targets = {} @@ -35,10 +37,10 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None): if targets is not None: # convert targets to path_infos, and capture any missing targets targets_path_infos, missing_targets[rev] = _targets_to_path_infos( - self, targets + repo_tree, targets ) - results[rev] = _paths_checksums(self, targets_path_infos) + results[rev] = _paths_checksums(self, repo_tree, targets_path_infos) if targets is not None: # check for overlapping missing targets between a_rev and b_rev @@ -57,7 +59,7 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None): if b_rev == "workspace": # missing status is only applicable when diffing local workspace # against a commit - missing = sorted(_filter_missing(self, deleted_or_missing)) + missing = sorted(_filter_missing(repo_tree, deleted_or_missing)) else: missing = [] deleted = sorted(deleted_or_missing - set(missing)) @@ -79,7 +81,7 @@ def diff(self, a_rev="HEAD", b_rev=None, targets=None): return ret if any(ret.values()) else {} -def _paths_checksums(repo, targets): +def _paths_checksums(repo, repo_tree, targets): """ A dictionary of checksums addressed by relpaths collected from the current tree outputs. @@ -91,11 +93,12 @@ def _paths_checksums(repo, targets): file: "data" """ - return dict(_output_paths(repo, targets)) + return dict(_output_paths(repo, repo_tree, targets)) -def _output_paths(repo, targets): - repo_tree = RepoTree(repo, stream=True) +def _output_paths(repo, repo_tree, targets): + from dvc.tree.local import LocalTree + on_working_tree = isinstance(repo.tree, LocalTree) def _exists(output): @@ -145,8 +148,7 @@ def _dir_output_paths(repo_tree, output, targets=None): logger.warning("dir cache entry for '%s' is missing", output) -def _filter_missing(repo, paths): - repo_tree = RepoTree(repo, stream=True) +def _filter_missing(repo_tree, paths): for path in paths: try: metadata = repo_tree.metadata(path) @@ -158,12 +160,10 @@ def _filter_missing(repo, paths): pass -def _targets_to_path_infos(repo, targets): +def _targets_to_path_infos(repo_tree, targets): path_infos = [] missing = [] - repo_tree = RepoTree(repo, stream=True) - for target in targets: if repo_tree.exists(target): path_infos.append(repo_tree.metadata(target).path_info) diff --git a/dvc/tree/repo.py b/dvc/tree/repo.py index 3bbe90c8b..21e319714 100644 --- a/dvc/tree/repo.py +++ b/dvc/tree/repo.py @@ -8,14 +8,9 @@ from typing import TYPE_CHECKING, Callable, Optional, Tuple, Type, Union from funcy import lfilter, wrap_with -from dvc.dvcfile import is_valid_filename from dvc.hash_info import HashInfo from dvc.path_info import PathInfo -from dvc.utils import file_md5, is_exec -from dvc.utils.collections import PathStringTrie -from dvc.utils.fs import copy_fobj_to_file, makedirs -from ._metadata import Metadata from .base import BaseTree from .dvc import DvcTree @@ -45,6 +40,8 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method ): super().__init__(repo, {"url": repo.root_dir}) + from dvc.utils.collections import PathStringTrie + if not repo_factory: from dvc.repo import Repo @@ -261,6 +258,8 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method yield from self._walk(tree_walk, dvc_walk, **kwargs) def _walk(self, repo_walk, dvc_walk=None, dvcfiles=False): + from dvc.dvcfile import is_valid_filename + assert repo_walk try: _, dvc_dirs, dvc_fnames = ( @@ -392,6 +391,8 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method will be used. If path_info is a git file, MD5 will be computed for the git object. """ + from dvc.utils import file_md5 + if not self.exists(path_info): raise FileNotFoundError _, dvc_tree = self._get_tree_pair(path_info) @@ -403,6 +404,8 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method return HashInfo(self.PARAM_CHECKSUM, file_md5(path_info, self)[0]) def copytree(self, top, dest): + from dvc.utils.fs import copy_fobj_to_file, makedirs + top = PathInfo(top) dest = PathInfo(dest) @@ -445,6 +448,8 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method if not stat_result and not dvc_meta: raise FileNotFoundError + from ._metadata import Metadata + meta = dvc_meta or Metadata( path_info=path_info, repo=self._get_repo(abspath) or self._main_repo, @@ -454,5 +459,7 @@ class RepoTree(BaseTree): # pylint:disable=abstract-method meta.isdir = meta.isdir or isdir if not dvc_meta: + from dvc.utils import is_exec + meta.is_exec = bool(stat_result) and is_exec(stat_result.st_mode) return meta