From 84e9c1b0debde9ae8192fb39df8f5bd124812446 Mon Sep 17 00:00:00 2001 From: Saugat Pachhai Date: Wed, 6 Jan 2021 21:26:01 +0545 Subject: [PATCH] lazy imports on repo command (#5221) --- dvc/exceptions.py | 4 ++-- dvc/repo/add.py | 9 ++++++--- dvc/repo/brancher.py | 7 ++++--- dvc/repo/commit.py | 6 ++++-- dvc/repo/get_url.py | 8 ++++---- dvc/repo/reproduce.py | 9 ++++++--- dvc/repo/run.py | 5 +++-- dvc/repo/update.py | 3 ++- dvc/updater.py | 13 ++++++++++--- 9 files changed, 41 insertions(+), 23 deletions(-) diff --git a/dvc/exceptions.py b/dvc/exceptions.py index 26461c253..7568ebc8e 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -1,8 +1,6 @@ """Exceptions raised by the dvc.""" from typing import List -from funcy import first - class DvcException(Exception): """Base class for all dvc exceptions.""" @@ -26,6 +24,8 @@ class OutputDuplicationError(DvcException): """ def __init__(self, output, stages): + from funcy import first + assert isinstance(output, str) assert all(hasattr(stage, "relpath") for stage in stages) if len(stages) == 1: diff --git a/dvc/repo/add.py b/dvc/repo/add.py index 397a33d92..ea1f39b31 100644 --- a/dvc/repo/add.py +++ b/dvc/repo/add.py @@ -3,15 +3,12 @@ import os import colorama -from dvc.dvcfile import Dvcfile, is_dvc_file - from ..exceptions import ( CacheLinkError, OutputDuplicationError, OverlappingOutputPathsError, RecursiveAddingWhileUsingFilename, ) -from ..output.base import OutputDoesNotExistError from ..progress import Tqdm from ..repo.scm_context import scm_context from ..utils import LARGE_DIR_SIZE, glob_targets, resolve_paths @@ -105,6 +102,9 @@ def add( def _process_stages(repo, stages, no_commit, pbar): link_failures = [] + from dvc.dvcfile import Dvcfile + + from ..output.base import OutputDoesNotExistError with Tqdm( total=len(stages), @@ -132,6 +132,8 @@ def _process_stages(repo, stages, no_commit, pbar): def _find_all_targets(repo, target, recursive): + from dvc.dvcfile import is_dvc_file + if os.path.isdir(target) and recursive: return [ os.fspath(path) @@ -152,6 +154,7 @@ def _find_all_targets(repo, target, recursive): def _create_stages( repo, targets, fname, pbar=None, external=False, glob=False, desc=None, ): + from dvc.dvcfile import Dvcfile from dvc.stage import Stage, create_stage, restore_meta expanded_targets = glob_targets(targets, glob=glob) diff --git a/dvc/repo/brancher.py b/dvc/repo/brancher.py index ed3bf9489..8e6503dea 100644 --- a/dvc/repo/brancher.py +++ b/dvc/repo/brancher.py @@ -1,8 +1,5 @@ from funcy import group_by -from dvc.repo.experiments.utils import exp_commits -from dvc.tree.local import LocalTree - def brancher( # noqa: E302 self, @@ -32,6 +29,8 @@ def brancher( # noqa: E302 yield "" return + from dvc.tree.local import LocalTree + saved_tree = self.tree revs = revs.copy() if revs else [] @@ -53,6 +52,8 @@ def brancher( # noqa: E302 revs.extend(scm.list_tags()) if all_experiments: + from dvc.repo.experiments.utils import exp_commits + revs.extend(exp_commits(scm)) try: diff --git a/dvc/repo/commit.py b/dvc/repo/commit.py index ba8452129..0773ccc1b 100644 --- a/dvc/repo/commit.py +++ b/dvc/repo/commit.py @@ -1,6 +1,4 @@ from dvc import prompt -from dvc.dvcfile import Dvcfile -from dvc.stage.exceptions import StageCommitError from . import locked @@ -28,6 +26,8 @@ def _prepare_message(stage, changes): def prompt_to_commit(stage, changes, force=False): + from dvc.stage.exceptions import StageCommitError + if not (force or prompt.confirm(_prepare_message(stage, changes))): raise StageCommitError( "unable to commit changed {}. Use `-f|--force` to " @@ -39,6 +39,8 @@ def prompt_to_commit(stage, changes, force=False): def commit( self, target, with_deps=False, recursive=False, force=False, ): + from dvc.dvcfile import Dvcfile + stages = self.stage.collect( target, with_deps=with_deps, recursive=recursive ) diff --git a/dvc/repo/get_url.py b/dvc/repo/get_url.py index acf46d648..48475a61f 100644 --- a/dvc/repo/get_url.py +++ b/dvc/repo/get_url.py @@ -1,11 +1,11 @@ import os -import dvc.dependency as dependency -import dvc.output as output -from dvc.utils import resolve_output - def get_url(url, out=None): + import dvc.dependency as dependency + import dvc.output as output + from dvc.utils import resolve_output + out = resolve_output(url, out) if os.path.exists(url): diff --git a/dvc/repo/reproduce.py b/dvc/repo/reproduce.py index e4872b9bd..8e74e58b7 100644 --- a/dvc/repo/reproduce.py +++ b/dvc/repo/reproduce.py @@ -4,11 +4,8 @@ from functools import partial from dvc.exceptions import ReproductionError from dvc.repo.scm_context import scm_context -from dvc.stage.run import CheckpointKilledError -from dvc.utils import relpath from . import locked -from .graph import get_pipeline, get_pipelines if typing.TYPE_CHECKING: from . import Repo @@ -60,6 +57,8 @@ def _dump_stage(stage): def _track_stage(stage): + from dvc.utils import relpath + stage.repo.scm.track_file(stage.dvcfile.relpath) for dep in stage.deps: if not dep.use_scm_ignore and dep.is_in_repo: @@ -86,6 +85,8 @@ def reproduce( all_pipelines=False, **kwargs, ): + from .graph import get_pipeline, get_pipelines + glob = kwargs.pop("glob", False) accept_group = not glob @@ -185,6 +186,8 @@ def _reproduce_stages( _repro_callback, checkpoint_func, unchanged ) + from dvc.stage.run import CheckpointKilledError + try: ret = _reproduce_stage(stage, **kwargs) diff --git a/dvc/repo/run.py b/dvc/repo/run.py index 08e1c92fb..1125383a9 100644 --- a/dvc/repo/run.py +++ b/dvc/repo/run.py @@ -4,7 +4,6 @@ from contextlib import suppress from funcy import concat, first, lfilter, without from dvc.exceptions import InvalidArgumentError -from dvc.stage import PipelineStage from dvc.stage.exceptions import ( DuplicateStageName, InvalidStageName, @@ -61,6 +60,8 @@ def _get_file_path(kwargs): def _check_stage_exists(dvcfile, stage): + from dvc.stage import PipelineStage + if not dvcfile.exists(): return @@ -79,7 +80,7 @@ def _check_stage_exists(dvcfile, stage): @scm_context def run(self, fname=None, no_exec=False, single_stage=False, **kwargs): from dvc.dvcfile import PIPELINE_FILE, Dvcfile - from dvc.stage import Stage, create_stage, restore_meta + from dvc.stage import PipelineStage, Stage, create_stage, restore_meta if not kwargs.get("cmd"): raise InvalidArgumentError("command is not specified") diff --git a/dvc/repo/update.py b/dvc/repo/update.py index e28c0f650..e0a08333e 100644 --- a/dvc/repo/update.py +++ b/dvc/repo/update.py @@ -1,9 +1,10 @@ -from ..dvcfile import Dvcfile from . import locked @locked def update(self, targets=None, rev=None, recursive=False): + from ..dvcfile import Dvcfile + if not targets: targets = [None] diff --git a/dvc/updater.py b/dvc/updater.py index 1eda2a4c8..92c69d2ac 100644 --- a/dvc/updater.py +++ b/dvc/updater.py @@ -7,9 +7,6 @@ import colorama from packaging import version from dvc import __version__ -from dvc.config import Config, to_bool -from dvc.lock import LockError, make_lock -from dvc.utils import boxify, env2bool from dvc.utils.pkg import PKG logger = logging.getLogger(__name__) @@ -22,6 +19,8 @@ class Updater: # pragma: no cover TIMEOUT_GET = 10 def __init__(self, tmp_dir, friendly=False, hardlink_lock=False): + from dvc.lock import make_lock + self.updater_file = os.path.join(tmp_dir, self.UPDATER_FILE) self.lock = make_lock( self.updater_file + ".lock", @@ -39,6 +38,8 @@ class Updater: # pragma: no cover return outdated def _with_lock(self, func, action): + from dvc.lock import LockError + try: with self.lock: func() @@ -47,6 +48,8 @@ class Updater: # pragma: no cover logger.debug(msg.format(self.lock.lockfile, action)) def check(self): + from dvc.utils import env2bool + if ( os.getenv("CI") or env2bool("DVC_TEST") @@ -120,6 +123,8 @@ class Updater: # pragma: no cover latest=latest, ) + from dvc.utils import boxify + logger.info(boxify(message, border_color="yellow")) def _get_update_instructions(self): @@ -153,6 +158,8 @@ class Updater: # pragma: no cover return instructions[package_manager] def is_enabled(self): + from dvc.config import Config, to_bool + enabled = to_bool( Config(validate=False).get("core", {}).get("check_update", "true") )