From 4c53e636bcbab2994cb813dd577977fec3102fde Mon Sep 17 00:00:00 2001 From: Saugat Pachhai Date: Wed, 6 Jan 2021 17:44:02 +0545 Subject: [PATCH] Move misc. imports from command, make them lazy (#5214) --- dvc/command/add.py | 6 +++++- dvc/command/dag.py | 3 ++- dvc/command/experiments.py | 7 ++++--- dvc/command/repro.py | 3 +-- dvc/command/version.py | 3 ++- dvc/dvcfile.py | 9 +++++---- dvc/exceptions.py | 14 ++++++++++++-- dvc/utils/flatten.py | 7 ++++--- 8 files changed, 35 insertions(+), 17 deletions(-) diff --git a/dvc/command/add.py b/dvc/command/add.py index 92d286599..db112162f 100644 --- a/dvc/command/add.py +++ b/dvc/command/add.py @@ -3,13 +3,17 @@ import logging from dvc.command import completion from dvc.command.base import CmdBase, append_doc_link -from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename logger = logging.getLogger(__name__) class CmdAdd(CmdBase): def run(self): + from dvc.exceptions import ( + DvcException, + RecursiveAddingWhileUsingFilename, + ) + try: if len(self.args.targets) > 1 and self.args.file: raise RecursiveAddingWhileUsingFilename() diff --git a/dvc/command/dag.py b/dvc/command/dag.py index c166a6745..21840eb9e 100644 --- a/dvc/command/dag.py +++ b/dvc/command/dag.py @@ -2,7 +2,6 @@ import argparse import logging from dvc.command.base import CmdBase, append_doc_link -from dvc.stage import Stage logger = logging.getLogger(__name__) @@ -53,6 +52,8 @@ def _collect_targets(repo, target, outs): def _transform(repo, outs): import networkx as nx + from dvc.stage import Stage + def _relabel(node) -> str: return node.addressing if isinstance(node, Stage) else str(node) diff --git a/dvc/command/experiments.py b/dvc/command/experiments.py index bb5290a2b..6f2190b9c 100644 --- a/dvc/command/experiments.py +++ b/dvc/command/experiments.py @@ -13,8 +13,6 @@ from dvc.command.metrics import DEFAULT_PRECISION from dvc.command.repro import CmdRepro from dvc.command.repro import add_arguments as add_repro_arguments from dvc.exceptions import DvcException, InvalidArgumentError -from dvc.repo.experiments import Experiments -from dvc.scm.git import Git from dvc.utils.flatten import flatten logger = logging.getLogger(__name__) @@ -109,6 +107,8 @@ def _collect_rows( sort_by=None, sort_order=None, ): + from dvc.scm.git import Git + if sort_by: if sort_by in metric_names: sort_type = "metrics" @@ -601,6 +601,7 @@ class CmdExperimentsPull(CmdBase): def add_parser(subparsers, parent_parser): EXPERIMENTS_HELP = "Commands to run and compare experiments." + LAST_CHECKPOINT = ":last" experiments_parser = subparsers.add_parser( "experiments", @@ -831,7 +832,7 @@ def add_parser(subparsers, parent_parser): "-r", "--rev", type=str, - default=Experiments.LAST_CHECKPOINT, + default=LAST_CHECKPOINT, dest="checkpoint_resume", help=( "Continue the specified checkpoint experiment. " diff --git a/dvc/command/repro.py b/dvc/command/repro.py index 5b3374172..e395546c8 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -5,7 +5,6 @@ from dvc.command import completion from dvc.command.base import CmdBase, append_doc_link from dvc.command.metrics import _show_metrics from dvc.command.status import CmdDataStatus -from dvc.dvcfile import PIPELINE_FILE logger = logging.getLogger(__name__) @@ -50,7 +49,7 @@ def add_arguments(repro_parser): repro_parser.add_argument( "targets", nargs="*", - help=f"Stages to reproduce. '{PIPELINE_FILE}' by default.", + help="Stages to reproduce. 'dvc.yaml' by default.", ).complete = completion.DVC_FILE repro_parser.add_argument( "-f", diff --git a/dvc/command/version.py b/dvc/command/version.py index 92c77209b..c2db8f31a 100644 --- a/dvc/command/version.py +++ b/dvc/command/version.py @@ -2,13 +2,14 @@ import argparse import logging from dvc.command.base import CmdBaseNoRepo, append_doc_link -from dvc.info import get_dvc_info logger = logging.getLogger(__name__) class CmdVersion(CmdBaseNoRepo): def run(self): + from dvc.info import get_dvc_info + dvc_info = get_dvc_info() logger.info(dvc_info) return 0 diff --git a/dvc/dvcfile.py b/dvc/dvcfile.py index 44207352c..6f0e5d8b7 100644 --- a/dvc/dvcfile.py +++ b/dvc/dvcfile.py @@ -14,7 +14,6 @@ from dvc.stage.exceptions import ( StageFileFormatError, StageFileIsNotDvcFileError, ) -from dvc.stage.loader import SingleStageLoader, StageLoader from dvc.utils import relpath from dvc.utils.collections import apply_diff from dvc.utils.serialize import ( @@ -137,16 +136,17 @@ class FileMixin: class SingleStageFile(FileMixin): from dvc.schema import COMPILED_SINGLE_STAGE_SCHEMA as SCHEMA + from dvc.stage.loader import SingleStageLoader as LOADER @property def stage(self): data, raw = self._load() - return SingleStageLoader.load_stage(self, data, raw) + return self.LOADER.load_stage(self, data, raw) @property def stages(self): data, raw = self._load() - return SingleStageLoader(self, data, raw) + return self.LOADER(self, data, raw) def dump(self, stage, **kwargs): """Dumps given stage appropriately in the dvcfile.""" @@ -177,6 +177,7 @@ class PipelineFile(FileMixin): """Abstraction for pipelines file, .yaml + .lock combined.""" from dvc.schema import COMPILED_MULTI_STAGE_SCHEMA as SCHEMA + from dvc.stage.loader import StageLoader as LOADER @property def _lockfile(self): @@ -239,7 +240,7 @@ class PipelineFile(FileMixin): def stages(self): data, _ = self._load() lockfile_data = self._lockfile.load() - return StageLoader(self, data, lockfile_data) + return self.LOADER(self, data, lockfile_data) def remove(self, force=False): if not force: diff --git a/dvc/exceptions.py b/dvc/exceptions.py index e89cd9c28..26461c253 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -3,8 +3,6 @@ from typing import List from funcy import first -from dvc.utils import error_link, format_link, relpath - class DvcException(Exception): """Base class for all dvc exceptions.""" @@ -51,6 +49,8 @@ class OutputNotFoundError(DvcException): """ def __init__(self, output, repo=None): + from dvc.utils import relpath + self.output = output self.repo = repo super().__init__( @@ -253,6 +253,8 @@ class DvcIgnoreInCollectedDirError(DvcException): class GitHookAlreadyExistsError(DvcException): def __init__(self, hook_name): + from dvc.utils import format_link + super().__init__( "Hook '{}' already exists. Please refer to {} for more " "info.".format( @@ -277,6 +279,8 @@ class UploadError(DvcException): class CheckoutError(DvcException): def __init__(self, target_infos, stats=None): + from dvc.utils import error_link + self.target_infos = target_infos self.stats = stats targets = [str(t) for t in target_infos] @@ -302,6 +306,8 @@ class NoRemoteInExternalRepoError(DvcException): class NoOutputInExternalRepoError(DvcException): def __init__(self, path, external_repo_path, external_repo_url): + from dvc.utils import relpath + super().__init__( "Output '{}' not found in target repository '{}'".format( relpath(path, external_repo_path), external_repo_url @@ -332,6 +338,8 @@ class PathMissingError(DvcException): class RemoteCacheRequiredError(DvcException): def __init__(self, path_info): + from dvc.utils import format_link + super().__init__( ( "Current operation was unsuccessful because '{}' requires " @@ -366,6 +374,8 @@ class MergeError(DvcException): class CacheLinkError(DvcException): + from dvc.utils import format_link + SUPPORT_LINK = "See {} for more information.".format( format_link( "https://dvc.org/doc/user-guide/troubleshooting#cache-types" diff --git a/dvc/utils/flatten.py b/dvc/utils/flatten.py index ff5f2c9f1..0d501eefe 100644 --- a/dvc/utils/flatten.py +++ b/dvc/utils/flatten.py @@ -1,9 +1,10 @@ -import flatten_dict - - def flatten(d): + import flatten_dict + return flatten_dict.flatten(d, reducer="dot") def unflatten(d): + import flatten_dict + return flatten_dict.unflatten(d, splitter="dot")