From f04ae2f74fa248aa6fce89f797283758b362de89 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Sun, 2 Dec 2018 04:33:48 +0200 Subject: [PATCH] dvc: optimize imports Fixes #903 Signed-off-by: Ruslan Kuprieiev --- dvc/analytics.py | 12 ++++++++---- dvc/cli.py | 7 +------ dvc/command/base.py | 3 ++- dvc/command/pipeline.py | 3 ++- dvc/exceptions.py | 11 +++++++++++ dvc/main.py | 3 +-- dvc/project.py | 26 ++++++++++++++++++-------- tests/test_cli.py | 14 -------------- 8 files changed, 43 insertions(+), 36 deletions(-) diff --git a/dvc/analytics.py b/dvc/analytics.py index 163ed6226..67c6d39fa 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -3,8 +3,6 @@ import json import errno from dvc import VERSION -from dvc.utils import is_binary -from dvc.lock import Lock, LockError from dvc.logger import Logger @@ -38,6 +36,7 @@ class Analytics(object): def __init__(self, info={}): from dvc.config import Config + from dvc.lock import Lock self.info = info @@ -80,6 +79,8 @@ class Analytics(object): return None def _get_user_id(self): + from dvc.lock import LockError + try: with self.user_id_file_lock: user_id = self._read_user_id() @@ -131,7 +132,9 @@ class Analytics(object): def collect(self): from dvc.scm import SCM - from dvc.project import Project, NotDvcProjectError + from dvc.utils import is_binary + from dvc.project import Project + from dvc.exceptions import NotDvcProjectError self.info[self.PARAM_DVC_VERSION] = VERSION self.info[self.PARAM_IS_BINARY] = is_binary() @@ -167,7 +170,8 @@ class Analytics(object): @staticmethod def _is_enabled(cmd=None): from dvc.config import Config - from dvc.project import Project, NotDvcProjectError + from dvc.project import Project + from dvc.exceptions import NotDvcProjectError from dvc.command.daemon import CmdDaemonBase if os.getenv('DVC_TEST'): diff --git a/dvc/cli.py b/dvc/cli.py index 607911670..4f3ceb90b 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -27,7 +27,7 @@ from dvc.command.root import CmdRoot from dvc.command.lock import CmdLock, CmdUnlock from dvc.command.pipeline import CmdPipelineShow, CmdPipelineList from dvc.command.daemon import CmdDaemonUpdater, CmdDaemonAnalytics -from dvc.exceptions import DvcException +from dvc.exceptions import DvcParserError from dvc.logger import Logger from dvc import VERSION @@ -42,11 +42,6 @@ def _fix_subparsers(subparsers): subparsers.dest = 'cmd' -class DvcParserError(DvcException): - def __init__(self): - super(DvcException, self).__init__("Parser error") - - class DvcParser(argparse.ArgumentParser): def error(self, message): sys.stderr.write('{}{}\n'.format(Logger.error_prefix(), message)) diff --git a/dvc/command/base.py b/dvc/command/base.py index 8befa169e..2d9be98c1 100644 --- a/dvc/command/base.py +++ b/dvc/command/base.py @@ -1,5 +1,4 @@ from dvc.logger import Logger -from dvc.lock import LockError class CmdBase(object): @@ -20,6 +19,8 @@ class CmdBase(object): Logger.be_verbose() def run_cmd(self): + from dvc.lock import LockError + try: with self.project.lock: return self.run() diff --git a/dvc/command/pipeline.py b/dvc/command/pipeline.py index 83eb40ebc..c17b02357 100644 --- a/dvc/command/pipeline.py +++ b/dvc/command/pipeline.py @@ -2,7 +2,6 @@ import os from dvc.exceptions import DvcException from dvc.command.base import CmdBase -from dvc.dagascii import Dagascii class CmdPipelineShow(CmdBase): @@ -70,6 +69,8 @@ class CmdPipelineShow(CmdBase): return nodes, edges def _show_ascii(self, target, commands, outs): + from dvc.dagascii import Dagascii + nodes, edges = self.__build_graph(target, commands, outs) if not nodes: diff --git a/dvc/exceptions.py b/dvc/exceptions.py index f4ac40ae4..d98019142 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -57,3 +57,14 @@ class MoveNotDataSourceError(DvcException): "it by hand, or remove '{path}' and create a new one at the " \ "desired location." super(MoveNotDataSourceError, self).__init__(msg.format(path=path)) + + +class NotDvcProjectError(DvcException): + def __init__(self, root): + msg = "Not a dvc repository (checked up to mount point {})" + super(NotDvcProjectError, self).__init__(msg.format(root)) + + +class DvcParserError(DvcException): + def __init__(self): + super(DvcException, self).__init__("Parser error") diff --git a/dvc/main.py b/dvc/main.py index 0560a127a..c95867770 100644 --- a/dvc/main.py +++ b/dvc/main.py @@ -2,8 +2,7 @@ from dvc.logger import Logger from dvc.cli import parse_args from dvc.command.base import CmdBase from dvc.analytics import Analytics -from dvc.cli import DvcParserError -from dvc.project import NotDvcProjectError +from dvc.exceptions import NotDvcProjectError, DvcParserError def main(argv=None): diff --git a/dvc/project.py b/dvc/project.py index 99749a95a..7703dea5c 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -2,7 +2,7 @@ import collections import os from dvc.exceptions import DvcException, MoveNotDataSourceError -from dvc.stage import Stage +from dvc.exceptions import NotDvcProjectError class InitError(DvcException): @@ -10,12 +10,6 @@ class InitError(DvcException): super(InitError, self).__init__(msg) -class NotDvcProjectError(DvcException): - def __init__(self, root): - msg = "Not a dvc repository (checked up to mount point {})" - super(NotDvcProjectError, self).__init__(msg.format(root)) - - class ReproductionError(DvcException): def __init__(self, dvc_file_name, ex): self.path = dvc_file_name @@ -227,6 +221,8 @@ class Project(object): raise CircularDependencyError(circular_dependencies.pop()) def add(self, fname, recursive=False): + from dvc.stage import Stage + fnames = [] if recursive and os.path.isdir(fname): fnames = [] @@ -263,6 +259,8 @@ class Project(object): return stages def remove(self, target, outs_only=False): + from dvc.stage import Stage + stage = Stage.load(self, target) if outs_only: stage.remove_outs() @@ -272,6 +270,8 @@ class Project(object): return stage def lock_stage(self, target, unlock=False): + from dvc.stage import Stage + stage = Stage.load(self, target) stage.locked = False if unlock else True stage.dump() @@ -280,6 +280,7 @@ class Project(object): def move(self, from_path, to_path): import dvc.output as Output + from dvc.stage import Stage from_out = Output.loads_from(Stage(self, cwd=os.curdir), [from_path])[0] @@ -359,10 +360,12 @@ class Project(object): outs=[], outs_no_cache=[], metrics_no_cache=[], - fname=Stage.STAGE_FILE, + fname=None, cwd=os.curdir, no_exec=False, overwrite=False): + from dvc.stage import Stage + stage = Stage.loads(project=self, fname=fname, cmd=cmd, @@ -391,6 +394,8 @@ class Project(object): return stage def imp(self, url, out): + from dvc.stage import Stage + stage = Stage.loads(project=self, cmd=None, deps=[url], @@ -434,6 +439,7 @@ class Project(object): interactive=False, pipeline=False, all_pipelines=False): + from dvc.stage import Stage if target is None and not all_pipelines: raise ValueError() @@ -482,6 +488,7 @@ class Project(object): dry=False, interactive=False): import networkx as nx + from dvc.stage import Stage stage = Stage.load(self, target) G = self.graph()[1] @@ -552,6 +559,7 @@ class Project(object): def _collect(self, target, with_deps=False): import networkx as nx + from dvc.stage import Stage stage = Stage.load(self, target) if not with_deps: @@ -1169,6 +1177,8 @@ class Project(object): NOTE: For large projects, this could be an expensive operation. Consider using some memorization. """ + from dvc.stage import Stage + stages = [] outs = [] for root, dirs, files in os.walk(self.root_dir): diff --git a/tests/test_cli.py b/tests/test_cli.py index 4aa22d47c..c31c58124 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -165,20 +165,6 @@ class TestCheckout(TestDvc): self.assertIsInstance(args.func(args), CmdCheckout) -class TestStartupTime(TestDvc): - def test(self): - import timeit - - start = timeit.default_timer() - try: - main([]) - except SystemExit: - pass - t = timeit.default_timer() - start - - self.assertTrue(t < 0.3) - - class TestFindRoot(TestDvc): def test(self): os.chdir("..")