Merge pull request #906 from efiop/master

dvc: optimize imports to speedup startup time

dvc/cli.py

@@ -25,7 +25,6 @@ from dvc.command.install import CmdInstall
 from dvc.command.root import CmdRoot
 from dvc.command.lock import CmdLock, CmdUnlock
 from dvc.command.pipeline import CmdPipelineShow
-from dvc.stage import Stage
 from dvc import VERSION
 
 
@@ -268,7 +267,7 @@ def parse_args(argv=None):
     repro_parser.add_argument(
                         'targets',
                         nargs='*',
-                        default=[Stage.STAGE_FILE],
+                        default=['Dvcfile'],
                         help='DVC file to reproduce.')
     repro_parser.add_argument('-f',
                         '--force',

dvc/command/common/base.py

@@ -1,6 +1,5 @@
 import os
 
-from dvc.project import Project
 from dvc.logger import Logger
 from dvc.exceptions import DvcException
 from dvc.lock import LockError
@@ -8,6 +7,8 @@ from dvc.lock import LockError
 
 class CmdBase(object):
     def __init__(self, args):
+        from dvc.project import Project
+
         self.project = Project(self._find_root())
         self.args = args
         self._set_loglevel(args)
@@ -20,6 +21,8 @@ class CmdBase(object):
             Logger.be_verbose()
 
     def _find_root(self):
+        from dvc.project import Project
+
         root = os.getcwd()
         while not os.path.ismount(root):
             dvc_dir = os.path.join(root, Project.DVC_DIR)

dvc/command/config.py

@@ -4,11 +4,12 @@ import configobj
 from dvc.command.common.base import CmdBase
 from dvc.logger import Logger
 from dvc.config import Config
-from dvc.project import Project
 
 
 class CmdConfig(CmdBase):
     def __init__(self, args):
+        from dvc.project import Project
+
         self.args = args
         root_dir = self._find_root()
         if args.local:

dvc/command/init.py

@@ -1,4 +1,3 @@
-from dvc.project import Project, InitError
 from dvc.logger import Logger
 
 
@@ -7,6 +6,8 @@ class CmdInit(object):
         self.args = args
 
     def run_cmd(self):
+        from dvc.project import Project, InitError
+
         try:
             Project.init('.', no_scm=self.args.no_scm, force=self.args.force)
         except InitError as e:

dvc/command/install.py

@@ -1,4 +1,3 @@
-from dvc.project import Project
 from dvc.logger import Logger
 from dvc.command.common.base import CmdBase
 

dvc/command/pipeline.py

@@ -1,13 +1,14 @@
 import os
-import networkx
 
 from dvc.exceptions import DvcException
 from dvc.command.common.base import CmdBase
-from dvc.project import NotDvcFileError
 
 
 class CmdPipelineShow(CmdBase):
     def _show(self, target, commands, outs):
+        import networkx
+        from dvc.project import NotDvcFileError
+
         G = self.project.graph()[0]
         stages = networkx.get_node_attributes(G, 'stage')
         node = os.path.relpath(os.path.abspath(target), self.project.root_dir)

dvc/command/repro.py

@@ -1,12 +1,13 @@
 import os
 
 from dvc.command.common.base import CmdBase
-from dvc.project import ReproductionError
-from dvc.stage import Stage
 
 
 class CmdRepro(CmdBase):
     def run(self):
+        from dvc.project import ReproductionError
+        from dvc.stage import Stage
+
         recursive = not self.args.single_item
         saved_dir = os.path.realpath(os.curdir)
         if self.args.cwd:

dvc/command/run.py

@@ -2,7 +2,6 @@ import os
 
 from dvc.command.common.base import CmdBase
 from dvc.logger import Logger
-from dvc.stage import Stage
 from dvc.exceptions import DvcException
 
 

dvc/project.py

@@ -1,23 +1,8 @@
 import os
-import csv
 import stat
-import json
-import shutil
-import networkx as nx
-from jsonpath_rw import parse
 
-import dvc.output as Output
-from dvc.logger import Logger
 from dvc.exceptions import DvcException
 from dvc.stage import Stage
-from dvc.config import Config
-from dvc.state import LinkState, State
-from dvc.lock import Lock
-from dvc.scm import SCM, Base
-from dvc.cache import Cache
-from dvc.data_cloud import DataCloud
-from dvc.system import System
-from dvc.updater import Updater
 
 
 class InitError(DvcException):
@@ -45,6 +30,15 @@ class Project(object):
     DVC_DIR = '.dvc'
 
     def __init__(self, root_dir):
+        from dvc.logger import Logger
+        from dvc.config import Config
+        from dvc.state import LinkState, State
+        from dvc.lock import Lock
+        from dvc.scm import SCM, Base
+        from dvc.cache import Cache
+        from dvc.data_cloud import DataCloud
+        from dvc.updater import Updater
+
         self.root_dir = os.path.abspath(os.path.realpath(root_dir))
         self.dvc_dir = os.path.join(self.root_dir, self.DVC_DIR)
 
@@ -78,6 +72,10 @@ class Project(object):
         Raises:
             KeyError: Raises an exception.
         """
+        import shutil
+        from dvc.scm import SCM, Base
+        from dvc.config import Config
+
         root_dir = os.path.abspath(root_dir)
         dvc_dir = os.path.join(root_dir, Project.DVC_DIR)
 
@@ -104,6 +102,8 @@ class Project(object):
         return proj
 
     def destroy(self):
+        import shutil
+
         for stage in self.stages():
             stage.remove()
 
@@ -160,6 +160,8 @@ class Project(object):
         return stage
 
     def move(self, from_path, to_path):
+        import dvc.output as Output
+
         from_out = Output.loads_from(Stage(self, cwd=os.curdir), [from_path])[0]
 
         found = False
@@ -236,6 +238,8 @@ class Project(object):
         return [stage]
 
     def reproduce(self, target, recursive=True, force=False):
+        import networkx as nx
+
         G = self.graph()[1]
         stages = nx.get_node_attributes(G, 'stage')
         node = os.path.relpath(os.path.abspath(target), self.root_dir)
@@ -248,6 +252,8 @@ class Project(object):
         return self._reproduce_stage(stages, node, force)
 
     def _reproduce_stages(self, G, stages, node, force):
+        import networkx as nx
+
         result = []
         for n in nx.dfs_postorder_nodes(G, node):
             try:
@@ -388,6 +394,9 @@ class Project(object):
         return self._local_status(target)
 
     def _read_metric_json(self, fd, json_path):
+        import json
+        from jsonpath_rw import parse
+
         parser = parse(json_path)
         return [x.value for x in parser.find(json.load(fd))]
 
@@ -401,12 +410,16 @@ class Project(object):
         return None
 
     def _read_metric_htsv(self, fd, htsv_path):
+        import csv
+
         col, row = htsv_path.split(',')
         row = int(row)
         reader = list(csv.DictReader(fd, delimiter='\t'))
         return self._do_read_metric_tsv(reader, row, col)
 
     def _read_metric_tsv(self, fd, tsv_path):
+        import csv
+
         col, row = tsv_path.split(',')
         row = int(row)
         col = int(col)
@@ -488,6 +501,8 @@ class Project(object):
         self._metrics_modify(path, False)
 
     def graph(self):
+        import networkx as nx
+
         G = nx.DiGraph()
         G_active = nx.DiGraph()
         stages = self.stages()
@@ -519,6 +534,8 @@ class Project(object):
         return G, G_active
 
     def pipelines(self):
+        import networkx as nx
+
         G, G_active = self.graph()
 
         if len(G.nodes()) == 0:
@@ -552,6 +569,8 @@ class Project(object):
         return stages
 
     def active_stages(self):
+        import networkx as nx
+
         stages = []
         for G in self.pipelines():
             stages.extend(list(nx.get_node_attributes(G, 'stage').values()))

tests/test_cli.py

@@ -1,6 +1,7 @@
 import os
 
 from dvc.cli import parse_args
+from dvc.main import main
 from dvc.command.init import CmdInit
 from dvc.command.run import CmdRun
 from dvc.command.data_sync import CmdDataPull, CmdDataPush
@@ -146,3 +147,17 @@ class TestCheckout(TestDvc):
     def test(self):
         args = parse_args(['checkout'])
         self.assertIsInstance(args.func(args), CmdCheckout)
+
+
+class TestStartupTime(TestDvc):
+    def test(self):
+        import timeit
+
+        start = timeit.default_timer()
+        try:
+            main([])
+        except SystemExit:
+            pass
+        t = timeit.default_timer() - start
+
+        self.assertTrue(t < 0.25)

tests/test_data_cloud.py

@@ -90,7 +90,7 @@ def _should_test_hdfs():
         return False
 
     try:
-        check_output(['hadoop', 'version'])
+        check_output(['hadoop', 'version'], shell=True, executable=os.getenv('SHELL'))
     except (CalledProcessError, IOError):
         return False