Merge pull request #433 from efiop/master

Misc enhancements and bug fixes

dvc/command/repro.py

@@ -12,6 +12,7 @@ class CmdRepro(CmdBase):
                                        force=self.args.force)
             except ReproductionError as ex:
                 self.project.logger.error(ex)
+                return 1
             except Exception as ex:
                 msg = 'Failed to reproduce \'{}\' - unexpected error: {}'.format(target, ex)
                 self.project.logger.error(msg)

dvc/project.py

@@ -3,7 +3,7 @@ import networkx as nx
 
 from dvc.logger import Logger
 from dvc.exceptions import DvcException
-from dvc.stage import Stage, Output, Dependency
+from dvc.stage import Stage, Output
 from dvc.config import Config
 from dvc.state import State
 from dvc.lock import Lock
@@ -116,15 +116,16 @@ class Project(object):
         return os.path.relpath(path, self.root_dir)
 
     def add(self, fname):
-        path = os.path.abspath(fname) + Stage.STAGE_FILE_SUFFIX
-        cwd = os.path.dirname(path)
-        outputs = [Output.loads(self, os.path.basename(fname), use_cache=True, cwd=cwd)]
-        stage = Stage(project=self,
-                      path=path,
-                      cmd=None,
-                      cwd=cwd,
-                      outs=outputs,
-                      deps=[])
+        out = os.path.basename(fname)
+        stage_fname = out + Stage.STAGE_FILE_SUFFIX
+        cwd = os.path.dirname(os.path.abspath(fname))
+        stage = Stage.loads(project=self,
+                            cmd=None,
+                            deps=[],
+                            outs=[out],
+                            fname=stage_fname,
+                            cwd=cwd)
+
         stage.save()
         stage.dump()
         return stage
@@ -153,44 +154,42 @@ class Project(object):
             fname=Stage.STAGE_FILE,
             cwd=os.curdir,
             no_exec=False):
-        cwd = os.path.abspath(cwd)
-        path = os.path.join(cwd, fname)
-        outputs = Output.loads_from(self, outs, use_cache=True, cwd=cwd)
-        outputs += Output.loads_from(self, outs_no_cache, use_cache=False, cwd=cwd)
-        deps = Dependency.loads_from(self, deps, cwd=cwd)
-
-        stage = Stage(project=self,
-                      path=path,
-                      cmd=cmd,
-                      cwd=cwd,
-                      outs=outputs,
-                      deps=deps)
+        stage = Stage.loads(project=self,
+                            fname=fname,
+                            cmd=cmd,
+                            cwd=cwd,
+                            outs=outs,
+                            outs_no_cache=outs_no_cache,
+                            deps=deps)
         if not no_exec:
             stage.run()
         stage.dump()
         return stage
 
+    def _reproduce_stage(self, stages, node, force):
+        if not stages[node].changed():
+            return []
+
+        stages[node].reproduce(force=force)
+        stages[node].dump()
+        return [stages[node]]
+
     def reproduce(self, target, recursive=True, force=False):
         stages = nx.get_node_attributes(self.graph(), 'stage')
         node = os.path.relpath(os.path.abspath(target), self.root_dir)
         if node not in stages:
             raise StageNotFoundError(target)
 
-        reproduced = self._reproduce_stages(force, node, stages) if recursive else []
+        if recursive:
+            return self._reproduce_stages(stages, node, force)
 
-        stages[node].reproduce(force=force)
-        stages[node].dump()
-        reproduced.append(stages[node])
+        return self._reproduce_stage(stages, node, force)
 
-        return reproduced
-
-    def _reproduce_stages(self, force, node, stages):
+    def _reproduce_stages(self, stages, node, force):
         result = []
         for n in nx.dfs_postorder_nodes(self.graph(), node):
             try:
-                stages[n].reproduce(force=force)
-                stages[n].dump()
-                result.append(stages[n])
+                result += self._reproduce_stage(stages, n, force)
             except Exception as ex:
                 raise ReproductionError(n, str(ex))
         return result

dvc/stage.py

@@ -324,6 +324,26 @@ class Stage(object):
                      cwd=cwd,
                      deps=deps,
                      outs=outs)
+    @staticmethod
+    def loads(project=None,
+              cmd=None,
+              deps=[],
+              outs=[],
+              outs_no_cache=[],
+              fname=None,
+              cwd=os.curdir):
+        cwd = os.path.abspath(cwd)
+        path = os.path.join(cwd, fname)
+        outputs = Output.loads_from(project, outs, use_cache=True, cwd=cwd)
+        outputs += Output.loads_from(project, outs_no_cache, use_cache=False, cwd=cwd)
+        dependencies = Dependency.loads_from(project, deps, cwd=cwd)
+
+        return Stage(project=project,
+                     path=path,
+                     cmd=cmd,
+                     cwd=cwd,
+                     outs=outputs,
+                     deps=dependencies)
 
     @staticmethod
     def load(project, fname):

tests/test_repro.py

@@ -31,18 +31,20 @@ class TestReproChangedCode(TestRepro):
         with open(self.CODE, 'a') as code:
             code.write("\nshutil.copyfile('{}', sys.argv[2])\n".format(self.BAR))
 
-        self.dvc.reproduce(self.file1_stage)
+        stages = self.dvc.reproduce(self.file1_stage)
 
         self.assertTrue(filecmp.cmp(self.file1, self.BAR))
+        self.assertEqual(len(stages), 1)
 
 
 class TestReproChangedData(TestRepro):
     def test(self):
         self.swap_foo_with_bar()
 
-        self.dvc.reproduce(self.file1_stage)
+        stages = self.dvc.reproduce(self.file1_stage)
 
         self.assertTrue(filecmp.cmp(self.file1, self.BAR))
+        self.assertEqual(len(stages), 2)
 
     def swap_foo_with_bar(self):
         os.chmod(self.FOO, stat.S_IWRITE)
@@ -61,10 +63,11 @@ class TestReproChangedDeepData(TestReproChangedData):
 
         self.swap_foo_with_bar()
 
-        self.dvc.reproduce(file2_stage)
+        stages = self.dvc.reproduce(file2_stage)
 
         self.assertTrue(filecmp.cmp(self.file1, self.BAR))
         self.assertTrue(filecmp.cmp(file2, self.BAR))
+        self.assertEqual(len(stages), 3)
 
 
 class TestReproPhony(TestReproChangedData):