diff --git a/dvc/command/repro.py b/dvc/command/repro.py index 08e5e7cfd..b049b8c8a 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -12,6 +12,7 @@ class CmdRepro(CmdBase): force=self.args.force) except ReproductionError as ex: self.project.logger.error(ex) + return 1 except Exception as ex: msg = 'Failed to reproduce \'{}\' - unexpected error: {}'.format(target, ex) self.project.logger.error(msg) diff --git a/dvc/project.py b/dvc/project.py index 099b240fa..fa7ea455a 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -3,7 +3,7 @@ import networkx as nx from dvc.logger import Logger from dvc.exceptions import DvcException -from dvc.stage import Stage, Output, Dependency +from dvc.stage import Stage, Output from dvc.config import Config from dvc.state import State from dvc.lock import Lock @@ -116,15 +116,16 @@ class Project(object): return os.path.relpath(path, self.root_dir) def add(self, fname): - path = os.path.abspath(fname) + Stage.STAGE_FILE_SUFFIX - cwd = os.path.dirname(path) - outputs = [Output.loads(self, os.path.basename(fname), use_cache=True, cwd=cwd)] - stage = Stage(project=self, - path=path, - cmd=None, - cwd=cwd, - outs=outputs, - deps=[]) + out = os.path.basename(fname) + stage_fname = out + Stage.STAGE_FILE_SUFFIX + cwd = os.path.dirname(os.path.abspath(fname)) + stage = Stage.loads(project=self, + cmd=None, + deps=[], + outs=[out], + fname=stage_fname, + cwd=cwd) + stage.save() stage.dump() return stage @@ -153,44 +154,42 @@ class Project(object): fname=Stage.STAGE_FILE, cwd=os.curdir, no_exec=False): - cwd = os.path.abspath(cwd) - path = os.path.join(cwd, fname) - outputs = Output.loads_from(self, outs, use_cache=True, cwd=cwd) - outputs += Output.loads_from(self, outs_no_cache, use_cache=False, cwd=cwd) - deps = Dependency.loads_from(self, deps, cwd=cwd) - - stage = Stage(project=self, - path=path, - cmd=cmd, - cwd=cwd, - outs=outputs, - deps=deps) + stage = Stage.loads(project=self, + fname=fname, + cmd=cmd, + cwd=cwd, + outs=outs, + outs_no_cache=outs_no_cache, + deps=deps) if not no_exec: stage.run() stage.dump() return stage + def _reproduce_stage(self, stages, node, force): + if not stages[node].changed(): + return [] + + stages[node].reproduce(force=force) + stages[node].dump() + return [stages[node]] + def reproduce(self, target, recursive=True, force=False): stages = nx.get_node_attributes(self.graph(), 'stage') node = os.path.relpath(os.path.abspath(target), self.root_dir) if node not in stages: raise StageNotFoundError(target) - reproduced = self._reproduce_stages(force, node, stages) if recursive else [] + if recursive: + return self._reproduce_stages(stages, node, force) - stages[node].reproduce(force=force) - stages[node].dump() - reproduced.append(stages[node]) + return self._reproduce_stage(stages, node, force) - return reproduced - - def _reproduce_stages(self, force, node, stages): + def _reproduce_stages(self, stages, node, force): result = [] for n in nx.dfs_postorder_nodes(self.graph(), node): try: - stages[n].reproduce(force=force) - stages[n].dump() - result.append(stages[n]) + result += self._reproduce_stage(stages, n, force) except Exception as ex: raise ReproductionError(n, str(ex)) return result diff --git a/dvc/stage.py b/dvc/stage.py index 3a49544ba..5a6d11946 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -324,6 +324,26 @@ class Stage(object): cwd=cwd, deps=deps, outs=outs) + @staticmethod + def loads(project=None, + cmd=None, + deps=[], + outs=[], + outs_no_cache=[], + fname=None, + cwd=os.curdir): + cwd = os.path.abspath(cwd) + path = os.path.join(cwd, fname) + outputs = Output.loads_from(project, outs, use_cache=True, cwd=cwd) + outputs += Output.loads_from(project, outs_no_cache, use_cache=False, cwd=cwd) + dependencies = Dependency.loads_from(project, deps, cwd=cwd) + + return Stage(project=project, + path=path, + cmd=cmd, + cwd=cwd, + outs=outputs, + deps=dependencies) @staticmethod def load(project, fname): diff --git a/tests/test_repro.py b/tests/test_repro.py index d90bb7bfe..d1305e1c3 100644 --- a/tests/test_repro.py +++ b/tests/test_repro.py @@ -31,18 +31,20 @@ class TestReproChangedCode(TestRepro): with open(self.CODE, 'a') as code: code.write("\nshutil.copyfile('{}', sys.argv[2])\n".format(self.BAR)) - self.dvc.reproduce(self.file1_stage) + stages = self.dvc.reproduce(self.file1_stage) self.assertTrue(filecmp.cmp(self.file1, self.BAR)) + self.assertEqual(len(stages), 1) class TestReproChangedData(TestRepro): def test(self): self.swap_foo_with_bar() - self.dvc.reproduce(self.file1_stage) + stages = self.dvc.reproduce(self.file1_stage) self.assertTrue(filecmp.cmp(self.file1, self.BAR)) + self.assertEqual(len(stages), 2) def swap_foo_with_bar(self): os.chmod(self.FOO, stat.S_IWRITE) @@ -61,10 +63,11 @@ class TestReproChangedDeepData(TestReproChangedData): self.swap_foo_with_bar() - self.dvc.reproduce(file2_stage) + stages = self.dvc.reproduce(file2_stage) self.assertTrue(filecmp.cmp(self.file1, self.BAR)) self.assertTrue(filecmp.cmp(file2, self.BAR)) + self.assertEqual(len(stages), 3) class TestReproPhony(TestReproChangedData):