From 7c2eb022092946163fc77b0d50f56c4fb0b4bdde Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Wed, 12 Dec 2018 16:59:28 +0200 Subject: [PATCH] repro: introduce --ignore-build-cache Fixes #1342 Signed-off-by: Ruslan Kuprieiev --- dvc/cli.py | 7 +++++++ dvc/command/repro.py | 15 ++++++++------- dvc/project.py | 41 +++++++++++++++++++++++++++++++---------- tests/test_repro.py | 41 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 87 insertions(+), 17 deletions(-) diff --git a/dvc/cli.py b/dvc/cli.py index 10dcc09d7..b43151a5b 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -524,6 +524,13 @@ def parse_args(argv=None): action='store_true', default=False, help='Reproduce all pipelines in the project.') + repro_parser.add_argument( + '--ignore-build-cache', + action='store_true', + default=False, + help='Reproduce all descendants of a changed stage' + 'even if their direct dependencies didn\'t ' + 'change.') repro_parser.set_defaults(func=CmdRepro) # Remove diff --git a/dvc/command/repro.py b/dvc/command/repro.py index 1d1f9c42d..726cbfeed 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -16,13 +16,14 @@ class CmdRepro(CmdBase): for target in self.args.targets: try: stages = self.project.reproduce( - target, - recursive=recursive, - force=self.args.force, - dry=self.args.dry, - interactive=self.args.interactive, - pipeline=self.args.pipeline, - all_pipelines=self.args.all_pipelines) + target, + recursive=recursive, + force=self.args.force, + dry=self.args.dry, + interactive=self.args.interactive, + pipeline=self.args.pipeline, + all_pipelines=self.args.all_pipelines, + ignore_build_cache=self.args.ignore_build_cache) if len(stages) == 0: self.project.logger.info(CmdDataStatus.UP_TO_DATE_MSG) diff --git a/dvc/project.py b/dvc/project.py index 628e66afb..dce1ade24 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -449,7 +449,8 @@ class Project(object): dry=False, interactive=False, pipeline=False, - all_pipelines=False): + all_pipelines=False, + ignore_build_cache=False): from dvc.stage import Stage if target is None and not all_pipelines: @@ -485,7 +486,8 @@ class Project(object): recursive=recursive, force=force, dry=dry, - interactive=interactive) + interactive=interactive, + ignore_build_cache=ignore_build_cache) ret.extend(stages) self._remind_to_git_add() @@ -497,7 +499,8 @@ class Project(object): recursive=True, force=False, dry=False, - interactive=False): + interactive=False, + ignore_build_cache=False): import networkx as nx from dvc.stage import Stage @@ -512,7 +515,8 @@ class Project(object): node, force, dry, - interactive) + interactive, + ignore_build_cache) else: ret = self._reproduce_stage(stages, node, @@ -522,17 +526,34 @@ class Project(object): return ret - def _reproduce_stages(self, G, stages, node, force, dry, interactive): + def _reproduce_stages(self, + G, + stages, + node, + force, + dry, + interactive, + ignore_build_cache): import networkx as nx result = [] for n in nx.dfs_postorder_nodes(G, node): try: - result += self._reproduce_stage(stages, - n, - force, - dry, - interactive) + ret = self._reproduce_stage(stages, + n, + force, + dry, + interactive) + + if len(ret) == 0 and ignore_build_cache: + # NOTE: we are walking our pipeline from the top to the + # bottom. If one stage is changed, it will be reproduced, + # which tells us that we should force reproducing all of + # the other stages down below, even if their direct + # dependencies didn't change. + force = True + + result += ret except Exception as ex: raise ReproductionError(stages[n].relpath, ex) return result diff --git a/tests/test_repro.py b/tests/test_repro.py index 29d227cdd..1d0c45edd 100644 --- a/tests/test_repro.py +++ b/tests/test_repro.py @@ -256,6 +256,47 @@ class TestReproChangedDeepData(TestReproChangedData): self.assertEqual(len(stages), 3) +class TestReproIgnoreBuildCache(TestDvc): + def test(self): + stages = self.dvc.add(self.FOO) + self.assertEqual(len(stages), 1) + foo_stage = stages[0] + self.assertTrue(foo_stage is not None) + + code1 = 'code1.py' + shutil.copyfile(self.CODE, code1) + file1 = 'file1' + file1_stage = file1 + '.dvc' + stage = self.dvc.run(fname=file1_stage, + outs=[file1], + deps=[self.FOO, code1], + cmd='python {} {} {}'.format(code1, + self.FOO, + file1)) + self.assertTrue(stage is not None) + + code2 = 'code2.py' + shutil.copyfile(self.CODE, code2) + file2 = 'file2' + file2_stage = file2 + '.dvc' + stage = self.dvc.run(fname=file2_stage, + outs=[file2], + deps=[file1, code2], + cmd='python {} {} {}'.format(code2, + file1, + file2)) + self.assertTrue(stage is not None) + + with open(code1, 'a') as fobj: + fobj.write('\n\n') + + stages = self.dvc.reproduce(file2_stage) + self.assertEqual(len(stages), 1) + + stages = self.dvc.reproduce(file2_stage, ignore_build_cache=True) + self.assertEqual(len(stages), 2) + + class TestReproPipeline(TestReproChangedDeepData): def test(self): stages = self.dvc.reproduce(self.file1_stage,