mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-10 15:28:34 +01:00
repro: introduce --ignore-build-cache
Fixes #1342 Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
@@ -524,6 +524,13 @@ def parse_args(argv=None):
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Reproduce all pipelines in the project.')
|
||||
repro_parser.add_argument(
|
||||
'--ignore-build-cache',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help='Reproduce all descendants of a changed stage'
|
||||
'even if their direct dependencies didn\'t '
|
||||
'change.')
|
||||
repro_parser.set_defaults(func=CmdRepro)
|
||||
|
||||
# Remove
|
||||
|
||||
@@ -16,13 +16,14 @@ class CmdRepro(CmdBase):
|
||||
for target in self.args.targets:
|
||||
try:
|
||||
stages = self.project.reproduce(
|
||||
target,
|
||||
recursive=recursive,
|
||||
force=self.args.force,
|
||||
dry=self.args.dry,
|
||||
interactive=self.args.interactive,
|
||||
pipeline=self.args.pipeline,
|
||||
all_pipelines=self.args.all_pipelines)
|
||||
target,
|
||||
recursive=recursive,
|
||||
force=self.args.force,
|
||||
dry=self.args.dry,
|
||||
interactive=self.args.interactive,
|
||||
pipeline=self.args.pipeline,
|
||||
all_pipelines=self.args.all_pipelines,
|
||||
ignore_build_cache=self.args.ignore_build_cache)
|
||||
|
||||
if len(stages) == 0:
|
||||
self.project.logger.info(CmdDataStatus.UP_TO_DATE_MSG)
|
||||
|
||||
+31
-10
@@ -449,7 +449,8 @@ class Project(object):
|
||||
dry=False,
|
||||
interactive=False,
|
||||
pipeline=False,
|
||||
all_pipelines=False):
|
||||
all_pipelines=False,
|
||||
ignore_build_cache=False):
|
||||
from dvc.stage import Stage
|
||||
|
||||
if target is None and not all_pipelines:
|
||||
@@ -485,7 +486,8 @@ class Project(object):
|
||||
recursive=recursive,
|
||||
force=force,
|
||||
dry=dry,
|
||||
interactive=interactive)
|
||||
interactive=interactive,
|
||||
ignore_build_cache=ignore_build_cache)
|
||||
ret.extend(stages)
|
||||
|
||||
self._remind_to_git_add()
|
||||
@@ -497,7 +499,8 @@ class Project(object):
|
||||
recursive=True,
|
||||
force=False,
|
||||
dry=False,
|
||||
interactive=False):
|
||||
interactive=False,
|
||||
ignore_build_cache=False):
|
||||
import networkx as nx
|
||||
from dvc.stage import Stage
|
||||
|
||||
@@ -512,7 +515,8 @@ class Project(object):
|
||||
node,
|
||||
force,
|
||||
dry,
|
||||
interactive)
|
||||
interactive,
|
||||
ignore_build_cache)
|
||||
else:
|
||||
ret = self._reproduce_stage(stages,
|
||||
node,
|
||||
@@ -522,17 +526,34 @@ class Project(object):
|
||||
|
||||
return ret
|
||||
|
||||
def _reproduce_stages(self, G, stages, node, force, dry, interactive):
|
||||
def _reproduce_stages(self,
|
||||
G,
|
||||
stages,
|
||||
node,
|
||||
force,
|
||||
dry,
|
||||
interactive,
|
||||
ignore_build_cache):
|
||||
import networkx as nx
|
||||
|
||||
result = []
|
||||
for n in nx.dfs_postorder_nodes(G, node):
|
||||
try:
|
||||
result += self._reproduce_stage(stages,
|
||||
n,
|
||||
force,
|
||||
dry,
|
||||
interactive)
|
||||
ret = self._reproduce_stage(stages,
|
||||
n,
|
||||
force,
|
||||
dry,
|
||||
interactive)
|
||||
|
||||
if len(ret) == 0 and ignore_build_cache:
|
||||
# NOTE: we are walking our pipeline from the top to the
|
||||
# bottom. If one stage is changed, it will be reproduced,
|
||||
# which tells us that we should force reproducing all of
|
||||
# the other stages down below, even if their direct
|
||||
# dependencies didn't change.
|
||||
force = True
|
||||
|
||||
result += ret
|
||||
except Exception as ex:
|
||||
raise ReproductionError(stages[n].relpath, ex)
|
||||
return result
|
||||
|
||||
@@ -256,6 +256,47 @@ class TestReproChangedDeepData(TestReproChangedData):
|
||||
self.assertEqual(len(stages), 3)
|
||||
|
||||
|
||||
class TestReproIgnoreBuildCache(TestDvc):
|
||||
def test(self):
|
||||
stages = self.dvc.add(self.FOO)
|
||||
self.assertEqual(len(stages), 1)
|
||||
foo_stage = stages[0]
|
||||
self.assertTrue(foo_stage is not None)
|
||||
|
||||
code1 = 'code1.py'
|
||||
shutil.copyfile(self.CODE, code1)
|
||||
file1 = 'file1'
|
||||
file1_stage = file1 + '.dvc'
|
||||
stage = self.dvc.run(fname=file1_stage,
|
||||
outs=[file1],
|
||||
deps=[self.FOO, code1],
|
||||
cmd='python {} {} {}'.format(code1,
|
||||
self.FOO,
|
||||
file1))
|
||||
self.assertTrue(stage is not None)
|
||||
|
||||
code2 = 'code2.py'
|
||||
shutil.copyfile(self.CODE, code2)
|
||||
file2 = 'file2'
|
||||
file2_stage = file2 + '.dvc'
|
||||
stage = self.dvc.run(fname=file2_stage,
|
||||
outs=[file2],
|
||||
deps=[file1, code2],
|
||||
cmd='python {} {} {}'.format(code2,
|
||||
file1,
|
||||
file2))
|
||||
self.assertTrue(stage is not None)
|
||||
|
||||
with open(code1, 'a') as fobj:
|
||||
fobj.write('\n\n')
|
||||
|
||||
stages = self.dvc.reproduce(file2_stage)
|
||||
self.assertEqual(len(stages), 1)
|
||||
|
||||
stages = self.dvc.reproduce(file2_stage, ignore_build_cache=True)
|
||||
self.assertEqual(len(stages), 2)
|
||||
|
||||
|
||||
class TestReproPipeline(TestReproChangedDeepData):
|
||||
def test(self):
|
||||
stages = self.dvc.reproduce(self.file1_stage,
|
||||
|
||||
Reference in New Issue
Block a user