Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

run/repro: rename "build cache" -> "run cache" (#3788)

Browse Source 
Part of #3777
This commit is contained in:
Ruslan Kuprieiev
2020-05-14 02:27:18 +03:00
committed by GitHub
parent 597ab98d0e
commit 974cf097dd
11 changed files with 45 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/command/repro.py
+12 -2
View File
@@ -33,10 +33,11 @@ class CmdRepro(CmdBase):
interactive=self.args.interactive,
pipeline=self.args.pipeline,
all_pipelines=self.args.all_pipelines,
ignore_build_cache=self.args.ignore_build_cache,
run_cache=not self.args.no_run_cache,
no_commit=self.args.no_commit,
downstream=self.args.downstream,
recursive=self.args.recursive,
force_downstream=self.args.force_downstream,
)
if len(stages) == 0:
@@ -136,7 +137,16 @@ def add_parser(subparsers, parent_parser):
help="Reproduce all stages in the specified directory.",
)
repro_parser.add_argument(
"--ignore-build-cache",
"--no-run-cache",
action="store_true",
default=False,
help=(
"Execute stage commands even if they have already been run with "
"the same command/dependencies/outputs/etc before."
),
)
repro_parser.add_argument(
"--force-downstream",
action="store_true",
default=False,
help="Reproduce all descendants of a changed stage even if their "
dvc/command/run.py
+6 -4
View File
@@ -42,7 +42,7 @@ class CmdRun(CmdBase):
wdir=self.args.wdir,
no_exec=self.args.no_exec,
overwrite=self.args.overwrite_dvcfile,
ignore_build_cache=self.args.ignore_build_cache,
run_cache=not self.args.no_run_cache,
no_commit=self.args.no_commit,
outs_persist=self.args.outs_persist,
outs_persist_no_cache=self.args.outs_persist_no_cache,
@@ -165,11 +165,13 @@ def add_parser(subparsers, parent_parser):
help="Overwrite existing DVC-file without asking for confirmation.",
)
run_parser.add_argument(
"--ignore-build-cache",
"--no-run-cache",
action="store_true",
default=False,
help="Run this stage even if it has been already ran with the same "
"command/dependencies/outputs/etc before.",
help=(
"Execute the command even if this stage has already been run "
"with the same command/dependencies/outputs/etc before."
),
)
run_parser.add_argument(
"--no-commit",
dvc/repo/reproduce.py
+2 -1
View File
@@ -162,12 +162,13 @@ def _reproduce_stages(
if stage not in pipeline:
pipeline.append(stage)
force_downstream = kwargs.pop("force_downstream", False)
result = []
for stage in pipeline:
try:
ret = _reproduce_stage(stage, **kwargs)
if len(ret) != 0 and kwargs.get("ignore_build_cache", False):
if len(ret) != 0 and force_downstream:
# NOTE: we are walking our pipeline from the top to the
# bottom. If one stage is changed, it will be reproduced,
# which tells us that we should force reproducing all of
dvc/repo/run.py
+1 -1
View File
@@ -85,7 +85,7 @@ def run(self, fname=None, no_exec=False, single_stage=False, **kwargs):
else:
stage.run(
no_commit=kwargs.get("no_commit", False),
ignore_build_cache=kwargs.get("ignore_build_cache", False),
run_cache=kwargs.get("run_cache", True),
)
dvcfile.dump(stage, update_pipeline=True)
dvc/stage/__init__.py
+5 -9
View File
@@ -70,13 +70,13 @@ def create_stage(cls, repo, path, **kwargs):
if stage and stage.dvcfile.exists():
has_persist_outs = any(out.persist for out in stage.outs)
ignore_build_cache = (
kwargs.get("ignore_build_cache", False) or has_persist_outs
ignore_run_cache = (
not kwargs.get("run_cache", True) or has_persist_outs
)
if has_persist_outs:
logger.warning("Build cache is ignored when persisting outputs.")
if not ignore_build_cache and stage.can_be_skipped:
if not ignore_run_cache and stage.can_be_skipped:
logger.info("Stage is cached, skipping.")
return None
@@ -623,9 +623,7 @@ class Stage(params.StageParams):
raise StageCmdFailedError(self, retcode)
@rwlocked(read=["deps"], write=["outs"])
def run(
self, dry=False, no_commit=False, force=False, ignore_build_cache=False
):
def run(self, dry=False, no_commit=False, force=False, run_cache=True):
if (self.cmd or self.is_import) and not self.locked and not dry:
self.remove_outs(ignore_remove=False, force=False)
@@ -670,9 +668,7 @@ class Stage(params.StageParams):
if not stage_cached:
self._save_deps()
use_build_cache = (
not force
and not ignore_build_cache
and stage_cache.is_cached(self)
not force and run_cache and stage_cache.is_cached(self)
)
if use_build_cache:
scripts/completion/dvc.bash
+2 -2
View File
@@ -68,10 +68,10 @@ _dvc_remote_modify='--global --system --local -u --unset'
_dvc_remote_remove='--global --system --local'
_dvc_remove='-o --outs -p --purge -f --force'
_dvc_remove_COMPGEN=_dvc_compgen_DVCFiles
_dvc_repro='-f --force -s --single-item -c --cwd -m --metrics --dry -i --interactive -p --pipeline -P --all-pipelines --ignore-build-cache --no-commit -R --recursive --downstream'
_dvc_repro='-f --force -s --single-item -c --cwd -m --metrics --dry -i --interactive -p --pipeline -P --all-pipelines --no-run-cache --force-downstream --no-commit -R --recursive --downstream'
_dvc_repro_COMPGEN=_dvc_compgen_DVCFiles
_dvc_root=''
_dvc_run='--no-exec -f --file -d --deps -o --outs -O --outs-no-cache --outs-persist --outs-persist-no-cache -m --metrics -M --metrics-no-cache --overwrite-dvcfile --ignore-build-cache --no-commit -w --wdir'
_dvc_run='--no-exec -f --file -d --deps -o --outs -O --outs-no-cache --outs-persist --outs-persist-no-cache -m --metrics -M --metrics-no-cache --overwrite-dvcfile --no-run-cache --no-commit -w --wdir'
_dvc_run_COMPGEN=_dvc_compgen_DVCFiles
_dvc_status='-j --jobs -r --remote -a --all-branches -T --all-tags -d --with-deps -c --cloud'
_dvc_status_COMPGEN=_dvc_compgen_DVCFiles
scripts/completion/dvc.zsh
+3 -2
View File
@@ -239,7 +239,8 @@ _dvc_repro=(
{-p,--pipeline}"[Reproduce the whole pipeline that the specified stage file belongs to.]"
{-P,--all-pipelines}"[Reproduce all pipelines in the repo.]"
{-R,--recursive}"[Reproduce all stages in the specified directory.]"
"--ignore-build-cache[Reproduce all descendants of a changed stage even if their direct dependencies didn't change.]"
"--force-downstream[Reproduce all descendants of a changed stage even if their direct dependencies didn't change.]"
"--no-run-cache[Run changed stage even if it has been already ran with the same command/dependencies/outputs/etc before.]"
"--no-commit[Don't put files/directories into cache.]"
"--downstream[Start from the specified stages when reproducing pipelines.]"
"*:Stages:_files -g '(*.dvc|Dvcfile)'"
@@ -259,7 +260,7 @@ _dvc_run=(
"--no-exec[Only create stage file without actually running it.]"
{-y,--yes}"[Deprecated, use --overwrite-dvcfile instead]"
"--overwrite-dvcfile[Overwrite existing DVC-file without asking for confirmation.]"
"--ignore-build-cache[Run this stage even if it has been already ran with the same command/dependencies/outputs/etc before.]"
"--no-run-cache[Run this stage even if it has been already ran with the same command/dependencies/outputs/etc before.]"
"--remove-outs[Deprecated, this is now the default behavior]"
"--no-commit[Don't put files/directories into cache.]"
"--outs-persist[Declare output file or directory that will not be removed upon repro.]:Output persistent:_files"
tests/func/test_repro.py
+2 -2
View File
@@ -436,7 +436,7 @@ class TestReproChangedDeepData(TestReproChangedData):
self.assertEqual(len(stages), 3)
class TestReproIgnoreBuildCache(TestDvc):
class TestReproForceDownstream(TestDvc):
def test(self):
stages = self.dvc.add(self.FOO)
self.assertEqual(len(stages), 1)
@@ -479,7 +479,7 @@ class TestReproIgnoreBuildCache(TestDvc):
with open(code2, "a") as fobj:
fobj.write("\n\n")
stages = self.dvc.reproduce(file3_stage.path, ignore_build_cache=True)
stages = self.dvc.reproduce(file3_stage.path, force_downstream=True)
self.assertEqual(len(stages), 2)
self.assertEqual(stages[0].path, file2_stage.path)
self.assertEqual(stages[1].path, file3_stage.path)
tests/func/test_run_single_stage.py
+6 -7
View File
@@ -343,7 +343,7 @@ class TestRunUnprotectOutsCopy(TestDvc):
[
"run",
"--overwrite-dvcfile",
"--ignore-build-cache",
"--no-run-cache",
"--single-stage",
"-d",
self.CODE,
@@ -401,7 +401,7 @@ class TestRunUnprotectOutsSymlink(TestDvc):
[
"run",
"--overwrite-dvcfile",
"--ignore-build-cache",
"--no-run-cache",
"--single-stage",
"-d",
self.CODE,
@@ -460,7 +460,7 @@ class TestRunUnprotectOutsHardlink(TestDvc):
[
"run",
"--overwrite-dvcfile",
"--ignore-build-cache",
"--no-run-cache",
"--single-stage",
"-d",
self.CODE,
@@ -545,7 +545,7 @@ class TestCmdRunOverwrite(TestDvc):
"-d",
self.CODE,
"--overwrite-dvcfile",
"--ignore-build-cache",
"--no-run-cache",
"--single-stage",
"-o",
"out",
@@ -664,8 +664,7 @@ def test_rerun_deterministic_ignore_cache(tmp_dir, run_copy):
assert run_copy("foo", "out", single_stage=True) is not None
assert (
run_copy("foo", "out", ignore_build_cache=True, single_stage=True)
is not None
run_copy("foo", "out", run_cache=False, single_stage=True) is not None
)
@@ -932,7 +931,7 @@ class TestShouldNotCheckoutUponCorruptedLocalHardlinkCache(TestDvc):
class TestPersistentOutput(TestDvc):
def test_ignore_build_cache(self):
def test_ignore_run_cache(self):
warning = "Build cache is ignored when persisting outputs."
with open("immutable", "w") as fobj:
tests/unit/command/test_repro.py
+2 -1
View File
@@ -7,12 +7,13 @@ default_arguments = {
"downstream": False,
"dry": False,
"force": False,
"ignore_build_cache": False,
"run_cache": True,
"interactive": False,
"no_commit": False,
"pipeline": False,
"single_item": False,
"recursive": False,
"force_downstream": False,
}
tests/unit/command/test_run.py
+4 -4
View File
@@ -24,7 +24,7 @@ def test_run(mocker, dvc):
"wdir",
"--no-exec",
"--overwrite-dvcfile",
"--ignore-build-cache",
"--no-run-cache",
"--no-commit",
"--outs-persist",
"outs-persist",
@@ -58,7 +58,7 @@ def test_run(mocker, dvc):
wdir="wdir",
no_exec=True,
overwrite=True,
ignore_build_cache=True,
run_cache=False,
no_commit=True,
always_changed=True,
cmd="command",
@@ -85,7 +85,7 @@ def test_run_args_from_cli(mocker, dvc):
wdir=None,
no_exec=False,
overwrite=False,
ignore_build_cache=False,
run_cache=True,
no_commit=False,
always_changed=False,
cmd="echo foo",
@@ -112,7 +112,7 @@ def test_run_args_with_spaces(mocker, dvc):
wdir=None,
no_exec=False,
overwrite=False,
ignore_build_cache=False,
run_cache=True,
no_commit=False,
always_changed=False,
cmd='echo "foo bar"',