dvcfile: serialize params on --no-exec (#3859)

* dvc.yaml: don't generate lockfile on --no-exec

* dvcfile: serialize params on --no-exec

Fixes #3851
This commit is contained in:
Ruslan Kuprieiev
2020-05-23 03:19:38 +03:00
committed by GitHub
parent 3e951162b8
commit 2fba2e030a
4 changed files with 66 additions and 8 deletions
+5 -2
View File
@@ -175,13 +175,16 @@ class PipelineFile(FileMixin):
def _lockfile(self):
return Lockfile(self.repo, os.path.splitext(self.path)[0] + ".lock")
def dump(self, stage, update_pipeline=False, **kwargs):
def dump(self, stage, update_pipeline=False, no_lock=False):
"""Dumps given stage appropriately in the dvcfile."""
from dvc.stage import PipelineStage
assert isinstance(stage, PipelineStage)
check_dvc_filename(self.path)
self._dump_lockfile(stage)
if not no_lock:
self._dump_lockfile(stage)
if update_pipeline and not stage.is_data_source:
self._dump_pipeline_file(stage)
+1 -1
View File
@@ -90,5 +90,5 @@ def run(self, fname=None, no_exec=False, single_stage=False, **kwargs):
run_cache=kwargs.get("run_cache", True),
)
dvcfile.dump(stage, update_pipeline=True)
dvcfile.dump(stage, update_pipeline=True, no_lock=no_exec)
return stage
+13 -5
View File
@@ -69,14 +69,22 @@ def _serialize_params(params: List[ParamsDependency]):
for param_dep in sort_by_path(params):
dump = param_dep.dumpd()
path, params = dump[PARAM_PATH], dump[PARAM_PARAMS]
k = list(params.keys())
if not k:
continue
key_vals[path] = OrderedDict([(key, params[key]) for key in sorted(k)])
if isinstance(params, dict):
k = list(params.keys())
if not k:
continue
key_vals[path] = OrderedDict(
[(key, params[key]) for key in sorted(k)]
)
else:
assert isinstance(params, list)
k = params
key_vals = OrderedDict()
# params from default file is always kept at the start of the `params:`
if path == DEFAULT_PARAMS_FILE:
keys = k + keys
key_vals.move_to_end(path, last=False)
if key_vals:
key_vals.move_to_end(path, last=False)
else:
# if it's not a default file, change the shape
# to: {path: k}
+47
View File
@@ -19,6 +19,30 @@ def test_run_with_name(tmp_dir, dvc, run_copy):
assert os.path.exists(PIPELINE_LOCK)
def test_run_no_exec(tmp_dir, dvc, run_copy):
from dvc.stage import PipelineStage
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
tmp_dir.dvc_gen("foo", "foo")
assert not os.path.exists(PIPELINE_FILE)
stage = run_copy("foo", "bar", name="copy-foo-to-bar", no_exec=True)
assert isinstance(stage, PipelineStage)
assert stage.name == "copy-foo-to-bar"
assert os.path.exists(PIPELINE_FILE)
assert not os.path.exists(PIPELINE_LOCK)
data, _ = stage.dvcfile._load()
assert data == {
"stages": {
"copy-foo-to-bar": {
"cmd": "python copy.py foo bar",
"deps": ["copy.py", "foo"],
"outs": ["bar"],
}
}
}
def test_run_with_multistage_and_single_stage(tmp_dir, dvc, run_copy):
from dvc.stage import PipelineStage, Stage
@@ -252,3 +276,26 @@ def test_run_params_custom_file(tmp_dir, dvc):
assert data["stages"]["read_params"]["params"] == [
{"params2.yaml": ["lists"]}
]
def test_run_params_no_exec(tmp_dir, dvc):
from dvc.dependency import ParamsDependency
with (tmp_dir / "params2.yaml").open("w+") as f:
yaml.dump(supported_params, f)
stage = dvc.run(
name="read_params",
params=["params2.yaml:lists"],
cmd="cat params2.yaml",
no_exec=True,
)
isinstance(stage.deps[0], ParamsDependency)
assert stage.deps[0].params == ["lists"]
assert not stage.dvcfile._lockfile.exists()
data, _ = stage.dvcfile._load()
assert data["stages"]["read_params"]["params"] == [
{"params2.yaml": ["lists"]}
]