dvc: implement params support for pipeline file (#3694)

* dvc: implement params support for pipeline file

* reorganize dump/load of params

* add tests for params

* fix order of dictionary in Python 3.5
This commit is contained in:
Saugat Pachhai
2020-04-29 14:35:26 +03:00
committed by GitHub
parent 1e65098819
commit 7edee84306
8 changed files with 277 additions and 7 deletions
+17
View File
@@ -102,3 +102,20 @@ def test_diff_with_unchanged(tmp_dir, scm, dvc):
"xyz": {"old": "val", "new": "val"},
}
}
def test_pipeline_tracked_params(tmp_dir, scm, dvc, run_copy):
from dvc.dvcfile import PIPELINE_FILE
tmp_dir.gen({"foo": "foo", "params.yaml": "foo: bar\nxyz: val"})
run_copy("foo", "bar", name="copy-foo-bar", params=["foo,xyz"])
scm.add(["params.yaml", PIPELINE_FILE])
scm.commit("add stage")
tmp_dir.scm_gen("params.yaml", "foo: baz\nxyz: val", commit="baz")
tmp_dir.scm_gen("params.yaml", "foo: qux\nxyz: val", commit="qux")
assert dvc.params.diff(a_rev="HEAD~2") == {
"params.yaml": {"foo": {"old": "bar", "new": "qux"}}
}
+16
View File
@@ -42,3 +42,19 @@ def test_show_branch(tmp_dir, scm, dvc):
"working tree": {"params.yaml": {"foo": "bar"}},
"branch": {"params.yaml": {"foo": "baz"}},
}
def test_pipeline_tracked_params(tmp_dir, scm, dvc, run_copy):
from dvc.dvcfile import PIPELINE_FILE
tmp_dir.gen({"foo": "foo", "params.yaml": "foo: bar\nxyz: val"})
run_copy("foo", "bar", name="copy-foo-bar", params=["foo,xyz"])
scm.add(["params.yaml", PIPELINE_FILE])
scm.commit("add stage")
tmp_dir.scm_gen("params.yaml", "foo: baz\nxyz: val", commit="baz")
tmp_dir.scm_gen("params.yaml", "foo: qux\nxyz: val", commit="qux")
assert dvc.params.show(revs=["master"]) == {
"master": {"params.yaml": {"foo": "qux", "xyz": "val"}}
}
+60
View File
@@ -1,7 +1,10 @@
import os
from copy import deepcopy
from textwrap import dedent
import pytest
import yaml
from funcy import lsplit
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
from dvc.exceptions import CyclicGraphError
@@ -485,3 +488,60 @@ def test_cyclic_graph_error(tmp_dir, dvc, run_copy):
dump_stage_file(PIPELINE_FILE, data)
with pytest.raises(CyclicGraphError):
dvc.reproduce(":copy-baz-foo")
def test_repro_multiple_params(tmp_dir, dvc):
from tests.func.test_run_multistage import supported_params
from dvc.serialize import get_params_deps
with (tmp_dir / "params2.yaml").open("w+") as f:
yaml.dump(supported_params, f)
with (tmp_dir / "params.yaml").open("w+") as f:
yaml.dump(supported_params, f)
(tmp_dir / "foo").write_text("foo")
stage = dvc.run(
name="read_params",
deps=["foo"],
outs=["bar"],
params=[
"params2.yaml:lists,floats,name",
"answer,floats,nested.nested1",
],
cmd="cat params2.yaml params.yaml > bar",
)
params, deps = get_params_deps(stage)
assert len(params) == 2
assert len(deps) == 1
assert len(stage.outs) == 1
lockfile = stage.dvcfile._lockfile
assert lockfile.load()["read_params"]["params"] == {
"params2.yaml": {
"lists": [42, 42.0, "42"],
"floats": 42.0,
"name": "Answer",
},
"params.yaml": {
"answer": 42,
"floats": 42.0,
"nested.nested1": {"nested2": "42", "nested2-2": 41.99999},
},
}
data, _ = stage.dvcfile._load()
params = data["stages"]["read_params"]["params"]
custom, defaults = lsplit(lambda v: isinstance(v, dict), params)
assert set(custom[0]["params2.yaml"]) == {"name", "lists", "floats"}
assert set(defaults) == {"answer", "floats", "nested.nested1"}
assert not dvc.reproduce(stage.addressing)
with (tmp_dir / "params.yaml").open("w+") as f:
params = deepcopy(supported_params)
params["answer"] = 43
yaml.dump(params, f)
assert dvc.reproduce(stage.addressing) == [stage]
+61
View File
@@ -1,6 +1,8 @@
import pytest
import os
import yaml
from dvc.stage.exceptions import InvalidStageName, DuplicateStageName
@@ -169,3 +171,62 @@ def test_run_already_exists(tmp_dir, dvc, run_copy):
run_copy("foo", "bar", name="copy")
with pytest.raises(DuplicateStageName):
run_copy("bar", "foobar", name="copy")
supported_params = {
"name": "Answer",
"answer": 42,
"floats": 42.0,
"lists": [42, 42.0, "42"],
"nested": {"nested1": {"nested2": "42", "nested2-2": 41.99999}},
}
def test_run_params_default(tmp_dir, dvc):
from dvc.dependency import ParamsDependency
with (tmp_dir / "params.yaml").open("w+") as f:
yaml.dump(supported_params, f)
stage = dvc.run(
name="read_params",
params=["nested.nested1.nested2"],
cmd="cat params.yaml",
)
isinstance(stage.deps[0], ParamsDependency)
assert stage.deps[0].params == ["nested.nested1.nested2"]
lockfile = stage.dvcfile._lockfile
assert lockfile.load()["read_params"]["params"] == {
"params.yaml": {"nested.nested1.nested2": "42"}
}
data, _ = stage.dvcfile._load()
assert data["stages"]["read_params"]["params"] == [
"nested.nested1.nested2"
]
def test_run_params_custom_file(tmp_dir, dvc):
from dvc.dependency import ParamsDependency
with (tmp_dir / "params2.yaml").open("w+") as f:
yaml.dump(supported_params, f)
stage = dvc.run(
name="read_params",
params=["params2.yaml:lists"],
cmd="cat params2.yaml",
)
isinstance(stage.deps[0], ParamsDependency)
assert stage.deps[0].params == ["lists"]
lockfile = stage.dvcfile._lockfile
assert lockfile.load()["read_params"]["params"] == {
"params2.yaml": {"lists": [42, 42.0, "42"]}
}
data, _ = stage.dvcfile._load()
assert data["stages"]["read_params"]["params"] == [
{"params2.yaml": ["lists"]}
]