mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 15:28:28 +01:00
dvc: implement params support for pipeline file (#3694)
* dvc: implement params support for pipeline file * reorganize dump/load of params * add tests for params * fix order of dictionary in Python 3.5
This commit is contained in:
@@ -102,3 +102,20 @@ def test_diff_with_unchanged(tmp_dir, scm, dvc):
|
||||
"xyz": {"old": "val", "new": "val"},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def test_pipeline_tracked_params(tmp_dir, scm, dvc, run_copy):
|
||||
from dvc.dvcfile import PIPELINE_FILE
|
||||
|
||||
tmp_dir.gen({"foo": "foo", "params.yaml": "foo: bar\nxyz: val"})
|
||||
run_copy("foo", "bar", name="copy-foo-bar", params=["foo,xyz"])
|
||||
|
||||
scm.add(["params.yaml", PIPELINE_FILE])
|
||||
scm.commit("add stage")
|
||||
|
||||
tmp_dir.scm_gen("params.yaml", "foo: baz\nxyz: val", commit="baz")
|
||||
tmp_dir.scm_gen("params.yaml", "foo: qux\nxyz: val", commit="qux")
|
||||
|
||||
assert dvc.params.diff(a_rev="HEAD~2") == {
|
||||
"params.yaml": {"foo": {"old": "bar", "new": "qux"}}
|
||||
}
|
||||
|
||||
@@ -42,3 +42,19 @@ def test_show_branch(tmp_dir, scm, dvc):
|
||||
"working tree": {"params.yaml": {"foo": "bar"}},
|
||||
"branch": {"params.yaml": {"foo": "baz"}},
|
||||
}
|
||||
|
||||
|
||||
def test_pipeline_tracked_params(tmp_dir, scm, dvc, run_copy):
|
||||
from dvc.dvcfile import PIPELINE_FILE
|
||||
|
||||
tmp_dir.gen({"foo": "foo", "params.yaml": "foo: bar\nxyz: val"})
|
||||
run_copy("foo", "bar", name="copy-foo-bar", params=["foo,xyz"])
|
||||
scm.add(["params.yaml", PIPELINE_FILE])
|
||||
scm.commit("add stage")
|
||||
|
||||
tmp_dir.scm_gen("params.yaml", "foo: baz\nxyz: val", commit="baz")
|
||||
tmp_dir.scm_gen("params.yaml", "foo: qux\nxyz: val", commit="qux")
|
||||
|
||||
assert dvc.params.show(revs=["master"]) == {
|
||||
"master": {"params.yaml": {"foo": "qux", "xyz": "val"}}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
import os
|
||||
from copy import deepcopy
|
||||
from textwrap import dedent
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from funcy import lsplit
|
||||
|
||||
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
|
||||
from dvc.exceptions import CyclicGraphError
|
||||
@@ -485,3 +488,60 @@ def test_cyclic_graph_error(tmp_dir, dvc, run_copy):
|
||||
dump_stage_file(PIPELINE_FILE, data)
|
||||
with pytest.raises(CyclicGraphError):
|
||||
dvc.reproduce(":copy-baz-foo")
|
||||
|
||||
|
||||
def test_repro_multiple_params(tmp_dir, dvc):
|
||||
from tests.func.test_run_multistage import supported_params
|
||||
|
||||
from dvc.serialize import get_params_deps
|
||||
|
||||
with (tmp_dir / "params2.yaml").open("w+") as f:
|
||||
yaml.dump(supported_params, f)
|
||||
|
||||
with (tmp_dir / "params.yaml").open("w+") as f:
|
||||
yaml.dump(supported_params, f)
|
||||
|
||||
(tmp_dir / "foo").write_text("foo")
|
||||
stage = dvc.run(
|
||||
name="read_params",
|
||||
deps=["foo"],
|
||||
outs=["bar"],
|
||||
params=[
|
||||
"params2.yaml:lists,floats,name",
|
||||
"answer,floats,nested.nested1",
|
||||
],
|
||||
cmd="cat params2.yaml params.yaml > bar",
|
||||
)
|
||||
|
||||
params, deps = get_params_deps(stage)
|
||||
assert len(params) == 2
|
||||
assert len(deps) == 1
|
||||
assert len(stage.outs) == 1
|
||||
|
||||
lockfile = stage.dvcfile._lockfile
|
||||
assert lockfile.load()["read_params"]["params"] == {
|
||||
"params2.yaml": {
|
||||
"lists": [42, 42.0, "42"],
|
||||
"floats": 42.0,
|
||||
"name": "Answer",
|
||||
},
|
||||
"params.yaml": {
|
||||
"answer": 42,
|
||||
"floats": 42.0,
|
||||
"nested.nested1": {"nested2": "42", "nested2-2": 41.99999},
|
||||
},
|
||||
}
|
||||
data, _ = stage.dvcfile._load()
|
||||
params = data["stages"]["read_params"]["params"]
|
||||
|
||||
custom, defaults = lsplit(lambda v: isinstance(v, dict), params)
|
||||
assert set(custom[0]["params2.yaml"]) == {"name", "lists", "floats"}
|
||||
assert set(defaults) == {"answer", "floats", "nested.nested1"}
|
||||
|
||||
assert not dvc.reproduce(stage.addressing)
|
||||
with (tmp_dir / "params.yaml").open("w+") as f:
|
||||
params = deepcopy(supported_params)
|
||||
params["answer"] = 43
|
||||
yaml.dump(params, f)
|
||||
|
||||
assert dvc.reproduce(stage.addressing) == [stage]
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import pytest
|
||||
import os
|
||||
|
||||
import yaml
|
||||
|
||||
from dvc.stage.exceptions import InvalidStageName, DuplicateStageName
|
||||
|
||||
|
||||
@@ -169,3 +171,62 @@ def test_run_already_exists(tmp_dir, dvc, run_copy):
|
||||
run_copy("foo", "bar", name="copy")
|
||||
with pytest.raises(DuplicateStageName):
|
||||
run_copy("bar", "foobar", name="copy")
|
||||
|
||||
|
||||
supported_params = {
|
||||
"name": "Answer",
|
||||
"answer": 42,
|
||||
"floats": 42.0,
|
||||
"lists": [42, 42.0, "42"],
|
||||
"nested": {"nested1": {"nested2": "42", "nested2-2": 41.99999}},
|
||||
}
|
||||
|
||||
|
||||
def test_run_params_default(tmp_dir, dvc):
|
||||
from dvc.dependency import ParamsDependency
|
||||
|
||||
with (tmp_dir / "params.yaml").open("w+") as f:
|
||||
yaml.dump(supported_params, f)
|
||||
|
||||
stage = dvc.run(
|
||||
name="read_params",
|
||||
params=["nested.nested1.nested2"],
|
||||
cmd="cat params.yaml",
|
||||
)
|
||||
isinstance(stage.deps[0], ParamsDependency)
|
||||
assert stage.deps[0].params == ["nested.nested1.nested2"]
|
||||
|
||||
lockfile = stage.dvcfile._lockfile
|
||||
assert lockfile.load()["read_params"]["params"] == {
|
||||
"params.yaml": {"nested.nested1.nested2": "42"}
|
||||
}
|
||||
|
||||
data, _ = stage.dvcfile._load()
|
||||
assert data["stages"]["read_params"]["params"] == [
|
||||
"nested.nested1.nested2"
|
||||
]
|
||||
|
||||
|
||||
def test_run_params_custom_file(tmp_dir, dvc):
|
||||
from dvc.dependency import ParamsDependency
|
||||
|
||||
with (tmp_dir / "params2.yaml").open("w+") as f:
|
||||
yaml.dump(supported_params, f)
|
||||
|
||||
stage = dvc.run(
|
||||
name="read_params",
|
||||
params=["params2.yaml:lists"],
|
||||
cmd="cat params2.yaml",
|
||||
)
|
||||
|
||||
isinstance(stage.deps[0], ParamsDependency)
|
||||
assert stage.deps[0].params == ["lists"]
|
||||
lockfile = stage.dvcfile._lockfile
|
||||
assert lockfile.load()["read_params"]["params"] == {
|
||||
"params2.yaml": {"lists": [42, 42.0, "42"]}
|
||||
}
|
||||
|
||||
data, _ = stage.dvcfile._load()
|
||||
assert data["stages"]["read_params"]["params"] == [
|
||||
{"params2.yaml": ["lists"]}
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user