dvc: introduce dvc params diff (#3573)

* params: use repo.tree

* remote: local: use `repo.tree` for `exists()`

* param: handle missing params

* stage: move status warning to debug

* dvc: introduce `dvc params diff`

Fixes #3528
This commit is contained in:
Ruslan Kuprieiev
2020-04-09 05:18:49 +03:00
committed by GitHub
parent 290a9fd434
commit c1ac6bb06b
19 changed files with 556 additions and 122 deletions
+2
View File
@@ -23,6 +23,7 @@ from .command import (
ls,
lock,
metrics,
params,
move,
pipeline,
remote,
@@ -62,6 +63,7 @@ COMMANDS = [
remote,
cache,
metrics,
params,
install,
root,
ls,
+7 -17
View File
@@ -114,21 +114,9 @@ class CmdMetricsRemove(CmdBase):
def _show_diff(diff):
from texttable import Texttable
from dvc.utils.diff import table
if not diff:
return ""
table = Texttable()
# disable automatic formatting
table.set_cols_dtype(["t", "t", "t", "t"])
# remove borders to make it easier for users to copy stuff
table.set_chars(("", "", "", ""))
table.set_deco(0)
rows = [["Path", "Metric", "Value", "Change"]]
rows = []
for fname, mdiff in diff.items():
for metric, change in mdiff.items():
rows.append(
@@ -139,8 +127,8 @@ def _show_diff(diff):
change.get("diff", "diff not supported"),
]
)
table.add_rows(rows)
return table.draw()
return table(["Path", "Metric", "Value", "Change"], rows)
class CmdMetricsDiff(CmdBase):
@@ -160,7 +148,9 @@ class CmdMetricsDiff(CmdBase):
logger.info(json.dumps(diff))
else:
logger.info(_show_diff(diff))
table = _show_diff(diff)
if table:
logger.info(table)
except DvcException:
logger.exception("failed to show metrics diff")
+90
View File
@@ -0,0 +1,90 @@
import argparse
import logging
from dvc.command.base import append_doc_link
from dvc.command.base import CmdBase
from dvc.command.base import fix_subparsers
from dvc.exceptions import DvcException
logger = logging.getLogger(__name__)
def _show_diff(diff):
from dvc.utils.diff import table
rows = []
for fname, mdiff in diff.items():
for param, change in mdiff.items():
rows.append([fname, param, change["old"], change["new"]])
return table(["Path", "Param", "Old", "New"], rows)
class CmdParamsDiff(CmdBase):
def run(self):
try:
diff = self.repo.params.diff(
a_rev=self.args.a_rev, b_rev=self.args.b_rev,
)
if self.args.show_json:
import json
logger.info(json.dumps(diff))
else:
table = _show_diff(diff)
if table:
logger.info(table)
except DvcException:
logger.exception("failed to show params diff")
return 1
return 0
def add_parser(subparsers, parent_parser):
PARAMS_HELP = "Commands to display params."
params_parser = subparsers.add_parser(
"params",
parents=[parent_parser],
description=append_doc_link(PARAMS_HELP, "params"),
help=PARAMS_HELP,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
params_subparsers = params_parser.add_subparsers(
dest="cmd",
help="Use `dvc params CMD --help` to display command-specific help.",
)
fix_subparsers(params_subparsers)
PARAMS_DIFF_HELP = (
"Show changes in params between commits in the DVC repository, or "
"between a commit and the workspace."
)
params_diff_parser = params_subparsers.add_parser(
"diff",
parents=[parent_parser],
description=append_doc_link(PARAMS_DIFF_HELP, "params/diff"),
help=PARAMS_DIFF_HELP,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
params_diff_parser.add_argument(
"a_rev", nargs="?", help="Old Git commit to compare (defaults to HEAD)"
)
params_diff_parser.add_argument(
"b_rev",
nargs="?",
help=("New Git commit to compare (defaults to the current workspace)"),
)
params_diff_parser.add_argument(
"--show-json",
action="store_true",
default=False,
help="Show output in JSON format.",
)
params_diff_parser.set_defaults(func=CmdParamsDiff)
+8 -5
View File
@@ -52,7 +52,7 @@ class DependencyPARAMS(DependencyLOCAL):
return status
status = defaultdict(dict)
info = self._get_info()
info = self.read_params()
for param in self.params:
if param not in info.keys():
st = "deleted"
@@ -74,11 +74,11 @@ class DependencyPARAMS(DependencyLOCAL):
self.PARAM_PARAMS: self.info or self.params,
}
def _get_info(self):
def read_params(self):
if not self.exists:
return {}
with open(fspath_py35(self.path_info), "r") as fobj:
with self.repo.tree.open(fspath_py35(self.path_info), "r") as fobj:
try:
config = yaml.safe_load(fobj)
except yaml.YAMLError as exc:
@@ -88,11 +88,14 @@ class DependencyPARAMS(DependencyLOCAL):
ret = {}
for param in self.params:
ret[param] = dpath.util.get(config, param, separator=".")
try:
ret[param] = dpath.util.get(config, param, separator=".")
except KeyError:
pass
return ret
def save_info(self):
info = self._get_info()
info = self.read_params()
missing_params = set(self.params) - set(info.keys())
if missing_params:
+2 -3
View File
@@ -76,10 +76,9 @@ class RemoteLOCAL(RemoteBASE):
return self.checksum_to_path_info(md5).url
@staticmethod
def exists(path_info):
def exists(self, path_info):
assert path_info.scheme == "local"
return os.path.lexists(fspath_py35(path_info))
return self.repo.tree.exists(fspath_py35(path_info))
def makedirs(self, path_info):
makedirs(path_info, exist_ok=True, mode=self._dir_mode)
+2
View File
@@ -67,6 +67,7 @@ class Repo(object):
from dvc.cache import Cache
from dvc.data_cloud import DataCloud
from dvc.repo.metrics import Metrics
from dvc.repo.params import Params
from dvc.scm.tree import WorkingTree
from dvc.repo.tag import Tag
from dvc.utils.fs import makedirs
@@ -102,6 +103,7 @@ class Repo(object):
self.cloud = DataCloud(self)
self.metrics = Metrics(self)
self.params = Params(self)
self.tag = Tag(self)
self._ignore()
+2 -83
View File
@@ -1,80 +1,7 @@
import json
from collections import defaultdict
from flatten_json import flatten
import dvc.utils.diff
from dvc.exceptions import NoMetricsError
def _parse(raw):
if raw is None or isinstance(raw, (dict, list, int, float)):
return raw
assert isinstance(raw, str)
try:
return json.loads(raw)
except json.JSONDecodeError:
return raw
def _diff_vals(old, new):
if (
isinstance(new, list)
and isinstance(old, list)
and len(old) == len(new) == 1
):
return _diff_vals(old[0], new[0])
if old == new:
return {}
res = {"old": old, "new": new}
if isinstance(new, (int, float)) and isinstance(old, (int, float)):
res["diff"] = new - old
return res
def _flatten(d):
if not d:
return defaultdict(lambda: None)
if isinstance(d, dict):
return defaultdict(lambda: None, flatten(d, "."))
return defaultdict(lambda: "unable to parse")
def _diff_dicts(old_dict, new_dict):
new = _flatten(new_dict)
old = _flatten(old_dict)
res = defaultdict(dict)
xpaths = set(old.keys())
xpaths.update(set(new.keys()))
for xpath in xpaths:
old_val = old[xpath]
new_val = new[xpath]
val_diff = _diff_vals(old_val, new_val)
if val_diff:
res[xpath] = val_diff
return dict(res)
def _diff(old_raw, new_raw):
old = _parse(old_raw)
new = _parse(new_raw)
if isinstance(new, dict) or isinstance(old, dict):
return _diff_dicts(old, new)
val_diff = _diff_vals(old, new)
if val_diff:
return {"": val_diff}
return {}
def _get_metrics(repo, *args, rev=None, **kwargs):
try:
metrics = repo.metrics.show(
@@ -89,12 +16,4 @@ def diff(repo, *args, a_rev=None, b_rev=None, **kwargs):
old = _get_metrics(repo, *args, **kwargs, rev=(a_rev or "HEAD"))
new = _get_metrics(repo, *args, **kwargs, rev=b_rev)
paths = set(old.keys())
paths.update(set(new.keys()))
res = defaultdict(dict)
for path in paths:
path_diff = _diff(old.get(path), new.get(path))
if path_diff:
res[path] = path_diff
return dict(res)
return dvc.utils.diff.diff(old, new)
+13
View File
@@ -0,0 +1,13 @@
class Params(object):
def __init__(self, repo):
self.repo = repo
def show(self, *args, **kwargs):
from .show import show
return show(self.repo, *args, **kwargs)
def diff(self, *args, **kwargs):
from .diff import diff
return diff(self.repo, *args, **kwargs)
+30
View File
@@ -0,0 +1,30 @@
import dvc.utils.diff
from .show import NoParamsError
def _get_params(repo, *args, rev=None, **kwargs):
try:
params = repo.params.show(*args, **kwargs, revs=[rev] if rev else None)
return params.get(rev or "", {})
except NoParamsError:
return {}
def _format(params):
ret = {}
for key, val in params.items():
if isinstance(val, dict):
new_val = _format(val)
elif isinstance(val, list):
new_val = str(val)
else:
new_val = val
ret[key] = new_val
return ret
def diff(repo, *args, a_rev=None, b_rev=None, **kwargs):
old = _get_params(repo, *args, **kwargs, rev=(a_rev or "HEAD"))
new = _get_params(repo, *args, **kwargs, rev=b_rev)
return dvc.utils.diff.diff(_format(old), _format(new))
+67
View File
@@ -0,0 +1,67 @@
import copy
from dvc.repo import locked
from dvc.exceptions import DvcException
from dvc.dependency.param import DependencyPARAMS
class NoParamsError(DvcException):
pass
def _collect_params(repo):
configs = {}
for stage in repo.stages:
for dep in stage.deps:
if not isinstance(dep, DependencyPARAMS):
continue
if dep.path_info not in configs.keys():
configs[dep.path_info] = copy.copy(dep)
continue
existing = set(configs[dep.path_info].params)
new = set(dep.params)
configs[dep.path_info].params = list(existing.update(new))
return configs.values()
def _read_params(deps):
res = {}
for dep in deps:
assert dep.scheme == "local"
params = dep.read_params()
if not params:
continue
res[str(dep.path_info)] = params
return res
@locked
def show(repo, revs=None):
res = {}
for branch in repo.brancher(revs=revs):
entries = _collect_params(repo)
params = _read_params(entries)
if params:
res[branch] = params
if not res:
raise NoParamsError("no parameter configs files in this repository")
# Hide working tree params if they are the same as in the active branch
try:
active_branch = repo.scm.active_branch()
except TypeError:
pass # Detached head
else:
if res.get("working tree") == res.get(active_branch):
res.pop("working tree", None)
return res
+1 -1
View File
@@ -47,7 +47,7 @@ class WorkingTree(BaseTree):
def exists(self, path):
"""Test whether a path exists."""
return os.path.exists(path)
return os.path.lexists(path)
def isdir(self, path):
"""Return true if the pathname refers to an existing directory."""
+2 -2
View File
@@ -309,7 +309,7 @@ class Stage(object):
for dep in self.deps:
status = dep.status()
if status:
logger.warning(
logger.debug(
"Dependency '{dep}' of '{stage}' changed because it is "
"'{status}'.".format(
dep=dep, stage=self.relpath, status=status[str(dep)]
@@ -323,7 +323,7 @@ class Stage(object):
for out in self.outs:
status = out.status()
if status:
logger.warning(
logger.debug(
"Output '{out}' of '{stage}' changed because it is "
"'{status}'".format(
out=out, stage=self.relpath, status=status[str(out)]
+105
View File
@@ -0,0 +1,105 @@
import json
from collections import defaultdict
from flatten_json import flatten
def _parse(raw):
if raw is None or isinstance(raw, (dict, list, int, float)):
return raw
assert isinstance(raw, str)
try:
return json.loads(raw)
except json.JSONDecodeError:
return raw
def _diff_vals(old, new):
if (
isinstance(new, list)
and isinstance(old, list)
and len(old) == len(new) == 1
):
return _diff_vals(old[0], new[0])
if old == new:
return {}
res = {"old": old, "new": new}
if isinstance(new, (int, float)) and isinstance(old, (int, float)):
res["diff"] = new - old
return res
def _flatten(d):
if not d:
return defaultdict(lambda: None)
if isinstance(d, dict):
return defaultdict(lambda: None, flatten(d, "."))
return defaultdict(lambda: "unable to parse")
def _diff_dicts(old_dict, new_dict):
new = _flatten(new_dict)
old = _flatten(old_dict)
res = defaultdict(dict)
xpaths = set(old.keys())
xpaths.update(set(new.keys()))
for xpath in xpaths:
old_val = old[xpath]
new_val = new[xpath]
val_diff = _diff_vals(old_val, new_val)
if val_diff:
res[xpath] = val_diff
return dict(res)
def _diff(old_raw, new_raw):
old = _parse(old_raw)
new = _parse(new_raw)
if isinstance(new, dict) or isinstance(old, dict):
return _diff_dicts(old, new)
val_diff = _diff_vals(old, new)
if val_diff:
return {"": val_diff}
return {}
def diff(old, new):
paths = set(old.keys())
paths.update(set(new.keys()))
res = defaultdict(dict)
for path in paths:
path_diff = _diff(old.get(path), new.get(path))
if path_diff:
res[path] = path_diff
return dict(res)
def table(header, rows):
from texttable import Texttable
if not rows:
return ""
t = Texttable()
# disable automatic formatting
t.set_cols_dtype(["t"] * len(header))
# remove borders to make it easier for users to copy stuff
t.set_chars([""] * len(header))
t.set_deco(0)
t.add_rows([header] + rows)
return t.draw()
View File
+87
View File
@@ -0,0 +1,87 @@
def test_diff_no_params(tmp_dir, scm, dvc):
assert dvc.params.diff() == {}
def test_diff_no_changes(tmp_dir, scm, dvc):
tmp_dir.gen("params.yaml", "foo: bar")
dvc.run(params=["foo"])
scm.add(["params.yaml", "Dvcfile"])
scm.commit("bar")
assert dvc.params.diff() == {}
def test_diff(tmp_dir, scm, dvc):
tmp_dir.gen("params.yaml", "foo: bar")
dvc.run(params=["foo"])
scm.add(["params.yaml", "Dvcfile"])
scm.commit("bar")
tmp_dir.scm_gen("params.yaml", "foo: baz", commit="baz")
tmp_dir.scm_gen("params.yaml", "foo: qux", commit="qux")
assert dvc.params.diff(a_rev="HEAD~2") == {
"params.yaml": {"foo": {"old": "bar", "new": "qux"}}
}
def test_diff_new(tmp_dir, scm, dvc):
tmp_dir.gen("params.yaml", "foo: bar")
dvc.run(params=["foo"])
assert dvc.params.diff() == {
"params.yaml": {"foo": {"old": None, "new": "bar"}}
}
def test_diff_deleted(tmp_dir, scm, dvc):
tmp_dir.gen("params.yaml", "foo: bar")
dvc.run(params=["foo"])
scm.add(["params.yaml", "Dvcfile"])
scm.commit("bar")
(tmp_dir / "Dvcfile").unlink()
assert dvc.params.diff() == {
"params.yaml": {"foo": {"old": "bar", "new": None}}
}
def test_diff_deleted_config(tmp_dir, scm, dvc):
tmp_dir.gen("params.yaml", "foo: bar")
dvc.run(params=["foo"])
scm.add(["params.yaml", "Dvcfile"])
scm.commit("bar")
(tmp_dir / "params.yaml").unlink()
assert dvc.params.diff() == {
"params.yaml": {"foo": {"old": "bar", "new": None}}
}
def test_diff_list(tmp_dir, scm, dvc):
tmp_dir.gen("params.yaml", "foo:\n- bar\n- baz")
dvc.run(params=["foo"])
scm.add(["params.yaml", "Dvcfile"])
scm.commit("foo")
tmp_dir.gen("params.yaml", "foo:\n- bar\n- baz\n- qux")
assert dvc.params.diff() == {
"params.yaml": {
"foo": {"old": "['bar', 'baz']", "new": "['bar', 'baz', 'qux']"}
}
}
def test_diff_dict(tmp_dir, scm, dvc):
tmp_dir.gen("params.yaml", "foo:\n bar: baz")
dvc.run(params=["foo"])
scm.add(["params.yaml", "Dvcfile"])
scm.commit("foo")
tmp_dir.gen("params.yaml", "foo:\n bar: qux")
assert dvc.params.diff() == {
"params.yaml": {"foo.bar": {"old": "baz", "new": "qux"}}
}
+35
View File
@@ -0,0 +1,35 @@
import pytest
from dvc.repo.params.show import NoParamsError
def test_show_empty(dvc):
with pytest.raises(NoParamsError):
dvc.params.show()
def test_show(tmp_dir, dvc):
tmp_dir.gen("params.yaml", "foo: bar")
dvc.run(params=["foo"])
assert dvc.params.show() == {"": {"params.yaml": {"foo": "bar"}}}
def test_show_list(tmp_dir, dvc):
tmp_dir.gen("params.yaml", "foo:\n- bar\n- baz\n")
dvc.run(params=["foo"])
assert dvc.params.show() == {"": {"params.yaml": {"foo": ["bar", "baz"]}}}
def test_show_branch(tmp_dir, scm, dvc):
tmp_dir.gen("params.yaml", "foo: bar")
dvc.run(params=["foo"])
scm.add(["params.yaml", "Dvcfile"])
scm.commit("init")
with tmp_dir.branch("branch", new=True):
tmp_dir.scm_gen("params.yaml", "foo: baz", commit="branch")
assert dvc.params.show(revs=["branch"]) == {
"working tree": {"params.yaml": {"foo": "bar"}},
"branch": {"params.yaml": {"foo": "baz"}},
}
+85
View File
@@ -0,0 +1,85 @@
import logging
from dvc.cli import parse_args
from dvc.command.params import CmdParamsDiff, _show_diff
def test_params_diff(dvc, mocker):
cli_args = parse_args(
["params", "diff", "HEAD~10", "HEAD~1", "--show-json"]
)
assert cli_args.func == CmdParamsDiff
cmd = cli_args.func(cli_args)
m = mocker.patch("dvc.repo.params.diff.diff", return_value={})
assert cmd.run() == 0
m.assert_called_once_with(
cmd.repo, a_rev="HEAD~10", b_rev="HEAD~1",
)
def test_params_diff_changed():
assert _show_diff({"params.yaml": {"a.b.c": {"old": 1, "new": 2}}}) == (
" Path Param Old New\n" "params.yaml a.b.c 1 2 "
)
def test_params_diff_list():
assert _show_diff(
{"params.yaml": {"a.b.c": {"old": 1, "new": [2, 3]}}}
) == (
" Path Param Old New \n"
"params.yaml a.b.c 1 [2, 3]"
)
def test_params_diff_unchanged():
assert _show_diff(
{"params.yaml": {"a.b.d": {"old": "old", "new": "new"}}}
) == (
" Path Param Old New\n" "params.yaml a.b.d old new"
)
def test_params_diff_no_changes():
assert _show_diff({}) == ""
def test_params_diff_new():
assert _show_diff(
{"params.yaml": {"a.b.d": {"old": None, "new": "new"}}}
) == (
" Path Param Old New\n" "params.yaml a.b.d None new"
)
def test_params_diff_deleted():
assert _show_diff(
{"params.yaml": {"a.b.d": {"old": "old", "new": None}}}
) == (
" Path Param Old New \n" "params.yaml a.b.d old None"
)
def test_params_diff_prec():
assert _show_diff(
{"params.yaml": {"train.lr": {"old": 0.0042, "new": 0.0043}}}
) == (
" Path Param Old New \n"
"params.yaml train.lr 0.0042 0.0043"
)
def test_params_diff_show_json(dvc, mocker, caplog):
cli_args = parse_args(
["params", "diff", "HEAD~10", "HEAD~1", "--show-json"]
)
cmd = cli_args.func(cli_args)
mocker.patch(
"dvc.repo.params.diff.diff", return_value={"params.yaml": {"a": "b"}}
)
with caplog.at_level(logging.INFO, logger="dvc"):
assert cmd.run() == 0
assert '{"params.yaml": {"a": "b"}}\n' in caplog.text
+14 -7
View File
@@ -57,27 +57,34 @@ def test_dumpd_without_info(dvc):
}
def test_get_info_nonexistent_file(dvc):
def test_read_params_nonexistent_file(dvc):
dep = DependencyPARAMS(Stage(dvc), None, ["foo"])
assert dep._get_info() == {}
assert dep.read_params() == {}
def test_get_info_unsupported_format(tmp_dir, dvc):
def test_read_params_unsupported_format(tmp_dir, dvc):
tmp_dir.gen("params.yaml", b"\0\1\2\3\4\5\6\7")
dep = DependencyPARAMS(Stage(dvc), None, ["foo"])
with pytest.raises(BadParamFileError):
dep._get_info()
dep.read_params()
def test_get_info_nested(tmp_dir, dvc):
def test_read_params_nested(tmp_dir, dvc):
tmp_dir.gen(
"params.yaml", yaml.dump({"some": {"path": {"foo": ["val1", "val2"]}}})
)
dep = DependencyPARAMS(Stage(dvc), None, ["some.path.foo"])
assert dep._get_info() == {"some.path.foo": ["val1", "val2"]}
assert dep.read_params() == {"some.path.foo": ["val1", "val2"]}
def test_save_info_missing_params(dvc):
def test_save_info_missing_config(dvc):
dep = DependencyPARAMS(Stage(dvc), None, ["foo"])
with pytest.raises(MissingParamsError):
dep.save_info()
def test_save_info_missing_param(tmp_dir, dvc):
tmp_dir.gen("params.yaml", "bar: baz")
dep = DependencyPARAMS(Stage(dvc), None, ["foo"])
with pytest.raises(MissingParamsError):
dep.save_info()
+4 -4
View File
@@ -8,12 +8,12 @@ from dvc.path_info import PathInfo
from dvc.remote.local import RemoteLOCAL
def test_status_download_optimization(mocker):
def test_status_download_optimization(mocker, dvc):
"""When comparing the status to pull a remote cache,
And the desired files to fetch are already on the local cache,
Don't check the existence of the desired files on the remote cache
"""
remote = RemoteLOCAL(None, {})
remote = RemoteLOCAL(dvc, {})
infos = NamedCache()
infos.add("local", "acbd18db4cc2f85cedef654fccc4a4d8", "foo")
@@ -32,8 +32,8 @@ def test_status_download_optimization(mocker):
@pytest.mark.parametrize("link_name", ["hardlink", "symlink"])
def test_is_protected(tmp_dir, link_name):
remote = RemoteLOCAL(None, {})
def test_is_protected(tmp_dir, dvc, link_name):
remote = RemoteLOCAL(dvc, {})
link_method = getattr(remote, link_name)
(tmp_dir / "foo").write_text("foo")