Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Show csv format of experiments (#6468)

Browse Source 
* Show csv format of experiments

fix #5989 and #5446 

1. add --show-csv to dvc exp show
2. add a new functional test for csv format
3. add a new unit test for show_experiements
4. solve name duplication in table common headers (Created,Parent, etc... ), metrics files and param files.

Co-authored-by: Peter Rowlands (변기호) <[email protected]>
This commit is contained in:
Gao
2021-09-08 16:59:01 +08:00
committed by GitHub
co-authored by Peter Rowlands
parent 73c493500b
commit 6106b1341b
4 changed files with 269 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/command/experiments.py
+69 -30
View File
@@ -141,6 +141,8 @@ def _collect_rows(
precision=DEFAULT_PRECISION,
sort_by=None,
sort_order=None,
fill_value=FILL_VALUE,
iso=False,
):
from dvc.scm.git import Git
@@ -161,8 +163,8 @@ def _collect_rows(
elif exp.get("queued"):
state = "Queued"
else:
state = FILL_VALUE
executor = exp.get("executor", FILL_VALUE)
state = fill_value
executor = exp.get("executor", fill_value)
is_baseline = rev == "baseline"
if is_baseline:
@@ -202,12 +204,12 @@ def _collect_rows(
exp_name,
name_rev,
typ,
_format_time(exp.get("timestamp")),
_format_time(exp.get("timestamp"), fill_value, iso),
parent,
state,
executor,
]
fill_value = FILL_VALUE_ERRORED if results.get("error") else FILL_VALUE
fill_value = FILL_VALUE_ERRORED if results.get("error") else fill_value
_extend_row(
row,
metric_names,
@@ -274,14 +276,18 @@ def _sort_exp(experiments, sort_path, sort_name, typ, reverse):
return ret
def _format_time(timestamp):
if timestamp is None:
return FILL_VALUE
if timestamp.date() == date.today():
def _format_time(datetime_obj, fill_value=FILL_VALUE, iso=False):
if datetime_obj is None:
return fill_value
if iso:
return datetime_obj.isoformat()
if datetime_obj.date() == date.today():
fmt = "%I:%M %p"
else:
fmt = "%b %d, %Y"
return timestamp.strftime(fmt)
return datetime_obj.strftime(fmt)
def _extend_row(row, names, items, precision, fill_value=FILL_VALUE):
@@ -320,6 +326,7 @@ def _parse_filter_list(param_list):
def experiments_table(
all_experiments,
headers,
metric_headers,
metric_names,
param_headers,
@@ -327,22 +334,15 @@ def experiments_table(
sort_by=None,
sort_order=None,
precision=DEFAULT_PRECISION,
fill_value=FILL_VALUE,
iso=False,
) -> "TabularData":
from funcy import lconcat
from dvc.compare import TabularData
headers = [
"Experiment",
"rev",
"typ",
"Created",
"parent",
"State",
"Executor",
]
td = TabularData(
lconcat(headers, metric_headers, param_headers), fill_value=FILL_VALUE
lconcat(headers, metric_headers, param_headers), fill_value=fill_value
)
for base_rev, experiments in all_experiments.items():
rows = _collect_rows(
@@ -353,6 +353,8 @@ def experiments_table(
sort_by=sort_by,
sort_order=sort_order,
precision=precision,
fill_value=fill_value,
iso=iso,
)
td.extend(rows)
@@ -385,8 +387,10 @@ def baseline_styler(typ):
def show_experiments(
all_experiments, pager=True, no_timestamp=False, **kwargs
all_experiments, pager=True, no_timestamp=False, show_csv=False, **kwargs
):
from funcy.seqs import flatten as flatten_list
include_metrics = _parse_filter_list(kwargs.pop("include_metrics", []))
exclude_metrics = _parse_filter_list(kwargs.pop("exclude_metrics", []))
include_params = _parse_filter_list(kwargs.pop("include_params", []))
@@ -399,11 +403,26 @@ def show_experiments(
include_params=include_params,
exclude_params=exclude_params,
)
metric_headers = _normalize_headers(metric_names)
param_headers = _normalize_headers(param_names)
headers = [
"Experiment",
"rev",
"typ",
"Created",
"parent",
"State",
"Executor",
]
names = {**metric_names, **param_names}
counter = Counter(flatten_list([list(a.keys()) for a in names.values()]))
counter.update(headers)
metric_headers = _normalize_headers(metric_names, counter)
param_headers = _normalize_headers(param_names, counter)
td = experiments_table(
all_experiments,
headers,
metric_headers,
metric_names,
param_headers,
@@ -411,6 +430,8 @@ def show_experiments(
kwargs.get("sort_by"),
kwargs.get("sort_order"),
kwargs.get("precision"),
kwargs.get("fill_value"),
kwargs.get("iso"),
)
if no_timestamp:
@@ -422,9 +443,12 @@ def show_experiments(
row_styles = lmap(baseline_styler, td.column("typ"))
merge_headers = ["Experiment", "rev", "typ", "parent"]
td.column("Experiment")[:] = map(prepare_exp_id, td.as_dict(merge_headers))
td.drop(*merge_headers[1:])
if not show_csv:
merge_headers = ["Experiment", "rev", "typ", "parent"]
td.column("Experiment")[:] = map(
prepare_exp_id, td.as_dict(merge_headers)
)
td.drop(*merge_headers[1:])
headers = {"metrics": metric_headers, "params": param_headers}
styles = {
@@ -453,13 +477,11 @@ def show_experiments(
rich_table=True,
header_styles=styles,
row_styles=row_styles,
show_csv=show_csv,
)
def _normalize_headers(names):
count = Counter(
name for path in names for name in names[path] for path in names
)
def _normalize_headers(names, count):
return [
name if count[name] == 1 else f"{path}:{name}"
for path in names
@@ -493,6 +515,14 @@ class CmdExperimentsShow(CmdBase):
ui.write(json.dumps(all_experiments, default=_format_json))
else:
precision = (
self.args.precision or None
if self.args.show_csv
else DEFAULT_PRECISION
)
fill_value = "" if self.args.show_csv else FILL_VALUE
iso = True if self.args.show_csv else False
show_experiments(
all_experiments,
include_metrics=self.args.include_metrics,
@@ -502,8 +532,11 @@ class CmdExperimentsShow(CmdBase):
no_timestamp=self.args.no_timestamp,
sort_by=self.args.sort_by,
sort_order=self.args.sort_order,
precision=self.args.precision or DEFAULT_PRECISION,
precision=precision,
fill_value=fill_value,
iso=iso,
pager=not self.args.no_pager,
show_csv=self.args.show_csv,
)
return 0
@@ -883,6 +916,12 @@ def add_parser(subparsers, parent_parser):
default=False,
help="Print output in JSON format instead of a human-readable table.",
)
experiments_show_parser.add_argument(
"--show-csv",
action="store_true",
default=False,
help="Print output in csv format instead of a human-readable table.",
)
experiments_show_parser.add_argument(
"--precision",
type=int,
dvc/compare.py
+4 -1
View File
@@ -168,7 +168,10 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
def render(self, **kwargs: Any):
from dvc.ui import ui
ui.table(self, headers=self.keys(), **kwargs)
if kwargs.pop("show_csv", False):
ui.write(self.to_csv(), end="")
else:
ui.table(self, headers=self.keys(), **kwargs)
def as_dict(
self, cols: Iterable[str] = None
tests/func/experiments/test_show.py
+68 -4
View File
@@ -9,6 +9,7 @@ from dvc.exceptions import InvalidArgumentError
from dvc.main import main
from dvc.repo.experiments.base import EXPS_STASH, ExpRefInfo
from dvc.repo.experiments.executor.base import BaseExecutor, ExecutorInfo
from dvc.repo.experiments.utils import exp_refs_by_rev
from dvc.utils.fs import makedirs
from dvc.utils.serialize import YAMLFileCorruptedError, dump_yaml
from tests.func.test_repro_multistage import COPY_SCRIPT
@@ -269,6 +270,10 @@ def test_show_filter(
included,
excluded,
):
from contextlib import contextmanager
from dvc.ui import ui
capsys.readouterr()
div = "│" if os.name == "nt" else "┃"
@@ -312,13 +317,30 @@ def test_show_filter(
if e_params is not None:
command.append(f"--exclude-params={e_params}")
assert main(command) == 0
@contextmanager
def console_with(console, width):
console_options = console.options
original = console_options.max_width
con_width = console._width
try:
console_options.max_width = width
console._width = width
yield
finally:
console_options.max_width = original
console._width = con_width
with console_with(ui.rich_console, 255):
assert main(command) == 0
cap = capsys.readouterr()
for i in included:
assert f"{div} {i} {div}" in cap.out
assert f"{div} params.yaml:{i} {div}" in cap.out
assert f"{div} metrics.yaml:{i} {div}" in cap.out
for e in excluded:
assert f"{div} {e} {div}" not in cap.out
assert f"{div} params.yaml:{e} {div}" not in cap.out
assert f"{div} metrics.yaml:{e} {div}" not in cap.out
def test_show_multiple_commits(tmp_dir, scm, dvc, exp_stage):
@@ -412,7 +434,6 @@ def test_show_running_checkpoint(
):
from dvc.repo.experiments.base import EXEC_BRANCH
from dvc.repo.experiments.executor.local import TempDirExecutor
from dvc.repo.experiments.utils import exp_refs_by_rev
baseline_rev = scm.get_rev()
dvc.experiments.run(
@@ -471,3 +492,46 @@ def test_show_with_broken_repo(tmp_dir, scm, dvc, exp_stage, caplog):
paths = ["workspace", "baseline", "error"]
assert isinstance(get_in(result, paths), YAMLFileCorruptedError)
def test_show_csv(tmp_dir, scm, dvc, exp_stage, capsys):
baseline_rev = scm.get_rev()
def _get_rev_isotimestamp(rev):
return datetime.fromtimestamp(
scm.gitpython.repo.rev_parse(rev).committed_date
).isoformat()
result1 = dvc.experiments.run(exp_stage.addressing, params=["foo=2"])
rev1 = first(result1)
ref_info1 = first(exp_refs_by_rev(scm, rev1))
result2 = dvc.experiments.run(exp_stage.addressing, params=["foo=3"])
rev2 = first(result2)
ref_info2 = first(exp_refs_by_rev(scm, rev2))
capsys.readouterr()
assert main(["exp", "show", "--show-csv"]) == 0
cap = capsys.readouterr()
assert (
"Experiment,rev,typ,Created,parent,metrics.yaml:foo,params.yaml:foo"
in cap.out
)
assert ",workspace,baseline,,,3,3" in cap.out
assert (
"master,{},baseline,{},,1,1".format(
baseline_rev[:7], _get_rev_isotimestamp(baseline_rev)
)
in cap.out
)
assert (
"{},{},branch_base,{},,2,2".format(
ref_info1.name, rev1[:7], _get_rev_isotimestamp(rev1)
)
in cap.out
)
assert (
"{},{},branch_commit,{},,3,3".format(
ref_info2.name, rev2[:7], _get_rev_isotimestamp(rev2)
)
in cap.out
)
tests/unit/command/test_experiments.py
+128
View File
@@ -1,3 +1,5 @@
from datetime import datetime
import pytest
from dvc.cli import parse_args
@@ -12,6 +14,7 @@ from dvc.command.experiments import (
CmdExperimentsRemove,
CmdExperimentsRun,
CmdExperimentsShow,
show_experiments,
)
from dvc.exceptions import InvalidArgumentError
@@ -277,3 +280,128 @@ def test_experiments_remove(dvc, scm, mocker, queue, clear_all, remote):
clear_all=clear_all,
remote="myremote" if remote else None,
)
all_experiments = {
"workspace": {
"baseline": {
"data": {
"timestamp": None,
"params": {
"params.yaml": {
"data": {
"featurize": {"max_features": 3000, "ngrams": 1},
"parent": 20170428,
"train": {
"n_est": 100,
"min_split": 36,
},
}
}
},
"queued": False,
"running": False,
"executor": None,
"metrics": {
"scores.json": {
"data": {
"featurize": {"max_features": 3000, "ngrams": 1},
"avg_prec": 0.5843640011189556,
"roc_auc": 0.9544670443829399,
}
}
},
}
}
},
"b05eecc666734e899f79af228ff49a7ae5a18cc0": {
"baseline": {
"data": {
"timestamp": datetime(2021, 8, 2, 16, 48, 14),
"params": {
"params.yaml": {
"data": {
"featurize": {"max_features": 3000, "ngrams": 1},
"parent": 20170428,
"train": {
"n_est": 100,
"min_split": 2,
},
}
}
},
"queued": False,
"running": False,
"executor": None,
"metrics": {
"scores.json": {
"data": {
"featurize": {"max_features": 3000, "ngrams": 1},
"avg_prec": 0.5325162867864254,
"roc_auc": 0.9106964878520005,
}
}
},
"name": "master",
}
},
"ae99936461d6c3092934160f8beafe66a294f98d": {
"data": {
"timestamp": datetime(2021, 8, 31, 14, 56, 55),
"params": {
"params.yaml": {
"data": {
"featurize": {"max_features": 3000, "ngrams": 1},
"parent": 20170428,
"train": {
"n_est": 100,
"min_split": 36,
},
}
}
},
"queued": True,
"running": True,
"executor": None,
"metrics": {
"scores.json": {
"data": {
"featurize": {"max_features": 3000, "ngrams": 1},
"avg_prec": 0.5843640011189556,
"roc_auc": 0.9544670443829399,
}
}
},
"name": "exp-44136",
}
},
},
}
def test_show_experiments(capsys):
show_experiments(
all_experiments, precision=None, fill_value="", iso=True, show_csv=True
)
cap = capsys.readouterr()
assert (
"Experiment,rev,typ,Created,parent,State,scores.json:"
"featurize.max_features,scores.json:featurize.ngrams,"
"avg_prec,roc_auc,params.yaml:featurize.max_features,"
"params.yaml:featurize.ngrams,params.yaml:parent,"
"train.n_est,train.min_split" in cap.out
)
assert (
",workspace,baseline,,,,3000,1,0.5843640011189556,0.9544670443829399,"
"3000,1,20170428,100,36" in cap.out
)
assert (
"master,b05eecc,baseline,2021-08-02T16:48:14,,,3000,1,"
"0.5325162867864254,0.9106964878520005,3000,1,20170428,100,2"
in cap.out
)
assert (
"exp-44136,ae99936,branch_base,2021-08-31T14:56:55,,Running,"
"3000,1,0.5843640011189556,0.9544670443829399,3000,1,20170428,100,36"
in cap.out
)