diff --git a/dvc/command/experiments.py b/dvc/command/experiments.py index 10c50691f..c96037921 100644 --- a/dvc/command/experiments.py +++ b/dvc/command/experiments.py @@ -141,6 +141,8 @@ def _collect_rows( precision=DEFAULT_PRECISION, sort_by=None, sort_order=None, + fill_value=FILL_VALUE, + iso=False, ): from dvc.scm.git import Git @@ -161,8 +163,8 @@ def _collect_rows( elif exp.get("queued"): state = "Queued" else: - state = FILL_VALUE - executor = exp.get("executor", FILL_VALUE) + state = fill_value + executor = exp.get("executor", fill_value) is_baseline = rev == "baseline" if is_baseline: @@ -202,12 +204,12 @@ def _collect_rows( exp_name, name_rev, typ, - _format_time(exp.get("timestamp")), + _format_time(exp.get("timestamp"), fill_value, iso), parent, state, executor, ] - fill_value = FILL_VALUE_ERRORED if results.get("error") else FILL_VALUE + fill_value = FILL_VALUE_ERRORED if results.get("error") else fill_value _extend_row( row, metric_names, @@ -274,14 +276,18 @@ def _sort_exp(experiments, sort_path, sort_name, typ, reverse): return ret -def _format_time(timestamp): - if timestamp is None: - return FILL_VALUE - if timestamp.date() == date.today(): +def _format_time(datetime_obj, fill_value=FILL_VALUE, iso=False): + if datetime_obj is None: + return fill_value + + if iso: + return datetime_obj.isoformat() + + if datetime_obj.date() == date.today(): fmt = "%I:%M %p" else: fmt = "%b %d, %Y" - return timestamp.strftime(fmt) + return datetime_obj.strftime(fmt) def _extend_row(row, names, items, precision, fill_value=FILL_VALUE): @@ -320,6 +326,7 @@ def _parse_filter_list(param_list): def experiments_table( all_experiments, + headers, metric_headers, metric_names, param_headers, @@ -327,22 +334,15 @@ def experiments_table( sort_by=None, sort_order=None, precision=DEFAULT_PRECISION, + fill_value=FILL_VALUE, + iso=False, ) -> "TabularData": from funcy import lconcat from dvc.compare import TabularData - headers = [ - "Experiment", - "rev", - "typ", - "Created", - "parent", - "State", - "Executor", - ] td = TabularData( - lconcat(headers, metric_headers, param_headers), fill_value=FILL_VALUE + lconcat(headers, metric_headers, param_headers), fill_value=fill_value ) for base_rev, experiments in all_experiments.items(): rows = _collect_rows( @@ -353,6 +353,8 @@ def experiments_table( sort_by=sort_by, sort_order=sort_order, precision=precision, + fill_value=fill_value, + iso=iso, ) td.extend(rows) @@ -385,8 +387,10 @@ def baseline_styler(typ): def show_experiments( - all_experiments, pager=True, no_timestamp=False, **kwargs + all_experiments, pager=True, no_timestamp=False, show_csv=False, **kwargs ): + from funcy.seqs import flatten as flatten_list + include_metrics = _parse_filter_list(kwargs.pop("include_metrics", [])) exclude_metrics = _parse_filter_list(kwargs.pop("exclude_metrics", [])) include_params = _parse_filter_list(kwargs.pop("include_params", [])) @@ -399,11 +403,26 @@ def show_experiments( include_params=include_params, exclude_params=exclude_params, ) - metric_headers = _normalize_headers(metric_names) - param_headers = _normalize_headers(param_names) + + headers = [ + "Experiment", + "rev", + "typ", + "Created", + "parent", + "State", + "Executor", + ] + + names = {**metric_names, **param_names} + counter = Counter(flatten_list([list(a.keys()) for a in names.values()])) + counter.update(headers) + metric_headers = _normalize_headers(metric_names, counter) + param_headers = _normalize_headers(param_names, counter) td = experiments_table( all_experiments, + headers, metric_headers, metric_names, param_headers, @@ -411,6 +430,8 @@ def show_experiments( kwargs.get("sort_by"), kwargs.get("sort_order"), kwargs.get("precision"), + kwargs.get("fill_value"), + kwargs.get("iso"), ) if no_timestamp: @@ -422,9 +443,12 @@ def show_experiments( row_styles = lmap(baseline_styler, td.column("typ")) - merge_headers = ["Experiment", "rev", "typ", "parent"] - td.column("Experiment")[:] = map(prepare_exp_id, td.as_dict(merge_headers)) - td.drop(*merge_headers[1:]) + if not show_csv: + merge_headers = ["Experiment", "rev", "typ", "parent"] + td.column("Experiment")[:] = map( + prepare_exp_id, td.as_dict(merge_headers) + ) + td.drop(*merge_headers[1:]) headers = {"metrics": metric_headers, "params": param_headers} styles = { @@ -453,13 +477,11 @@ def show_experiments( rich_table=True, header_styles=styles, row_styles=row_styles, + show_csv=show_csv, ) -def _normalize_headers(names): - count = Counter( - name for path in names for name in names[path] for path in names - ) +def _normalize_headers(names, count): return [ name if count[name] == 1 else f"{path}:{name}" for path in names @@ -493,6 +515,14 @@ class CmdExperimentsShow(CmdBase): ui.write(json.dumps(all_experiments, default=_format_json)) else: + precision = ( + self.args.precision or None + if self.args.show_csv + else DEFAULT_PRECISION + ) + fill_value = "" if self.args.show_csv else FILL_VALUE + iso = True if self.args.show_csv else False + show_experiments( all_experiments, include_metrics=self.args.include_metrics, @@ -502,8 +532,11 @@ class CmdExperimentsShow(CmdBase): no_timestamp=self.args.no_timestamp, sort_by=self.args.sort_by, sort_order=self.args.sort_order, - precision=self.args.precision or DEFAULT_PRECISION, + precision=precision, + fill_value=fill_value, + iso=iso, pager=not self.args.no_pager, + show_csv=self.args.show_csv, ) return 0 @@ -883,6 +916,12 @@ def add_parser(subparsers, parent_parser): default=False, help="Print output in JSON format instead of a human-readable table.", ) + experiments_show_parser.add_argument( + "--show-csv", + action="store_true", + default=False, + help="Print output in csv format instead of a human-readable table.", + ) experiments_show_parser.add_argument( "--precision", type=int, diff --git a/dvc/compare.py b/dvc/compare.py index 7f4b0798e..ff51c2ce1 100644 --- a/dvc/compare.py +++ b/dvc/compare.py @@ -168,7 +168,10 @@ class TabularData(MutableSequence[Sequence["CellT"]]): def render(self, **kwargs: Any): from dvc.ui import ui - ui.table(self, headers=self.keys(), **kwargs) + if kwargs.pop("show_csv", False): + ui.write(self.to_csv(), end="") + else: + ui.table(self, headers=self.keys(), **kwargs) def as_dict( self, cols: Iterable[str] = None diff --git a/tests/func/experiments/test_show.py b/tests/func/experiments/test_show.py index 81be6f4b1..55cfc11ad 100644 --- a/tests/func/experiments/test_show.py +++ b/tests/func/experiments/test_show.py @@ -9,6 +9,7 @@ from dvc.exceptions import InvalidArgumentError from dvc.main import main from dvc.repo.experiments.base import EXPS_STASH, ExpRefInfo from dvc.repo.experiments.executor.base import BaseExecutor, ExecutorInfo +from dvc.repo.experiments.utils import exp_refs_by_rev from dvc.utils.fs import makedirs from dvc.utils.serialize import YAMLFileCorruptedError, dump_yaml from tests.func.test_repro_multistage import COPY_SCRIPT @@ -269,6 +270,10 @@ def test_show_filter( included, excluded, ): + from contextlib import contextmanager + + from dvc.ui import ui + capsys.readouterr() div = "│" if os.name == "nt" else "┃" @@ -312,13 +317,30 @@ def test_show_filter( if e_params is not None: command.append(f"--exclude-params={e_params}") - assert main(command) == 0 + @contextmanager + def console_with(console, width): + console_options = console.options + original = console_options.max_width + con_width = console._width + + try: + console_options.max_width = width + console._width = width + yield + finally: + console_options.max_width = original + console._width = con_width + + with console_with(ui.rich_console, 255): + assert main(command) == 0 cap = capsys.readouterr() for i in included: - assert f"{div} {i} {div}" in cap.out + assert f"{div} params.yaml:{i} {div}" in cap.out + assert f"{div} metrics.yaml:{i} {div}" in cap.out for e in excluded: - assert f"{div} {e} {div}" not in cap.out + assert f"{div} params.yaml:{e} {div}" not in cap.out + assert f"{div} metrics.yaml:{e} {div}" not in cap.out def test_show_multiple_commits(tmp_dir, scm, dvc, exp_stage): @@ -412,7 +434,6 @@ def test_show_running_checkpoint( ): from dvc.repo.experiments.base import EXEC_BRANCH from dvc.repo.experiments.executor.local import TempDirExecutor - from dvc.repo.experiments.utils import exp_refs_by_rev baseline_rev = scm.get_rev() dvc.experiments.run( @@ -471,3 +492,46 @@ def test_show_with_broken_repo(tmp_dir, scm, dvc, exp_stage, caplog): paths = ["workspace", "baseline", "error"] assert isinstance(get_in(result, paths), YAMLFileCorruptedError) + + +def test_show_csv(tmp_dir, scm, dvc, exp_stage, capsys): + baseline_rev = scm.get_rev() + + def _get_rev_isotimestamp(rev): + return datetime.fromtimestamp( + scm.gitpython.repo.rev_parse(rev).committed_date + ).isoformat() + + result1 = dvc.experiments.run(exp_stage.addressing, params=["foo=2"]) + rev1 = first(result1) + ref_info1 = first(exp_refs_by_rev(scm, rev1)) + result2 = dvc.experiments.run(exp_stage.addressing, params=["foo=3"]) + rev2 = first(result2) + ref_info2 = first(exp_refs_by_rev(scm, rev2)) + + capsys.readouterr() + assert main(["exp", "show", "--show-csv"]) == 0 + cap = capsys.readouterr() + assert ( + "Experiment,rev,typ,Created,parent,metrics.yaml:foo,params.yaml:foo" + in cap.out + ) + assert ",workspace,baseline,,,3,3" in cap.out + assert ( + "master,{},baseline,{},,1,1".format( + baseline_rev[:7], _get_rev_isotimestamp(baseline_rev) + ) + in cap.out + ) + assert ( + "{},{},branch_base,{},,2,2".format( + ref_info1.name, rev1[:7], _get_rev_isotimestamp(rev1) + ) + in cap.out + ) + assert ( + "{},{},branch_commit,{},,3,3".format( + ref_info2.name, rev2[:7], _get_rev_isotimestamp(rev2) + ) + in cap.out + ) diff --git a/tests/unit/command/test_experiments.py b/tests/unit/command/test_experiments.py index d843a49b5..2fb23ac03 100644 --- a/tests/unit/command/test_experiments.py +++ b/tests/unit/command/test_experiments.py @@ -1,3 +1,5 @@ +from datetime import datetime + import pytest from dvc.cli import parse_args @@ -12,6 +14,7 @@ from dvc.command.experiments import ( CmdExperimentsRemove, CmdExperimentsRun, CmdExperimentsShow, + show_experiments, ) from dvc.exceptions import InvalidArgumentError @@ -277,3 +280,128 @@ def test_experiments_remove(dvc, scm, mocker, queue, clear_all, remote): clear_all=clear_all, remote="myremote" if remote else None, ) + + +all_experiments = { + "workspace": { + "baseline": { + "data": { + "timestamp": None, + "params": { + "params.yaml": { + "data": { + "featurize": {"max_features": 3000, "ngrams": 1}, + "parent": 20170428, + "train": { + "n_est": 100, + "min_split": 36, + }, + } + } + }, + "queued": False, + "running": False, + "executor": None, + "metrics": { + "scores.json": { + "data": { + "featurize": {"max_features": 3000, "ngrams": 1}, + "avg_prec": 0.5843640011189556, + "roc_auc": 0.9544670443829399, + } + } + }, + } + } + }, + "b05eecc666734e899f79af228ff49a7ae5a18cc0": { + "baseline": { + "data": { + "timestamp": datetime(2021, 8, 2, 16, 48, 14), + "params": { + "params.yaml": { + "data": { + "featurize": {"max_features": 3000, "ngrams": 1}, + "parent": 20170428, + "train": { + "n_est": 100, + "min_split": 2, + }, + } + } + }, + "queued": False, + "running": False, + "executor": None, + "metrics": { + "scores.json": { + "data": { + "featurize": {"max_features": 3000, "ngrams": 1}, + "avg_prec": 0.5325162867864254, + "roc_auc": 0.9106964878520005, + } + } + }, + "name": "master", + } + }, + "ae99936461d6c3092934160f8beafe66a294f98d": { + "data": { + "timestamp": datetime(2021, 8, 31, 14, 56, 55), + "params": { + "params.yaml": { + "data": { + "featurize": {"max_features": 3000, "ngrams": 1}, + "parent": 20170428, + "train": { + "n_est": 100, + "min_split": 36, + }, + } + } + }, + "queued": True, + "running": True, + "executor": None, + "metrics": { + "scores.json": { + "data": { + "featurize": {"max_features": 3000, "ngrams": 1}, + "avg_prec": 0.5843640011189556, + "roc_auc": 0.9544670443829399, + } + } + }, + "name": "exp-44136", + } + }, + }, +} + + +def test_show_experiments(capsys): + show_experiments( + all_experiments, precision=None, fill_value="", iso=True, show_csv=True + ) + cap = capsys.readouterr() + assert ( + "Experiment,rev,typ,Created,parent,State,scores.json:" + "featurize.max_features,scores.json:featurize.ngrams," + "avg_prec,roc_auc,params.yaml:featurize.max_features," + "params.yaml:featurize.ngrams,params.yaml:parent," + "train.n_est,train.min_split" in cap.out + ) + assert ( + ",workspace,baseline,,,,3000,1,0.5843640011189556,0.9544670443829399," + "3000,1,20170428,100,36" in cap.out + ) + assert ( + "master,b05eecc,baseline,2021-08-02T16:48:14,,,3000,1," + "0.5325162867864254,0.9106964878520005,3000,1,20170428,100,2" + in cap.out + ) + assert ( + "exp-44136,ae99936,branch_base,2021-08-31T14:56:55,,Running," + "3000,1,0.5843640011189556,0.9544670443829399,3000,1,20170428,100,36" + in cap.out + )