From 8e5f4a8f35a15e741031bf1c9dbe4728fa55021d Mon Sep 17 00:00:00 2001 From: skshetry Date: Fri, 9 Sep 2022 17:36:24 +0545 Subject: [PATCH] data ls: new command to show metadata with outputs (#8252) data ls: add new command to show metadata for the outputs --- dvc/cli/actions.py | 9 ++ dvc/commands/data.py | 112 ++++++++++++++++++++++++- dvc/repo/data.py | 17 ++++ tests/func/test_data_ls.py | 31 +++++++ tests/unit/command/test_compat_flag.py | 1 + tests/unit/command/test_data_ls.py | 96 +++++++++++++++++++++ 6 files changed, 265 insertions(+), 1 deletion(-) create mode 100644 tests/func/test_data_ls.py create mode 100644 tests/unit/command/test_data_ls.py diff --git a/dvc/cli/actions.py b/dvc/cli/actions.py index 279b53680..12430bea9 100644 --- a/dvc/cli/actions.py +++ b/dvc/cli/actions.py @@ -1,6 +1,15 @@ from argparse import _AppendAction +class CommaSeparatedArgs(_AppendAction): # pylint: disable=protected-access + def __call__(self, parser, namespace, values, option_string=None): + from funcy import ldistinct + + items = getattr(namespace, self.dest) or [] + items.extend(map(str.strip, values.split(","))) + setattr(namespace, self.dest, ldistinct(items)) + + class KeyValueArgs(_AppendAction): def __call__(self, parser, namespace, values, option_string=None): items = getattr(namespace, self.dest) or {} diff --git a/dvc/commands/data.py b/dvc/commands/data.py index 13ad7dbcd..1fae632fb 100644 --- a/dvc/commands/data.py +++ b/dvc/commands/data.py @@ -1,9 +1,12 @@ import argparse import logging -from typing import TYPE_CHECKING +from operator import itemgetter +from typing import TYPE_CHECKING, Any, Dict, Iterable, Set from funcy import chunks, compact, log_durations +from dvc.cli import completion +from dvc.cli.actions import CommaSeparatedArgs from dvc.cli.command import CmdBase from dvc.cli.utils import append_doc_link, fix_subparsers from dvc.ui import ui @@ -124,6 +127,65 @@ class CmdDataStatus(CmdBase): return self._show_status(status) +class CmdDataLs(CmdBase): + @staticmethod + def _show_table( + d: Iterable[Dict[str, Any]], + filter_types: Set[str], + filter_labels: Set[str], + markdown: bool = False, + ) -> None: + from rich.style import Style + + from dvc.compare import TabularData + + td = TabularData( + columns=["Path", "Type", "Labels", "Description"], fill_value="-" + ) + for entry in sorted(d, key=itemgetter("path")): + typ = entry.get("type", "") + desc = entry.get("desc", "-") + labels = entry.get("labels", []) + + if filter_types and typ not in filter_types: + continue + if filter_labels and filter_labels.isdisjoint(labels): + continue + + rich_label = ui.rich_text() + for index, label in enumerate(labels): + if index: + rich_label.append(",") + style = Style(bold=label in filter_labels, color="green") + rich_label.append(label, style=style) + + if markdown and desc: + desc = desc.partition("\n")[0] + + path = ui.rich_text(entry["path"], style="cyan") + type_style = Style(bold=typ in filter_types, color="yellow") + typ = ui.rich_text(entry.get("type", ""), style=type_style) + td.append([path, typ or "-", rich_label or "-", desc]) + + td.render(markdown=markdown, rich_table=True) + + def run(self): + from dvc.repo.data import ls + + filter_labels = set(self.args.labels) + filter_types = set(self.args.type) + d = ls( + self.repo, targets=self.args.targets, recursive=self.args.recursive + ) + self._show_table( + d, + filter_labels=filter_labels, + filter_types=filter_types, + markdown=self.args.markdown, + ) + return 0 + + def add_parser(subparsers, parent_parser): data_parser = subparsers.add_parser( "data", @@ -181,3 +243,51 @@ def add_parser(subparsers, parent_parser): help="Show untracked files.", ) data_status_parser.set_defaults(func=CmdDataStatus) + + DATA_LS_HELP = "List data tracked by DVC with its metadata." + data_ls_parser = data_subparsers.add_parser( + "ls", + aliases=["list"], + parents=[parent_parser], + description=append_doc_link(DATA_LS_HELP, "data/ls"), + formatter_class=argparse.RawDescriptionHelpFormatter, + help=DATA_LS_HELP, + ) + data_ls_parser.add_argument( + "--md", + "--show-md", + dest="markdown", + action="store_true", + default=False, + help="Show tabulated output in the Markdown format (GFM).", + ) + data_ls_parser.add_argument( + "--type", + action=CommaSeparatedArgs, + default=[], + help="Comma-separated list of type to filter.", + ) + data_ls_parser.add_argument( + "--labels", + action=CommaSeparatedArgs, + default=[], + help="Comma-separated list of labels to filter.", + ) + data_ls_parser.add_argument( + "-R", + "--recursive", + action="store_true", + default=False, + help="Recursively list from the specified directories.", + ) + data_ls_parser.add_argument( + "targets", + default=None, + nargs="*", + help=( + "Limit command scope to these tracked files/directories, " + ".dvc files, or stage names." + ), + ).complete = completion.DVCFILES_AND_STAGE + + data_ls_parser.set_defaults(func=CmdDataLs) diff --git a/dvc/repo/data.py b/dvc/repo/data.py index d91144b50..c9f6390a8 100644 --- a/dvc/repo/data.py +++ b/dvc/repo/data.py @@ -1,11 +1,13 @@ import os import posixpath from collections import defaultdict +from itertools import chain from typing import ( TYPE_CHECKING, Any, Dict, Iterable, + Iterator, List, Optional, Tuple, @@ -305,3 +307,18 @@ def status(repo: "Repo", untracked_files: str = "no", **kwargs: Any) -> Status: unchanged=list(unchanged), git=git_info, ) + + +def ls( + repo: "Repo", + targets: List[Optional[str]] = None, + recursive: bool = False, +) -> Iterator[Dict[str, Any]]: + targets = targets or [None] + pairs = chain.from_iterable( + repo.stage.collect_granular(target, recursive=recursive) + for target in targets + ) + for stage, filter_info in pairs: + for out in stage.filter_outs(filter_info): + yield {"path": str(out), **out.annot.to_dict()} diff --git a/tests/func/test_data_ls.py b/tests/func/test_data_ls.py new file mode 100644 index 000000000..d39e6d928 --- /dev/null +++ b/tests/func/test_data_ls.py @@ -0,0 +1,31 @@ +from operator import itemgetter + +from dvc.repo.data import ls + + +def test_data_ls(tmp_dir, dvc): + assert not list(ls(dvc)) + + tmp_dir.dvc_gen("bar", "bar") + tmp_dir.gen("foo", "foo") + dvc.add( + "foo", + meta={"key": "value"}, + labels=["l1", "l2"], + type="t1", + desc="foo", + ) + + foo_entry = { + "path": "foo", + "desc": "foo", + "type": "t1", + "labels": ["l1", "l2"], + "meta": {"key": "value"}, + } + assert sorted(ls(dvc), key=itemgetter("path")) == [ + {"path": "bar"}, + foo_entry, + ] + assert list(ls(dvc, targets=["foo"])) == [foo_entry] + assert list(ls(dvc, targets=["foo"], recursive=True)) == [foo_entry] diff --git a/tests/unit/command/test_compat_flag.py b/tests/unit/command/test_compat_flag.py index f5aa78eb2..1adb9d01a 100644 --- a/tests/unit/command/test_compat_flag.py +++ b/tests/unit/command/test_compat_flag.py @@ -15,6 +15,7 @@ def _id_gen(val) -> str: "args, key", [ (["dag", "--show-md"], "markdown"), + (["data", "ls", "--show-md"], "markdown"), (["diff", "--show-json"], "json"), (["diff", "--show-md"], "markdown"), (["experiments", "diff", "--show-json"], "json"), diff --git a/tests/unit/command/test_data_ls.py b/tests/unit/command/test_data_ls.py new file mode 100644 index 000000000..ca16a4bc9 --- /dev/null +++ b/tests/unit/command/test_data_ls.py @@ -0,0 +1,96 @@ +import pytest + +from dvc.cli import parse_args +from dvc.commands.data import CmdDataLs + + +def test_cli(mocker): + mocker.patch("dvc.repo.Repo") + ls = mocker.patch("dvc.repo.data.ls", return_value={}) + show_table_spy = mocker.spy(CmdDataLs, "_show_table") + + cli_args = parse_args( + [ + "data", + "ls", + "--labels", + "label1,label2", + "--labels", + "label3", + "--type", + "type1,type2", + "--type", + "type3", + "target", + "--recursive", + "--md", + ] + ) + + assert cli_args.func == CmdDataLs + cmd = cli_args.func(cli_args) + assert cmd.run() == 0 + + ls.assert_called_once_with( + cmd.repo, + targets=["target"], + recursive=True, + ) + show_table_spy.assert_called_once_with( + {}, + filter_labels={"label1", "label2", "label3"}, + filter_types={"type1", "type2", "type3"}, + markdown=True, + ) + + +EXAMPLE_DATA = [ + { + "path": "model.pkl", + "desc": "desc", + "type": "model", + "labels": ["get-started", "dataset-registry"], + "meta": {"key": "value", "key1": "value1"}, + }, + { + "path": "model.pkl", + "desc": "desc", + "type": "model", + "labels": ["get-started", "example"], + }, + {"path": "unlabeled.txt"}, +] + + +@pytest.mark.parametrize( + "markdown, expected_output", + [ + ( + True, + """\ +| Path | Type | Labels | Description | +|-----------|--------|------------------------------|---------------| +| model.pkl | model | get-started,dataset-registry | desc | +| model.pkl | model | get-started,example | desc |\n""", + ), + ( + False, + """\ + Path Type Labels Description + model.pkl model get-started,dataset-registry desc + model.pkl model get-started,example desc""", + ), + ], + ids=["markdown", "default"], +) +def test_ls(capsys, markdown, expected_output): + CmdDataLs._show_table( + EXAMPLE_DATA, + filter_types={"model"}, + filter_labels={"get-started", "dataset-registry"}, + markdown=markdown, + ) + out, err = capsys.readouterr() + assert not err + out = "\n".join(line.rstrip() for line in out.splitlines()) + assert out == expected_output