Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

data ls: new command to show metadata with outputs (#8252)

Browse Source 
data ls: add new command to show metadata for the outputs
This commit is contained in:
skshetry
2022-09-09 17:36:24 +05:45
committed by GitHub
parent cb3159e586
commit 8e5f4a8f35
6 changed files with 265 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/cli/actions.py
+9
View File
@@ -1,6 +1,15 @@
from argparse import _AppendAction
class CommaSeparatedArgs(_AppendAction): # pylint: disable=protected-access
def __call__(self, parser, namespace, values, option_string=None):
from funcy import ldistinct
items = getattr(namespace, self.dest) or []
items.extend(map(str.strip, values.split(",")))
setattr(namespace, self.dest, ldistinct(items))
class KeyValueArgs(_AppendAction):
def __call__(self, parser, namespace, values, option_string=None):
items = getattr(namespace, self.dest) or {}
dvc/commands/data.py
+111 -1
View File
@@ -1,9 +1,12 @@
import argparse
import logging
from typing import TYPE_CHECKING
from operator import itemgetter
from typing import TYPE_CHECKING, Any, Dict, Iterable, Set
from funcy import chunks, compact, log_durations
from dvc.cli import completion
from dvc.cli.actions import CommaSeparatedArgs
from dvc.cli.command import CmdBase
from dvc.cli.utils import append_doc_link, fix_subparsers
from dvc.ui import ui
@@ -124,6 +127,65 @@ class CmdDataStatus(CmdBase):
return self._show_status(status)
class CmdDataLs(CmdBase):
@staticmethod
def _show_table(
d: Iterable[Dict[str, Any]],
filter_types: Set[str],
filter_labels: Set[str],
markdown: bool = False,
) -> None:
from rich.style import Style
from dvc.compare import TabularData
td = TabularData(
columns=["Path", "Type", "Labels", "Description"], fill_value="-"
)
for entry in sorted(d, key=itemgetter("path")):
typ = entry.get("type", "")
desc = entry.get("desc", "-")
labels = entry.get("labels", [])
if filter_types and typ not in filter_types:
continue
if filter_labels and filter_labels.isdisjoint(labels):
continue
rich_label = ui.rich_text()
for index, label in enumerate(labels):
if index:
rich_label.append(",")
style = Style(bold=label in filter_labels, color="green")
rich_label.append(label, style=style)
if markdown and desc:
desc = desc.partition("\n")[0]
path = ui.rich_text(entry["path"], style="cyan")
type_style = Style(bold=typ in filter_types, color="yellow")
typ = ui.rich_text(entry.get("type", ""), style=type_style)
td.append([path, typ or "-", rich_label or "-", desc])
td.render(markdown=markdown, rich_table=True)
def run(self):
from dvc.repo.data import ls
filter_labels = set(self.args.labels)
filter_types = set(self.args.type)
d = ls(
self.repo, targets=self.args.targets, recursive=self.args.recursive
)
self._show_table(
d,
filter_labels=filter_labels,
filter_types=filter_types,
markdown=self.args.markdown,
)
return 0
def add_parser(subparsers, parent_parser):
data_parser = subparsers.add_parser(
"data",
@@ -181,3 +243,51 @@ def add_parser(subparsers, parent_parser):
help="Show untracked files.",
)
data_status_parser.set_defaults(func=CmdDataStatus)
DATA_LS_HELP = "List data tracked by DVC with its metadata."
data_ls_parser = data_subparsers.add_parser(
"ls",
aliases=["list"],
parents=[parent_parser],
description=append_doc_link(DATA_LS_HELP, "data/ls"),
formatter_class=argparse.RawDescriptionHelpFormatter,
help=DATA_LS_HELP,
)
data_ls_parser.add_argument(
"--md",
"--show-md",
dest="markdown",
action="store_true",
default=False,
help="Show tabulated output in the Markdown format (GFM).",
)
data_ls_parser.add_argument(
"--type",
action=CommaSeparatedArgs,
default=[],
help="Comma-separated list of type to filter.",
)
data_ls_parser.add_argument(
"--labels",
action=CommaSeparatedArgs,
default=[],
help="Comma-separated list of labels to filter.",
)
data_ls_parser.add_argument(
"-R",
"--recursive",
action="store_true",
default=False,
help="Recursively list from the specified directories.",
)
data_ls_parser.add_argument(
"targets",
default=None,
nargs="*",
help=(
"Limit command scope to these tracked files/directories, "
".dvc files, or stage names."
),
).complete = completion.DVCFILES_AND_STAGE
data_ls_parser.set_defaults(func=CmdDataLs)
dvc/repo/data.py
+17
View File
@@ -1,11 +1,13 @@
import os
import posixpath
from collections import defaultdict
from itertools import chain
from typing import (
TYPE_CHECKING,
Any,
Dict,
Iterable,
Iterator,
List,
Optional,
Tuple,
@@ -305,3 +307,18 @@ def status(repo: "Repo", untracked_files: str = "no", **kwargs: Any) -> Status:
unchanged=list(unchanged),
git=git_info,
)
def ls(
repo: "Repo",
targets: List[Optional[str]] = None,
recursive: bool = False,
) -> Iterator[Dict[str, Any]]:
targets = targets or [None]
pairs = chain.from_iterable(
repo.stage.collect_granular(target, recursive=recursive)
for target in targets
)
for stage, filter_info in pairs:
for out in stage.filter_outs(filter_info):
yield {"path": str(out), **out.annot.to_dict()}
tests/func/test_data_ls.py
+31
View File
@@ -0,0 +1,31 @@
from operator import itemgetter
from dvc.repo.data import ls
def test_data_ls(tmp_dir, dvc):
assert not list(ls(dvc))
tmp_dir.dvc_gen("bar", "bar")
tmp_dir.gen("foo", "foo")
dvc.add(
"foo",
meta={"key": "value"},
labels=["l1", "l2"],
type="t1",
desc="foo",
)
foo_entry = {
"path": "foo",
"desc": "foo",
"type": "t1",
"labels": ["l1", "l2"],
"meta": {"key": "value"},
}
assert sorted(ls(dvc), key=itemgetter("path")) == [
{"path": "bar"},
foo_entry,
]
assert list(ls(dvc, targets=["foo"])) == [foo_entry]
assert list(ls(dvc, targets=["foo"], recursive=True)) == [foo_entry]
tests/unit/command/test_compat_flag.py
+1
View File
@@ -15,6 +15,7 @@ def _id_gen(val) -> str:
"args, key",
[
(["dag", "--show-md"], "markdown"),
(["data", "ls", "--show-md"], "markdown"),
(["diff", "--show-json"], "json"),
(["diff", "--show-md"], "markdown"),
(["experiments", "diff", "--show-json"], "json"),
tests/unit/command/test_data_ls.py
+96
View File
@@ -0,0 +1,96 @@
import pytest
from dvc.cli import parse_args
from dvc.commands.data import CmdDataLs
def test_cli(mocker):
mocker.patch("dvc.repo.Repo")
ls = mocker.patch("dvc.repo.data.ls", return_value={})
show_table_spy = mocker.spy(CmdDataLs, "_show_table")
cli_args = parse_args(
[
"data",
"ls",
"--labels",
"label1,label2",
"--labels",
"label3",
"--type",
"type1,type2",
"--type",
"type3",
"target",
"--recursive",
"--md",
]
)
assert cli_args.func == CmdDataLs
cmd = cli_args.func(cli_args)
assert cmd.run() == 0
ls.assert_called_once_with(
cmd.repo,
targets=["target"],
recursive=True,
)
show_table_spy.assert_called_once_with(
{},
filter_labels={"label1", "label2", "label3"},
filter_types={"type1", "type2", "type3"},
markdown=True,
)
EXAMPLE_DATA = [
{
"path": "model.pkl",
"desc": "desc",
"type": "model",
"labels": ["get-started", "dataset-registry"],
"meta": {"key": "value", "key1": "value1"},
},
{
"path": "model.pkl",
"desc": "desc",
"type": "model",
"labels": ["get-started", "example"],
},
{"path": "unlabeled.txt"},
]
@pytest.mark.parametrize(
"markdown, expected_output",
[
(
True,
"""\
| Path | Type | Labels | Description |
|-----------|--------|------------------------------|---------------|
| model.pkl | model | get-started,dataset-registry | desc |
| model.pkl | model | get-started,example | desc |\n""",
),
(
False,
"""\
Path Type Labels Description
model.pkl model get-started,dataset-registry desc
model.pkl model get-started,example desc""",
),
],
ids=["markdown", "default"],
)
def test_ls(capsys, markdown, expected_output):
CmdDataLs._show_table(
EXAMPLE_DATA,
filter_types={"model"},
filter_labels={"get-started", "dataset-registry"},
markdown=markdown,
)
out, err = capsys.readouterr()
assert not err
out = "\n".join(line.rstrip() for line in out.splitlines())
assert out == expected_output