Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

remove data ls and support for --meta/--desc/--label/--type cli flags (#9481)

Browse Source
This commit is contained in:
skshetry
2023-05-19 20:10:34 +05:45
committed by GitHub
parent 02f7fcae73
commit e089ee0365
19 changed files with 3 additions and 528 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/annotations.py
-6
View File
@@ -17,12 +17,6 @@ class Annotation:
labels: List[str] = field(default_factory=list)
meta: Dict[str, Any] = field(default_factory=dict)
def update(self, **kwargs) -> "Annotation":
for attr, value in kwargs.items():
if value and hasattr(self, attr):
setattr(self, attr, value)
return self
def to_dict(self) -> Dict[str, str]:
return compact(asdict(self))
dvc/cli/actions.py
-20
View File
@@ -1,20 +0,0 @@
from argparse import _AppendAction
class CommaSeparatedArgs(_AppendAction): # pylint: disable=protected-access
def __call__(self, parser, namespace, values, option_string=None): # noqa: ARG002
from funcy import ldistinct
items = getattr(namespace, self.dest) or []
items.extend(map(str.strip, values.split(",")))
setattr(namespace, self.dest, ldistinct(items))
class KeyValueArgs(_AppendAction):
def __call__(self, parser, namespace, values, option_string=None): # noqa: ARG002
items = getattr(namespace, self.dest) or {}
for value in filter(bool, values):
key, _, value = value.partition("=")
items[key.strip()] = value
setattr(namespace, self.dest, items)
dvc/commands/add.py
-37
View File
@@ -2,43 +2,12 @@ import argparse
import logging
from dvc.cli import completion
from dvc.cli.actions import KeyValueArgs
from dvc.cli.command import CmdBase
from dvc.cli.utils import append_doc_link
logger = logging.getLogger(__name__)
def _add_annotating_args(parser: argparse.ArgumentParser) -> None:
parser.add_argument(
"--desc",
type=str,
metavar="<text>",
help="User description of the data.",
)
parser.add_argument(
"--meta",
metavar="key=value",
nargs=1,
action=KeyValueArgs,
help="Custom metadata to add to the data.",
)
parser.add_argument(
"--label",
dest="labels",
type=str,
action="append",
metavar="<str>",
help="Label for the data.",
)
parser.add_argument(
"--type",
type=str,
metavar="<str>",
help="Type of the data.",
)
class CmdAdd(CmdBase):
def run(self):
from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename
@@ -54,11 +23,7 @@ class CmdAdd(CmdBase):
fname=self.args.file,
external=self.args.external,
glob=self.args.glob,
desc=self.args.desc,
out=self.args.out,
type=self.args.type,
labels=self.args.labels,
meta=self.args.meta,
remote=self.args.remote,
to_remote=self.args.to_remote,
jobs=self.args.jobs,
@@ -150,8 +115,6 @@ def add_parser(subparsers, parent_parser):
default=False,
help="Override local file or folder if exists.",
)
_add_annotating_args(parser)
parser.add_argument(
"targets", nargs="+", help="Input files/directories to add."
).complete = completion.FILE
dvc/commands/data.py
+2 -109
View File
@@ -1,14 +1,11 @@
import argparse
import logging
from operator import itemgetter
from typing import TYPE_CHECKING, Any, Dict, Iterable, Set
from typing import TYPE_CHECKING
from funcy import chunks, compact, log_durations
from dvc.cli import completion
from dvc.cli.actions import CommaSeparatedArgs
from dvc.cli.command import CmdBase
from dvc.cli.utils import append_doc_link, fix_subparsers, hide_subparsers_from_help
from dvc.cli.utils import append_doc_link, fix_subparsers
from dvc.ui import ui
from dvc.utils import colorize
@@ -132,63 +129,6 @@ class CmdDataStatus(CmdBase):
return self._show_status(status)
class CmdDataLs(CmdBase):
@staticmethod
def _show_table(
d: Iterable[Dict[str, Any]],
filter_types: Set[str],
filter_labels: Set[str],
markdown: bool = False,
) -> None:
from rich.style import Style
from dvc.compare import TabularData
td = TabularData(
columns=["Path", "Type", "Labels", "Description"], fill_value="-"
)
for entry in sorted(d, key=itemgetter("path")):
typ = entry.get("type", "")
desc = entry.get("desc", "-")
labels = entry.get("labels", [])
if filter_types and typ not in filter_types:
continue
if filter_labels and filter_labels.isdisjoint(labels):
continue
rich_label = ui.rich_text()
for index, label in enumerate(labels):
if index:
rich_label.append(",")
style = Style(bold=label in filter_labels, color="green")
rich_label.append(label, style=style)
if markdown and desc:
desc = desc.partition("\n")[0]
path = ui.rich_text(entry["path"], style="cyan")
type_style = Style(bold=typ in filter_types, color="yellow")
typ = ui.rich_text(entry.get("type", ""), style=type_style)
td.append([path, typ or "-", rich_label or "-", desc])
td.render(markdown=markdown, rich_table=True)
def run(self):
from dvc.repo.data import ls
filter_labels = set(self.args.labels)
filter_types = set(self.args.type)
d = ls(self.repo, targets=self.args.targets, recursive=self.args.recursive)
self._show_table(
d,
filter_labels=filter_labels,
filter_types=filter_types,
markdown=self.args.markdown,
)
return 0
def add_parser(subparsers, parent_parser):
data_parser = subparsers.add_parser(
"data",
@@ -250,50 +190,3 @@ def add_parser(subparsers, parent_parser):
help="Use cached remote index (don't check remote).",
)
data_status_parser.set_defaults(func=CmdDataStatus)
DATA_LS_HELP = "List data tracked by DVC with its metadata."
data_ls_parser = data_subparsers.add_parser(
"ls",
aliases=["list"],
parents=[parent_parser],
description=append_doc_link(DATA_LS_HELP, "data/ls"),
formatter_class=argparse.RawDescriptionHelpFormatter,
add_help=False,
)
data_ls_parser.add_argument(
"--md",
dest="markdown",
action="store_true",
default=False,
help="Show tabulated output in the Markdown format (GFM).",
)
data_ls_parser.add_argument(
"--type",
action=CommaSeparatedArgs,
default=[],
help="Comma-separated list of type to filter.",
)
data_ls_parser.add_argument(
"--labels",
action=CommaSeparatedArgs,
default=[],
help="Comma-separated list of labels to filter.",
)
data_ls_parser.add_argument(
"-R",
"--recursive",
action="store_true",
default=False,
help="Recursively list from the specified directories.",
)
data_ls_parser.add_argument(
"targets",
default=None,
nargs="*",
help=(
"Limit command scope to these tracked files/directories, "
".dvc files, or stage names."
),
).complete = completion.DVCFILES_AND_STAGE
data_ls_parser.set_defaults(func=CmdDataLs)
hide_subparsers_from_help(data_subparsers)
dvc/commands/imp.py
-8
View File
@@ -22,10 +22,6 @@ class CmdImport(CmdBase):
rev=self.args.rev,
no_exec=self.args.no_exec,
no_download=self.args.no_download,
desc=self.args.desc,
type=self.args.type,
labels=self.args.labels,
meta=self.args.meta,
jobs=self.args.jobs,
)
except CloneError:
@@ -42,8 +38,6 @@ class CmdImport(CmdBase):
def add_parser(subparsers, parent_parser):
from .add import _add_annotating_args
IMPORT_HELP = (
"Download file or directory tracked by DVC or by Git "
"into the workspace, and track it."
@@ -106,6 +100,4 @@ def add_parser(subparsers, parent_parser):
),
metavar="<number>",
)
_add_annotating_args(import_parser)
import_parser.set_defaults(func=CmdImport)
dvc/commands/imp_url.py
-7
View File
@@ -20,10 +20,6 @@ class CmdImportUrl(CmdBase):
no_download=self.args.no_download,
remote=self.args.remote,
to_remote=self.args.to_remote,
desc=self.args.desc,
type=self.args.type,
labels=self.args.labels,
meta=self.args.meta,
jobs=self.args.jobs,
force=self.args.force,
version_aware=self.args.version_aware,
@@ -41,8 +37,6 @@ class CmdImportUrl(CmdBase):
def add_parser(subparsers, parent_parser):
from .add import _add_annotating_args
IMPORT_HELP = "Download or copy file from URL and take it under DVC control."
import_parser = subparsers.add_parser(
@@ -126,5 +120,4 @@ def add_parser(subparsers, parent_parser):
default=False,
help="Import using cloud versioning. Implied if the URL contains a version ID.",
)
_add_annotating_args(import_parser)
import_parser.set_defaults(func=CmdImportUrl)
dvc/repo/add.py
-3
View File
@@ -297,7 +297,4 @@ def create_stages(
force=force,
)
output_exists = False
out_obj = stage.outs[0]
out_obj.annot.update(**kwargs)
yield StageInfo(stage, output_exists)
dvc/repo/data.py
+1 -26
View File
@@ -1,17 +1,6 @@
import os
import posixpath
from itertools import chain
from typing import (
TYPE_CHECKING,
Any,
Dict,
Iterable,
Iterator,
List,
Optional,
TypedDict,
Union,
)
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, TypedDict, Union
from dvc.ui import ui
@@ -251,17 +240,3 @@ def status(
unchanged=list(unchanged),
git=git_info,
)
def ls(
repo: "Repo",
targets: Optional[List[Optional[str]]] = None,
recursive: bool = False,
) -> Iterator[Dict[str, Any]]:
targets = targets or [None]
pairs = chain.from_iterable(
repo.stage.collect_granular(target, recursive=recursive) for target in targets
)
for stage, filter_info in pairs:
for out in stage.filter_outs(filter_info):
yield {"path": str(out), **out.annot.to_dict()}
dvc/repo/imp_url.py
-7
View File
@@ -25,10 +25,6 @@ def imp_url( # noqa: C901, PLR0913
no_exec=False,
remote=None,
to_remote=False,
desc=None,
type=None, # noqa: A002, pylint: disable=redefined-builtin
labels=None,
meta=None,
jobs=None,
force=False,
fs_config=None,
@@ -70,9 +66,6 @@ def imp_url( # noqa: C901, PLR0913
fs_config=fs_config,
)
out_obj = stage.outs[0]
out_obj.annot.update(desc=desc, type=type, labels=labels, meta=meta)
try:
self.check_graph(stages={stage})
except OutputDuplicationError as exc:
tests/func/test_add.py
-21
View File
@@ -11,7 +11,6 @@ import pytest
import dvc as dvc_module
import dvc_data
from dvc.annotations import Annotation
from dvc.cachemgr import CacheManager
from dvc.cli import main
from dvc.config import ConfigError
@@ -1124,26 +1123,6 @@ def test_add_ignore_duplicated_targets(tmp_dir, dvc, capsys):
assert "ignoring duplicated targets: foo, bar" in err
def test_add_with_annotations(M, tmp_dir, dvc):
tmp_dir.gen("foo", "foo")
annot = {
"desc": "foo desc",
"labels": ["l1", "l2"],
"type": "t1",
"meta": {"key": "value"},
}
(stage,) = dvc.add("foo", **annot)
assert stage.outs[0].annot == Annotation(**annot)
assert (tmp_dir / "foo.dvc").parse() == M.dict(outs=[M.dict(**annot)])
# try to selectively update/overwrite some annotations
annot = {**annot, "type": "t2"}
(stage,) = dvc.add("foo", type="t2")
assert stage.outs[0].annot == Annotation(**annot)
assert (tmp_dir / "foo.dvc").parse() == M.dict(outs=[M.dict(**annot)])
def test_add_updates_to_cloud_versioning_dir(tmp_dir, dvc):
data_dvc = tmp_dir / "data.dvc"
data_dvc.dump(
tests/func/test_data_ls.py
-31
View File
@@ -1,31 +0,0 @@
from operator import itemgetter
from dvc.repo.data import ls
def test_data_ls(tmp_dir, dvc):
assert not list(ls(dvc))
tmp_dir.dvc_gen("bar", "bar")
tmp_dir.gen("foo", "foo")
dvc.add(
"foo",
meta={"key": "value"},
labels=["l1", "l2"],
type="t1",
desc="foo",
)
foo_entry = {
"path": "foo",
"desc": "foo",
"type": "t1",
"labels": ["l1", "l2"],
"meta": {"key": "value"},
}
assert sorted(ls(dvc), key=itemgetter("path")) == [
{"path": "bar"},
foo_entry,
]
assert list(ls(dvc, targets=["foo"])) == [foo_entry]
assert list(ls(dvc, targets=["foo"], recursive=True)) == [foo_entry]
tests/func/test_import.py
-22
View File
@@ -4,7 +4,6 @@ import os
import pytest
from funcy import first
from dvc.annotations import Annotation
from dvc.cachemgr import CacheManager
from dvc.config import NoRemoteError
from dvc.dvcfile import load_file
@@ -637,24 +636,3 @@ def test_parameterized_repo(tmp_dir, dvc, scm, erepo_dir, paths):
"url": os.fspath(erepo_dir),
"rev_lock": erepo_dir.scm.get_rev(),
}
def test_import_with_annotations(M, tmp_dir, scm, dvc, erepo_dir):
with erepo_dir.chdir():
erepo_dir.dvc_gen("foo", "foo content", commit="create foo")
annot = {
"desc": "foo desc",
"labels": ["l1", "l2"],
"type": "t1",
"meta": {"key": "value"},
}
stage = dvc.imp(os.fspath(erepo_dir), "foo", "foo", no_exec=True, **annot)
assert stage.outs[0].annot == Annotation(**annot)
assert (tmp_dir / "foo.dvc").parse() == M.dict(outs=[M.dict(**annot)])
# try to selectively update/overwrite some annotations
annot = {**annot, "type": "t2"}
stage = dvc.imp(os.fspath(erepo_dir), "foo", "foo", no_exec=True, type="t2")
assert stage.outs[0].annot == Annotation(**annot)
assert (tmp_dir / "foo.dvc").parse() == M.dict(outs=[M.dict(**annot)])
tests/func/test_import_url.py
-30
View File
@@ -4,7 +4,6 @@ from uuid import uuid4
import pytest
from dvc.annotations import Annotation
from dvc.cli import main
from dvc.dependency.base import Dependency, DependencyDoesNotExistError
from dvc.dvcfile import load_file
@@ -253,35 +252,6 @@ def test_partial_import_pull(tmp_dir, scm, dvc, local_workspace):
assert stage.outs[0].meta.size == 12
def test_imp_url_with_annotations(M, tmp_dir, dvc, local_workspace):
local_workspace.gen("foo", "foo")
annot = {
"desc": "foo desc",
"labels": ["l1", "l2"],
"type": "t1",
"meta": {"key": "value"},
}
stage = dvc.imp_url(
"remote://workspace/foo",
os.fspath(tmp_dir / "foo"),
no_exec=True,
**annot,
)
assert stage.outs[0].annot == Annotation(**annot)
assert (tmp_dir / "foo.dvc").parse() == M.dict(outs=[M.dict(**annot)])
# try to selectively update/overwrite some annotations
annot = {**annot, "type": "t2"}
stage = dvc.imp_url(
"remote://workspace/foo",
os.fspath(tmp_dir / "foo"),
no_exec=True,
type="t2",
)
assert stage.outs[0].annot == Annotation(**annot)
assert (tmp_dir / "foo.dvc").parse() == M.dict(outs=[M.dict(**annot)])
def test_import_url_fs_config(tmp_dir, dvc, workspace, mocker):
import dvc.fs as dvc_fs
tests/unit/command/test_add.py
-10
View File
@@ -14,8 +14,6 @@ def test_add(mocker, dvc):
"--glob",
"--file",
"file",
"--desc",
"stage description",
"data",
]
)
@@ -36,10 +34,6 @@ def test_add(mocker, dvc):
out=None,
remote=None,
to_remote=False,
desc="stage description",
type=None,
labels=None,
meta=None,
jobs=None,
force=False,
)
@@ -74,10 +68,6 @@ def test_add_to_remote(mocker):
out="bar",
remote="remote",
to_remote=True,
desc=None,
type=None,
labels=None,
meta=None,
jobs=None,
force=False,
)
tests/unit/command/test_annotations_flags.py
-47
View File
@@ -1,47 +0,0 @@
import pytest
from dvc.cli import parse_args
from tests.utils.asserts import called_once_with_subset
@pytest.mark.parametrize(
"func, args",
[
("add", ("add", "model.pkl")),
("imp", ("import", "https://github.com/org/repo", "model.pkl")),
("imp_url", ("import-url", "https://dvc.org/data.xml", "data.xml")),
],
ids=["add", "import", "import-url"],
)
def test_add_annotations(mocker, func, args):
mocker.patch("dvc.repo.Repo")
cli_args = parse_args(
[
*args,
"--label",
"example-get-started",
"--label",
"model-registry",
"--type",
"model",
"--desc",
"description",
"--meta",
"key1=value1",
"--meta",
"key2=value2",
]
)
_, *posargs = args
cmd = cli_args.func(cli_args)
assert cmd.run() == 0
assert called_once_with_subset(
getattr(cmd.repo, func),
posargs,
desc="description",
type="model",
labels=["example-get-started", "model-registry"],
meta={"key1": "value1", "key2": "value2"},
)
tests/unit/command/test_data_ls.py
-96
View File
@@ -1,96 +0,0 @@
import pytest
from dvc.cli import parse_args
from dvc.commands.data import CmdDataLs
def test_cli(mocker):
mocker.patch("dvc.repo.Repo")
ls = mocker.patch("dvc.repo.data.ls", return_value={})
show_table_spy = mocker.spy(CmdDataLs, "_show_table")
cli_args = parse_args(
[
"data",
"ls",
"--labels",
"label1,label2",
"--labels",
"label3",
"--type",
"type1,type2",
"--type",
"type3",
"target",
"--recursive",
"--md",
]
)
assert cli_args.func == CmdDataLs
cmd = cli_args.func(cli_args)
assert cmd.run() == 0
ls.assert_called_once_with(
cmd.repo,
targets=["target"],
recursive=True,
)
show_table_spy.assert_called_once_with(
{},
filter_labels={"label1", "label2", "label3"},
filter_types={"type1", "type2", "type3"},
markdown=True,
)
EXAMPLE_DATA = [
{
"path": "model.pkl",
"desc": "desc",
"type": "model",
"labels": ["get-started", "dataset-registry"],
"meta": {"key": "value", "key1": "value1"},
},
{
"path": "model.pkl",
"desc": "desc",
"type": "model",
"labels": ["get-started", "example"],
},
{"path": "unlabeled.txt"},
]
@pytest.mark.parametrize(
"markdown, expected_output",
[
(
True,
"""\
| Path | Type | Labels | Description |
|-----------|--------|------------------------------|---------------|
| model.pkl | model | get-started,dataset-registry | desc |
| model.pkl | model | get-started,example | desc |\n""",
),
(
False,
"""\
Path Type Labels Description
model.pkl model get-started,dataset-registry desc
model.pkl model get-started,example desc""",
),
],
ids=["markdown", "default"],
)
def test_ls(capsys, markdown, expected_output):
CmdDataLs._show_table(
EXAMPLE_DATA,
filter_types={"model"},
filter_labels={"get-started", "dataset-registry"},
markdown=markdown,
)
out, err = capsys.readouterr()
assert not err
out = "\n".join(line.rstrip() for line in out.splitlines())
assert out == expected_output
tests/unit/command/test_imp.py
-18
View File
@@ -14,8 +14,6 @@ def test_import(mocker):
"file",
"--rev",
"version",
"--desc",
"description",
"--jobs",
"3",
]
@@ -35,10 +33,6 @@ def test_import(mocker):
rev="version",
no_exec=False,
no_download=False,
desc="description",
type=None,
labels=None,
meta=None,
jobs=3,
)
@@ -56,8 +50,6 @@ def test_import_no_exec(mocker):
"--rev",
"version",
"--no-exec",
"--desc",
"description",
]
)
@@ -74,10 +66,6 @@ def test_import_no_exec(mocker):
rev="version",
no_exec=True,
no_download=False,
desc="description",
type=None,
labels=None,
meta=None,
jobs=None,
)
@@ -95,8 +83,6 @@ def test_import_no_download(mocker):
"--rev",
"version",
"--no-download",
"--desc",
"description",
]
)
@@ -113,9 +99,5 @@ def test_import_no_download(mocker):
rev="version",
no_exec=False,
no_download=True,
desc="description",
type=None,
labels=None,
meta=None,
jobs=None,
)
tests/unit/command/test_imp_url.py
-18
View File
@@ -17,8 +17,6 @@ def test_import_url(mocker):
"file",
"--jobs",
"4",
"--desc",
"description",
]
)
assert cli_args.func == CmdImportUrl
@@ -36,10 +34,6 @@ def test_import_url(mocker):
no_download=False,
remote=None,
to_remote=False,
desc="description",
type=None,
labels=None,
meta=None,
jobs=4,
force=False,
version_aware=False,
@@ -78,8 +72,6 @@ def test_import_url_no_exec_download_flags(mocker, flag, expected):
"out",
"--file",
"file",
"--desc",
"description",
]
)
@@ -94,10 +86,6 @@ def test_import_url_no_exec_download_flags(mocker, flag, expected):
fname="file",
remote=None,
to_remote=False,
desc="description",
type=None,
labels=None,
meta=None,
jobs=None,
force=False,
version_aware=False,
@@ -114,8 +102,6 @@ def test_import_url_to_remote(mocker):
"--to-remote",
"--remote",
"remote",
"--desc",
"description",
]
)
assert cli_args.func == CmdImportUrl
@@ -133,10 +119,6 @@ def test_import_url_to_remote(mocker):
no_download=False,
remote="remote",
to_remote=True,
desc="description",
type=None,
labels=None,
meta=None,
jobs=None,
force=False,
version_aware=False,
tests/unit/output/test_annotations.py
-12
View File
@@ -13,15 +13,3 @@ from dvc.annotations import Annotation
def test_annotation_to_dict(kwargs):
annot = Annotation(**kwargs)
assert annot.to_dict() == kwargs
def test_annotation_update():
annot = Annotation(desc="desc", labels=["label1", "label2"])
annot.update(labels=["label"], type="type", new="new", meta={"key": "value"})
assert vars(annot) == {
"desc": "desc",
"type": "type",
"labels": ["label"],
"meta": {"key": "value"},
}