metadata: support custom metadata in .dvc files

This adds support for custom metadata in outputs under `meta` key.
Also adds `--meta` flag to `add`/`import`/`import-url` that supports
`key=value` values.

dvc/annotations.py

@@ -1,5 +1,5 @@
 from dataclasses import asdict, dataclass, field, fields
-from typing import ClassVar, Dict, List, Optional
+from typing import Any, ClassVar, Dict, List, Optional
 
 from funcy import compact
 
@@ -9,10 +9,12 @@ class Annotation:
     PARAM_DESC: ClassVar[str] = "desc"
     PARAM_TYPE: ClassVar[str] = "type"
     PARAM_LABELS: ClassVar[str] = "labels"
+    PARAM_META: ClassVar[str] = "meta"
 
     desc: Optional[str] = None
     type: Optional[str] = None
     labels: List[str] = field(default_factory=list)
+    meta: Dict[str, Any] = field(default_factory=dict)
 
     def update(self, **kwargs) -> "Annotation":
         for attr, value in kwargs.items():
@@ -29,4 +31,5 @@ ANNOTATION_SCHEMA = {
     Annotation.PARAM_DESC: str,
     Annotation.PARAM_TYPE: str,
     Annotation.PARAM_LABELS: [str],
+    Annotation.PARAM_META: object,
 }

dvc/cli/actions.py

@@ -0,0 +1,11 @@
+from argparse import _AppendAction
+
+
+class KeyValueArgs(_AppendAction):
+    def __call__(self, parser, namespace, values, option_string=None):
+        items = getattr(namespace, self.dest) or {}
+        for value in filter(bool, values):
+            key, _, value = value.partition("=")
+            items[key.strip()] = value
+
+        setattr(namespace, self.dest, items)

dvc/commands/add.py

@@ -2,6 +2,7 @@ import argparse
 import logging
 
 from dvc.cli import completion
+from dvc.cli.actions import KeyValueArgs
 from dvc.cli.command import CmdBase
 from dvc.cli.utils import append_doc_link
 
@@ -18,6 +19,13 @@ def _add_annotating_args(parser: argparse.ArgumentParser) -> None:
             "This doesn't affect any DVC operations."
         ),
     )
+    parser.add_argument(
+        "--meta",
+        metavar="key=value",
+        nargs=1,
+        action=KeyValueArgs,
+        help="Custom metadata to add to the data",
+    )
     parser.add_argument(
         "--label",
         dest="labels",
@@ -56,6 +64,7 @@ class CmdAdd(CmdBase):
                 out=self.args.out,
                 type=self.args.type,
                 labels=self.args.labels,
+                meta=self.args.meta,
                 remote=self.args.remote,
                 to_remote=self.args.to_remote,
                 jobs=self.args.jobs,

dvc/commands/imp.py

@@ -23,6 +23,7 @@ class CmdImport(CmdBase):
                 desc=self.args.desc,
                 type=self.args.type,
                 labels=self.args.labels,
+                meta=self.args.meta,
                 jobs=self.args.jobs,
             )
         except DvcException:

dvc/commands/imp_url.py

@@ -23,6 +23,7 @@ class CmdImportUrl(CmdBase):
                 desc=self.args.desc,
                 type=self.args.type,
                 labels=self.args.labels,
+                meta=self.args.meta,
                 jobs=self.args.jobs,
             )
         except DvcException:

dvc/output.py

@@ -287,10 +287,13 @@ class Output:
         desc=None,
         type=None,  # pylint: disable=redefined-builtin
         labels=None,
+        meta=None,
         remote=None,
         repo=None,
     ):
-        self.annot = Annotation(desc=desc, type=type, labels=labels or [])
+        self.annot = Annotation(
+            desc=desc, type=type, labels=labels or [], meta=meta or {}
+        )
         self.repo = stage.repo if not repo and stage else repo
         meta = Meta.from_dict(info or {})
         # NOTE: when version_aware is not passed into get_cloud_fs, it will be

dvc/repo/imp_url.py

@@ -24,6 +24,7 @@ def imp_url(
     desc=None,
     type=None,  # pylint: disable=redefined-builtin
     labels=None,
+    meta=None,
     jobs=None,
 ):
     from dvc.dvcfile import Dvcfile
@@ -64,7 +65,7 @@ def imp_url(
     restore_fields(stage)
 
     out_obj = stage.outs[0]
-    out_obj.annot.update(desc=desc, type=type, labels=labels)
+    out_obj.annot.update(desc=desc, type=type, labels=labels, meta=meta)
     dvcfile = Dvcfile(self, stage.path)
     dvcfile.remove()
 

tests/func/test_add.py

@@ -1175,7 +1175,12 @@ def test_add_ignore_duplicated_targets(tmp_dir, dvc, capsys):
 def test_add_with_annotations(M, tmp_dir, dvc):
     tmp_dir.gen("foo", "foo")
 
-    annot = {"desc": "foo desc", "labels": ["l1", "l2"], "type": "t1"}
+    annot = {
+        "desc": "foo desc",
+        "labels": ["l1", "l2"],
+        "type": "t1",
+        "meta": {"key": "value"},
+    }
     (stage,) = dvc.add("foo", **annot)
     assert stage.outs[0].annot == Annotation(**annot)
     assert (tmp_dir / "foo.dvc").parse() == M.dict(outs=[M.dict(**annot)])

tests/func/test_commit.py

@@ -51,6 +51,9 @@ def test_commit_preserve_fields(tmp_dir, dvc):
           labels:
           - label1
           - label2
+          meta:
+            key1: value1
+            key2: value2
           remote: testremote
         meta: some metadata
     """
@@ -69,6 +72,9 @@ def test_commit_preserve_fields(tmp_dir, dvc):
           labels:
           - label1
           - label2
+          meta:
+            key1: value1
+            key2: value2
           remote: testremote
           md5: acbd18db4cc2f85cedef654fccc4a4d8
           size: 3

tests/func/test_import.py

@@ -634,7 +634,12 @@ def test_import_with_annotations(M, tmp_dir, scm, dvc, erepo_dir):
     with erepo_dir.chdir():
         erepo_dir.dvc_gen("foo", "foo content", commit="create foo")
 
-    annot = {"desc": "foo desc", "labels": ["l1", "l2"], "type": "t1"}
+    annot = {
+        "desc": "foo desc",
+        "labels": ["l1", "l2"],
+        "type": "t1",
+        "meta": {"key": "value"},
+    }
     stage = dvc.imp(os.fspath(erepo_dir), "foo", "foo", no_exec=True, **annot)
     assert stage.outs[0].annot == Annotation(**annot)
     assert (tmp_dir / "foo.dvc").parse() == M.dict(outs=[M.dict(**annot)])

tests/func/test_import_url.py

@@ -147,6 +147,8 @@ def test_import_url_preserve_fields(tmp_dir, dvc):
           labels:
           - label1
           - label2
+          meta:
+            key: value
         meta: some metadata
     """
     )
@@ -169,6 +171,8 @@ def test_import_url_preserve_fields(tmp_dir, dvc):
           labels:
           - label1
           - label2
+          meta:
+            key: value
           md5: acbd18db4cc2f85cedef654fccc4a4d8
           size: 3
         meta: some metadata
@@ -301,7 +305,12 @@ def test_import_url_no_download(tmp_dir, dvc, local_workspace):
 
 def test_imp_url_with_annotations(M, tmp_dir, dvc, local_workspace):
     local_workspace.gen("foo", "foo")
-    annot = {"desc": "foo desc", "labels": ["l1", "l2"], "type": "t1"}
+    annot = {
+        "desc": "foo desc",
+        "labels": ["l1", "l2"],
+        "type": "t1",
+        "meta": {"key": "value"},
+    }
     stage = dvc.imp_url(
         "remote://workspace/foo",
         os.fspath(tmp_dir / "foo"),

tests/func/test_run_single_stage.py

@@ -976,6 +976,8 @@ def test_run_force_preserves_comments_and_meta(tmp_dir, dvc, run_copy):
         labels:
         - label1
         - label2
+        meta:
+          key: value
       meta:
         name: copy-foo-bar
     """
@@ -1004,6 +1006,8 @@ def test_run_force_preserves_comments_and_meta(tmp_dir, dvc, run_copy):
           labels:
           - label1
           - label2
+          meta:
+            key: value
           md5: acbd18db4cc2f85cedef654fccc4a4d8
           size: 3
         meta:

tests/func/test_stage.py

@@ -179,6 +179,7 @@ def test_md5_ignores_annotations(tmp_dir, dvc):
                 "desc": "foo desc",
                 "type": "mytype",
                 "labels": ["get-started", "dataset-registry"],
+                "meta": {"key1": "value1"},
             }
         ],
     }
@@ -209,6 +210,7 @@ def test_meta_desc_is_preserved(tmp_dir, dvc):
                 "desc": "foo desc",
                 "type": "mytype",
                 "labels": ["get-started", "dataset-registry"],
+                "meta": {"key": "value"},
             }
         ],
     }
@@ -221,6 +223,7 @@ def test_meta_desc_is_preserved(tmp_dir, dvc):
         desc="foo desc",
         type="mytype",
         labels=["get-started", "dataset-registry"],
+        meta={"key": "value"},
     )
 
     # sanity check

tests/unit/command/test_add.py

@@ -39,6 +39,7 @@ def test_add(mocker, dvc):
         desc="stage description",
         type=None,
         labels=None,
+        meta=None,
         jobs=None,
     )
 
@@ -75,6 +76,7 @@ def test_add_to_remote(mocker):
         desc=None,
         type=None,
         labels=None,
+        meta=None,
         jobs=None,
     )
 

tests/unit/command/test_annotations_flags.py

@@ -26,6 +26,10 @@ def test_add_annotations(mocker, func, args):
             "model",
             "--desc",
             "description",
+            "--meta",
+            "key1=value1",
+            "--meta",
+            "key2=value2",
         ]
     )
 
@@ -39,4 +43,5 @@ def test_add_annotations(mocker, func, args):
         desc="description",
         type="model",
         labels=["example-get-started", "model-registry"],
+        meta={"key1": "value1", "key2": "value2"},
     )

tests/unit/command/test_imp.py

@@ -38,6 +38,7 @@ def test_import(mocker):
         desc="description",
         type=None,
         labels=None,
+        meta=None,
         jobs=3,
     )
 
@@ -76,6 +77,7 @@ def test_import_no_exec(mocker):
         desc="description",
         type=None,
         labels=None,
+        meta=None,
         jobs=None,
     )
 
@@ -114,5 +116,6 @@ def test_import_no_download(mocker):
         desc="description",
         type=None,
         labels=None,
+        meta=None,
         jobs=None,
     )

tests/unit/command/test_imp_url.py

@@ -39,6 +39,7 @@ def test_import_url(mocker):
         desc="description",
         type=None,
         labels=None,
+        meta=None,
         jobs=4,
     )
 
@@ -96,6 +97,7 @@ def test_import_url_no_exec_download_flags(mocker, flag, expected):
         jobs=None,
         type=None,
         labels=None,
+        meta=None,
         **expected
     )
 
@@ -131,6 +133,7 @@ def test_import_url_to_remote(mocker):
         desc="description",
         type=None,
         labels=None,
+        meta=None,
         jobs=None,
     )
 

tests/unit/output/test_annotations.py

@@ -7,7 +7,7 @@ from dvc.annotations import Annotation
     "kwargs",
     [
         {"desc": "desc", "type": "type", "labels": ["label1", "label2"]},
-        {"desc": "desc", "type": "type"},
+        {"desc": "desc", "type": "type", "meta": {"key": "value"}},
     ],
 )
 def test_annotation_to_dict(kwargs):
@@ -17,6 +17,13 @@ def test_annotation_to_dict(kwargs):
 
 def test_annotation_update():
     annot = Annotation(desc="desc", labels=["label1", "label2"])
-    annot.update(labels=["label"], type="type", new="new")
+    annot.update(
+        labels=["label"], type="type", new="new", meta={"key": "value"}
+    )
 
-    assert vars(annot) == {"desc": "desc", "type": "type", "labels": ["label"]}
+    assert vars(annot) == {
+        "desc": "desc",
+        "type": "type",
+        "labels": ["label"],
+        "meta": {"key": "value"},
+    }