From 251a3269c2c5f7e39938274b06fca319de1bf79b Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Wed, 18 Nov 2020 22:06:52 +0200 Subject: [PATCH] dvcfile: add desc field for outs/stages (#4901) Outs desc will be used by viewer right away. Stage desc is for people to start using now, but we won't be using it it viewer until DAG support is added there. --- dvc/command/add.py | 10 +++++++++ dvc/command/imp.py | 10 +++++++++ dvc/command/imp_url.py | 10 +++++++++ dvc/command/run.py | 10 +++++++++ dvc/dependency/__init__.py | 1 + dvc/output/__init__.py | 7 ++++++ dvc/output/base.py | 6 +++++ dvc/repo/add.py | 19 +++++----------- dvc/repo/imp.py | 4 ++-- dvc/repo/imp_url.py | 12 +++++++++- dvc/schema.py | 3 +++ dvc/stage/__init__.py | 3 +++ dvc/stage/loader.py | 1 + dvc/stage/params.py | 1 + dvc/stage/serialize.py | 4 ++++ dvc/stage/utils.py | 1 + tests/func/test_dvcfile.py | 21 ++++++++++++++++++ tests/func/test_stage.py | 19 ++++++++++++++++ tests/unit/command/test_add.py | 35 ++++++++++++++++++++++++++++++ tests/unit/command/test_imp.py | 6 +++++ tests/unit/command/test_imp_url.py | 23 ++++++++++++++++---- tests/unit/command/test_run.py | 5 +++++ 22 files changed, 190 insertions(+), 21 deletions(-) create mode 100644 tests/unit/command/test_add.py diff --git a/dvc/command/add.py b/dvc/command/add.py index fb70ef279..92d286599 100644 --- a/dvc/command/add.py +++ b/dvc/command/add.py @@ -21,6 +21,7 @@ class CmdAdd(CmdBase): fname=self.args.file, external=self.args.external, glob=self.args.glob, + desc=self.args.desc, ) except DvcException: @@ -69,6 +70,15 @@ def add_parser(subparsers, parent_parser): help="Specify name of the DVC-file this command will generate.", metavar="", ) + parser.add_argument( + "--desc", + type=str, + metavar="", + help=( + "User description of the data (optional). " + "This doesn't affect any DVC operations." + ), + ) parser.add_argument( "targets", nargs="+", help="Input files/directories to add.", ).complete = completion.FILE diff --git a/dvc/command/imp.py b/dvc/command/imp.py index c5c371d85..9528282a0 100644 --- a/dvc/command/imp.py +++ b/dvc/command/imp.py @@ -18,6 +18,7 @@ class CmdImport(CmdBase): fname=self.args.file, rev=self.args.rev, no_exec=self.args.no_exec, + desc=self.args.desc, ) except DvcException: logger.exception( @@ -72,4 +73,13 @@ def add_parser(subparsers, parent_parser): default=False, help="Only create DVC-file without actually downloading it.", ) + import_parser.add_argument( + "--desc", + type=str, + metavar="", + help=( + "User description of the data (optional). " + "This doesn't affect any DVC operations." + ), + ) import_parser.set_defaults(func=CmdImport) diff --git a/dvc/command/imp_url.py b/dvc/command/imp_url.py index eb19896f7..a5b5767f7 100644 --- a/dvc/command/imp_url.py +++ b/dvc/command/imp_url.py @@ -16,6 +16,7 @@ class CmdImportUrl(CmdBase): out=self.args.out, fname=self.args.file, no_exec=self.args.no_exec, + desc=self.args.desc, ) except DvcException: logger.exception( @@ -67,4 +68,13 @@ def add_parser(subparsers, parent_parser): default=False, help="Only create DVC-file without actually downloading it.", ) + import_parser.add_argument( + "--desc", + type=str, + metavar="", + help=( + "User description of the data (optional). " + "This doesn't affect any DVC operations." + ), + ) import_parser.set_defaults(func=CmdImportUrl) diff --git a/dvc/command/run.py b/dvc/command/run.py index f5be09dba..211cc1de7 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -57,6 +57,7 @@ class CmdRun(CmdBase): name=self.args.name, single_stage=self.args.single_stage, external=self.args.external, + desc=self.args.desc, ) except DvcException: logger.exception("") @@ -240,6 +241,15 @@ def add_parser(subparsers, parent_parser): default=False, help="Allow outputs that are outside of the DVC repository.", ) + run_parser.add_argument( + "--desc", + type=str, + metavar="", + help=( + "User description of the stage (optional). " + "This doesn't affect any DVC operations." + ), + ) run_parser.add_argument( "command", nargs=argparse.REMAINDER, help="Command to execute." ) diff --git a/dvc/dependency/__init__.py b/dvc/dependency/__init__.py index 22db00bae..681fe9a48 100644 --- a/dvc/dependency/__init__.py +++ b/dvc/dependency/__init__.py @@ -56,6 +56,7 @@ DEP_MAP = { SCHEMA = output.SCHEMA.copy() del SCHEMA[BaseOutput.PARAM_CACHE] del SCHEMA[BaseOutput.PARAM_METRIC] +del SCHEMA[BaseOutput.PARAM_DESC] SCHEMA.update(RepoDependency.REPO_SCHEMA) SCHEMA.update(ParamsDependency.PARAM_SCHEMA) diff --git a/dvc/output/__init__.py b/dvc/output/__init__.py index 9b230015d..c3bf75153 100644 --- a/dvc/output/__init__.py +++ b/dvc/output/__init__.py @@ -68,6 +68,7 @@ SCHEMA[BaseOutput.PARAM_PERSIST] = bool SCHEMA[BaseOutput.PARAM_CHECKPOINT] = bool SCHEMA[HashInfo.PARAM_SIZE] = int SCHEMA[HashInfo.PARAM_NFILES] = int +SCHEMA[BaseOutput.PARAM_DESC] = str def _get( @@ -79,6 +80,7 @@ def _get( plot=False, persist=False, checkpoint=False, + desc=None, ): parsed = urlparse(p) @@ -94,6 +96,7 @@ def _get( plot=plot, persist=persist, checkpoint=checkpoint, + desc=desc, ) for o in OUTS: @@ -108,6 +111,7 @@ def _get( plot=plot, persist=persist, checkpoint=checkpoint, + desc=desc, ) return LocalOutput( stage, @@ -119,6 +123,7 @@ def _get( plot=plot, persist=persist, checkpoint=checkpoint, + desc=desc, ) @@ -131,6 +136,7 @@ def loadd_from(stage, d_list): plot = d.pop(BaseOutput.PARAM_PLOT, False) persist = d.pop(BaseOutput.PARAM_PERSIST, False) checkpoint = d.pop(BaseOutput.PARAM_CHECKPOINT, False) + desc = d.pop(BaseOutput.PARAM_DESC, False) ret.append( _get( stage, @@ -141,6 +147,7 @@ def loadd_from(stage, d_list): plot=plot, persist=persist, checkpoint=checkpoint, + desc=desc, ) ) return ret diff --git a/dvc/output/base.py b/dvc/output/base.py index 7855397d0..a095973c0 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -74,6 +74,7 @@ class BaseOutput: PARAM_PLOT_TITLE = "title" PARAM_PLOT_HEADER = "header" PARAM_PERSIST = "persist" + PARAM_DESC = "desc" METRIC_SCHEMA = Any( None, @@ -102,6 +103,7 @@ class BaseOutput: plot=False, persist=False, checkpoint=False, + desc=None, ): self._validate_output_path(path, stage) # This output (and dependency) objects have too many paths/urls @@ -127,6 +129,7 @@ class BaseOutput: self.plot = False if self.IS_DEPENDENCY else plot self.persist = persist self.checkpoint = checkpoint + self.desc = desc self.path_info = self._parse_path(tree, path) if self.use_cache and self.cache is None: @@ -299,6 +302,9 @@ class BaseOutput: if self.IS_DEPENDENCY: return ret + if self.desc: + ret[self.PARAM_DESC] = self.desc + if not self.use_cache: ret[self.PARAM_CACHE] = self.use_cache diff --git a/dvc/repo/add.py b/dvc/repo/add.py index b78a81a7a..2c009bb90 100644 --- a/dvc/repo/add.py +++ b/dvc/repo/add.py @@ -23,13 +23,7 @@ logger = logging.getLogger(__name__) @locked @scm_context def add( - repo, - targets, - recursive=False, - no_commit=False, - fname=None, - external=False, - glob=False, + repo, targets, recursive=False, no_commit=False, fname=None, **kwargs, ): if recursive and fname: raise RecursiveAddingWhileUsingFilename() @@ -63,12 +57,7 @@ def add( ) stages = _create_stages( - repo, - sub_targets, - fname, - pbar=pbar, - external=external, - glob=glob, + repo, sub_targets, fname, pbar=pbar, **kwargs, ) try: @@ -161,7 +150,7 @@ def _find_all_targets(repo, target, recursive): def _create_stages( - repo, targets, fname, pbar=None, external=False, glob=False + repo, targets, fname, pbar=None, external=False, glob=False, desc=None, ): from glob import iglob @@ -194,6 +183,8 @@ def _create_stages( ) if stage: Dvcfile(repo, stage.path).remove() + if desc: + stage.outs[0].desc = desc repo._reset() # pylint: disable=protected-access diff --git a/dvc/repo/imp.py b/dvc/repo/imp.py index 2c10b57c3..5fc572543 100644 --- a/dvc/repo/imp.py +++ b/dvc/repo/imp.py @@ -1,8 +1,8 @@ -def imp(self, url, path, out=None, fname=None, rev=None, no_exec=False): +def imp(self, url, path, out=None, fname=None, rev=None, **kwargs): erepo = {"url": url} if rev is not None: erepo["rev"] = rev return self.imp_url( - path, out=out, fname=fname, erepo=erepo, frozen=True, no_exec=no_exec + path, out=out, fname=fname, erepo=erepo, frozen=True, **kwargs ) diff --git a/dvc/repo/imp_url.py b/dvc/repo/imp_url.py index c20e57e65..974cd8b83 100644 --- a/dvc/repo/imp_url.py +++ b/dvc/repo/imp_url.py @@ -11,7 +11,14 @@ from . import locked @locked @scm_context def imp_url( - self, url, out=None, fname=None, erepo=None, frozen=True, no_exec=False + self, + url, + out=None, + fname=None, + erepo=None, + frozen=True, + no_exec=False, + desc=None, ): from dvc.dvcfile import Dvcfile from dvc.stage import Stage, create_stage @@ -40,6 +47,9 @@ def imp_url( if stage is None: return None + if desc: + stage.outs[0].desc = desc + dvcfile = Dvcfile(self, stage.path) dvcfile.remove() diff --git a/dvc/schema.py b/dvc/schema.py index b58d30dcd..9dfe918e2 100644 --- a/dvc/schema.py +++ b/dvc/schema.py @@ -17,6 +17,7 @@ SINGLE_STAGE_SCHEMA = { StageParams.PARAM_FROZEN: bool, StageParams.PARAM_META: object, StageParams.PARAM_ALWAYS_CHANGED: bool, + StageParams.PARAM_DESC: str, } DATA_SCHEMA = { @@ -38,6 +39,7 @@ OUT_PSTAGE_DETAILED_SCHEMA = { BaseOutput.PARAM_CACHE: bool, BaseOutput.PARAM_PERSIST: bool, BaseOutput.PARAM_CHECKPOINT: bool, + BaseOutput.PARAM_DESC: str, } } @@ -68,6 +70,7 @@ STAGE_DEFINITION = { ], Optional(StageParams.PARAM_FROZEN): bool, Optional(StageParams.PARAM_META): object, + Optional(StageParams.PARAM_DESC): str, Optional(StageParams.PARAM_ALWAYS_CHANGED): bool, Optional(StageParams.PARAM_OUTS): [Any(str, OUT_PSTAGE_DETAILED_SCHEMA)], Optional(StageParams.PARAM_METRICS): [ diff --git a/dvc/stage/__init__.py b/dvc/stage/__init__.py index abbdb6133..ed040c4cd 100644 --- a/dvc/stage/__init__.py +++ b/dvc/stage/__init__.py @@ -52,6 +52,7 @@ def loads_from(cls, repo, path, wdir, data): Stage.PARAM_FROZEN, Stage.PARAM_ALWAYS_CHANGED, Stage.PARAM_MD5, + Stage.PARAM_DESC, "name", ], ), @@ -111,6 +112,7 @@ class Stage(params.StageParams): always_changed=False, stage_text=None, dvcfile=None, + desc=None, ): if deps is None: deps = [] @@ -128,6 +130,7 @@ class Stage(params.StageParams): self.always_changed = always_changed self._stage_text = stage_text self._dvcfile = dvcfile + self.desc = desc @property def path(self): diff --git a/dvc/stage/loader.py b/dvc/stage/loader.py index 947737c75..5f952aa93 100644 --- a/dvc/stage/loader.py +++ b/dvc/stage/loader.py @@ -58,6 +58,7 @@ class StageLoader(Mapping): ) stage = loads_from(PipelineStage, dvcfile.repo, path, wdir, stage_data) stage.name = name + stage.desc = stage_data.get(Stage.PARAM_DESC) deps = project(stage_data, [stage.PARAM_DEPS, stage.PARAM_PARAMS]) fill_stage_dependencies(stage, **deps) diff --git a/dvc/stage/params.py b/dvc/stage/params.py index 30cc7b8be..14ca18b25 100644 --- a/dvc/stage/params.py +++ b/dvc/stage/params.py @@ -11,3 +11,4 @@ class StageParams: PARAM_PARAMS = "params" PARAM_METRICS = "metrics" PARAM_PLOTS = "plots" + PARAM_DESC = "desc" diff --git a/dvc/stage/serialize.py b/dvc/stage/serialize.py index fef9edb30..1c6ffe8e0 100644 --- a/dvc/stage/serialize.py +++ b/dvc/stage/serialize.py @@ -27,6 +27,7 @@ PARAM_METRIC = BaseOutput.PARAM_METRIC PARAM_PLOT = BaseOutput.PARAM_PLOT PARAM_PERSIST = BaseOutput.PARAM_PERSIST PARAM_CHECKPOINT = BaseOutput.PARAM_CHECKPOINT +PARAM_DESC = BaseOutput.PARAM_DESC DEFAULT_PARAMS_FILE = ParamsDependency.DEFAULT_PARAMS_FILE @@ -36,6 +37,8 @@ sort_by_path = partial(sorted, key=attrgetter("def_path")) @post_processing(OrderedDict) def _get_flags(out): + if out.desc: + yield PARAM_DESC, out.desc if not out.use_cache: yield PARAM_CACHE, False if out.checkpoint: @@ -119,6 +122,7 @@ def to_pipeline_file(stage: "PipelineStage"): outs, metrics, plots = _serialize_outs(stage.outs) res = [ + (stage.PARAM_DESC, stage.desc), (stage.PARAM_CMD, stage.cmd), (stage.PARAM_WDIR, wdir), (stage.PARAM_DEPS, deps), diff --git a/dvc/stage/utils.py b/dvc/stage/utils.py index 134305293..d743adcd6 100644 --- a/dvc/stage/utils.py +++ b/dvc/stage/utils.py @@ -199,6 +199,7 @@ def get_dump(stage): return { key: value for key, value in { + stage.PARAM_DESC: stage.desc, stage.PARAM_MD5: stage.md5, stage.PARAM_CMD: stage.cmd, stage.PARAM_WDIR: resolve_wdir(stage.wdir, stage.path), diff --git a/tests/func/test_dvcfile.py b/tests/func/test_dvcfile.py index cccbc825b..82b708b2b 100644 --- a/tests/func/test_dvcfile.py +++ b/tests/func/test_dvcfile.py @@ -317,6 +317,27 @@ def test_dvcfile_dump_preserves_meta(tmp_dir, dvc, run_copy): assert dvcfile._load()[0]["stages"]["run_copy"]["meta"] == metadata +def test_dvcfile_dump_preserves_desc(tmp_dir, dvc, run_copy): + tmp_dir.gen("foo", "foo") + stage_desc = "test stage description" + out_desc = "test out description" + + stage = run_copy("foo", "bar", name="run_copy", desc=stage_desc) + dvcfile = stage.dvcfile + + data = dvcfile._load()[0] + data["stages"]["run_copy"]["outs"][0] = {"bar": {"desc": out_desc}} + dump_yaml(dvcfile.path, data) + + assert stage.desc == stage_desc + stage.outs[0].desc = out_desc + dvcfile.dump(stage) + loaded = dvcfile._load()[0] + assert loaded == data + assert loaded["stages"]["run_copy"]["desc"] == stage_desc + assert loaded["stages"]["run_copy"]["outs"][0]["bar"]["desc"] == out_desc + + def test_dvcfile_dump_preserves_comments(tmp_dir, dvc): text = textwrap.dedent( """\ diff --git a/tests/func/test_stage.py b/tests/func/test_stage.py index 362c5aa7a..df17bf440 100644 --- a/tests/func/test_stage.py +++ b/tests/func/test_stage.py @@ -181,6 +181,25 @@ def test_meta_is_preserved(tmp_dir, dvc): assert new_data["meta"] == data["meta"] +def test_desc_is_preserved(tmp_dir, dvc): + (stage,) = tmp_dir.dvc_gen("foo", "foo content") + + data = load_yaml(stage.path) + stage_desc = "test stage description" + out_desc = "test out description" + data["desc"] = stage_desc + data["outs"][0]["desc"] = out_desc + dump_yaml(stage.path, data) + + dvcfile = SingleStageFile(dvc, stage.path) + new_stage = dvcfile.stage + dvcfile.dump(new_stage) + + new_data = load_yaml(stage.path) + assert new_data["desc"] == stage_desc + assert new_data["outs"][0]["desc"] == out_desc + + def test_parent_repo_collect_stages(tmp_dir, scm, dvc): tmp_dir.gen({"subdir": {}}) tmp_dir.gen({"deep": {"dir": {}}}) diff --git a/tests/unit/command/test_add.py b/tests/unit/command/test_add.py new file mode 100644 index 000000000..22b57c6cb --- /dev/null +++ b/tests/unit/command/test_add.py @@ -0,0 +1,35 @@ +from dvc.cli import parse_args +from dvc.command.add import CmdAdd + + +def test_add(mocker, dvc): + cli_args = parse_args( + [ + "add", + "--recursive", + "--no-commit", + "--external", + "--glob", + "--file", + "file", + "--desc", + "stage description", + "data", + ] + ) + assert cli_args.func == CmdAdd + + cmd = cli_args.func(cli_args) + m = mocker.patch.object(cmd.repo, "add", autospec=True) + + assert cmd.run() == 0 + + m.assert_called_once_with( + ["data"], + recursive=True, + no_commit=True, + glob=True, + fname="file", + external=True, + desc="stage description", + ) diff --git a/tests/unit/command/test_imp.py b/tests/unit/command/test_imp.py index b6b5a924b..ff04e6503 100644 --- a/tests/unit/command/test_imp.py +++ b/tests/unit/command/test_imp.py @@ -14,6 +14,8 @@ def test_import(mocker): "file", "--rev", "version", + "--desc", + "description", ] ) assert cli_args.func == CmdImport @@ -30,6 +32,7 @@ def test_import(mocker): fname="file", rev="version", no_exec=False, + desc="description", ) @@ -46,6 +49,8 @@ def test_import_no_exec(mocker): "--rev", "version", "--no-exec", + "--desc", + "description", ] ) @@ -61,4 +66,5 @@ def test_import_no_exec(mocker): fname="file", rev="version", no_exec=True, + desc="description", ) diff --git a/tests/unit/command/test_imp_url.py b/tests/unit/command/test_imp_url.py index 6b5b5f784..3eb6dcc0d 100644 --- a/tests/unit/command/test_imp_url.py +++ b/tests/unit/command/test_imp_url.py @@ -6,7 +6,9 @@ from dvc.exceptions import DvcException def test_import_url(mocker): - cli_args = parse_args(["import-url", "src", "out", "--file", "file"]) + cli_args = parse_args( + ["import-url", "src", "out", "--file", "file", "--desc", "description"] + ) assert cli_args.func == CmdImportUrl cmd = cli_args.func(cli_args) @@ -14,7 +16,9 @@ def test_import_url(mocker): assert cmd.run() == 0 - m.assert_called_once_with("src", out="out", fname="file", no_exec=False) + m.assert_called_once_with( + "src", out="out", fname="file", no_exec=False, desc="description" + ) def test_failed_import_url(mocker, caplog): @@ -35,7 +39,16 @@ def test_failed_import_url(mocker, caplog): def test_import_url_no_exec(mocker): cli_args = parse_args( - ["import-url", "--no-exec", "src", "out", "--file", "file"] + [ + "import-url", + "--no-exec", + "src", + "out", + "--file", + "file", + "--desc", + "description", + ] ) cmd = cli_args.func(cli_args) @@ -43,4 +56,6 @@ def test_import_url_no_exec(mocker): assert cmd.run() == 0 - m.assert_called_once_with("src", out="out", fname="file", no_exec=True) + m.assert_called_once_with( + "src", out="out", fname="file", no_exec=True, desc="description" + ) diff --git a/tests/unit/command/test_run.py b/tests/unit/command/test_run.py index e0f82d1e3..6a2939099 100644 --- a/tests/unit/command/test_run.py +++ b/tests/unit/command/test_run.py @@ -42,6 +42,8 @@ def test_run(mocker, dvc): "--params", "param3", "--external", + "--desc", + "description", "command", ] ) @@ -75,6 +77,7 @@ def test_run(mocker, dvc): name="nam", single_stage=False, external=True, + desc="description", ) @@ -106,6 +109,7 @@ def test_run_args_from_cli(mocker, dvc): name=None, single_stage=False, external=False, + desc=None, ) @@ -137,4 +141,5 @@ def test_run_args_with_spaces(mocker, dvc): name=None, single_stage=False, external=False, + desc=None, )