diff --git a/dvc/command/add.py b/dvc/command/add.py index fb70ef279..92d286599 100644 --- a/dvc/command/add.py +++ b/dvc/command/add.py @@ -21,6 +21,7 @@ class CmdAdd(CmdBase): fname=self.args.file, external=self.args.external, glob=self.args.glob, + desc=self.args.desc, ) except DvcException: @@ -69,6 +70,15 @@ def add_parser(subparsers, parent_parser): help="Specify name of the DVC-file this command will generate.", metavar="", ) + parser.add_argument( + "--desc", + type=str, + metavar="", + help=( + "User description of the data (optional). " + "This doesn't affect any DVC operations." + ), + ) parser.add_argument( "targets", nargs="+", help="Input files/directories to add.", ).complete = completion.FILE diff --git a/dvc/command/imp.py b/dvc/command/imp.py index c5c371d85..9528282a0 100644 --- a/dvc/command/imp.py +++ b/dvc/command/imp.py @@ -18,6 +18,7 @@ class CmdImport(CmdBase): fname=self.args.file, rev=self.args.rev, no_exec=self.args.no_exec, + desc=self.args.desc, ) except DvcException: logger.exception( @@ -72,4 +73,13 @@ def add_parser(subparsers, parent_parser): default=False, help="Only create DVC-file without actually downloading it.", ) + import_parser.add_argument( + "--desc", + type=str, + metavar="", + help=( + "User description of the data (optional). " + "This doesn't affect any DVC operations." + ), + ) import_parser.set_defaults(func=CmdImport) diff --git a/dvc/command/imp_url.py b/dvc/command/imp_url.py index eb19896f7..a5b5767f7 100644 --- a/dvc/command/imp_url.py +++ b/dvc/command/imp_url.py @@ -16,6 +16,7 @@ class CmdImportUrl(CmdBase): out=self.args.out, fname=self.args.file, no_exec=self.args.no_exec, + desc=self.args.desc, ) except DvcException: logger.exception( @@ -67,4 +68,13 @@ def add_parser(subparsers, parent_parser): default=False, help="Only create DVC-file without actually downloading it.", ) + import_parser.add_argument( + "--desc", + type=str, + metavar="", + help=( + "User description of the data (optional). " + "This doesn't affect any DVC operations." + ), + ) import_parser.set_defaults(func=CmdImportUrl) diff --git a/dvc/command/run.py b/dvc/command/run.py index f5be09dba..211cc1de7 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -57,6 +57,7 @@ class CmdRun(CmdBase): name=self.args.name, single_stage=self.args.single_stage, external=self.args.external, + desc=self.args.desc, ) except DvcException: logger.exception("") @@ -240,6 +241,15 @@ def add_parser(subparsers, parent_parser): default=False, help="Allow outputs that are outside of the DVC repository.", ) + run_parser.add_argument( + "--desc", + type=str, + metavar="", + help=( + "User description of the stage (optional). " + "This doesn't affect any DVC operations." + ), + ) run_parser.add_argument( "command", nargs=argparse.REMAINDER, help="Command to execute." ) diff --git a/dvc/dependency/__init__.py b/dvc/dependency/__init__.py index 22db00bae..681fe9a48 100644 --- a/dvc/dependency/__init__.py +++ b/dvc/dependency/__init__.py @@ -56,6 +56,7 @@ DEP_MAP = { SCHEMA = output.SCHEMA.copy() del SCHEMA[BaseOutput.PARAM_CACHE] del SCHEMA[BaseOutput.PARAM_METRIC] +del SCHEMA[BaseOutput.PARAM_DESC] SCHEMA.update(RepoDependency.REPO_SCHEMA) SCHEMA.update(ParamsDependency.PARAM_SCHEMA) diff --git a/dvc/output/__init__.py b/dvc/output/__init__.py index 9b230015d..c3bf75153 100644 --- a/dvc/output/__init__.py +++ b/dvc/output/__init__.py @@ -68,6 +68,7 @@ SCHEMA[BaseOutput.PARAM_PERSIST] = bool SCHEMA[BaseOutput.PARAM_CHECKPOINT] = bool SCHEMA[HashInfo.PARAM_SIZE] = int SCHEMA[HashInfo.PARAM_NFILES] = int +SCHEMA[BaseOutput.PARAM_DESC] = str def _get( @@ -79,6 +80,7 @@ def _get( plot=False, persist=False, checkpoint=False, + desc=None, ): parsed = urlparse(p) @@ -94,6 +96,7 @@ def _get( plot=plot, persist=persist, checkpoint=checkpoint, + desc=desc, ) for o in OUTS: @@ -108,6 +111,7 @@ def _get( plot=plot, persist=persist, checkpoint=checkpoint, + desc=desc, ) return LocalOutput( stage, @@ -119,6 +123,7 @@ def _get( plot=plot, persist=persist, checkpoint=checkpoint, + desc=desc, ) @@ -131,6 +136,7 @@ def loadd_from(stage, d_list): plot = d.pop(BaseOutput.PARAM_PLOT, False) persist = d.pop(BaseOutput.PARAM_PERSIST, False) checkpoint = d.pop(BaseOutput.PARAM_CHECKPOINT, False) + desc = d.pop(BaseOutput.PARAM_DESC, False) ret.append( _get( stage, @@ -141,6 +147,7 @@ def loadd_from(stage, d_list): plot=plot, persist=persist, checkpoint=checkpoint, + desc=desc, ) ) return ret diff --git a/dvc/output/base.py b/dvc/output/base.py index 7855397d0..a095973c0 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -74,6 +74,7 @@ class BaseOutput: PARAM_PLOT_TITLE = "title" PARAM_PLOT_HEADER = "header" PARAM_PERSIST = "persist" + PARAM_DESC = "desc" METRIC_SCHEMA = Any( None, @@ -102,6 +103,7 @@ class BaseOutput: plot=False, persist=False, checkpoint=False, + desc=None, ): self._validate_output_path(path, stage) # This output (and dependency) objects have too many paths/urls @@ -127,6 +129,7 @@ class BaseOutput: self.plot = False if self.IS_DEPENDENCY else plot self.persist = persist self.checkpoint = checkpoint + self.desc = desc self.path_info = self._parse_path(tree, path) if self.use_cache and self.cache is None: @@ -299,6 +302,9 @@ class BaseOutput: if self.IS_DEPENDENCY: return ret + if self.desc: + ret[self.PARAM_DESC] = self.desc + if not self.use_cache: ret[self.PARAM_CACHE] = self.use_cache diff --git a/dvc/repo/add.py b/dvc/repo/add.py index b78a81a7a..2c009bb90 100644 --- a/dvc/repo/add.py +++ b/dvc/repo/add.py @@ -23,13 +23,7 @@ logger = logging.getLogger(__name__) @locked @scm_context def add( - repo, - targets, - recursive=False, - no_commit=False, - fname=None, - external=False, - glob=False, + repo, targets, recursive=False, no_commit=False, fname=None, **kwargs, ): if recursive and fname: raise RecursiveAddingWhileUsingFilename() @@ -63,12 +57,7 @@ def add( ) stages = _create_stages( - repo, - sub_targets, - fname, - pbar=pbar, - external=external, - glob=glob, + repo, sub_targets, fname, pbar=pbar, **kwargs, ) try: @@ -161,7 +150,7 @@ def _find_all_targets(repo, target, recursive): def _create_stages( - repo, targets, fname, pbar=None, external=False, glob=False + repo, targets, fname, pbar=None, external=False, glob=False, desc=None, ): from glob import iglob @@ -194,6 +183,8 @@ def _create_stages( ) if stage: Dvcfile(repo, stage.path).remove() + if desc: + stage.outs[0].desc = desc repo._reset() # pylint: disable=protected-access diff --git a/dvc/repo/imp.py b/dvc/repo/imp.py index 2c10b57c3..5fc572543 100644 --- a/dvc/repo/imp.py +++ b/dvc/repo/imp.py @@ -1,8 +1,8 @@ -def imp(self, url, path, out=None, fname=None, rev=None, no_exec=False): +def imp(self, url, path, out=None, fname=None, rev=None, **kwargs): erepo = {"url": url} if rev is not None: erepo["rev"] = rev return self.imp_url( - path, out=out, fname=fname, erepo=erepo, frozen=True, no_exec=no_exec + path, out=out, fname=fname, erepo=erepo, frozen=True, **kwargs ) diff --git a/dvc/repo/imp_url.py b/dvc/repo/imp_url.py index c20e57e65..974cd8b83 100644 --- a/dvc/repo/imp_url.py +++ b/dvc/repo/imp_url.py @@ -11,7 +11,14 @@ from . import locked @locked @scm_context def imp_url( - self, url, out=None, fname=None, erepo=None, frozen=True, no_exec=False + self, + url, + out=None, + fname=None, + erepo=None, + frozen=True, + no_exec=False, + desc=None, ): from dvc.dvcfile import Dvcfile from dvc.stage import Stage, create_stage @@ -40,6 +47,9 @@ def imp_url( if stage is None: return None + if desc: + stage.outs[0].desc = desc + dvcfile = Dvcfile(self, stage.path) dvcfile.remove() diff --git a/dvc/schema.py b/dvc/schema.py index b58d30dcd..9dfe918e2 100644 --- a/dvc/schema.py +++ b/dvc/schema.py @@ -17,6 +17,7 @@ SINGLE_STAGE_SCHEMA = { StageParams.PARAM_FROZEN: bool, StageParams.PARAM_META: object, StageParams.PARAM_ALWAYS_CHANGED: bool, + StageParams.PARAM_DESC: str, } DATA_SCHEMA = { @@ -38,6 +39,7 @@ OUT_PSTAGE_DETAILED_SCHEMA = { BaseOutput.PARAM_CACHE: bool, BaseOutput.PARAM_PERSIST: bool, BaseOutput.PARAM_CHECKPOINT: bool, + BaseOutput.PARAM_DESC: str, } } @@ -68,6 +70,7 @@ STAGE_DEFINITION = { ], Optional(StageParams.PARAM_FROZEN): bool, Optional(StageParams.PARAM_META): object, + Optional(StageParams.PARAM_DESC): str, Optional(StageParams.PARAM_ALWAYS_CHANGED): bool, Optional(StageParams.PARAM_OUTS): [Any(str, OUT_PSTAGE_DETAILED_SCHEMA)], Optional(StageParams.PARAM_METRICS): [ diff --git a/dvc/stage/__init__.py b/dvc/stage/__init__.py index abbdb6133..ed040c4cd 100644 --- a/dvc/stage/__init__.py +++ b/dvc/stage/__init__.py @@ -52,6 +52,7 @@ def loads_from(cls, repo, path, wdir, data): Stage.PARAM_FROZEN, Stage.PARAM_ALWAYS_CHANGED, Stage.PARAM_MD5, + Stage.PARAM_DESC, "name", ], ), @@ -111,6 +112,7 @@ class Stage(params.StageParams): always_changed=False, stage_text=None, dvcfile=None, + desc=None, ): if deps is None: deps = [] @@ -128,6 +130,7 @@ class Stage(params.StageParams): self.always_changed = always_changed self._stage_text = stage_text self._dvcfile = dvcfile + self.desc = desc @property def path(self): diff --git a/dvc/stage/loader.py b/dvc/stage/loader.py index 947737c75..5f952aa93 100644 --- a/dvc/stage/loader.py +++ b/dvc/stage/loader.py @@ -58,6 +58,7 @@ class StageLoader(Mapping): ) stage = loads_from(PipelineStage, dvcfile.repo, path, wdir, stage_data) stage.name = name + stage.desc = stage_data.get(Stage.PARAM_DESC) deps = project(stage_data, [stage.PARAM_DEPS, stage.PARAM_PARAMS]) fill_stage_dependencies(stage, **deps) diff --git a/dvc/stage/params.py b/dvc/stage/params.py index 30cc7b8be..14ca18b25 100644 --- a/dvc/stage/params.py +++ b/dvc/stage/params.py @@ -11,3 +11,4 @@ class StageParams: PARAM_PARAMS = "params" PARAM_METRICS = "metrics" PARAM_PLOTS = "plots" + PARAM_DESC = "desc" diff --git a/dvc/stage/serialize.py b/dvc/stage/serialize.py index fef9edb30..1c6ffe8e0 100644 --- a/dvc/stage/serialize.py +++ b/dvc/stage/serialize.py @@ -27,6 +27,7 @@ PARAM_METRIC = BaseOutput.PARAM_METRIC PARAM_PLOT = BaseOutput.PARAM_PLOT PARAM_PERSIST = BaseOutput.PARAM_PERSIST PARAM_CHECKPOINT = BaseOutput.PARAM_CHECKPOINT +PARAM_DESC = BaseOutput.PARAM_DESC DEFAULT_PARAMS_FILE = ParamsDependency.DEFAULT_PARAMS_FILE @@ -36,6 +37,8 @@ sort_by_path = partial(sorted, key=attrgetter("def_path")) @post_processing(OrderedDict) def _get_flags(out): + if out.desc: + yield PARAM_DESC, out.desc if not out.use_cache: yield PARAM_CACHE, False if out.checkpoint: @@ -119,6 +122,7 @@ def to_pipeline_file(stage: "PipelineStage"): outs, metrics, plots = _serialize_outs(stage.outs) res = [ + (stage.PARAM_DESC, stage.desc), (stage.PARAM_CMD, stage.cmd), (stage.PARAM_WDIR, wdir), (stage.PARAM_DEPS, deps), diff --git a/dvc/stage/utils.py b/dvc/stage/utils.py index 134305293..d743adcd6 100644 --- a/dvc/stage/utils.py +++ b/dvc/stage/utils.py @@ -199,6 +199,7 @@ def get_dump(stage): return { key: value for key, value in { + stage.PARAM_DESC: stage.desc, stage.PARAM_MD5: stage.md5, stage.PARAM_CMD: stage.cmd, stage.PARAM_WDIR: resolve_wdir(stage.wdir, stage.path), diff --git a/tests/func/test_dvcfile.py b/tests/func/test_dvcfile.py index cccbc825b..82b708b2b 100644 --- a/tests/func/test_dvcfile.py +++ b/tests/func/test_dvcfile.py @@ -317,6 +317,27 @@ def test_dvcfile_dump_preserves_meta(tmp_dir, dvc, run_copy): assert dvcfile._load()[0]["stages"]["run_copy"]["meta"] == metadata +def test_dvcfile_dump_preserves_desc(tmp_dir, dvc, run_copy): + tmp_dir.gen("foo", "foo") + stage_desc = "test stage description" + out_desc = "test out description" + + stage = run_copy("foo", "bar", name="run_copy", desc=stage_desc) + dvcfile = stage.dvcfile + + data = dvcfile._load()[0] + data["stages"]["run_copy"]["outs"][0] = {"bar": {"desc": out_desc}} + dump_yaml(dvcfile.path, data) + + assert stage.desc == stage_desc + stage.outs[0].desc = out_desc + dvcfile.dump(stage) + loaded = dvcfile._load()[0] + assert loaded == data + assert loaded["stages"]["run_copy"]["desc"] == stage_desc + assert loaded["stages"]["run_copy"]["outs"][0]["bar"]["desc"] == out_desc + + def test_dvcfile_dump_preserves_comments(tmp_dir, dvc): text = textwrap.dedent( """\ diff --git a/tests/func/test_stage.py b/tests/func/test_stage.py index 362c5aa7a..df17bf440 100644 --- a/tests/func/test_stage.py +++ b/tests/func/test_stage.py @@ -181,6 +181,25 @@ def test_meta_is_preserved(tmp_dir, dvc): assert new_data["meta"] == data["meta"] +def test_desc_is_preserved(tmp_dir, dvc): + (stage,) = tmp_dir.dvc_gen("foo", "foo content") + + data = load_yaml(stage.path) + stage_desc = "test stage description" + out_desc = "test out description" + data["desc"] = stage_desc + data["outs"][0]["desc"] = out_desc + dump_yaml(stage.path, data) + + dvcfile = SingleStageFile(dvc, stage.path) + new_stage = dvcfile.stage + dvcfile.dump(new_stage) + + new_data = load_yaml(stage.path) + assert new_data["desc"] == stage_desc + assert new_data["outs"][0]["desc"] == out_desc + + def test_parent_repo_collect_stages(tmp_dir, scm, dvc): tmp_dir.gen({"subdir": {}}) tmp_dir.gen({"deep": {"dir": {}}}) diff --git a/tests/unit/command/test_add.py b/tests/unit/command/test_add.py new file mode 100644 index 000000000..22b57c6cb --- /dev/null +++ b/tests/unit/command/test_add.py @@ -0,0 +1,35 @@ +from dvc.cli import parse_args +from dvc.command.add import CmdAdd + + +def test_add(mocker, dvc): + cli_args = parse_args( + [ + "add", + "--recursive", + "--no-commit", + "--external", + "--glob", + "--file", + "file", + "--desc", + "stage description", + "data", + ] + ) + assert cli_args.func == CmdAdd + + cmd = cli_args.func(cli_args) + m = mocker.patch.object(cmd.repo, "add", autospec=True) + + assert cmd.run() == 0 + + m.assert_called_once_with( + ["data"], + recursive=True, + no_commit=True, + glob=True, + fname="file", + external=True, + desc="stage description", + ) diff --git a/tests/unit/command/test_imp.py b/tests/unit/command/test_imp.py index b6b5a924b..ff04e6503 100644 --- a/tests/unit/command/test_imp.py +++ b/tests/unit/command/test_imp.py @@ -14,6 +14,8 @@ def test_import(mocker): "file", "--rev", "version", + "--desc", + "description", ] ) assert cli_args.func == CmdImport @@ -30,6 +32,7 @@ def test_import(mocker): fname="file", rev="version", no_exec=False, + desc="description", ) @@ -46,6 +49,8 @@ def test_import_no_exec(mocker): "--rev", "version", "--no-exec", + "--desc", + "description", ] ) @@ -61,4 +66,5 @@ def test_import_no_exec(mocker): fname="file", rev="version", no_exec=True, + desc="description", ) diff --git a/tests/unit/command/test_imp_url.py b/tests/unit/command/test_imp_url.py index 6b5b5f784..3eb6dcc0d 100644 --- a/tests/unit/command/test_imp_url.py +++ b/tests/unit/command/test_imp_url.py @@ -6,7 +6,9 @@ from dvc.exceptions import DvcException def test_import_url(mocker): - cli_args = parse_args(["import-url", "src", "out", "--file", "file"]) + cli_args = parse_args( + ["import-url", "src", "out", "--file", "file", "--desc", "description"] + ) assert cli_args.func == CmdImportUrl cmd = cli_args.func(cli_args) @@ -14,7 +16,9 @@ def test_import_url(mocker): assert cmd.run() == 0 - m.assert_called_once_with("src", out="out", fname="file", no_exec=False) + m.assert_called_once_with( + "src", out="out", fname="file", no_exec=False, desc="description" + ) def test_failed_import_url(mocker, caplog): @@ -35,7 +39,16 @@ def test_failed_import_url(mocker, caplog): def test_import_url_no_exec(mocker): cli_args = parse_args( - ["import-url", "--no-exec", "src", "out", "--file", "file"] + [ + "import-url", + "--no-exec", + "src", + "out", + "--file", + "file", + "--desc", + "description", + ] ) cmd = cli_args.func(cli_args) @@ -43,4 +56,6 @@ def test_import_url_no_exec(mocker): assert cmd.run() == 0 - m.assert_called_once_with("src", out="out", fname="file", no_exec=True) + m.assert_called_once_with( + "src", out="out", fname="file", no_exec=True, desc="description" + ) diff --git a/tests/unit/command/test_run.py b/tests/unit/command/test_run.py index e0f82d1e3..6a2939099 100644 --- a/tests/unit/command/test_run.py +++ b/tests/unit/command/test_run.py @@ -42,6 +42,8 @@ def test_run(mocker, dvc): "--params", "param3", "--external", + "--desc", + "description", "command", ] ) @@ -75,6 +77,7 @@ def test_run(mocker, dvc): name="nam", single_stage=False, external=True, + desc="description", ) @@ -106,6 +109,7 @@ def test_run_args_from_cli(mocker, dvc): name=None, single_stage=False, external=False, + desc=None, ) @@ -137,4 +141,5 @@ def test_run_args_with_spaces(mocker, dvc): name=None, single_stage=False, external=False, + desc=None, )