dvcfile: add desc field for outs/stages (#4901)

Outs desc will be used by viewer right away. Stage desc is for people to
start using now, but we won't be using it it viewer until DAG support is
added there.
This commit is contained in:
Ruslan Kuprieiev
2020-11-18 22:06:52 +02:00
committed by GitHub
parent b340254556
commit 251a3269c2
22 changed files with 190 additions and 21 deletions
+10
View File
@@ -21,6 +21,7 @@ class CmdAdd(CmdBase):
fname=self.args.file,
external=self.args.external,
glob=self.args.glob,
desc=self.args.desc,
)
except DvcException:
@@ -69,6 +70,15 @@ def add_parser(subparsers, parent_parser):
help="Specify name of the DVC-file this command will generate.",
metavar="<filename>",
)
parser.add_argument(
"--desc",
type=str,
metavar="<text>",
help=(
"User description of the data (optional). "
"This doesn't affect any DVC operations."
),
)
parser.add_argument(
"targets", nargs="+", help="Input files/directories to add.",
).complete = completion.FILE
+10
View File
@@ -18,6 +18,7 @@ class CmdImport(CmdBase):
fname=self.args.file,
rev=self.args.rev,
no_exec=self.args.no_exec,
desc=self.args.desc,
)
except DvcException:
logger.exception(
@@ -72,4 +73,13 @@ def add_parser(subparsers, parent_parser):
default=False,
help="Only create DVC-file without actually downloading it.",
)
import_parser.add_argument(
"--desc",
type=str,
metavar="<text>",
help=(
"User description of the data (optional). "
"This doesn't affect any DVC operations."
),
)
import_parser.set_defaults(func=CmdImport)
+10
View File
@@ -16,6 +16,7 @@ class CmdImportUrl(CmdBase):
out=self.args.out,
fname=self.args.file,
no_exec=self.args.no_exec,
desc=self.args.desc,
)
except DvcException:
logger.exception(
@@ -67,4 +68,13 @@ def add_parser(subparsers, parent_parser):
default=False,
help="Only create DVC-file without actually downloading it.",
)
import_parser.add_argument(
"--desc",
type=str,
metavar="<text>",
help=(
"User description of the data (optional). "
"This doesn't affect any DVC operations."
),
)
import_parser.set_defaults(func=CmdImportUrl)
+10
View File
@@ -57,6 +57,7 @@ class CmdRun(CmdBase):
name=self.args.name,
single_stage=self.args.single_stage,
external=self.args.external,
desc=self.args.desc,
)
except DvcException:
logger.exception("")
@@ -240,6 +241,15 @@ def add_parser(subparsers, parent_parser):
default=False,
help="Allow outputs that are outside of the DVC repository.",
)
run_parser.add_argument(
"--desc",
type=str,
metavar="<text>",
help=(
"User description of the stage (optional). "
"This doesn't affect any DVC operations."
),
)
run_parser.add_argument(
"command", nargs=argparse.REMAINDER, help="Command to execute."
)
+1
View File
@@ -56,6 +56,7 @@ DEP_MAP = {
SCHEMA = output.SCHEMA.copy()
del SCHEMA[BaseOutput.PARAM_CACHE]
del SCHEMA[BaseOutput.PARAM_METRIC]
del SCHEMA[BaseOutput.PARAM_DESC]
SCHEMA.update(RepoDependency.REPO_SCHEMA)
SCHEMA.update(ParamsDependency.PARAM_SCHEMA)
+7
View File
@@ -68,6 +68,7 @@ SCHEMA[BaseOutput.PARAM_PERSIST] = bool
SCHEMA[BaseOutput.PARAM_CHECKPOINT] = bool
SCHEMA[HashInfo.PARAM_SIZE] = int
SCHEMA[HashInfo.PARAM_NFILES] = int
SCHEMA[BaseOutput.PARAM_DESC] = str
def _get(
@@ -79,6 +80,7 @@ def _get(
plot=False,
persist=False,
checkpoint=False,
desc=None,
):
parsed = urlparse(p)
@@ -94,6 +96,7 @@ def _get(
plot=plot,
persist=persist,
checkpoint=checkpoint,
desc=desc,
)
for o in OUTS:
@@ -108,6 +111,7 @@ def _get(
plot=plot,
persist=persist,
checkpoint=checkpoint,
desc=desc,
)
return LocalOutput(
stage,
@@ -119,6 +123,7 @@ def _get(
plot=plot,
persist=persist,
checkpoint=checkpoint,
desc=desc,
)
@@ -131,6 +136,7 @@ def loadd_from(stage, d_list):
plot = d.pop(BaseOutput.PARAM_PLOT, False)
persist = d.pop(BaseOutput.PARAM_PERSIST, False)
checkpoint = d.pop(BaseOutput.PARAM_CHECKPOINT, False)
desc = d.pop(BaseOutput.PARAM_DESC, False)
ret.append(
_get(
stage,
@@ -141,6 +147,7 @@ def loadd_from(stage, d_list):
plot=plot,
persist=persist,
checkpoint=checkpoint,
desc=desc,
)
)
return ret
+6
View File
@@ -74,6 +74,7 @@ class BaseOutput:
PARAM_PLOT_TITLE = "title"
PARAM_PLOT_HEADER = "header"
PARAM_PERSIST = "persist"
PARAM_DESC = "desc"
METRIC_SCHEMA = Any(
None,
@@ -102,6 +103,7 @@ class BaseOutput:
plot=False,
persist=False,
checkpoint=False,
desc=None,
):
self._validate_output_path(path, stage)
# This output (and dependency) objects have too many paths/urls
@@ -127,6 +129,7 @@ class BaseOutput:
self.plot = False if self.IS_DEPENDENCY else plot
self.persist = persist
self.checkpoint = checkpoint
self.desc = desc
self.path_info = self._parse_path(tree, path)
if self.use_cache and self.cache is None:
@@ -299,6 +302,9 @@ class BaseOutput:
if self.IS_DEPENDENCY:
return ret
if self.desc:
ret[self.PARAM_DESC] = self.desc
if not self.use_cache:
ret[self.PARAM_CACHE] = self.use_cache
+5 -14
View File
@@ -23,13 +23,7 @@ logger = logging.getLogger(__name__)
@locked
@scm_context
def add(
repo,
targets,
recursive=False,
no_commit=False,
fname=None,
external=False,
glob=False,
repo, targets, recursive=False, no_commit=False, fname=None, **kwargs,
):
if recursive and fname:
raise RecursiveAddingWhileUsingFilename()
@@ -63,12 +57,7 @@ def add(
)
stages = _create_stages(
repo,
sub_targets,
fname,
pbar=pbar,
external=external,
glob=glob,
repo, sub_targets, fname, pbar=pbar, **kwargs,
)
try:
@@ -161,7 +150,7 @@ def _find_all_targets(repo, target, recursive):
def _create_stages(
repo, targets, fname, pbar=None, external=False, glob=False
repo, targets, fname, pbar=None, external=False, glob=False, desc=None,
):
from glob import iglob
@@ -194,6 +183,8 @@ def _create_stages(
)
if stage:
Dvcfile(repo, stage.path).remove()
if desc:
stage.outs[0].desc = desc
repo._reset() # pylint: disable=protected-access
+2 -2
View File
@@ -1,8 +1,8 @@
def imp(self, url, path, out=None, fname=None, rev=None, no_exec=False):
def imp(self, url, path, out=None, fname=None, rev=None, **kwargs):
erepo = {"url": url}
if rev is not None:
erepo["rev"] = rev
return self.imp_url(
path, out=out, fname=fname, erepo=erepo, frozen=True, no_exec=no_exec
path, out=out, fname=fname, erepo=erepo, frozen=True, **kwargs
)
+11 -1
View File
@@ -11,7 +11,14 @@ from . import locked
@locked
@scm_context
def imp_url(
self, url, out=None, fname=None, erepo=None, frozen=True, no_exec=False
self,
url,
out=None,
fname=None,
erepo=None,
frozen=True,
no_exec=False,
desc=None,
):
from dvc.dvcfile import Dvcfile
from dvc.stage import Stage, create_stage
@@ -40,6 +47,9 @@ def imp_url(
if stage is None:
return None
if desc:
stage.outs[0].desc = desc
dvcfile = Dvcfile(self, stage.path)
dvcfile.remove()
+3
View File
@@ -17,6 +17,7 @@ SINGLE_STAGE_SCHEMA = {
StageParams.PARAM_FROZEN: bool,
StageParams.PARAM_META: object,
StageParams.PARAM_ALWAYS_CHANGED: bool,
StageParams.PARAM_DESC: str,
}
DATA_SCHEMA = {
@@ -38,6 +39,7 @@ OUT_PSTAGE_DETAILED_SCHEMA = {
BaseOutput.PARAM_CACHE: bool,
BaseOutput.PARAM_PERSIST: bool,
BaseOutput.PARAM_CHECKPOINT: bool,
BaseOutput.PARAM_DESC: str,
}
}
@@ -68,6 +70,7 @@ STAGE_DEFINITION = {
],
Optional(StageParams.PARAM_FROZEN): bool,
Optional(StageParams.PARAM_META): object,
Optional(StageParams.PARAM_DESC): str,
Optional(StageParams.PARAM_ALWAYS_CHANGED): bool,
Optional(StageParams.PARAM_OUTS): [Any(str, OUT_PSTAGE_DETAILED_SCHEMA)],
Optional(StageParams.PARAM_METRICS): [
+3
View File
@@ -52,6 +52,7 @@ def loads_from(cls, repo, path, wdir, data):
Stage.PARAM_FROZEN,
Stage.PARAM_ALWAYS_CHANGED,
Stage.PARAM_MD5,
Stage.PARAM_DESC,
"name",
],
),
@@ -111,6 +112,7 @@ class Stage(params.StageParams):
always_changed=False,
stage_text=None,
dvcfile=None,
desc=None,
):
if deps is None:
deps = []
@@ -128,6 +130,7 @@ class Stage(params.StageParams):
self.always_changed = always_changed
self._stage_text = stage_text
self._dvcfile = dvcfile
self.desc = desc
@property
def path(self):
+1
View File
@@ -58,6 +58,7 @@ class StageLoader(Mapping):
)
stage = loads_from(PipelineStage, dvcfile.repo, path, wdir, stage_data)
stage.name = name
stage.desc = stage_data.get(Stage.PARAM_DESC)
deps = project(stage_data, [stage.PARAM_DEPS, stage.PARAM_PARAMS])
fill_stage_dependencies(stage, **deps)
+1
View File
@@ -11,3 +11,4 @@ class StageParams:
PARAM_PARAMS = "params"
PARAM_METRICS = "metrics"
PARAM_PLOTS = "plots"
PARAM_DESC = "desc"
+4
View File
@@ -27,6 +27,7 @@ PARAM_METRIC = BaseOutput.PARAM_METRIC
PARAM_PLOT = BaseOutput.PARAM_PLOT
PARAM_PERSIST = BaseOutput.PARAM_PERSIST
PARAM_CHECKPOINT = BaseOutput.PARAM_CHECKPOINT
PARAM_DESC = BaseOutput.PARAM_DESC
DEFAULT_PARAMS_FILE = ParamsDependency.DEFAULT_PARAMS_FILE
@@ -36,6 +37,8 @@ sort_by_path = partial(sorted, key=attrgetter("def_path"))
@post_processing(OrderedDict)
def _get_flags(out):
if out.desc:
yield PARAM_DESC, out.desc
if not out.use_cache:
yield PARAM_CACHE, False
if out.checkpoint:
@@ -119,6 +122,7 @@ def to_pipeline_file(stage: "PipelineStage"):
outs, metrics, plots = _serialize_outs(stage.outs)
res = [
(stage.PARAM_DESC, stage.desc),
(stage.PARAM_CMD, stage.cmd),
(stage.PARAM_WDIR, wdir),
(stage.PARAM_DEPS, deps),
+1
View File
@@ -199,6 +199,7 @@ def get_dump(stage):
return {
key: value
for key, value in {
stage.PARAM_DESC: stage.desc,
stage.PARAM_MD5: stage.md5,
stage.PARAM_CMD: stage.cmd,
stage.PARAM_WDIR: resolve_wdir(stage.wdir, stage.path),
+21
View File
@@ -317,6 +317,27 @@ def test_dvcfile_dump_preserves_meta(tmp_dir, dvc, run_copy):
assert dvcfile._load()[0]["stages"]["run_copy"]["meta"] == metadata
def test_dvcfile_dump_preserves_desc(tmp_dir, dvc, run_copy):
tmp_dir.gen("foo", "foo")
stage_desc = "test stage description"
out_desc = "test out description"
stage = run_copy("foo", "bar", name="run_copy", desc=stage_desc)
dvcfile = stage.dvcfile
data = dvcfile._load()[0]
data["stages"]["run_copy"]["outs"][0] = {"bar": {"desc": out_desc}}
dump_yaml(dvcfile.path, data)
assert stage.desc == stage_desc
stage.outs[0].desc = out_desc
dvcfile.dump(stage)
loaded = dvcfile._load()[0]
assert loaded == data
assert loaded["stages"]["run_copy"]["desc"] == stage_desc
assert loaded["stages"]["run_copy"]["outs"][0]["bar"]["desc"] == out_desc
def test_dvcfile_dump_preserves_comments(tmp_dir, dvc):
text = textwrap.dedent(
"""\
+19
View File
@@ -181,6 +181,25 @@ def test_meta_is_preserved(tmp_dir, dvc):
assert new_data["meta"] == data["meta"]
def test_desc_is_preserved(tmp_dir, dvc):
(stage,) = tmp_dir.dvc_gen("foo", "foo content")
data = load_yaml(stage.path)
stage_desc = "test stage description"
out_desc = "test out description"
data["desc"] = stage_desc
data["outs"][0]["desc"] = out_desc
dump_yaml(stage.path, data)
dvcfile = SingleStageFile(dvc, stage.path)
new_stage = dvcfile.stage
dvcfile.dump(new_stage)
new_data = load_yaml(stage.path)
assert new_data["desc"] == stage_desc
assert new_data["outs"][0]["desc"] == out_desc
def test_parent_repo_collect_stages(tmp_dir, scm, dvc):
tmp_dir.gen({"subdir": {}})
tmp_dir.gen({"deep": {"dir": {}}})
+35
View File
@@ -0,0 +1,35 @@
from dvc.cli import parse_args
from dvc.command.add import CmdAdd
def test_add(mocker, dvc):
cli_args = parse_args(
[
"add",
"--recursive",
"--no-commit",
"--external",
"--glob",
"--file",
"file",
"--desc",
"stage description",
"data",
]
)
assert cli_args.func == CmdAdd
cmd = cli_args.func(cli_args)
m = mocker.patch.object(cmd.repo, "add", autospec=True)
assert cmd.run() == 0
m.assert_called_once_with(
["data"],
recursive=True,
no_commit=True,
glob=True,
fname="file",
external=True,
desc="stage description",
)
+6
View File
@@ -14,6 +14,8 @@ def test_import(mocker):
"file",
"--rev",
"version",
"--desc",
"description",
]
)
assert cli_args.func == CmdImport
@@ -30,6 +32,7 @@ def test_import(mocker):
fname="file",
rev="version",
no_exec=False,
desc="description",
)
@@ -46,6 +49,8 @@ def test_import_no_exec(mocker):
"--rev",
"version",
"--no-exec",
"--desc",
"description",
]
)
@@ -61,4 +66,5 @@ def test_import_no_exec(mocker):
fname="file",
rev="version",
no_exec=True,
desc="description",
)
+19 -4
View File
@@ -6,7 +6,9 @@ from dvc.exceptions import DvcException
def test_import_url(mocker):
cli_args = parse_args(["import-url", "src", "out", "--file", "file"])
cli_args = parse_args(
["import-url", "src", "out", "--file", "file", "--desc", "description"]
)
assert cli_args.func == CmdImportUrl
cmd = cli_args.func(cli_args)
@@ -14,7 +16,9 @@ def test_import_url(mocker):
assert cmd.run() == 0
m.assert_called_once_with("src", out="out", fname="file", no_exec=False)
m.assert_called_once_with(
"src", out="out", fname="file", no_exec=False, desc="description"
)
def test_failed_import_url(mocker, caplog):
@@ -35,7 +39,16 @@ def test_failed_import_url(mocker, caplog):
def test_import_url_no_exec(mocker):
cli_args = parse_args(
["import-url", "--no-exec", "src", "out", "--file", "file"]
[
"import-url",
"--no-exec",
"src",
"out",
"--file",
"file",
"--desc",
"description",
]
)
cmd = cli_args.func(cli_args)
@@ -43,4 +56,6 @@ def test_import_url_no_exec(mocker):
assert cmd.run() == 0
m.assert_called_once_with("src", out="out", fname="file", no_exec=True)
m.assert_called_once_with(
"src", out="out", fname="file", no_exec=True, desc="description"
)
+5
View File
@@ -42,6 +42,8 @@ def test_run(mocker, dvc):
"--params",
"param3",
"--external",
"--desc",
"description",
"command",
]
)
@@ -75,6 +77,7 @@ def test_run(mocker, dvc):
name="nam",
single_stage=False,
external=True,
desc="description",
)
@@ -106,6 +109,7 @@ def test_run_args_from_cli(mocker, dvc):
name=None,
single_stage=False,
external=False,
desc=None,
)
@@ -137,4 +141,5 @@ def test_run_args_with_spaces(mocker, dvc):
name=None,
single_stage=False,
external=False,
desc=None,
)