mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 23:11:44 +01:00
dvcfile: add desc field for outs/stages (#4901)
Outs desc will be used by viewer right away. Stage desc is for people to start using now, but we won't be using it it viewer until DAG support is added there.
This commit is contained in:
@@ -21,6 +21,7 @@ class CmdAdd(CmdBase):
|
||||
fname=self.args.file,
|
||||
external=self.args.external,
|
||||
glob=self.args.glob,
|
||||
desc=self.args.desc,
|
||||
)
|
||||
|
||||
except DvcException:
|
||||
@@ -69,6 +70,15 @@ def add_parser(subparsers, parent_parser):
|
||||
help="Specify name of the DVC-file this command will generate.",
|
||||
metavar="<filename>",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--desc",
|
||||
type=str,
|
||||
metavar="<text>",
|
||||
help=(
|
||||
"User description of the data (optional). "
|
||||
"This doesn't affect any DVC operations."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"targets", nargs="+", help="Input files/directories to add.",
|
||||
).complete = completion.FILE
|
||||
|
||||
@@ -18,6 +18,7 @@ class CmdImport(CmdBase):
|
||||
fname=self.args.file,
|
||||
rev=self.args.rev,
|
||||
no_exec=self.args.no_exec,
|
||||
desc=self.args.desc,
|
||||
)
|
||||
except DvcException:
|
||||
logger.exception(
|
||||
@@ -72,4 +73,13 @@ def add_parser(subparsers, parent_parser):
|
||||
default=False,
|
||||
help="Only create DVC-file without actually downloading it.",
|
||||
)
|
||||
import_parser.add_argument(
|
||||
"--desc",
|
||||
type=str,
|
||||
metavar="<text>",
|
||||
help=(
|
||||
"User description of the data (optional). "
|
||||
"This doesn't affect any DVC operations."
|
||||
),
|
||||
)
|
||||
import_parser.set_defaults(func=CmdImport)
|
||||
|
||||
@@ -16,6 +16,7 @@ class CmdImportUrl(CmdBase):
|
||||
out=self.args.out,
|
||||
fname=self.args.file,
|
||||
no_exec=self.args.no_exec,
|
||||
desc=self.args.desc,
|
||||
)
|
||||
except DvcException:
|
||||
logger.exception(
|
||||
@@ -67,4 +68,13 @@ def add_parser(subparsers, parent_parser):
|
||||
default=False,
|
||||
help="Only create DVC-file without actually downloading it.",
|
||||
)
|
||||
import_parser.add_argument(
|
||||
"--desc",
|
||||
type=str,
|
||||
metavar="<text>",
|
||||
help=(
|
||||
"User description of the data (optional). "
|
||||
"This doesn't affect any DVC operations."
|
||||
),
|
||||
)
|
||||
import_parser.set_defaults(func=CmdImportUrl)
|
||||
|
||||
@@ -57,6 +57,7 @@ class CmdRun(CmdBase):
|
||||
name=self.args.name,
|
||||
single_stage=self.args.single_stage,
|
||||
external=self.args.external,
|
||||
desc=self.args.desc,
|
||||
)
|
||||
except DvcException:
|
||||
logger.exception("")
|
||||
@@ -240,6 +241,15 @@ def add_parser(subparsers, parent_parser):
|
||||
default=False,
|
||||
help="Allow outputs that are outside of the DVC repository.",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--desc",
|
||||
type=str,
|
||||
metavar="<text>",
|
||||
help=(
|
||||
"User description of the stage (optional). "
|
||||
"This doesn't affect any DVC operations."
|
||||
),
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"command", nargs=argparse.REMAINDER, help="Command to execute."
|
||||
)
|
||||
|
||||
@@ -56,6 +56,7 @@ DEP_MAP = {
|
||||
SCHEMA = output.SCHEMA.copy()
|
||||
del SCHEMA[BaseOutput.PARAM_CACHE]
|
||||
del SCHEMA[BaseOutput.PARAM_METRIC]
|
||||
del SCHEMA[BaseOutput.PARAM_DESC]
|
||||
SCHEMA.update(RepoDependency.REPO_SCHEMA)
|
||||
SCHEMA.update(ParamsDependency.PARAM_SCHEMA)
|
||||
|
||||
|
||||
@@ -68,6 +68,7 @@ SCHEMA[BaseOutput.PARAM_PERSIST] = bool
|
||||
SCHEMA[BaseOutput.PARAM_CHECKPOINT] = bool
|
||||
SCHEMA[HashInfo.PARAM_SIZE] = int
|
||||
SCHEMA[HashInfo.PARAM_NFILES] = int
|
||||
SCHEMA[BaseOutput.PARAM_DESC] = str
|
||||
|
||||
|
||||
def _get(
|
||||
@@ -79,6 +80,7 @@ def _get(
|
||||
plot=False,
|
||||
persist=False,
|
||||
checkpoint=False,
|
||||
desc=None,
|
||||
):
|
||||
parsed = urlparse(p)
|
||||
|
||||
@@ -94,6 +96,7 @@ def _get(
|
||||
plot=plot,
|
||||
persist=persist,
|
||||
checkpoint=checkpoint,
|
||||
desc=desc,
|
||||
)
|
||||
|
||||
for o in OUTS:
|
||||
@@ -108,6 +111,7 @@ def _get(
|
||||
plot=plot,
|
||||
persist=persist,
|
||||
checkpoint=checkpoint,
|
||||
desc=desc,
|
||||
)
|
||||
return LocalOutput(
|
||||
stage,
|
||||
@@ -119,6 +123,7 @@ def _get(
|
||||
plot=plot,
|
||||
persist=persist,
|
||||
checkpoint=checkpoint,
|
||||
desc=desc,
|
||||
)
|
||||
|
||||
|
||||
@@ -131,6 +136,7 @@ def loadd_from(stage, d_list):
|
||||
plot = d.pop(BaseOutput.PARAM_PLOT, False)
|
||||
persist = d.pop(BaseOutput.PARAM_PERSIST, False)
|
||||
checkpoint = d.pop(BaseOutput.PARAM_CHECKPOINT, False)
|
||||
desc = d.pop(BaseOutput.PARAM_DESC, False)
|
||||
ret.append(
|
||||
_get(
|
||||
stage,
|
||||
@@ -141,6 +147,7 @@ def loadd_from(stage, d_list):
|
||||
plot=plot,
|
||||
persist=persist,
|
||||
checkpoint=checkpoint,
|
||||
desc=desc,
|
||||
)
|
||||
)
|
||||
return ret
|
||||
|
||||
@@ -74,6 +74,7 @@ class BaseOutput:
|
||||
PARAM_PLOT_TITLE = "title"
|
||||
PARAM_PLOT_HEADER = "header"
|
||||
PARAM_PERSIST = "persist"
|
||||
PARAM_DESC = "desc"
|
||||
|
||||
METRIC_SCHEMA = Any(
|
||||
None,
|
||||
@@ -102,6 +103,7 @@ class BaseOutput:
|
||||
plot=False,
|
||||
persist=False,
|
||||
checkpoint=False,
|
||||
desc=None,
|
||||
):
|
||||
self._validate_output_path(path, stage)
|
||||
# This output (and dependency) objects have too many paths/urls
|
||||
@@ -127,6 +129,7 @@ class BaseOutput:
|
||||
self.plot = False if self.IS_DEPENDENCY else plot
|
||||
self.persist = persist
|
||||
self.checkpoint = checkpoint
|
||||
self.desc = desc
|
||||
|
||||
self.path_info = self._parse_path(tree, path)
|
||||
if self.use_cache and self.cache is None:
|
||||
@@ -299,6 +302,9 @@ class BaseOutput:
|
||||
if self.IS_DEPENDENCY:
|
||||
return ret
|
||||
|
||||
if self.desc:
|
||||
ret[self.PARAM_DESC] = self.desc
|
||||
|
||||
if not self.use_cache:
|
||||
ret[self.PARAM_CACHE] = self.use_cache
|
||||
|
||||
|
||||
+5
-14
@@ -23,13 +23,7 @@ logger = logging.getLogger(__name__)
|
||||
@locked
|
||||
@scm_context
|
||||
def add(
|
||||
repo,
|
||||
targets,
|
||||
recursive=False,
|
||||
no_commit=False,
|
||||
fname=None,
|
||||
external=False,
|
||||
glob=False,
|
||||
repo, targets, recursive=False, no_commit=False, fname=None, **kwargs,
|
||||
):
|
||||
if recursive and fname:
|
||||
raise RecursiveAddingWhileUsingFilename()
|
||||
@@ -63,12 +57,7 @@ def add(
|
||||
)
|
||||
|
||||
stages = _create_stages(
|
||||
repo,
|
||||
sub_targets,
|
||||
fname,
|
||||
pbar=pbar,
|
||||
external=external,
|
||||
glob=glob,
|
||||
repo, sub_targets, fname, pbar=pbar, **kwargs,
|
||||
)
|
||||
|
||||
try:
|
||||
@@ -161,7 +150,7 @@ def _find_all_targets(repo, target, recursive):
|
||||
|
||||
|
||||
def _create_stages(
|
||||
repo, targets, fname, pbar=None, external=False, glob=False
|
||||
repo, targets, fname, pbar=None, external=False, glob=False, desc=None,
|
||||
):
|
||||
from glob import iglob
|
||||
|
||||
@@ -194,6 +183,8 @@ def _create_stages(
|
||||
)
|
||||
if stage:
|
||||
Dvcfile(repo, stage.path).remove()
|
||||
if desc:
|
||||
stage.outs[0].desc = desc
|
||||
|
||||
repo._reset() # pylint: disable=protected-access
|
||||
|
||||
|
||||
+2
-2
@@ -1,8 +1,8 @@
|
||||
def imp(self, url, path, out=None, fname=None, rev=None, no_exec=False):
|
||||
def imp(self, url, path, out=None, fname=None, rev=None, **kwargs):
|
||||
erepo = {"url": url}
|
||||
if rev is not None:
|
||||
erepo["rev"] = rev
|
||||
|
||||
return self.imp_url(
|
||||
path, out=out, fname=fname, erepo=erepo, frozen=True, no_exec=no_exec
|
||||
path, out=out, fname=fname, erepo=erepo, frozen=True, **kwargs
|
||||
)
|
||||
|
||||
+11
-1
@@ -11,7 +11,14 @@ from . import locked
|
||||
@locked
|
||||
@scm_context
|
||||
def imp_url(
|
||||
self, url, out=None, fname=None, erepo=None, frozen=True, no_exec=False
|
||||
self,
|
||||
url,
|
||||
out=None,
|
||||
fname=None,
|
||||
erepo=None,
|
||||
frozen=True,
|
||||
no_exec=False,
|
||||
desc=None,
|
||||
):
|
||||
from dvc.dvcfile import Dvcfile
|
||||
from dvc.stage import Stage, create_stage
|
||||
@@ -40,6 +47,9 @@ def imp_url(
|
||||
if stage is None:
|
||||
return None
|
||||
|
||||
if desc:
|
||||
stage.outs[0].desc = desc
|
||||
|
||||
dvcfile = Dvcfile(self, stage.path)
|
||||
dvcfile.remove()
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ SINGLE_STAGE_SCHEMA = {
|
||||
StageParams.PARAM_FROZEN: bool,
|
||||
StageParams.PARAM_META: object,
|
||||
StageParams.PARAM_ALWAYS_CHANGED: bool,
|
||||
StageParams.PARAM_DESC: str,
|
||||
}
|
||||
|
||||
DATA_SCHEMA = {
|
||||
@@ -38,6 +39,7 @@ OUT_PSTAGE_DETAILED_SCHEMA = {
|
||||
BaseOutput.PARAM_CACHE: bool,
|
||||
BaseOutput.PARAM_PERSIST: bool,
|
||||
BaseOutput.PARAM_CHECKPOINT: bool,
|
||||
BaseOutput.PARAM_DESC: str,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -68,6 +70,7 @@ STAGE_DEFINITION = {
|
||||
],
|
||||
Optional(StageParams.PARAM_FROZEN): bool,
|
||||
Optional(StageParams.PARAM_META): object,
|
||||
Optional(StageParams.PARAM_DESC): str,
|
||||
Optional(StageParams.PARAM_ALWAYS_CHANGED): bool,
|
||||
Optional(StageParams.PARAM_OUTS): [Any(str, OUT_PSTAGE_DETAILED_SCHEMA)],
|
||||
Optional(StageParams.PARAM_METRICS): [
|
||||
|
||||
@@ -52,6 +52,7 @@ def loads_from(cls, repo, path, wdir, data):
|
||||
Stage.PARAM_FROZEN,
|
||||
Stage.PARAM_ALWAYS_CHANGED,
|
||||
Stage.PARAM_MD5,
|
||||
Stage.PARAM_DESC,
|
||||
"name",
|
||||
],
|
||||
),
|
||||
@@ -111,6 +112,7 @@ class Stage(params.StageParams):
|
||||
always_changed=False,
|
||||
stage_text=None,
|
||||
dvcfile=None,
|
||||
desc=None,
|
||||
):
|
||||
if deps is None:
|
||||
deps = []
|
||||
@@ -128,6 +130,7 @@ class Stage(params.StageParams):
|
||||
self.always_changed = always_changed
|
||||
self._stage_text = stage_text
|
||||
self._dvcfile = dvcfile
|
||||
self.desc = desc
|
||||
|
||||
@property
|
||||
def path(self):
|
||||
|
||||
@@ -58,6 +58,7 @@ class StageLoader(Mapping):
|
||||
)
|
||||
stage = loads_from(PipelineStage, dvcfile.repo, path, wdir, stage_data)
|
||||
stage.name = name
|
||||
stage.desc = stage_data.get(Stage.PARAM_DESC)
|
||||
|
||||
deps = project(stage_data, [stage.PARAM_DEPS, stage.PARAM_PARAMS])
|
||||
fill_stage_dependencies(stage, **deps)
|
||||
|
||||
@@ -11,3 +11,4 @@ class StageParams:
|
||||
PARAM_PARAMS = "params"
|
||||
PARAM_METRICS = "metrics"
|
||||
PARAM_PLOTS = "plots"
|
||||
PARAM_DESC = "desc"
|
||||
|
||||
@@ -27,6 +27,7 @@ PARAM_METRIC = BaseOutput.PARAM_METRIC
|
||||
PARAM_PLOT = BaseOutput.PARAM_PLOT
|
||||
PARAM_PERSIST = BaseOutput.PARAM_PERSIST
|
||||
PARAM_CHECKPOINT = BaseOutput.PARAM_CHECKPOINT
|
||||
PARAM_DESC = BaseOutput.PARAM_DESC
|
||||
|
||||
DEFAULT_PARAMS_FILE = ParamsDependency.DEFAULT_PARAMS_FILE
|
||||
|
||||
@@ -36,6 +37,8 @@ sort_by_path = partial(sorted, key=attrgetter("def_path"))
|
||||
|
||||
@post_processing(OrderedDict)
|
||||
def _get_flags(out):
|
||||
if out.desc:
|
||||
yield PARAM_DESC, out.desc
|
||||
if not out.use_cache:
|
||||
yield PARAM_CACHE, False
|
||||
if out.checkpoint:
|
||||
@@ -119,6 +122,7 @@ def to_pipeline_file(stage: "PipelineStage"):
|
||||
|
||||
outs, metrics, plots = _serialize_outs(stage.outs)
|
||||
res = [
|
||||
(stage.PARAM_DESC, stage.desc),
|
||||
(stage.PARAM_CMD, stage.cmd),
|
||||
(stage.PARAM_WDIR, wdir),
|
||||
(stage.PARAM_DEPS, deps),
|
||||
|
||||
@@ -199,6 +199,7 @@ def get_dump(stage):
|
||||
return {
|
||||
key: value
|
||||
for key, value in {
|
||||
stage.PARAM_DESC: stage.desc,
|
||||
stage.PARAM_MD5: stage.md5,
|
||||
stage.PARAM_CMD: stage.cmd,
|
||||
stage.PARAM_WDIR: resolve_wdir(stage.wdir, stage.path),
|
||||
|
||||
@@ -317,6 +317,27 @@ def test_dvcfile_dump_preserves_meta(tmp_dir, dvc, run_copy):
|
||||
assert dvcfile._load()[0]["stages"]["run_copy"]["meta"] == metadata
|
||||
|
||||
|
||||
def test_dvcfile_dump_preserves_desc(tmp_dir, dvc, run_copy):
|
||||
tmp_dir.gen("foo", "foo")
|
||||
stage_desc = "test stage description"
|
||||
out_desc = "test out description"
|
||||
|
||||
stage = run_copy("foo", "bar", name="run_copy", desc=stage_desc)
|
||||
dvcfile = stage.dvcfile
|
||||
|
||||
data = dvcfile._load()[0]
|
||||
data["stages"]["run_copy"]["outs"][0] = {"bar": {"desc": out_desc}}
|
||||
dump_yaml(dvcfile.path, data)
|
||||
|
||||
assert stage.desc == stage_desc
|
||||
stage.outs[0].desc = out_desc
|
||||
dvcfile.dump(stage)
|
||||
loaded = dvcfile._load()[0]
|
||||
assert loaded == data
|
||||
assert loaded["stages"]["run_copy"]["desc"] == stage_desc
|
||||
assert loaded["stages"]["run_copy"]["outs"][0]["bar"]["desc"] == out_desc
|
||||
|
||||
|
||||
def test_dvcfile_dump_preserves_comments(tmp_dir, dvc):
|
||||
text = textwrap.dedent(
|
||||
"""\
|
||||
|
||||
@@ -181,6 +181,25 @@ def test_meta_is_preserved(tmp_dir, dvc):
|
||||
assert new_data["meta"] == data["meta"]
|
||||
|
||||
|
||||
def test_desc_is_preserved(tmp_dir, dvc):
|
||||
(stage,) = tmp_dir.dvc_gen("foo", "foo content")
|
||||
|
||||
data = load_yaml(stage.path)
|
||||
stage_desc = "test stage description"
|
||||
out_desc = "test out description"
|
||||
data["desc"] = stage_desc
|
||||
data["outs"][0]["desc"] = out_desc
|
||||
dump_yaml(stage.path, data)
|
||||
|
||||
dvcfile = SingleStageFile(dvc, stage.path)
|
||||
new_stage = dvcfile.stage
|
||||
dvcfile.dump(new_stage)
|
||||
|
||||
new_data = load_yaml(stage.path)
|
||||
assert new_data["desc"] == stage_desc
|
||||
assert new_data["outs"][0]["desc"] == out_desc
|
||||
|
||||
|
||||
def test_parent_repo_collect_stages(tmp_dir, scm, dvc):
|
||||
tmp_dir.gen({"subdir": {}})
|
||||
tmp_dir.gen({"deep": {"dir": {}}})
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
from dvc.cli import parse_args
|
||||
from dvc.command.add import CmdAdd
|
||||
|
||||
|
||||
def test_add(mocker, dvc):
|
||||
cli_args = parse_args(
|
||||
[
|
||||
"add",
|
||||
"--recursive",
|
||||
"--no-commit",
|
||||
"--external",
|
||||
"--glob",
|
||||
"--file",
|
||||
"file",
|
||||
"--desc",
|
||||
"stage description",
|
||||
"data",
|
||||
]
|
||||
)
|
||||
assert cli_args.func == CmdAdd
|
||||
|
||||
cmd = cli_args.func(cli_args)
|
||||
m = mocker.patch.object(cmd.repo, "add", autospec=True)
|
||||
|
||||
assert cmd.run() == 0
|
||||
|
||||
m.assert_called_once_with(
|
||||
["data"],
|
||||
recursive=True,
|
||||
no_commit=True,
|
||||
glob=True,
|
||||
fname="file",
|
||||
external=True,
|
||||
desc="stage description",
|
||||
)
|
||||
@@ -14,6 +14,8 @@ def test_import(mocker):
|
||||
"file",
|
||||
"--rev",
|
||||
"version",
|
||||
"--desc",
|
||||
"description",
|
||||
]
|
||||
)
|
||||
assert cli_args.func == CmdImport
|
||||
@@ -30,6 +32,7 @@ def test_import(mocker):
|
||||
fname="file",
|
||||
rev="version",
|
||||
no_exec=False,
|
||||
desc="description",
|
||||
)
|
||||
|
||||
|
||||
@@ -46,6 +49,8 @@ def test_import_no_exec(mocker):
|
||||
"--rev",
|
||||
"version",
|
||||
"--no-exec",
|
||||
"--desc",
|
||||
"description",
|
||||
]
|
||||
)
|
||||
|
||||
@@ -61,4 +66,5 @@ def test_import_no_exec(mocker):
|
||||
fname="file",
|
||||
rev="version",
|
||||
no_exec=True,
|
||||
desc="description",
|
||||
)
|
||||
|
||||
@@ -6,7 +6,9 @@ from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
def test_import_url(mocker):
|
||||
cli_args = parse_args(["import-url", "src", "out", "--file", "file"])
|
||||
cli_args = parse_args(
|
||||
["import-url", "src", "out", "--file", "file", "--desc", "description"]
|
||||
)
|
||||
assert cli_args.func == CmdImportUrl
|
||||
|
||||
cmd = cli_args.func(cli_args)
|
||||
@@ -14,7 +16,9 @@ def test_import_url(mocker):
|
||||
|
||||
assert cmd.run() == 0
|
||||
|
||||
m.assert_called_once_with("src", out="out", fname="file", no_exec=False)
|
||||
m.assert_called_once_with(
|
||||
"src", out="out", fname="file", no_exec=False, desc="description"
|
||||
)
|
||||
|
||||
|
||||
def test_failed_import_url(mocker, caplog):
|
||||
@@ -35,7 +39,16 @@ def test_failed_import_url(mocker, caplog):
|
||||
|
||||
def test_import_url_no_exec(mocker):
|
||||
cli_args = parse_args(
|
||||
["import-url", "--no-exec", "src", "out", "--file", "file"]
|
||||
[
|
||||
"import-url",
|
||||
"--no-exec",
|
||||
"src",
|
||||
"out",
|
||||
"--file",
|
||||
"file",
|
||||
"--desc",
|
||||
"description",
|
||||
]
|
||||
)
|
||||
|
||||
cmd = cli_args.func(cli_args)
|
||||
@@ -43,4 +56,6 @@ def test_import_url_no_exec(mocker):
|
||||
|
||||
assert cmd.run() == 0
|
||||
|
||||
m.assert_called_once_with("src", out="out", fname="file", no_exec=True)
|
||||
m.assert_called_once_with(
|
||||
"src", out="out", fname="file", no_exec=True, desc="description"
|
||||
)
|
||||
|
||||
@@ -42,6 +42,8 @@ def test_run(mocker, dvc):
|
||||
"--params",
|
||||
"param3",
|
||||
"--external",
|
||||
"--desc",
|
||||
"description",
|
||||
"command",
|
||||
]
|
||||
)
|
||||
@@ -75,6 +77,7 @@ def test_run(mocker, dvc):
|
||||
name="nam",
|
||||
single_stage=False,
|
||||
external=True,
|
||||
desc="description",
|
||||
)
|
||||
|
||||
|
||||
@@ -106,6 +109,7 @@ def test_run_args_from_cli(mocker, dvc):
|
||||
name=None,
|
||||
single_stage=False,
|
||||
external=False,
|
||||
desc=None,
|
||||
)
|
||||
|
||||
|
||||
@@ -137,4 +141,5 @@ def test_run_args_with_spaces(mocker, dvc):
|
||||
name=None,
|
||||
single_stage=False,
|
||||
external=False,
|
||||
desc=None,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user