tag: getting rid of it (#3699)

* tags: get rid of it

* tests: adjust for tags and allow @ in stage name

* disallow @ on stage name

* fix ds issue, leftover tag in Stage
This commit is contained in:
Saugat Pachhai
2020-04-29 16:57:31 +03:00
committed by GitHub
parent 31fe77848f
commit 907853b985
19 changed files with 62 additions and 437 deletions
-2
View File
@@ -31,7 +31,6 @@ from .command import (
repro,
root,
run,
tag,
unprotect,
update,
version,
@@ -71,7 +70,6 @@ COMMANDS = [
pipeline,
daemon,
commit,
tag,
diff,
version,
update,
+4 -4
View File
@@ -14,8 +14,8 @@ class CmdPipelineShow(CmdBase):
from dvc import dvcfile
from dvc.utils import parse_target
path, name, tag = parse_target(target)
stage = dvcfile.Dvcfile(self.repo, path, tag=tag).stages[name]
path, name = parse_target(target)
stage = dvcfile.Dvcfile(self.repo, path).stages[name]
G = self.repo.graph
stages = networkx.dfs_postorder_nodes(G, stage)
if locked:
@@ -38,8 +38,8 @@ class CmdPipelineShow(CmdBase):
from dvc.repo.graph import get_pipeline
from dvc.utils import parse_target
path, name, tag = parse_target(target)
target_stage = dvcfile.Dvcfile(self.repo, path, tag=tag).stages[name]
path, name = parse_target(target)
target_stage = dvcfile.Dvcfile(self.repo, path).stages[name]
G = get_pipeline(self.repo.pipelines, target_stage)
nodes = set()
-156
View File
@@ -1,156 +0,0 @@
import logging
from dvc.command.base import append_doc_link
from dvc.command.base import CmdBase
from dvc.command.base import fix_subparsers
from dvc.exceptions import DvcException
from dvc.utils import to_yaml_string
logger = logging.getLogger(__name__)
class CmdTagAdd(CmdBase):
def run(self):
for target in self.args.targets:
try:
self.repo.tag.add(
self.args.tag,
target=target,
with_deps=self.args.with_deps,
recursive=self.args.recursive,
)
except DvcException:
logger.exception("failed to add tag")
return 1
return 0
class CmdTagRemove(CmdBase):
def run(self):
for target in self.args.targets:
try:
self.repo.tag.remove(
self.args.tag,
target=target,
with_deps=self.args.with_deps,
recursive=self.args.recursive,
)
except DvcException:
logger.exception("failed to remove tag")
return 1
return 0
class CmdTagList(CmdBase):
def run(self):
for target in self.args.targets:
try:
tags = self.repo.tag.list(
target,
with_deps=self.args.with_deps,
recursive=self.args.recursive,
)
if tags:
logger.info(to_yaml_string(tags))
except DvcException:
logger.exception("failed list tags")
return 1
return 0
def add_parser(subparsers, parent_parser):
TAG_HELP = "A set of commands to manage DVC tags."
tag_parser = subparsers.add_parser(
"tag",
parents=[parent_parser],
description=append_doc_link(TAG_HELP, "tag"),
add_help=False,
)
tag_subparsers = tag_parser.add_subparsers(
dest="cmd",
help="Use DVC tag CMD --help to display command-specific help.",
)
fix_subparsers(tag_subparsers)
TAG_ADD_HELP = "Add DVC tag."
tag_add_parser = tag_subparsers.add_parser(
"add",
parents=[parent_parser],
description=append_doc_link(TAG_ADD_HELP, "tag-add"),
help=TAG_ADD_HELP,
)
tag_add_parser.add_argument("tag", help="Dvc tag.")
tag_add_parser.add_argument(
"targets", nargs="*", default=[None], help="DVC-files."
)
tag_add_parser.add_argument(
"-d",
"--with-deps",
action="store_true",
default=False,
help="Add tag for all dependencies of the specified DVC-file.",
)
tag_add_parser.add_argument(
"-R",
"--recursive",
action="store_true",
default=False,
help="Add tag for subdirectories of the specified directory.",
)
tag_add_parser.set_defaults(func=CmdTagAdd)
TAG_REMOVE_HELP = "Remove DVC tag."
tag_remove_parser = tag_subparsers.add_parser(
"remove",
parents=[parent_parser],
description=append_doc_link(TAG_REMOVE_HELP, "tag-remove"),
help=TAG_REMOVE_HELP,
)
tag_remove_parser.add_argument("tag", help="Dvc tag.")
tag_remove_parser.add_argument(
"targets", nargs="*", default=[None], help="DVC-files."
)
tag_remove_parser.add_argument(
"-d",
"--with-deps",
action="store_true",
default=False,
help="Remove tag for all dependencies of the specified DVC-file.",
)
tag_remove_parser.add_argument(
"-R",
"--recursive",
action="store_true",
default=False,
help="Remove tag for subdirectories of the specified directory.",
)
tag_remove_parser.set_defaults(func=CmdTagRemove)
TAG_LIST_HELP = "List DVC tags."
tag_list_parser = tag_subparsers.add_parser(
"list",
parents=[parent_parser],
description=append_doc_link(TAG_LIST_HELP, "tag-list"),
help=TAG_LIST_HELP,
)
tag_list_parser.add_argument(
"targets", nargs="*", default=[None], help="DVC-files."
)
tag_list_parser.add_argument(
"-d",
"--with-deps",
action="store_true",
default=False,
help="List tags for all dependencies of the specified DVC-file.",
)
tag_list_parser.add_argument(
"-R",
"--recursive",
action="store_true",
default=False,
help="List tags for subdirectories of the specified directory.",
)
tag_list_parser.set_defaults(func=CmdTagList)
+3 -4
View File
@@ -128,19 +128,18 @@ class FileMixin:
class SingleStageFile(FileMixin):
from dvc.schema import COMPILED_SINGLE_STAGE_SCHEMA as SCHEMA
def __init__(self, repo, path, tag=None):
def __init__(self, repo, path):
super().__init__(repo, path)
self.tag = tag
@property
def stage(self):
data, raw = self._load()
return SingleStageLoader.load_stage(self, data, raw, tag=self.tag)
return SingleStageLoader.load_stage(self, data, raw)
@property
def stages(self):
data, raw = self._load()
return SingleStageLoader(self, data, raw, tag=self.tag)
return SingleStageLoader(self, data, raw)
def dump(self, stage, **kwargs):
"""Dumps given stage appropriately in the dvcfile."""
+2 -15
View File
@@ -49,17 +49,14 @@ CHECKSUMS_SCHEMA = {
HDFSRemote.PARAM_CHECKSUM: CHECKSUM_SCHEMA,
}
TAGS_SCHEMA = {str: CHECKSUMS_SCHEMA}
SCHEMA = CHECKSUMS_SCHEMA.copy()
SCHEMA[Required(BaseOutput.PARAM_PATH)] = str
SCHEMA[BaseOutput.PARAM_CACHE] = bool
SCHEMA[BaseOutput.PARAM_METRIC] = BaseOutput.METRIC_SCHEMA
SCHEMA[BaseOutput.PARAM_TAGS] = TAGS_SCHEMA
SCHEMA[BaseOutput.PARAM_PERSIST] = bool
def _get(stage, p, info, cache, metric, persist=False, tags=None):
def _get(stage, p, info, cache, metric, persist=False):
parsed = urlparse(p)
if parsed.scheme == "remote":
@@ -72,7 +69,6 @@ def _get(stage, p, info, cache, metric, persist=False, tags=None):
remote=remote,
metric=metric,
persist=persist,
tags=tags,
)
for o in OUTS:
@@ -85,7 +81,6 @@ def _get(stage, p, info, cache, metric, persist=False, tags=None):
remote=None,
metric=metric,
persist=persist,
tags=tags,
)
return LocalOutput(
stage,
@@ -95,7 +90,6 @@ def _get(stage, p, info, cache, metric, persist=False, tags=None):
remote=None,
metric=metric,
persist=persist,
tags=tags,
)
@@ -106,16 +100,9 @@ def loadd_from(stage, d_list):
cache = d.pop(BaseOutput.PARAM_CACHE, True)
metric = d.pop(BaseOutput.PARAM_METRIC, False)
persist = d.pop(BaseOutput.PARAM_PERSIST, False)
tags = d.pop(BaseOutput.PARAM_TAGS, None)
ret.append(
_get(
stage,
p,
info=d,
cache=cache,
metric=metric,
persist=persist,
tags=tags,
stage, p, info=d, cache=cache, metric=metric, persist=persist,
)
)
return ret
+1 -14
View File
@@ -59,8 +59,6 @@ class BaseOutput(object):
},
)
PARAM_TAGS = "tags"
DoesNotExistError = OutputDoesNotExistError
IsNotFileOrDirError = OutputIsNotFileOrDirError
IsStageFileError = OutputIsStageFileError
@@ -76,7 +74,6 @@ class BaseOutput(object):
cache=True,
metric=False,
persist=False,
tags=None,
):
self._validate_output_path(path)
# This output (and dependency) objects have too many paths/urls
@@ -97,7 +94,6 @@ class BaseOutput(object):
self.use_cache = False if self.IS_DEPENDENCY else cache
self.metric = False if self.IS_DEPENDENCY else metric
self.persist = persist
self.tags = None if self.IS_DEPENDENCY else (tags or {})
self.path_info = self._parse_path(remote, path)
if self.use_cache and self.cache is None:
@@ -274,9 +270,6 @@ class BaseOutput(object):
ret[self.PARAM_METRIC] = self.metric
ret[self.PARAM_PERSIST] = self.persist
if self.tags:
ret[self.PARAM_TAGS] = self.tags
return ret
def verify_metric(self):
@@ -291,7 +284,6 @@ class BaseOutput(object):
self,
force=False,
progress_callback=None,
tag=None,
relink=False,
filter_info=None,
):
@@ -302,14 +294,9 @@ class BaseOutput(object):
)
return None
if tag:
info = self.tags[tag]
else:
info = self.info
return self.cache.checkout(
self.path_info,
info,
self.info,
force=force,
progress_callback=progress_callback,
relink=relink,
+8 -7
View File
@@ -1,3 +1,4 @@
import logging
import os
from contextlib import contextmanager
from functools import wraps
@@ -70,7 +71,6 @@ class Repo(object):
from dvc.repo.metrics import Metrics
from dvc.repo.params import Params
from dvc.scm.tree import WorkingTree
from dvc.repo.tag import Tag
from dvc.utils.fs import makedirs
root_dir = self.find_root(root_dir)
@@ -106,7 +106,6 @@ class Repo(object):
self.metrics = Metrics(self)
self.params = Params(self)
self.tag = Tag(self)
self._ignore()
@@ -211,8 +210,8 @@ class Repo(object):
os.path.abspath(target), graph or self.graph
)
file, name, tag = parse_target(target)
dvcfile = Dvcfile(self, file, tag=tag)
path, name = parse_target(target)
dvcfile = Dvcfile(self, path)
stages = list(dvcfile.stages.filter(name).values())
if not with_deps:
return stages
@@ -229,10 +228,10 @@ class Repo(object):
if not target:
return [(stage, None) for stage in self.stages]
file, name, tag = parse_target(target)
file, name = parse_target(target)
if is_valid_filename(file) and not kwargs.get("with_deps"):
# Optimization: do not collect the graph for a specific .dvc target
stages = Dvcfile(self, file, tag=tag).stages.filter(name)
stages = Dvcfile(self, file).stages.filter(name)
return [(stage, None) for stage in stages.values()]
try:
@@ -433,7 +432,9 @@ class Repo(object):
for root, dirs, files in self.tree.walk(self.root_dir):
for file_name in filter(is_valid_filename, files):
path = os.path.join(root, file_name)
stages.extend(list(Dvcfile(self, path).stages.values()))
stage_loader = Dvcfile(self, path).stages
with stage_loader.log_level(at=logging.DEBUG):
stages.extend(stage_loader.values())
outs.update(
out.fspath
for stage in stages
+2 -2
View File
@@ -6,8 +6,8 @@ def lock(self, target, unlock=False):
from .. import dvcfile
from dvc.utils import parse_target
path, name, tag = parse_target(target)
dvcfile = dvcfile.Dvcfile(self, path, tag=tag)
path, name = parse_target(target)
dvcfile = dvcfile.Dvcfile(self, path)
stage = dvcfile.stages[name]
stage.locked = False if unlock else True
dvcfile.dump(stage, update_pipeline=True)
+1 -1
View File
@@ -10,7 +10,7 @@ logger = logging.getLogger(__name__)
def remove(self, target, outs_only=False):
from ..dvcfile import Dvcfile
path, name, _ = parse_target(target)
path, name = parse_target(target)
dvcfile = Dvcfile(self, path)
stages = list(dvcfile.stages.filter(name).values())
for stage in stages:
+2 -2
View File
@@ -76,12 +76,12 @@ def reproduce(
active_graph = _get_active_graph(self.graph)
active_pipelines = get_pipelines(active_graph)
path, name, tag = parse_target(target)
path, name = parse_target(target)
if pipeline or all_pipelines:
if all_pipelines:
pipelines = active_pipelines
else:
dvcfile = Dvcfile(self, path, tag=tag)
dvcfile = Dvcfile(self, path)
stage = dvcfile.stages[name]
pipelines = [get_pipeline(active_pipelines, stage)]
-18
View File
@@ -1,18 +0,0 @@
class Tag(object):
def __init__(self, repo):
self.repo = repo
def add(self, *args, **kwargs):
from dvc.repo.tag.add import add
return add(self.repo, *args, **kwargs)
def list(self, *args, **kwargs):
from dvc.repo.tag.list import list
return list(self.repo, *args, **kwargs)
def remove(self, *args, **kwargs):
from dvc.repo.tag.remove import remove
return remove(self.repo, *args, **kwargs)
-24
View File
@@ -1,24 +0,0 @@
import logging
from copy import copy
from dvc.repo import locked
from dvc.dvcfile import Dvcfile
logger = logging.getLogger(__name__)
@locked
def add(self, tag, target=None, with_deps=False, recursive=False):
stages = self.collect(target, with_deps=with_deps, recursive=recursive)
for stage in stages:
changed = False
for out in stage.outs:
if not out.info:
logger.warning("missing checksum info for '{}'".format(out))
continue
out.tags[tag] = copy(out.info)
changed = True
if changed:
dvcfile = Dvcfile(self, stage.path)
dvcfile.dump(stage)
-15
View File
@@ -1,15 +0,0 @@
from dvc.repo import locked
@locked
def list(self, target=None, with_deps=False, recursive=False):
ret = {}
stages = self.collect(target, with_deps=with_deps, recursive=recursive)
for stage in stages:
outs_tags = {}
for out in stage.outs:
if out.tags:
outs_tags[str(out)] = out.tags.copy()
if outs_tags:
ret.update({stage.relpath: outs_tags})
return ret
-23
View File
@@ -1,23 +0,0 @@
import logging
from dvc.repo import locked
from dvc.dvcfile import Dvcfile
logger = logging.getLogger(__name__)
@locked
def remove(self, tag, target=None, with_deps=False, recursive=False):
stages = self.collect(target, with_deps=with_deps, recursive=recursive)
for stage in stages:
changed = False
for out in stage.outs:
if tag not in out.tags.keys():
logger.warning("tag '{}' not found for '{}'".format(tag, out))
continue
del out.tags[tag]
changed = True
if changed:
dvcfile = Dvcfile(self, stage.path)
dvcfile.dump(stage)
-4
View File
@@ -94,7 +94,6 @@ class Stage(params.StageParams):
outs=None,
md5=None,
locked=False,
tag=None,
always_changed=False,
stage_text=None,
dvcfile=None,
@@ -112,7 +111,6 @@ class Stage(params.StageParams):
self.deps = deps
self.md5 = md5
self.locked = locked
self.tag = tag
self.always_changed = always_changed
self._stage_text = stage_text
self._dvcfile = dvcfile
@@ -478,7 +476,6 @@ class Stage(params.StageParams):
exclude=[
self.PARAM_LOCKED,
BaseOutput.PARAM_METRIC,
BaseOutput.PARAM_TAGS,
BaseOutput.PARAM_PERSIST,
],
)
@@ -694,7 +691,6 @@ class Stage(params.StageParams):
try:
result = out.checkout(
force=force,
tag=self.tag,
progress_callback=progress_callback,
relink=relink,
filter_info=filter_info,
+23 -7
View File
@@ -1,5 +1,6 @@
import logging
import os
from contextlib import contextmanager
from copy import deepcopy
from collections import defaultdict, Mapping
@@ -29,6 +30,14 @@ class StageLoader(Mapping):
self.dvcfile = dvcfile
self.stages_data = stages_data or {}
self.lockfile_data = lockfile_data or {}
self._log_level = logging.WARNING
@contextmanager
def log_level(self, at):
"""Change log_level temporarily for StageLoader."""
self._log_level, level = at, self._log_level
yield
self._log_level = level
def filter(self, item=None):
if not item:
@@ -147,8 +156,11 @@ class StageLoader(Mapping):
raise StageNotFound(self.dvcfile, name)
if not self.lockfile_data.get(name):
logger.warning(
"No lock entry found for '%s:%s'", self.dvcfile.relpath, name
logger.log(
self._log_level,
"No lock entry found for '%s:%s'",
self.dvcfile.relpath,
name,
)
return self.load_stage(
@@ -169,15 +181,19 @@ class StageLoader(Mapping):
class SingleStageLoader(Mapping):
def __init__(self, dvcfile, stage_data, stage_text=None, tag=None):
def __init__(self, dvcfile, stage_data, stage_text=None):
self.dvcfile = dvcfile
self.stage_data = stage_data or {}
self.stage_text = stage_text
self.tag = tag
def filter(self, item=None):
return self
@contextmanager
def log_level(self, *args, **kwargs):
"""No-op context manager."""
yield
def __getitem__(self, item):
if item:
logger.warning(
@@ -188,16 +204,16 @@ class SingleStageLoader(Mapping):
# during `load`, we remove attributes from stage data, so as to
# not duplicate, therefore, for MappingView, we need to deepcopy.
return self.load_stage(
self.dvcfile, deepcopy(self.stage_data), self.stage_text, self.tag
self.dvcfile, deepcopy(self.stage_data), self.stage_text
)
@classmethod
def load_stage(cls, dvcfile, d, stage_text, tag=None):
def load_stage(cls, dvcfile, d, stage_text):
from dvc.stage import Stage, loads_from
path, wdir = resolve_paths(dvcfile.path, d.get(Stage.PARAM_WDIR))
stage = loads_from(Stage, dvcfile.repo, path, wdir, d)
stage._stage_text, stage.tag = stage_text, tag
stage._stage_text = stage_text
stage.deps = dependency.loadd_from(
stage, d.get(Stage.PARAM_DEPS) or []
)
+6 -9
View File
@@ -23,9 +23,7 @@ logger = logging.getLogger(__name__)
LOCAL_CHUNK_SIZE = 2 ** 20 # 1 MB
LARGE_FILE_SIZE = 2 ** 30 # 1 GB
LARGE_DIR_SIZE = 100
TARGET_REGEX = re.compile(
r"(?P<path>.*?)(:(?P<name>[^\\/@:]*))??(@(?P<tag>[^\\/@:]*))??$"
)
TARGET_REGEX = re.compile(r"(?P<path>.*?)(:(?P<name>[^\\/:]*))??$")
def dos2unix(data):
@@ -384,24 +382,23 @@ def parse_target(target, default=None):
from dvc.exceptions import DvcException
if not target:
return None, None, None
return None, None
match = TARGET_REGEX.match(target)
if not match:
return target, None, None
return target, None
path, name, tag = (
path, name = (
match.group("path"),
match.group("name"),
match.group("tag"),
)
if not path:
path = default or PIPELINE_FILE
logger.warning("Assuming file to be '%s'", path)
logger.debug("Assuming file to be '%s'", path)
if os.path.basename(path) == PIPELINE_LOCK:
raise DvcException(
"Did you mean: `{}`?".format(target.replace(".lock", ".yaml", 1))
)
return path, name, tag
return path, name
-120
View File
@@ -1,120 +0,0 @@
import filecmp
import logging
import os
import shutil
from dvc.main import main
from dvc.utils import from_yaml_string
from tests.basic_env import TestDvc
class TestTag(TestDvc):
def test(self):
fname = "file"
shutil.copyfile(self.FOO, fname)
stages = self.dvc.add(fname)
self.assertEqual(len(stages), 1)
stage = stages[0]
self.assertTrue(stage is not None)
ret = main(["tag", "add", "v1", stage.path])
self.assertEqual(ret, 0)
os.unlink(fname)
shutil.copyfile(self.BAR, fname)
ret = main(["repro", stage.path])
self.assertEqual(ret, 0)
ret = main(["tag", "add", "v2", stage.path])
self.assertEqual(ret, 0)
ret = main(["tag", "list", stage.path])
self.assertEqual(ret, 0)
ret = main(["checkout", stage.path + "@v1"])
self.assertEqual(ret, 0)
self.assertTrue(filecmp.cmp(fname, self.FOO, shallow=False))
ret = main(["checkout", stage.path + "@v2"])
self.assertEqual(ret, 0)
self.assertTrue(filecmp.cmp(fname, self.BAR, shallow=False))
class TestTagAll(TestDvc):
def test(self):
with self._caplog.at_level(logging.INFO, logger="dvc"):
ret = main(["add", self.FOO, self.BAR])
self.assertEqual(ret, 0)
self._caplog.clear()
ret = main(["tag", "list"])
self.assertEqual(ret, 0)
self.assertEqual("", self._caplog.text)
ret = main(["tag", "add", "v1"])
self.assertEqual(ret, 0)
self._caplog.clear()
ret = main(["tag", "list"])
self.assertEqual(ret, 0)
expected = {
"foo.dvc": {
"foo": {"v1": {"md5": "acbd18db4cc2f85cedef654fccc4a4d8"}}
},
"bar.dvc": {
"bar": {"v1": {"md5": "8978c98bb5a48c2fb5f2c4c905768afa"}}
},
}
stdout = "\n".join(
record.message for record in self._caplog.records
)
received = from_yaml_string(stdout)
assert expected == received
ret = main(["tag", "remove", "v1"])
self.assertEqual(ret, 0)
self._caplog.clear()
ret = main(["tag", "list"])
self.assertEqual(ret, 0)
self.assertEqual("", self._caplog.text)
class TestTagAddNoChecksumInfo(TestDvc):
def test(self):
ret = main(["run", "-o", self.FOO, "--no-exec"])
self.assertEqual(ret, 0)
self._caplog.clear()
with self._caplog.at_level(logging.WARNING, logger="dvc"):
ret = main(["tag", "add", "v1", "foo.dvc"])
self.assertEqual(ret, 0)
assert "missing checksum info for 'foo'" in self._caplog.text
class TestTagRemoveNoTag(TestDvc):
def test(self):
ret = main(["add", self.FOO])
self.assertEqual(ret, 0)
self._caplog.clear()
with self._caplog.at_level(logging.WARNING, logger="dvc"):
ret = main(["tag", "remove", "v1", "foo.dvc"])
self.assertEqual(ret, 0)
assert "tag 'v1' not found for 'foo'" in self._caplog.text
+10 -10
View File
@@ -137,14 +137,14 @@ def test_resolve_output(inp, out, is_dir, expected, mocker):
@pytest.mark.parametrize(
"inp,out, default",
[
["pipelines.yaml", ("pipelines.yaml", None, None), None],
["pipelines.yaml:name", ("pipelines.yaml", "name", None), None],
[":name", ("pipelines.yaml", "name", None), None],
["stage.dvc", ("stage.dvc", None, None), None],
["pipelines.yaml:name@v1", ("pipelines.yaml", "name", "v1"), None],
["../models/stage.dvc", ("../models/stage.dvc", None, None), "def"],
[":name", ("default", "name", None), "default"],
[":name@v2", ("default", "name", "v2"), "default"],
["pipelines.yaml", ("pipelines.yaml", None), None],
["pipelines.yaml:name", ("pipelines.yaml", "name"), None],
[":name", ("pipelines.yaml", "name"), None],
["stage.dvc", ("stage.dvc", None), None],
["pipelines.yaml:name", ("pipelines.yaml", "name"), None],
["../models/stage.dvc", ("../models/stage.dvc", None), "def"],
[":name", ("default", "name"), "default"],
[":name", ("default", "name"), "default"],
],
)
def test_parse_target(inp, out, default):
@@ -153,5 +153,5 @@ def test_parse_target(inp, out, default):
def test_hint_on_lockfile():
with pytest.raises(Exception) as exc:
assert parse_target("pipelines.lock:name@v223")
assert "pipelines.yaml:name@v223" in str(exc.value)
assert parse_target("pipelines.lock:name")
assert "pipelines.yaml:name" in str(exc.value)