From aaa9167a00acd7cb37c6b63992116196c8176ddc Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 28 Mar 2019 00:17:19 +0200 Subject: [PATCH 1/6] dvc: fix remote.checkout() to follow same conventions Currently `checkout()` accepts `OutputBase*` class as an argument, which doesn't follow conventions from all other methods, which accept path_info and/or checksum_info. To fix that, cache for dir info is moved from output to the underlying remote. Signed-off-by: Ruslan Kuprieiev --- dvc/output/base.py | 2 +- dvc/output/local.py | 7 +------ dvc/remote/base.py | 32 ++++++++++++++++++-------------- dvc/remote/local.py | 24 +++++++++++++++--------- 4 files changed, 35 insertions(+), 30 deletions(-) diff --git a/dvc/output/base.py b/dvc/output/base.py index 1aa7e1488..ff3d8de24 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -212,7 +212,7 @@ class OutputBase(object): return getattr(self.repo.cache, self.scheme).checkout( - output=self, force=force, progress_callback=progress_callback + self.path_info, self.info, force=force, progress_callback=progress_callback ) def remove(self, ignore_remove=False): diff --git a/dvc/output/local.py b/dvc/output/local.py index 70728eb04..6a76cdd8e 100644 --- a/dvc/output/local.py +++ b/dvc/output/local.py @@ -155,12 +155,7 @@ class OutputLOCAL(OutputBase): @property def dir_cache(self): - if self.checksum not in self._dir_cache.keys(): - self._dir_cache[ - self.checksum - ] = self.repo.cache.local.load_dir_cache(self.checksum) - - return self._dir_cache[self.checksum] + return self.repo.cache.local.load_dir_cache(self.checksum) def get_files_number(self): if self.cache is None: diff --git a/dvc/remote/base.py b/dvc/remote/base.py index 714421968..cdba4c620 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -322,10 +322,9 @@ class RemoteBase(object): self.remove(path_info) - def do_checkout(self, output, force=False, progress_callback=None): - path_info = output.path_info - checksum = output.info.get(self.PARAM_CHECKSUM) - + def do_checkout( + self, path_info, checksum, force=False, progress_callback=None + ): if self.exists(path_info): msg = "data '{}' exists. Removing before checkout." logger.warning(msg.format(str(path_info))) @@ -334,33 +333,38 @@ class RemoteBase(object): from_info = self.checksum_to_path_info(checksum) self.copy(from_info, path_info) - def checkout(self, output, force=False, progress_callback=None): - scheme = output.path_info["scheme"] + def checkout( + self, path_info, checksum_info, force=False, progress_callback=None + ): + scheme = path_info["scheme"] if scheme not in ["", "local"] and scheme != self.scheme: raise NotImplementedError - checksum = output.info.get(self.PARAM_CHECKSUM) + checksum = checksum_info.get(self.PARAM_CHECKSUM) if not checksum: msg = "No checksum info for '{}'." - logger.debug(msg.format(str(output.path_info))) + logger.debug(msg.format(str(path_info))) return - if not self.changed(output.path_info, output.info): + if not self.changed(path_info, checksum_info): msg = "Data '{}' didn't change." - logger.debug(msg.format(str(output.path_info))) + logger.debug(msg.format(str(path_info))) return if self.changed_cache(checksum): msg = "Cache '{}' not found. File '{}' won't be created." - logger.warning(msg.format(checksum, str(output.path_info))) - self.safe_remove(output.path_info, force=force) + logger.warning(msg.format(checksum, str(path_info))) + self.safe_remove(path_info, force=force) return msg = "Checking out '{}' with cache '{}'." - logger.debug(msg.format(str(output.path_info), checksum)) + logger.debug(msg.format(str(path_info), checksum)) self.do_checkout( - output, force=force, progress_callback=progress_callback + path_info, + checksum, + force=force, + progress_callback=progress_callback, ) @staticmethod diff --git a/dvc/remote/local.py b/dvc/remote/local.py index b59e64109..702f34ad2 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -78,6 +78,8 @@ class RemoteLOCAL(RemoteBase): self.path_info = {"scheme": "local"} + self._dir_info = {} + @staticmethod def compat_config(config): ret = config.copy() @@ -256,9 +258,12 @@ class RemoteLOCAL(RemoteBase): def load_dir_cache(self, md5): path = self.get(md5) - assert self.is_dir_cache(path) + dir_info = self._dir_info.get(md5) + if dir_info: + return dir_info + try: with open(path, "r") as fd: d = json.load(fd) @@ -275,6 +280,8 @@ class RemoteLOCAL(RemoteBase): for info in d: info["relpath"] = self.to_ospath(info["relpath"]) + self._dir_info[md5] = d + return d def dump_dir_cache(self, md5, dir_info): @@ -299,15 +306,13 @@ class RemoteLOCAL(RemoteBase): def is_dir_cache(cls, cache): return cache.endswith(cls.MD5_DIR_SUFFIX) - def do_checkout(self, output, force=False, progress_callback=None): - path_info = output.path_info - checksum = output.info.get(self.PARAM_CHECKSUM) - + def do_checkout( + self, path_info, checksum, force=False, progress_callback=None + ): path = path_info["path"] - md5 = checksum - cache = self.get(md5) + cache = self.get(checksum) - if not output.is_dir_cache: + if not self.is_dir_cache(cache): if os.path.exists(path): self.safe_remove(path_info, force=force) @@ -322,11 +327,12 @@ class RemoteLOCAL(RemoteBase): if not os.path.exists(path): os.makedirs(path) - dir_info = output.dir_cache dir_relpath = os.path.relpath(path) logger.debug("Linking directory '{}'.".format(dir_relpath)) + dir_info = self.load_dir_cache(checksum) + for processed, entry in enumerate(dir_info): relpath = entry[self.PARAM_RELPATH] m = entry[self.PARAM_CHECKSUM] From 76c15ab80df85212628bf1f4d2f5322cb2a08d53 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 28 Mar 2019 00:24:39 +0200 Subject: [PATCH 2/6] dvc: repo: unify collect() method Signed-off-by: Ruslan Kuprieiev --- dvc/repo/__init__.py | 5 ++++- dvc/repo/checkout.py | 17 ++++++++--------- dvc/repo/commit.py | 6 +----- dvc/repo/status.py | 5 +---- 4 files changed, 14 insertions(+), 19 deletions(-) diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 69fecde6f..a169f3a57 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -143,10 +143,13 @@ class Repo(object): assert len(pipelines) == 1 return pipelines[0] - def collect(self, target, with_deps=False): + def collect(self, target, with_deps=False, recursive=False): import networkx as nx from dvc.stage import Stage + if not target or recursive: + return self.active_stages(target) + stage = Stage.load(self, target) if not with_deps: return [stage] diff --git a/dvc/repo/checkout.py b/dvc/repo/checkout.py index 479e05cbc..883f21c4c 100644 --- a/dvc/repo/checkout.py +++ b/dvc/repo/checkout.py @@ -29,15 +29,14 @@ def checkout(self, target=None, with_deps=False, force=False, recursive=False): all_stages = self.stages() - if target and not recursive: - try: - stages = self.collect(target, with_deps=with_deps) - except (StageFileDoesNotExistError, StageFileBadNameError) as exc: - raise DvcException( - str(exc) + " Did you mean 'git checkout {}'?".format(target) - ) - else: - stages = self.active_stages(target) + try: + stages = self.collect(target, with_deps=with_deps, recursive=recursive) + except (StageFileDoesNotExistError, StageFileBadNameError) as exc: + if not target: + raise + raise DvcException( + str(exc) + " Did you mean 'git checkout {}'?".format(target) + ) with self.state: _cleanup_unused_links(self, all_stages) diff --git a/dvc/repo/commit.py b/dvc/repo/commit.py index ae4b9007c..eb297493a 100644 --- a/dvc/repo/commit.py +++ b/dvc/repo/commit.py @@ -1,9 +1,5 @@ def commit(self, target, with_deps=False, recursive=False, force=False): - if target and not recursive: - stages = self.collect(target, with_deps=with_deps) - else: - stages = self.active_stages(target) - + stages = self.collect(target, with_deps=with_deps, recursive=recursive) with self.state: for stage in stages: stage.check_can_commit(force=force) diff --git a/dvc/repo/status.py b/dvc/repo/status.py index 9a9013a46..b50130134 100644 --- a/dvc/repo/status.py +++ b/dvc/repo/status.py @@ -6,10 +6,7 @@ import dvc.logger as logger def _local_status(self, target=None, with_deps=False): status = {} - if target: - stages = self.collect(target, with_deps=with_deps) - else: - stages = self.active_stages() + stages = self.collect(target, with_deps=with_deps) for stage in stages: if stage.locked: From df971ec32dc292aaa1d4b7624d132187d1bbbc95 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 14 Mar 2019 11:57:55 +0200 Subject: [PATCH 3/6] dvc: introduce `dvc tag` Fixes #1766 Signed-off-by: Ruslan Kuprieiev --- dvc/cli.py | 2 + dvc/command/tag.py | 151 +++++++++++++++++++++++++++++++++++++++ dvc/output/__init__.py | 57 ++++++++++----- dvc/output/base.py | 19 ++++- dvc/output/hdfs.py | 2 + dvc/output/local.py | 2 + dvc/output/s3.py | 2 + dvc/output/ssh.py | 2 + dvc/repo/__init__.py | 2 + dvc/repo/tag/__init__.py | 18 +++++ dvc/repo/tag/add.py | 15 ++++ dvc/repo/tag/list.py | 11 +++ dvc/repo/tag/remove.py | 15 ++++ dvc/stage.py | 28 +++++++- 14 files changed, 306 insertions(+), 20 deletions(-) create mode 100644 dvc/command/tag.py create mode 100644 dvc/repo/tag/__init__.py create mode 100644 dvc/repo/tag/add.py create mode 100644 dvc/repo/tag/list.py create mode 100644 dvc/repo/tag/remove.py diff --git a/dvc/cli.py b/dvc/cli.py index 63d674c75..d8af5717a 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -31,6 +31,7 @@ import dvc.command.lock as lock import dvc.command.pipeline as pipeline import dvc.command.daemon as daemon import dvc.command.commit as commit +import dvc.command.tag as tag from dvc.exceptions import DvcParserError from dvc import VERSION @@ -60,6 +61,7 @@ COMMANDS = [ pipeline, daemon, commit, + tag, ] diff --git a/dvc/command/tag.py b/dvc/command/tag.py new file mode 100644 index 000000000..0db770fc7 --- /dev/null +++ b/dvc/command/tag.py @@ -0,0 +1,151 @@ +import yaml + +import dvc.logger as logger +from dvc.exceptions import DvcException +from dvc.command.base import CmdBase, fix_subparsers + + +class CmdTagAdd(CmdBase): + def run(self): + for target in self.args.targets: + try: + self.repo.tag.add( + self.args.tag, + target=target, + with_deps=self.args.with_deps, + recursive=self.args.recursive, + ) + except DvcException: + logger.error("failed to add tag") + return 1 + return 0 + + +class CmdTagRemove(CmdBase): + def run(self): + for target in self.args.targets: + try: + self.repo.tag.remove( + self.args.tag, + target=target, + with_deps=self.args.with_deps, + recursive=self.args.recursive, + ) + except DvcException: + logger.error("failed to remove tag") + return 1 + return 0 + + +class CmdTagList(CmdBase): + def run(self): + for target in self.args.targets: + try: + tags = self.repo.tag.list( + target, + with_deps=self.args.with_deps, + recursive=self.args.recursive, + ) + if tags: + logger.info(yaml.dump(tags, default_flow_style=False)) + except DvcException: + logger.error("failed list tags") + return 1 + return 0 + + +def add_parser(subparsers, parent_parser): + TAG_HELP = "A set of commands to manage dvc tags." + tag_parser = subparsers.add_parser( + "tag", + parents=[parent_parser], + description=append_doc_link(TAG_HELP, "tag"), + add_help=TAG_HELP, + ) + + tag_subparsers = tag_parser.add_subparsers( + dest="cmd", + help="Use dvc tag CMD --help to display command-specific help.", + ) + + fix_subparsers(tag_subparsers) + + TAG_ADD_HELP = "Add dvc tag." + tag_add_parser = tag_subparsers.add_parser( + "add", + parents=[parent_parser], + description=TAG_ADD_HELP, + help=TAG_ADD_HELP, + ) + tag_add_parser.add_argument("tag", help="Dvc tag.") + tag_add_parser.add_argument( + "targets", nargs="*", default=[None], help="Dvc files." + ) + tag_add_parser.add_argument( + "-d", + "--with-deps", + action="store_true", + default=False, + help="Add tag for all dependencies of the specified dvc file.", + ) + tag_add_parser.add_argument( + "-R", + "--recursive", + action="store_true", + default=False, + help="Add tag for subdirectories of the specified directory.", + ) + tag_add_parser.set_defaults(func=CmdTagAdd) + + TAG_REMOVE_HELP = "Remove dvc tag." + tag_remove_parser = tag_subparsers.add_parser( + "remove", + parents=[parent_parser], + description=TAG_REMOVE_HELP, + help=TAG_REMOVE_HELP, + ) + tag_remove_parser.add_argument("tag", help="Dvc tag.") + tag_remove_parser.add_argument( + "targets", nargs="*", default=[None], help="Dvc files." + ) + tag_remove_parser.add_argument( + "-d", + "--with-deps", + action="store_true", + default=False, + help="Remove tag for all dependencies of the specified dvc file.", + ) + tag_remove_parser.add_argument( + "-R", + "--recursive", + action="store_true", + default=False, + help="Remove tag for subdirectories of the specified directory.", + ) + tag_remove_parser.set_defaults(func=CmdTagRemove) + + TAG_LIST_HELP = "List dvc tags." + tag_list_parser = tag_subparsers.add_parser( + "list", + parents=[parent_parser], + description=TAG_LIST_HELP, + help=TAG_LIST_HELP, + ) + tag_list_parser.add_argument( + "targets", nargs="*", default=[None], help="Dvc files." + ) + tag_list_parser.add_argument( + "-d", + "--with-deps", + action="store_true", + default=False, + help="List tags for all dependencies of the specified dvc file.", + ) + tag_list_parser.add_argument( + "-R", + "--recursive", + action="store_true", + default=False, + help="List tags for subdirectories of the specified directory.", + ) + tag_list_parser.set_defaults(func=CmdTagList) diff --git a/dvc/output/__init__.py b/dvc/output/__init__.py index 4f7058782..2eb642205 100644 --- a/dvc/output/__init__.py +++ b/dvc/output/__init__.py @@ -33,26 +33,31 @@ OUTS_MAP = { "local": OutputLOCAL, } -SCHEMA = { - OutputBase.PARAM_PATH: str, - # NOTE: currently there are only 3 possible checksum names: - # - # 1) md5 (LOCAL, SSH, GS); - # 2) etag (S3); - # 3) checksum (HDFS); - # - # so when a few types of outputs share the same name, we only need - # specify it once. +# NOTE: currently there are only 3 possible checksum names: +# +# 1) md5 (LOCAL, SSH, GS); +# 2) etag (S3); +# 3) checksum (HDFS); +# +# so when a few types of outputs share the same name, we only need +# specify it once. +CHECKSUM_SCHEMA = { schema.Optional(RemoteLOCAL.PARAM_CHECKSUM): schema.Or(str, None), schema.Optional(RemoteS3.PARAM_CHECKSUM): schema.Or(str, None), schema.Optional(RemoteHDFS.PARAM_CHECKSUM): schema.Or(str, None), - schema.Optional(OutputBase.PARAM_CACHE): bool, - schema.Optional(OutputBase.PARAM_METRIC): OutputBase.METRIC_SCHEMA, - schema.Optional(OutputBase.PARAM_PERSIST): bool, } +TAGS_SCHEMA = {schema.Optional(str): CHECKSUM_SCHEMA} -def _get(stage, p, info, cache, metric, persist): +SCHEMA = CHECKSUM_SCHEMA.copy() +SCHEMA[OutputBase.PARAM_PATH] = str +SCHEMA[schema.Optional(OutputBase.PARAM_CACHE)] = bool +SCHEMA[schema.Optional(OutputBase.PARAM_METRIC)] = OutputBase.METRIC_SCHEMA +SCHEMA[schema.Optional(OutputBase.PARAM_TAGS)] = TAGS_SCHEMA +SCHEMA[schema.Optional(OutputBase.PARAM_PERSIST)] = bool + + +def _get(stage, p, info, cache, metric, persist=False, tags=None): parsed = urlparse(p) if parsed.scheme == "remote": name = Config.SECTION_REMOTE_FMT.format(parsed.netloc) @@ -66,11 +71,21 @@ def _get(stage, p, info, cache, metric, persist): remote=remote, metric=metric, persist=persist, + tags=tags, ) for o in OUTS: if o.supported(p): - return o(stage, p, info, cache=cache, remote=None, metric=metric) + return o( + stage, + p, + info, + cache=cache, + remote=None, + metric=metric, + persist=persist, + tags=tags, + ) return OutputLOCAL( stage, p, @@ -79,6 +94,7 @@ def _get(stage, p, info, cache, metric, persist): remote=None, metric=metric, persist=persist, + tags=tags, ) @@ -89,8 +105,17 @@ def loadd_from(stage, d_list): cache = d.pop(OutputBase.PARAM_CACHE, True) metric = d.pop(OutputBase.PARAM_METRIC, False) persist = d.pop(OutputBase.PARAM_PERSIST, False) + tags = d.pop(OutputBase.PARAM_TAGS, None) ret.append( - _get(stage, p, info=d, cache=cache, metric=metric, persist=persist) + _get( + stage, + p, + info=d, + cache=cache, + metric=metric, + persist=persist, + tags=tags, + ) ) return ret diff --git a/dvc/output/base.py b/dvc/output/base.py index ff3d8de24..f262a1b80 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -47,6 +47,8 @@ class OutputBase(object): }, ) + PARAM_TAGS = "tags" + DoesNotExistError = OutputDoesNotExistError IsNotFileOrDirError = OutputIsNotFileOrDirError @@ -59,6 +61,7 @@ class OutputBase(object): cache=True, metric=False, persist=False, + tags=None, ): self.stage = stage self.repo = stage.repo @@ -68,6 +71,7 @@ class OutputBase(object): self.use_cache = False if self.IS_DEPENDENCY else cache self.metric = False if self.IS_DEPENDENCY else metric self.persist = persist + self.tags = None if self.IS_DEPENDENCY else (tags or {}) if ( self.use_cache @@ -197,6 +201,9 @@ class OutputBase(object): ret[self.PARAM_METRIC] = self.metric ret[self.PARAM_PERSIST] = self.persist + if self.tags: + ret[self.PARAM_TAGS] = self.tags + return ret def verify_metric(self): @@ -207,12 +214,20 @@ class OutputBase(object): def download(self, to_info, resume=False): self.remote.download([self.path_info], [to_info], resume=resume) - def checkout(self, force=False, progress_callback=None): + def checkout(self, force=False, progress_callback=None, tag=None): if not self.use_cache: return + if tag: + info = self.tags[tag] + else: + info = self.info + getattr(self.repo.cache, self.scheme).checkout( - self.path_info, self.info, force=force, progress_callback=progress_callback + self.path_info, + info, + force=force, + progress_callback=progress_callback, ) def remove(self, ignore_remove=False): diff --git a/dvc/output/hdfs.py b/dvc/output/hdfs.py index 4e6b7b8b3..0b076054f 100644 --- a/dvc/output/hdfs.py +++ b/dvc/output/hdfs.py @@ -19,6 +19,7 @@ class OutputHDFS(OutputBase): cache=True, metric=False, persist=False, + tags=None, ): super(OutputHDFS, self).__init__( stage, @@ -28,6 +29,7 @@ class OutputHDFS(OutputBase): cache=cache, metric=metric, persist=persist, + tags=tags, ) if remote: path = posixpath.join(remote.url, urlparse(path).path.lstrip("/")) diff --git a/dvc/output/local.py b/dvc/output/local.py index 6a76cdd8e..d4bcabf5d 100644 --- a/dvc/output/local.py +++ b/dvc/output/local.py @@ -22,6 +22,7 @@ class OutputLOCAL(OutputBase): cache=True, metric=False, persist=False, + tags=None, ): super(OutputLOCAL, self).__init__( stage, @@ -31,6 +32,7 @@ class OutputLOCAL(OutputBase): cache=cache, metric=metric, persist=persist, + tags=tags, ) if remote: p = os.path.join( diff --git a/dvc/output/s3.py b/dvc/output/s3.py index 6eebef7b0..20c38ded7 100644 --- a/dvc/output/s3.py +++ b/dvc/output/s3.py @@ -19,6 +19,7 @@ class OutputS3(OutputBase): cache=True, metric=False, persist=False, + tags=None, ): super(OutputS3, self).__init__( stage, @@ -28,6 +29,7 @@ class OutputS3(OutputBase): cache=cache, metric=metric, persist=persist, + tags=tags, ) bucket = remote.bucket if remote else urlparse(path).netloc path = urlparse(path).path.lstrip("/") diff --git a/dvc/output/ssh.py b/dvc/output/ssh.py index c0ed25dfa..ace06d2ec 100644 --- a/dvc/output/ssh.py +++ b/dvc/output/ssh.py @@ -20,6 +20,7 @@ class OutputSSH(OutputBase): cache=True, metric=False, persist=False, + tags=None, ): super(OutputSSH, self).__init__( stage, @@ -29,6 +30,7 @@ class OutputSSH(OutputBase): cache=cache, metric=metric, persist=persist, + tags=tags, ) parsed = urlparse(path) host = remote.host if remote else parsed.hostname diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index a169f3a57..43ecbe600 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -45,6 +45,7 @@ class Repo(object): from dvc.updater import Updater from dvc.repo.metrics import Metrics from dvc.scm.tree import WorkingTree + from dvc.repo.tag import Tag root_dir = self.find_root(root_dir) @@ -70,6 +71,7 @@ class Repo(object): self.updater = Updater(self.dvc_dir) self.metrics = Metrics(self) + self.tag = Tag(self) self._ignore() diff --git a/dvc/repo/tag/__init__.py b/dvc/repo/tag/__init__.py new file mode 100644 index 000000000..7cd64dea8 --- /dev/null +++ b/dvc/repo/tag/__init__.py @@ -0,0 +1,18 @@ +class Tag(object): + def __init__(self, repo): + self.repo = repo + + def add(self, *args, **kwargs): + from dvc.repo.tag.add import add + + return add(self.repo, *args, **kwargs) + + def list(self, *args, **kwargs): + from dvc.repo.tag.list import list + + return list(self.repo, *args, **kwargs) + + def remove(self, *args, **kwargs): + from dvc.repo.tag.remove import remove + + return remove(self.repo, *args, **kwargs) diff --git a/dvc/repo/tag/add.py b/dvc/repo/tag/add.py new file mode 100644 index 000000000..5a06f2129 --- /dev/null +++ b/dvc/repo/tag/add.py @@ -0,0 +1,15 @@ +import dvc.logger as logger + + +def add(self, tag, target=None, with_deps=False, recursive=False): + stages = self.collect(target, with_deps=with_deps, recursive=recursive) + for stage in stages: + changed = False + for out in stage.outs: + if not out.info: + logger.warning("missing checksum info for '{}'".format(out)) + continue + out.tags[tag] = out.info.copy() + changed = True + if changed: + stage.dump() diff --git a/dvc/repo/tag/list.py b/dvc/repo/tag/list.py new file mode 100644 index 000000000..cad7083ad --- /dev/null +++ b/dvc/repo/tag/list.py @@ -0,0 +1,11 @@ +def list(self, target=None, with_deps=False, recursive=False): + ret = {} + stages = self.collect(target, with_deps=with_deps, recursive=recursive) + for stage in stages: + outs_tags = {} + for out in stage.outs: + if out.tags: + outs_tags[str(out)] = out.tags.copy() + if outs_tags: + ret.update({stage.relpath: outs_tags}) + return ret diff --git a/dvc/repo/tag/remove.py b/dvc/repo/tag/remove.py new file mode 100644 index 000000000..f5f29f8bc --- /dev/null +++ b/dvc/repo/tag/remove.py @@ -0,0 +1,15 @@ +import dvc.logger as logger + + +def remove(self, tag, target=None, with_deps=False, recursive=False): + stages = self.collect(target, with_deps=with_deps, recursive=recursive) + for stage in stages: + changed = False + for out in stage.outs: + if tag not in out.tags.keys(): + logger.warning("tag '{}' not found for '{}'".format(tag, out)) + continue + del out.tags[tag] + changed = True + if changed: + stage.dump() diff --git a/dvc/stage.py b/dvc/stage.py index 913b21e0a..454ad838e 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals from dvc.utils.compat import str, open +import re import os import yaml import subprocess @@ -129,6 +130,8 @@ class Stage(object): Optional(PARAM_LOCKED): bool, } + TAG_REGEX = r"^(?P.*)@(?P[^\\/@:]*)$" + def __init__( self, repo, @@ -139,6 +142,7 @@ class Stage(object): outs=None, md5=None, locked=False, + tag=None, ): if deps is None: deps = [] @@ -153,6 +157,7 @@ class Stage(object): self.deps = deps self.md5 = md5 self.locked = locked + self.tag = tag def __repr__(self): return "Stage: '{path}'".format( @@ -536,8 +541,17 @@ class Stage(object): if not repo.tree.isfile(fname): raise StageFileIsNotDvcFileError(fname) + @classmethod + def _get_path_tag(cls, s): + regex = re.compile(cls.TAG_REGEX) + match = regex.match(s) + if not match: + return s, None + return match.group("path"), match.group("tag") + @staticmethod def load(repo, fname): + fname, tag = Stage._get_path_tag(fname) # it raises the proper exceptions by priority: # 1. when the file doesn't exists @@ -563,6 +577,7 @@ class Stage(object): cmd=d.get(Stage.PARAM_CMD), md5=d.get(Stage.PARAM_MD5), locked=d.get(Stage.PARAM_LOCKED, False), + tag=tag, ) stage.deps = dependency.loadd_from(stage, d.get(Stage.PARAM_DEPS, [])) @@ -624,7 +639,14 @@ class Stage(object): # NOTE: excluding parameters that don't affect the state of the # pipeline. Not excluding `OutputLOCAL.PARAM_CACHE`, because if # it has changed, we might not have that output in our cache. - m = dict_md5(d, exclude=[self.PARAM_LOCKED, OutputLOCAL.PARAM_METRIC]) + m = dict_md5( + d, + exclude=[ + self.PARAM_LOCKED, + OutputLOCAL.PARAM_METRIC, + OutputLOCAL.PARAM_TAGS, + ], + ) logger.debug("Computed stage '{}' md5: '{}'".format(self.relpath, m)) return m @@ -791,7 +813,9 @@ class Stage(object): def checkout(self, force=False, progress_callback=None): for out in self.outs: - out.checkout(force=force, progress_callback=progress_callback) + out.checkout( + force=force, tag=self.tag, progress_callback=progress_callback + ) @staticmethod def _status(entries): From 6e4d74282a89ee7a7730c003af2be308902859b4 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 28 Mar 2019 17:42:27 +0200 Subject: [PATCH 4/6] dvc: temporarily hide `dvc tag` This feature is not yet ready for public use. Signed-off-by: Ruslan Kuprieiev --- dvc/command/tag.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/dvc/command/tag.py b/dvc/command/tag.py index 0db770fc7..ed2367d67 100644 --- a/dvc/command/tag.py +++ b/dvc/command/tag.py @@ -2,7 +2,7 @@ import yaml import dvc.logger as logger from dvc.exceptions import DvcException -from dvc.command.base import CmdBase, fix_subparsers +from dvc.command.base import CmdBase, fix_subparsers, append_doc_link class CmdTagAdd(CmdBase): @@ -55,26 +55,26 @@ class CmdTagList(CmdBase): def add_parser(subparsers, parent_parser): - TAG_HELP = "A set of commands to manage dvc tags." + TAG_HELP = "A set of commands to manage DVC tags." tag_parser = subparsers.add_parser( "tag", parents=[parent_parser], description=append_doc_link(TAG_HELP, "tag"), - add_help=TAG_HELP, + add_help=False, ) tag_subparsers = tag_parser.add_subparsers( dest="cmd", - help="Use dvc tag CMD --help to display command-specific help.", + help="Use DVC tag CMD --help to display command-specific help.", ) fix_subparsers(tag_subparsers) - TAG_ADD_HELP = "Add dvc tag." + TAG_ADD_HELP = "Add DVC tag." tag_add_parser = tag_subparsers.add_parser( "add", parents=[parent_parser], - description=TAG_ADD_HELP, + description=append_doc_link(TAG_ADD_HELP, "tag-add"), help=TAG_ADD_HELP, ) tag_add_parser.add_argument("tag", help="Dvc tag.") @@ -86,7 +86,7 @@ def add_parser(subparsers, parent_parser): "--with-deps", action="store_true", default=False, - help="Add tag for all dependencies of the specified dvc file.", + help="Add tag for all dependencies of the specified DVC file.", ) tag_add_parser.add_argument( "-R", @@ -97,11 +97,11 @@ def add_parser(subparsers, parent_parser): ) tag_add_parser.set_defaults(func=CmdTagAdd) - TAG_REMOVE_HELP = "Remove dvc tag." + TAG_REMOVE_HELP = "Remove DVC tag." tag_remove_parser = tag_subparsers.add_parser( "remove", parents=[parent_parser], - description=TAG_REMOVE_HELP, + description=append_doc_link(TAG_REMOVE_HELP, "tag-remove"), help=TAG_REMOVE_HELP, ) tag_remove_parser.add_argument("tag", help="Dvc tag.") @@ -113,7 +113,7 @@ def add_parser(subparsers, parent_parser): "--with-deps", action="store_true", default=False, - help="Remove tag for all dependencies of the specified dvc file.", + help="Remove tag for all dependencies of the specified DVC file.", ) tag_remove_parser.add_argument( "-R", @@ -124,11 +124,11 @@ def add_parser(subparsers, parent_parser): ) tag_remove_parser.set_defaults(func=CmdTagRemove) - TAG_LIST_HELP = "List dvc tags." + TAG_LIST_HELP = "List DVC tags." tag_list_parser = tag_subparsers.add_parser( "list", parents=[parent_parser], - description=TAG_LIST_HELP, + description=append_doc_link(TAG_LIST_HELP, "tag-list"), help=TAG_LIST_HELP, ) tag_list_parser.add_argument( @@ -139,7 +139,7 @@ def add_parser(subparsers, parent_parser): "--with-deps", action="store_true", default=False, - help="List tags for all dependencies of the specified dvc file.", + help="List tags for all dependencies of the specified DVC file.", ) tag_list_parser.add_argument( "-R", From c03b1c16938572aea70dd22b838459aeec585b0d Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 28 Mar 2019 02:53:05 +0200 Subject: [PATCH 5/6] test: add tests for `dvc tag` Signed-off-by: Ruslan Kuprieiev --- tests/test_tag.py | 128 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 tests/test_tag.py diff --git a/tests/test_tag.py b/tests/test_tag.py new file mode 100644 index 000000000..07244746e --- /dev/null +++ b/tests/test_tag.py @@ -0,0 +1,128 @@ +import os +import shutil +import filecmp + +from dvc.main import main +from dvc.logger import logger + +from tests.basic_env import TestDvc + +from tests.utils import reset_logger_standard_output, reset_logger_error_output +from tests.utils.logger import MockLoggerHandlers, ConsoleFontColorsRemover + + +class TestTag(TestDvc): + def test(self): + fname = "file" + shutil.copyfile(self.FOO, fname) + + stages = self.dvc.add(fname) + self.assertEqual(len(stages), 1) + stage = stages[0] + self.assertTrue(stage is not None) + + ret = main(["tag", "add", "v1", stage.path]) + self.assertEqual(ret, 0) + + os.unlink(fname) + shutil.copyfile(self.BAR, fname) + + ret = main(["repro", stage.path]) + self.assertEqual(ret, 0) + + ret = main(["tag", "add", "v2", stage.path]) + self.assertEqual(ret, 0) + + ret = main(["tag", "list", stage.path]) + self.assertEqual(ret, 0) + + ret = main(["checkout", stage.path + "@v1"]) + self.assertEqual(ret, 0) + + self.assertTrue(filecmp.cmp(fname, self.FOO, shallow=False)) + + ret = main(["checkout", stage.path + "@v2"]) + self.assertEqual(ret, 0) + + self.assertTrue(filecmp.cmp(fname, self.BAR, shallow=False)) + + +class TestTagAll(TestDvc): + def test(self): + ret = main(["add", self.FOO, self.BAR]) + self.assertEqual(ret, 0) + + with MockLoggerHandlers(logger), ConsoleFontColorsRemover(): + reset_logger_standard_output() + + ret = main(["tag", "list"]) + self.assertEqual(ret, 0) + + self.assertEqual("", logger.handlers[0].stream.getvalue()) + + ret = main(["tag", "add", "v1"]) + self.assertEqual(ret, 0) + + with MockLoggerHandlers(logger), ConsoleFontColorsRemover(): + reset_logger_standard_output() + + ret = main(["tag", "list"]) + self.assertEqual(ret, 0) + + self.assertEqual( + logger.handlers[0].stream.getvalue(), + "bar.dvc:\n" + " bar:\n" + " v1:\n" + " md5: 8978c98bb5a48c2fb5f2c4c905768afa\n" + "foo.dvc:\n" + " foo:\n" + " v1:\n" + " md5: acbd18db4cc2f85cedef654fccc4a4d8\n" + "\n", + ) + + ret = main(["tag", "remove", "v1"]) + self.assertEqual(ret, 0) + + with MockLoggerHandlers(logger), ConsoleFontColorsRemover(): + reset_logger_standard_output() + + ret = main(["tag", "list"]) + self.assertEqual(ret, 0) + + self.assertEqual("", logger.handlers[0].stream.getvalue()) + + +class TestTagAddNoChecksumInfo(TestDvc): + def test(self): + ret = main(["run", "-o", self.FOO, "--no-exec"]) + self.assertEqual(ret, 0) + + with MockLoggerHandlers(logger), ConsoleFontColorsRemover(): + reset_logger_error_output() + + ret = main(["tag", "add", "v1", "foo.dvc"]) + self.assertEqual(ret, 0) + + self.assertEqual( + "Warning: missing checksum info for 'foo'\n", + logger.handlers[1].stream.getvalue(), + ) + + +class TestTagRemoveNoTag(TestDvc): + def test(self): + ret = main(["add", self.FOO]) + self.assertEqual(ret, 0) + + with MockLoggerHandlers(logger), ConsoleFontColorsRemover(): + reset_logger_error_output() + + ret = main(["tag", "remove", "v1", "foo.dvc"]) + self.assertEqual(ret, 0) + + self.assertEqual( + "Warning: tag 'v1' not found for 'foo'\n", + logger.handlers[1].stream.getvalue(), + ) From 4c5cb7afaed9413a39e15664a729c6dffafb0c18 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Fri, 29 Mar 2019 12:56:37 +0200 Subject: [PATCH 6/6] checkout: reorganize "`git checkout` hint" exception Signed-off-by: Ruslan Kuprieiev --- dvc/exceptions.py | 7 +++++++ dvc/repo/checkout.py | 6 ++---- tests/test_checkout.py | 29 ++++++++--------------------- 3 files changed, 17 insertions(+), 25 deletions(-) diff --git a/dvc/exceptions.py b/dvc/exceptions.py index 57410a892..92bdb9169 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -248,3 +248,10 @@ class TargetNotDirectoryError(DvcException): super(TargetNotDirectoryError, self).__init__( "Target: {} is not a directory".format(path) ) + + +class CheckoutErrorSuggestGit(DvcException): + def __init__(self, target, cause): + super(CheckoutErrorSuggestGit, self).__init__( + "Did you mean 'git checkout {}'?".format(target), cause=cause + ) diff --git a/dvc/repo/checkout.py b/dvc/repo/checkout.py index 883f21c4c..d5365a893 100644 --- a/dvc/repo/checkout.py +++ b/dvc/repo/checkout.py @@ -1,7 +1,7 @@ from __future__ import unicode_literals import dvc.logger as logger -from dvc.exceptions import DvcException +from dvc.exceptions import CheckoutErrorSuggestGit from dvc.progress import ProgressCallback @@ -34,9 +34,7 @@ def checkout(self, target=None, with_deps=False, force=False, recursive=False): except (StageFileDoesNotExistError, StageFileBadNameError) as exc: if not target: raise - raise DvcException( - str(exc) + " Did you mean 'git checkout {}'?".format(target) - ) + raise CheckoutErrorSuggestGit(target, exc) with self.state: _cleanup_unused_links(self, all_stages) diff --git a/tests/test_checkout.py b/tests/test_checkout.py index a07d24bc6..852223da1 100644 --- a/tests/test_checkout.py +++ b/tests/test_checkout.py @@ -15,12 +15,13 @@ from dvc.system import System from dvc.utils import walk_files from tests.basic_env import TestDvc from tests.test_repro import TestRepro -from dvc.stage import Stage +from dvc.stage import Stage, StageFileBadNameError, StageFileDoesNotExistError from dvc.remote.local import RemoteLOCAL from dvc.exceptions import ( DvcException, ConfirmRemoveError, TargetNotDirectoryError, + CheckoutErrorSuggestGit, ) from mock import patch @@ -385,30 +386,16 @@ class TestCheckoutSuggestGit(TestRepro): try: self.dvc.checkout(target="gitbranch") except DvcException as exc: - msg = str(exc) - self.assertEqual( - msg, - ( - "'gitbranch' does not exist. " - "Did you mean 'git checkout gitbranch'?" - ), - ) + self.assertIsInstance(exc, CheckoutErrorSuggestGit) + self.assertIsInstance(exc.cause, StageFileDoesNotExistError) + self.assertIsNone(exc.cause.cause) try: self.dvc.checkout(target=self.FOO) except DvcException as exc: - msg = str(exc) - self.assertEqual( - msg, - ( - "bad stage filename '{}'." - " Stage files should be named 'Dvcfile'" - " or have a '.dvc' suffix (e.g. '{}.dvc')." - " Did you mean 'git checkout {}'?".format( - self.FOO, self.FOO, self.FOO - ) - ), - ) + self.assertIsInstance(exc, CheckoutErrorSuggestGit) + self.assertIsInstance(exc.cause, StageFileBadNameError) + self.assertIsNone(exc.cause.cause) class TestCheckoutShouldHaveSelfClearingProgressBar(TestDvc):