From cc2ebb2099cf6ede3ad2b4496abafa07420d02ae Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Mon, 1 Oct 2018 12:13:17 +0300 Subject: [PATCH] dvc: introduce `-T|--all-tags` option Fixes #1150 Signed-off-by: Ruslan Kuprieiev --- dvc/cli.py | 37 +++++++++++++++++++++++++++++++++++ dvc/command/data_sync.py | 3 +++ dvc/command/gc.py | 1 + dvc/command/metrics.py | 3 ++- dvc/command/status.py | 1 + dvc/project.py | 42 +++++++++++++++++++++++++++------------- dvc/scm.py | 37 ++++++++++++++++++++++++++++------- 7 files changed, 103 insertions(+), 21 deletions(-) diff --git a/dvc/cli.py b/dvc/cli.py index 71ee097c5..1b37eb72c 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -287,6 +287,12 @@ def parse_args(argv=None): action='store_true', default=False, help='Fetch cache for all branches.') + pull_parser.add_argument( + '-T', + '--all-tags', + action='store_true', + default=False, + help='Fetch cache for all tags.') pull_parser.add_argument( '-d', '--with-deps', @@ -313,6 +319,12 @@ def parse_args(argv=None): action='store_true', default=False, help='Push cache for all branches.') + push_parser.add_argument( + '-T', + '--all-tags', + action='store_true', + default=False, + help='Push cache for all tags.') push_parser.add_argument( '-d', '--with-deps', @@ -339,6 +351,12 @@ def parse_args(argv=None): action='store_true', default=False, help='Fetch cache for all branches.') + fetch_parser.add_argument( + '-T', + '--all-tags', + action='store_true', + default=False, + help='Fetch cache for all tags.') fetch_parser.add_argument( '-d', '--with-deps', @@ -373,6 +391,13 @@ def parse_args(argv=None): default=False, help='Show status of a local cache compared to a ' 'remote repository for all branches.') + status_parser.add_argument( + '-T', + '--all-tags', + action='store_true', + default=False, + help='Show status of a local cache compared to a ' + 'remote repository for all tags.') status_parser.add_argument( '-d', '--with-deps', @@ -501,6 +526,12 @@ def parse_args(argv=None): action='store_true', default=False, help='Collect garbage for all branches.') + gc_parser.add_argument( + '-T', + '--all-tags', + action='store_true', + default=False, + help='Collect garbage for all tags.') gc_parser.add_argument( '-c', '--cloud', @@ -694,6 +725,12 @@ def parse_args(argv=None): action='store_true', default=False, help='Show metrics for all branches.') + metrics_show_parser.add_argument( + '-T', + '--all-tags', + action='store_true', + default=False, + help='Show metrics for all tags.') metrics_show_parser.set_defaults(func=CmdMetricsShow) METRICS_ADD_HELP = 'Add metrics.' diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py index 9fcaef872..839da86f4 100644 --- a/dvc/command/data_sync.py +++ b/dvc/command/data_sync.py @@ -24,6 +24,7 @@ class CmdDataPull(CmdDataBase): remote=self.args.remote, show_checksums=self.args.show_checksums, all_branches=self.args.all_branches, + all_tags=self.args.all_tags, with_deps=self.args.with_deps) except Exception as exc: self.project.logger.error('Failed to pull data from the cloud', @@ -40,6 +41,7 @@ class CmdDataPush(CmdDataBase): remote=self.args.remote, show_checksums=self.args.show_checksums, all_branches=self.args.all_branches, + all_tags=self.args.all_tags, with_deps=self.args.with_deps) except Exception as exc: self.project.logger.error('Failed to push data to the cloud', exc) @@ -55,6 +57,7 @@ class CmdDataFetch(CmdDataBase): remote=self.args.remote, show_checksums=self.args.show_checksums, all_branches=self.args.all_branches, + all_tags=self.args.all_tags, with_deps=self.args.with_deps) except Exception as exc: self.project.logger.error('Failed to fetch data from the cloud', diff --git a/dvc/command/gc.py b/dvc/command/gc.py index 2bd9fcaff..a43a05bb9 100644 --- a/dvc/command/gc.py +++ b/dvc/command/gc.py @@ -4,6 +4,7 @@ from dvc.command.base import CmdBase class CmdGC(CmdBase): def run(self): self.project.gc(all_branches=self.args.all_branches, + all_tags=self.args.all_tags, cloud=self.args.cloud, remote=self.args.remote) return 0 diff --git a/dvc/command/metrics.py b/dvc/command/metrics.py index e9eac7324..e0da68767 100644 --- a/dvc/command/metrics.py +++ b/dvc/command/metrics.py @@ -28,7 +28,8 @@ class CmdMetricsShow(CmdBase): self.project.metrics_show(self.args.path, typ=typ, xpath=xpath, - all_branches=self.args.all_branches) + all_branches=self.args.all_branches, + all_tags=self.args.all_tags) except DvcException as exc: self.project.logger.error('Failed to show metrics', exc) return 1 diff --git a/dvc/command/status.py b/dvc/command/status.py index 5b0bce2da..aeae8652f 100644 --- a/dvc/command/status.py +++ b/dvc/command/status.py @@ -31,6 +31,7 @@ class CmdDataStatus(CmdDataBase): show_checksums=self.args.show_checksums, remote=self.args.remote, all_branches=self.args.all_branches, + all_tags=self.args.all_tags, with_deps=self.args.with_deps) if st: self._show(st, indent) diff --git a/dvc/project.py b/dvc/project.py index 91d92fca2..0bb795044 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -451,7 +451,8 @@ class Project(object): target=None, all_branches=False, active=True, - with_deps=False): + with_deps=False, + all_tags=False): cache = {} cache['local'] = [] cache['s3'] = [] @@ -459,7 +460,8 @@ class Project(object): cache['hdfs'] = [] cache['ssh'] = [] - for branch in self.scm.brancher(all_branches=all_branches): + for branch in self.scm.brancher(all_branches=all_branches, + all_tags=all_tags): if target: stages = self._collect(target, with_deps=with_deps) @@ -494,12 +496,14 @@ class Project(object): all_branches=False, cloud=False, remote=None, - with_deps=False): + with_deps=False, + all_tags=False): with self.state: clist = self._used_cache(target=None, all_branches=all_branches, active=False, - with_deps=with_deps) + with_deps=with_deps, + all_tags=all_tags) self._do_gc('local', self.cache.local.gc, clist) if self.cache.s3: @@ -527,10 +531,12 @@ class Project(object): remote=None, all_branches=False, show_checksums=False, - with_deps=False): + with_deps=False, + all_tags=False): with self.state: used = self._used_cache(target, all_branches=all_branches, + all_tags=all_tags, with_deps=with_deps)['local'] self.cloud.push(used, jobs, @@ -543,10 +549,12 @@ class Project(object): remote=None, all_branches=False, show_checksums=False, - with_deps=False): + with_deps=False, + all_tags=False): with self.state: used = self._used_cache(target, all_branches=all_branches, + all_tags=all_tags, with_deps=with_deps)['local'] self.cloud.pull(used, jobs, @@ -559,11 +567,13 @@ class Project(object): remote=None, all_branches=False, show_checksums=False, - with_deps=False): + with_deps=False, + all_tags=False): self.fetch(target, jobs, remote=remote, all_branches=all_branches, + all_tags=all_tags, show_checksums=show_checksums, with_deps=with_deps) self.checkout(target=target) @@ -592,11 +602,13 @@ class Project(object): remote=None, show_checksums=False, all_branches=False, - with_deps=False): + with_deps=False, + all_tags=False): import dvc.remote.base as cloud used = self._used_cache(target, all_branches=all_branches, + all_tags=all_tags, with_deps=with_deps)['local'] status = {} @@ -623,7 +635,8 @@ class Project(object): remote=None, show_checksums=False, all_branches=False, - with_deps=False): + with_deps=False, + all_tags=False): with self.state: if cloud: return self._cloud_status(target, @@ -631,7 +644,8 @@ class Project(object): remote=remote, show_checksums=show_checksums, all_branches=all_branches, - with_deps=with_deps) + with_deps=with_deps, + all_tags=all_tags) return self._local_status(target) def _read_metric_json(self, fd, json_path): @@ -712,9 +726,11 @@ class Project(object): path=None, typ=None, xpath=None, - all_branches=False): + all_branches=False, + all_tags=False): res = {} - for branch in self.scm.brancher(all_branches=all_branches): + for branch in self.scm.brancher(all_branches=all_branches, + all_tags=all_tags): astages = self.active_stages() outs = [out for stage in astages for out in stage.outs] @@ -759,7 +775,7 @@ class Project(object): res[branch][rel] = metric for branch, val in res.items(): - if all_branches: + if all_branches or all_tags: self.logger.info('{}:'.format(branch)) for fname, metric in val.items(): self.logger.info('\t{}: {}'.format(fname, metric)) diff --git a/dvc/scm.py b/dvc/scm.py index 63333c716..2c7f35b1d 100644 --- a/dvc/scm.py +++ b/dvc/scm.py @@ -50,17 +50,34 @@ class Base(object): def branch(self, branch): pass - def brancher(self, branches=None, all_branches=False): - if not branches and not all_branches: + def brancher(self, + branches=None, + all_branches=False, + tags=None, + all_tags=False): + if not branches and not all_branches \ + and not tags and not all_tags: yield '' return saved = self.active_branch() - if not branches: - branches = self.list_branches() if all_branches else [saved] - for branch in branches: - self.checkout(branch) - yield branch + revs = [] + + if branches is not None: + revs.extend(branches) + elif all_branches: + revs.extend(self.list_branches()) + elif tags is not None: + revs.extend(tags) + elif all_tags: + revs.extend(self.list_tags()) + else: + revs.extend([saved]) + + for rev in revs: + self.checkout(rev) + yield rev + self.checkout(saved) def untracked_files(self): @@ -75,6 +92,9 @@ class Base(object): def list_branches(self): pass + def list_tags(self): + pass + def install(self): pass @@ -208,6 +228,9 @@ class Git(Base): def list_branches(self): return [h.name for h in self.repo.heads] + def list_tags(self): + return [t.name for t in self.repo.tags] + def install(self): hook = os.path.join(self.root_dir, self.GIT_DIR,