From 30341438e00ed3bc89908a68af991dd317fbbdba Mon Sep 17 00:00:00 2001 From: Alexander Schepanovski Date: Wed, 23 Oct 2019 22:00:31 +0700 Subject: [PATCH] gc: added --all-commits (#2643) * dvc: fix branch suffixes in status/push/pull * gc: add --all-commits flag * gc: fix gc warning Say that we always leave things used in the working tree --- dvc/cache.py | 5 ++++- dvc/command/gc.py | 29 +++++++++++++++++++---------- dvc/repo/__init__.py | 5 ++++- dvc/repo/brancher.py | 28 ++++++++++++++++++---------- dvc/repo/gc.py | 2 ++ dvc/scm/base.py | 4 ++++ dvc/scm/git/__init__.py | 3 +++ tests/func/test_gc.py | 35 +++++++++++++++++++++++++++++++++++ 8 files changed, 89 insertions(+), 22 deletions(-) diff --git a/dvc/cache.py b/dvc/cache.py index 8379a3a0f..0b4c90404 100644 --- a/dvc/cache.py +++ b/dvc/cache.py @@ -127,6 +127,9 @@ class NamedCache(object): for scheme, src in cache._items.items(): dst = self._items[scheme] for checksum, names in src.items(): - dst[checksum].update(names) + if suffix: + dst[checksum].update(n + suffix for n in names) + else: + dst[checksum].update(names) self.repo.extend(cache.repo) diff --git a/dvc/command/gc.py b/dvc/command/gc.py index 27e756c54..699f88304 100644 --- a/dvc/command/gc.py +++ b/dvc/command/gc.py @@ -13,17 +13,19 @@ logger = logging.getLogger(__name__) class CmdGC(CmdBase): def run(self): - msg = "this will remove all cache except the cache that is used in " - if not self.args.all_branches and not self.args.all_tags: - msg += "the current git branch" - elif self.args.all_branches and not self.args.all_tags: - msg += "all git branches" - elif not self.args.all_branches and self.args.all_tags: - msg += "all git tags" - else: - msg += "all git branches and all git tags" + msg = "This will remove all cache except items used in " - if self.args.repos is not None and len(self.args.repos) > 0: + msg += "the working tree" + if self.args.all_commits: + msg += " and all git commits" + elif self.args.all_branches and self.args.all_tags: + msg += " and all git branches and tags" + elif self.args.all_branches: + msg += " and all git branches" + elif self.args.all_tags: + msg += " and all git tags" + + if self.args.repos: msg += " of the current and the following repos:" for repo_path in self.args.repos: @@ -40,6 +42,7 @@ class CmdGC(CmdBase): self.repo.gc( all_branches=self.args.all_branches, all_tags=self.args.all_tags, + all_commits=self.args.all_commits, cloud=self.args.cloud, remote=self.args.remote, force=self.args.force, @@ -76,6 +79,12 @@ def add_parser(subparsers, parent_parser): default=False, help="Keep data files for all git tags.", ) + gc_parser.add_argument( + "--all-commits", + action="store_true", + default=False, + help="Keep data files for all commits.", + ) gc_parser.add_argument( "-c", "--cloud", diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index c632fc1b7..97dfa3103 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -213,6 +213,7 @@ class Repo(object): all_branches=False, with_deps=False, all_tags=False, + all_commits=False, remote=None, force=False, jobs=None, @@ -236,7 +237,9 @@ class Repo(object): cache = NamedCache() for branch in self.brancher( - all_branches=all_branches, all_tags=all_tags + all_branches=all_branches, + all_tags=all_tags, + all_commits=all_commits, ): if targets: stages = [] diff --git a/dvc/repo/brancher.py b/dvc/repo/brancher.py index 0073a04ac..ce60fbd2f 100644 --- a/dvc/repo/brancher.py +++ b/dvc/repo/brancher.py @@ -3,7 +3,12 @@ from dvc.scm.tree import WorkingTree def brancher( # noqa: E302 - self, branches=None, all_branches=False, tags=None, all_tags=False + self, + branches=None, + all_branches=False, + tags=None, + all_tags=False, + all_commits=False, ): """Generator that iterates over specified revisions. @@ -19,7 +24,7 @@ def brancher( # noqa: E302 - empty string it there is no branches to iterate over - "Working Tree" if there are uncommited changes in the SCM repo """ - if not any([branches, all_branches, tags, all_tags]): + if not any([branches, all_branches, tags, all_tags, all_commits]): yield "" return @@ -31,17 +36,20 @@ def brancher( # noqa: E302 self.tree = WorkingTree(self.root_dir) yield "working tree" - if all_branches: - branches = scm.list_branches() + if all_commits: + revs = scm.list_all_commits() + else: + if all_branches: + branches = scm.list_branches() - if all_tags: - tags = scm.list_tags() + if all_tags: + tags = scm.list_tags() - if branches is not None: - revs.extend(branches) + if branches is not None: + revs.extend(branches) - if tags is not None: - revs.extend(tags) + if tags is not None: + revs.extend(tags) # NOTE: it might be a good idea to wrap this loop in try/finally block # to don't leave the tree on some unexpected branch after the diff --git a/dvc/repo/gc.py b/dvc/repo/gc.py index 4480232ab..a7f4f2ab3 100644 --- a/dvc/repo/gc.py +++ b/dvc/repo/gc.py @@ -23,6 +23,7 @@ def gc( remote=None, with_deps=False, all_tags=False, + all_commits=False, force=False, jobs=None, repos=None, @@ -47,6 +48,7 @@ def gc( all_branches=all_branches, with_deps=with_deps, all_tags=all_tags, + all_commits=all_commits, remote=remote, force=force, jobs=jobs, diff --git a/dvc/scm/base.py b/dvc/scm/base.py index d7ad9d2a5..e529745c2 100644 --- a/dvc/scm/base.py +++ b/dvc/scm/base.py @@ -120,6 +120,10 @@ class Base(object): """Returns a list of available tags in the repo.""" return [] + def list_all_commits(self): # pylint: disable=no-self-use + """Returns a list of commits in the repo.""" + return [] + def install(self): """Adds dvc commands to SCM hooks for the repo.""" diff --git a/dvc/scm/git/__init__.py b/dvc/scm/git/__init__.py index 980835e6c..05bb2faf1 100644 --- a/dvc/scm/git/__init__.py +++ b/dvc/scm/git/__init__.py @@ -245,6 +245,9 @@ class Git(Base): def list_tags(self): return [t.name for t in self.repo.tags] + def list_all_commits(self): + return [c.hexsha for c in self.repo.iter_commits("--all")] + def _install_hook(self, name, cmd): command = ( '[ "$3" = "0" ]' diff --git a/tests/func/test_gc.py b/tests/func/test_gc.py index deece6adb..d9a558617 100644 --- a/tests/func/test_gc.py +++ b/tests/func/test_gc.py @@ -1,8 +1,10 @@ +from __future__ import unicode_literals import os import configobj from git import Repo +from dvc.utils.compat import pathlib from dvc.main import main from dvc.repo import Repo as DvcRepo @@ -173,3 +175,36 @@ class TestGCMultipleDvcRepos(TestDvcGit): self.dvc.gc() self._check_cache(2) + + +def test_all_commits(git, dvc_repo): + def add_and_commit(): + stages = dvc_repo.add(str(testfile)) + dvc_repo.scm.add([s.relpath for s in stages]) + dvc_repo.scm.commit("message") + + cache_dir = os.path.join(dvc_repo.root_dir, ".dvc", "cache") + testfile = pathlib.Path("testfile") + + testfile.write_text("uncommited") + dvc_repo.add(str(testfile)) + + testfile.write_text("commited") + add_and_commit() + + testfile.write_text("modified") + add_and_commit() + + testfile.write_text("workspace") + dvc_repo.add(str(testfile)) + + N = _count_files(cache_dir) + + dvc_repo.gc(all_commits=True) + + # Only one uncommited file should go away + assert _count_files(cache_dir) == N - 1 + + +def _count_files(path): + return sum(len(files) for _, _, files in os.walk(path))