diff --git a/dvc/cli.py b/dvc/cli.py index 39ec74680..a49460f6f 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -591,6 +591,13 @@ def parse_args(argv=None): type=int, default=None, help='Number of jobs to run simultaneously.') + gc_parser.add_argument( + '-p', + '--projects', + type=str, + nargs='*', + default=None, + help='Collect garbage for all given projects.') gc_parser.set_defaults(func=CmdGC) # Config diff --git a/dvc/command/gc.py b/dvc/command/gc.py index f0b25ae49..a802bf08c 100644 --- a/dvc/command/gc.py +++ b/dvc/command/gc.py @@ -1,3 +1,5 @@ +import os + from dvc.command.base import CmdBase @@ -5,13 +7,21 @@ class CmdGC(CmdBase): def run(self): msg = 'This will remove all cache except the cache that is used in ' if not self.args.all_branches and not self.args.all_tags: - msg += 'the current git branch.' + msg += 'the current git branch' elif self.args.all_branches and not self.args.all_tags: - msg += 'all git branches.' + msg += 'all git branches' elif not self.args.all_branches and self.args.all_tags: - msg += 'all git tags.' + msg += 'all git tags' else: - msg += 'all git branches and all git tags.' + msg += 'all git branches and all git tags' + + if self.args.projects is not None and len(self.args.projects) > 0: + msg += ' of the current and the following projects:' + + for project_path in self.args.projects: + msg += '\n - %s' % os.path.abspath(project_path) + else: + msg += ' of the current project.' self.project.logger.warn(msg) @@ -24,5 +34,6 @@ class CmdGC(CmdBase): cloud=self.args.cloud, remote=self.args.remote, force=self.args.force, - jobs=self.args.jobs) + jobs=self.args.jobs, + projects=self.args.projects) return 0 diff --git a/dvc/project.py b/dvc/project.py index 0828dbce4..1c62a3a81 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -1,3 +1,4 @@ +import collections import os from dvc.exceptions import DvcException, MoveNotDataSourceError @@ -108,6 +109,19 @@ class Project(object): return proj + @staticmethod + def load_all(projects_paths): + """ + Instantiate all projects in the given list of paths. + + Args: + projects_paths: List of paths to projects. + + Returns: + List of Project instances in the same order of the given paths. + """ + return [Project(path) for path in projects_paths] + def destroy(self): import shutil @@ -602,6 +616,45 @@ class Project(object): return cache + @staticmethod + def merge_cache_lists(clists): + merged_cache = collections.defaultdict(list) + + for cache_list in clists: + for scheme, cache in cache_list.items(): + for item in cache: + if item not in merged_cache[scheme]: + merged_cache[scheme].append(item) + + return merged_cache + + @staticmethod + def load_all_used_cache(projects, + target=None, + all_branches=False, + active=True, + with_deps=False, + all_tags=False, + remote=None, + force=False, + jobs=None): + clists = [] + + for project in projects: + with project.state: + project_clist = project._used_cache(target=None, + all_branches=all_branches, + active=False, + with_deps=with_deps, + all_tags=all_tags, + remote=remote, + force=force, + jobs=jobs) + + clists.append(project_clist) + + return clists + def _do_gc(self, typ, func, clist): removed = func(clist) if not removed: @@ -614,16 +667,30 @@ class Project(object): with_deps=False, all_tags=False, force=False, - jobs=None): + jobs=None, + projects=None): + + all_projects = [self] + + if projects is not None and len(projects) > 0: + all_projects.extend(Project.load_all(projects)) + + all_clists = Project.load_all_used_cache(all_projects, + target=None, + all_branches=all_branches, + active=False, + with_deps=with_deps, + all_tags=all_tags, + remote=remote, + force=force, + jobs=jobs) + + if len(all_clists) > 1: + clist = Project.merge_cache_lists(all_clists) + else: + clist = all_clists[0] + with self.state: - clist = self._used_cache(target=None, - all_branches=all_branches, - active=False, - with_deps=with_deps, - all_tags=all_tags, - remote=remote, - force=force, - jobs=jobs) self._do_gc('local', self.cache.local.gc, clist) if self.cache.s3: diff --git a/tests/test_cli.py b/tests/test_cli.py index ee438b4e3..4aa22d47c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -129,6 +129,20 @@ class TestGC(TestDvc): self.assertIsInstance(args.func(args), CmdGC) +class TestGCMultipleProjects(TestDvc): + def test(self): + args = parse_args([ + 'gc', + '-p', + '/tmp/asdf', + '/tmp/xyz' + ]) + + self.assertIsInstance(args.func(args), CmdGC) + + self.assertEqual(args.projects, ['/tmp/asdf', '/tmp/xyz']) + + class TestConfig(TestDvc): def test(self): name = 'param' diff --git a/tests/test_gc.py b/tests/test_gc.py index dcafd6005..4467f0986 100644 --- a/tests/test_gc.py +++ b/tests/test_gc.py @@ -1,13 +1,18 @@ import os import shutil import filecmp +import configobj + +from git import Repo from dvc.main import main from dvc.utils import file_md5 from dvc.stage import Stage +from dvc.project import Project from dvc.command.gc import CmdGC from tests.basic_env import TestDvc +from tests.basic_env import TestDir class TestGC(TestDvc): @@ -16,7 +21,8 @@ class TestGC(TestDvc): self.dvc.add(self.FOO) self.dvc.add(self.DATA_DIR) - self.good_cache = [self.dvc.cache.local.get(md5) for md5 in self.dvc.cache.local.all()] + self.good_cache = [self.dvc.cache.local.get( + md5) for md5 in self.dvc.cache.local.all()] self.bad_cache = [] for i in ['123', '234', '345']: @@ -100,3 +106,75 @@ class TestGCBranchesTags(TestDvc): self.dvc.gc(all_tags=True, all_branches=False) self._check_cache(1) + + +class TestGCMultipleProjects(TestDvc): + def _check_cache(self, num): + total = 0 + for root, dirs, files in os.walk(os.path.join('.dvc', 'cache')): + total += len(files) + self.assertEqual(total, num) + + def setUp(self): + super(TestGCMultipleProjects, self).setUp() + self.additional_path = TestDir.mkdtemp() + self.additional_git = Repo.init(self.additional_path) + self.additional_dvc = Project.init(self.additional_path) + + cache_path = os.path.join(self._root_dir, '.dvc', 'cache') + config_path = os.path.join(self.additional_path, '.dvc', + 'config.local') + cfg = configobj.ConfigObj() + cfg.filename = config_path + cfg['cache'] = { + 'dir': cache_path + } + cfg.write() + + self.additional_dvc = Project(self.additional_path) + + def test(self): + + # ADD FILE ONLY IN MAIN PROJECT + fname = 'only_in_first' + with open(fname, 'w+') as fobj: + fobj.write('only in main project') + + stages = self.dvc.add(fname) + self.assertEqual(len(stages), 1) + + # ADD FILE IN MAIN PROJECT THAT IS ALSO IN SECOND PROJECT + fname = 'in_both' + with open(fname, 'w+') as fobj: + fobj.write('in both projects') + + stages = self.dvc.add(fname) + self.assertEqual(len(stages), 1) + + cwd = os.getcwd() + os.chdir(self.additional_path) + # ADD FILE ONLY IN SECOND PROJECT + fname = os.path.join(self.additional_path, 'only_in_second') + with open(fname, 'w+') as fobj: + fobj.write('only in additional project') + + stages = self.additional_dvc.add(fname) + self.assertEqual(len(stages), 1) + + # ADD FILE IN SECOND PROJECT THAT IS ALSO IN MAIN PROJECT + fname = os.path.join(self.additional_path, 'in_both') + with open(fname, 'w+') as fobj: + fobj.write('in both projects') + + stages = self.additional_dvc.add(fname) + self.assertEqual(len(stages), 1) + + os.chdir(cwd) + + self._check_cache(3) + + self.dvc.gc(projects=[self.additional_path]) + self._check_cache(3) + + self.dvc.gc() + self._check_cache(2)