gc: add option to collect cache for multiple projects

dvc/project.py

@@ -1,3 +1,4 @@
+import collections
 import os
 
 from dvc.exceptions import DvcException, MoveNotDataSourceError
@@ -108,6 +109,19 @@ class Project(object):
 
         return proj
 
+    @staticmethod
+    def load_all(projects_paths):
+        """
+        Instantiate all projects in the given list of paths.
+
+        Args:
+            projects_paths: List of paths to projects.
+
+        Returns:
+            List of Project instances in the same order of the given paths.
+        """
+        return [Project(path) for path in projects_paths]
+
     def destroy(self):
         import shutil
 
@@ -602,6 +616,45 @@ class Project(object):
 
         return cache
 
+    @staticmethod
+    def merge_cache_lists(clists):
+        merged_cache = collections.defaultdict(list)
+
+        for cache_list in clists:
+            for scheme, cache in cache_list.items():
+                for item in cache:
+                    if item not in merged_cache[scheme]:
+                        merged_cache[scheme].append(item)
+
+        return merged_cache
+
+    @staticmethod
+    def load_all_used_cache(projects,
+                            target=None,
+                            all_branches=False,
+                            active=True,
+                            with_deps=False,
+                            all_tags=False,
+                            remote=None,
+                            force=False,
+                            jobs=None):
+        clists = []
+
+        for project in projects:
+            with project.state:
+                project_clist = project._used_cache(target=None,
+                                                    all_branches=all_branches,
+                                                    active=False,
+                                                    with_deps=with_deps,
+                                                    all_tags=all_tags,
+                                                    remote=remote,
+                                                    force=force,
+                                                    jobs=jobs)
+
+                clists.append(project_clist)
+
+        return clists
+
     def _do_gc(self, typ, func, clist):
         removed = func(clist)
         if not removed:
@@ -614,16 +667,30 @@ class Project(object):
            with_deps=False,
            all_tags=False,
            force=False,
-           jobs=None):
+           jobs=None,
+           projects=None):
+
+        all_projects = [self]
+
+        if projects is not None and len(projects) > 0:
+            all_projects.extend(Project.load_all(projects))
+
+        all_clists = Project.load_all_used_cache(all_projects,
+                                                 target=None,
+                                                 all_branches=all_branches,
+                                                 active=False,
+                                                 with_deps=with_deps,
+                                                 all_tags=all_tags,
+                                                 remote=remote,
+                                                 force=force,
+                                                 jobs=jobs)
+
+        if len(all_clists) > 1:
+            clist = Project.merge_cache_lists(all_clists)
+        else:
+            clist = all_clists[0]
+
         with self.state:
-            clist = self._used_cache(target=None,
-                                     all_branches=all_branches,
-                                     active=False,
-                                     with_deps=with_deps,
-                                     all_tags=all_tags,
-                                     remote=remote,
-                                     force=force,
-                                     jobs=jobs)
             self._do_gc('local', self.cache.local.gc, clist)
 
             if self.cache.s3:

tests/test_gc.py

@@ -1,13 +1,18 @@
 import os
 import shutil
 import filecmp
+import configobj
+
+from git import Repo
 
 from dvc.main import main
 from dvc.utils import file_md5
 from dvc.stage import Stage
+from dvc.project import Project
 from dvc.command.gc import CmdGC
 
 from tests.basic_env import TestDvc
+from tests.basic_env import TestDir
 
 
 class TestGC(TestDvc):
@@ -16,7 +21,8 @@ class TestGC(TestDvc):
 
         self.dvc.add(self.FOO)
         self.dvc.add(self.DATA_DIR)
-        self.good_cache = [self.dvc.cache.local.get(md5) for md5 in self.dvc.cache.local.all()]
+        self.good_cache = [self.dvc.cache.local.get(
+            md5) for md5 in self.dvc.cache.local.all()]
 
         self.bad_cache = []
         for i in ['123', '234', '345']:
@@ -100,3 +106,75 @@ class TestGCBranchesTags(TestDvc):
         self.dvc.gc(all_tags=True, all_branches=False)
 
         self._check_cache(1)
+
+
+class TestGCMultipleProjects(TestDvc):
+    def _check_cache(self, num):
+        total = 0
+        for root, dirs, files in os.walk(os.path.join('.dvc', 'cache')):
+            total += len(files)
+        self.assertEqual(total, num)
+
+    def setUp(self):
+        super(TestGCMultipleProjects, self).setUp()
+        self.additional_path = TestDir.mkdtemp()
+        self.additional_git = Repo.init(self.additional_path)
+        self.additional_dvc = Project.init(self.additional_path)
+
+        cache_path = os.path.join(self._root_dir, '.dvc', 'cache')
+        config_path = os.path.join(self.additional_path, '.dvc',
+                                   'config.local')
+        cfg = configobj.ConfigObj()
+        cfg.filename = config_path
+        cfg['cache'] = {
+            'dir': cache_path
+        }
+        cfg.write()
+
+        self.additional_dvc = Project(self.additional_path)
+
+    def test(self):
+
+        # ADD FILE ONLY IN MAIN PROJECT
+        fname = 'only_in_first'
+        with open(fname, 'w+') as fobj:
+            fobj.write('only in main project')
+
+        stages = self.dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+
+        # ADD FILE IN MAIN PROJECT THAT IS ALSO IN SECOND PROJECT
+        fname = 'in_both'
+        with open(fname, 'w+') as fobj:
+            fobj.write('in both projects')
+
+        stages = self.dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+
+        cwd = os.getcwd()
+        os.chdir(self.additional_path)
+        # ADD FILE ONLY IN SECOND PROJECT
+        fname = os.path.join(self.additional_path, 'only_in_second')
+        with open(fname, 'w+') as fobj:
+            fobj.write('only in additional project')
+
+        stages = self.additional_dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+
+        # ADD FILE IN SECOND PROJECT THAT IS ALSO IN MAIN PROJECT
+        fname = os.path.join(self.additional_path, 'in_both')
+        with open(fname, 'w+') as fobj:
+            fobj.write('in both projects')
+
+        stages = self.additional_dvc.add(fname)
+        self.assertEqual(len(stages), 1)
+
+        os.chdir(cwd)
+
+        self._check_cache(3)
+
+        self.dvc.gc(projects=[self.additional_path])
+        self._check_cache(3)
+
+        self.dvc.gc()
+        self._check_cache(2)