Merge pull request #878 from efiop/master

gc: introduce -c|--cloud

dvc/cache.py

@@ -30,6 +30,7 @@ class Cache(object):
         self.gs = self._get_remote(project, config, Config.SECTION_CACHE_GS)
         self.ssh = self._get_remote(project, config, Config.SECTION_CACHE_SSH)
         self.hdfs = self._get_remote(project, config, Config.SECTION_CACHE_HDFS)
+        self.azure = self._get_remote(project, config, Config.SECTION_CACHE_AZURE)
 
     def _get_remote(self, project, config, name):
         from dvc.remote import Remote

dvc/cli.py

@@ -314,6 +314,14 @@ def parse_args(argv=None):
                         action='store_true',
                         default=False,
                         help='Collect garbage for all branches.')
+    gc_parser.add_argument('-c',
+                        '--cloud',
+                        action='store_true',
+                        default=False,
+                        help='Collect garbage in remote repository')
+    gc_parser.add_argument('-r',
+                        '--remote',
+                        help='Remote repository to collect garbage in')
     gc_parser.set_defaults(func=CmdGC)
 
     # Config

dvc/command/gc.py

@@ -3,5 +3,7 @@ from dvc.command.common.base import CmdBase
 
 class CmdGC(CmdBase):
     def run(self):
-        self.project.gc()
+        self.project.gc(all_branches=self.args.all_branches,
+                        cloud=self.args.cloud,
+                        remote=self.args.remote)
         return 0

dvc/config.py

@@ -46,12 +46,14 @@ class Config(object):
     SECTION_CACHE_GS = 'gs'
     SECTION_CACHE_SSH = 'ssh'
     SECTION_CACHE_HDFS = 'hdfs'
+    SECTION_CACHE_AZURE = 'azure'
     SECTION_CACHE_SCHEMA = {
         schema.Optional(SECTION_CACHE_LOCAL): str,
         schema.Optional(SECTION_CACHE_S3): str,
         schema.Optional(SECTION_CACHE_GS): str,
         schema.Optional(SECTION_CACHE_HDFS): str,
         schema.Optional(SECTION_CACHE_SSH): str,
+        schema.Optional(SECTION_CACHE_AZURE): str,
 
         # backward compatibility
         schema.Optional(SECTION_CACHE_DIR, default='cache'): str,

dvc/project.py

@@ -291,21 +291,27 @@ class Project(object):
 
         return cache
 
-    def gc(self, all_branches=False):
+    def gc(self, all_branches=False, cloud=False, remote=None):
         clist = self._used_cache(target=None, all_branches=all_branches)
-        self.cache.local.gc(clist['local'])
+        self.cache.local.gc(clist)
 
         if self.cache.s3:
-            self.cache.s3.gc(clist['s3'])
+            self.cache.s3.gc(clist)
 
         if self.cache.gs:
-            self.cache.gs.gc(clist['gs'])
+            self.cache.gs.gc(clist)
 
         if self.cache.ssh:
-            self.cache.ssh.gc(clist['ssh'])
+            self.cache.ssh.gc(clist)
 
         if self.cache.hdfs:
-            self.cache.hdfs.gc(clist['hdfs'])
+            self.cache.hdfs.gc(clist)
+
+        if self.cache.azure:
+            self.cache.azure.gc(clist)
+
+        if cloud:
+            self.cloud._get_cloud(remote).gc(clist)
 
     def push(self, target=None, jobs=1, remote=None, all_branches=False):
         self.cloud.push(self._used_cache(target, all_branches)['local'], jobs, remote=remote)

dvc/remote/azure.py

@@ -17,6 +17,7 @@ from dvc.logger import Logger
 from dvc.progress import progress
 from dvc.config import Config
 from dvc.remote.base import RemoteBase
+from dvc.remote.local import RemoteLOCAL
 
 
 class Callback(object):
@@ -211,7 +212,8 @@ class RemoteAzure(RemoteBase):
                     progress.finish_target(name)
 
     def gc(self, checksum_infos):
-        used_etags = [info[self.PARAM_ETAG] for info in checksum_infos]
+        used_etags = [info[self.PARAM_ETAG] for info in checksum_infos['azure']]
+        used_etags += [info[RemoteLOCAL.PARAM_MD5] for info in checksum_infos['local']]
 
         all_blobs = self.blob_service.list_blobs(self.bucket)
 

dvc/remote/gs.py

@@ -13,6 +13,7 @@ except ImportError:
 
 from dvc.logger import Logger
 from dvc.remote.base import RemoteBase
+from dvc.remote.local import RemoteLOCAL
 from dvc.config import Config
 from dvc.progress import progress
 
@@ -189,7 +190,8 @@ class RemoteGS(RemoteBase):
         return [self._path_to_etag(blob.name) for blob in blobs]
 
     def gc(self, checksum_infos):
-        used_etags = [info[self.PARAM_ETAG] for info in checksum_infos]
+        used_etags = [info[self.PARAM_ETAG] for info in checksum_infos['gs']]
+        used_etags += [info[RemoteLOCAL.PARAM_MD5] for info in checksum_infos['local']]
 
         for etag in self._all_etags():
             if etag in used_etags:

dvc/remote/hdfs.py

@@ -11,6 +11,7 @@ except ImportError:
 
 from dvc.config import Config
 from dvc.remote.base import RemoteBase
+from dvc.remote.local import RemoteLOCAL
 from dvc.exceptions import DvcException
 from dvc.logger import Logger
 
@@ -173,7 +174,8 @@ class RemoteHDFS(RemoteBase):
         return [self._path_to_checksum(path) for path in flist]
 
     def gc(self, checksum_infos):
-        used_checksums = [info[self.PARAM_CHECKSUM] for info in checksum_infos]
+        used_checksums = [info[self.PARAM_CHECKSUM] for info in checksum_infos['hdfs']]
+        used_checksums += [info[RemoteLOCAL.PARAM_MD5] for info in checksum_infos['local']]
 
         for checksum in self._all_checksums():
             if checksum in used_checksums:

dvc/remote/local.py

@@ -327,7 +327,7 @@ class RemoteLOCAL(RemoteBase):
         return collected, missing
 
     def gc(self, checksum_infos):
-        used_md5s = [info[self.PARAM_MD5] for info in self._collect(checksum_infos)[0]]
+        used_md5s = [info[self.PARAM_MD5] for info in self._collect(checksum_infos['local'])[0]]
 
         for md5 in self.all():
             if md5 in used_md5s:

dvc/remote/s3.py

@@ -17,6 +17,7 @@ from dvc.logger import Logger
 from dvc.progress import progress
 from dvc.config import Config
 from dvc.remote.base import RemoteBase
+from dvc.remote.local import RemoteLOCAL
 
 
 class Callback(object):
@@ -229,7 +230,8 @@ class RemoteS3(RemoteBase):
         return [self._path_to_etag(obj.key) for obj in objects]
 
     def gc(self, checksum_infos):
-        used_etags = [info[self.PARAM_ETAG] for info in checksum_infos]
+        used_etags = [info[self.PARAM_ETAG] for info in checksum_infos['s3']]
+        used_etags += [info[RemoteLOCAL.PARAM_MD5] for info in checksum_infos['local']]
 
         for etag in self._all():
             if etag in used_etags:

dvc/remote/ssh.py

@@ -13,6 +13,7 @@ from dvc.logger import Logger
 from dvc.utils import copyfile, file_md5
 from dvc.progress import progress
 from dvc.remote.base import RemoteBase
+from dvc.remote.local import RemoteLOCAL
 from dvc.config import Config
 from dvc.exceptions import DvcException
 
@@ -255,7 +256,8 @@ class RemoteSSH(RemoteBase):
         return [self._path_to_md5(path) for path in flist]
 
     def gc(self, checksum_infos):
-        used_md5s = [info[self.PARAM_MD5] for info in checksum_infos]
+        used_md5s = [info[self.PARAM_MD5] for info in checksum_infos['ssh']]
+        used_md5s += [info[RemoteLOCAL.PARAM_MD5] for info in checksum_infos['local']]
 
         for md5 in self._all_md5s():
             if md5 in used_md5s: