diff --git a/dvc/cache.py b/dvc/cache.py index 0262136fa..14596d488 100644 --- a/dvc/cache.py +++ b/dvc/cache.py @@ -30,6 +30,7 @@ class Cache(object): self.gs = self._get_remote(project, config, Config.SECTION_CACHE_GS) self.ssh = self._get_remote(project, config, Config.SECTION_CACHE_SSH) self.hdfs = self._get_remote(project, config, Config.SECTION_CACHE_HDFS) + self.azure = self._get_remote(project, config, Config.SECTION_CACHE_AZURE) def _get_remote(self, project, config, name): from dvc.remote import Remote diff --git a/dvc/cli.py b/dvc/cli.py index aaddf1e73..a8f18de8c 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -314,6 +314,14 @@ def parse_args(argv=None): action='store_true', default=False, help='Collect garbage for all branches.') + gc_parser.add_argument('-c', + '--cloud', + action='store_true', + default=False, + help='Collect garbage in remote repository') + gc_parser.add_argument('-r', + '--remote', + help='Remote repository to collect garbage in') gc_parser.set_defaults(func=CmdGC) # Config diff --git a/dvc/command/gc.py b/dvc/command/gc.py index a4230f58d..19c4c0036 100644 --- a/dvc/command/gc.py +++ b/dvc/command/gc.py @@ -3,5 +3,7 @@ from dvc.command.common.base import CmdBase class CmdGC(CmdBase): def run(self): - self.project.gc() + self.project.gc(all_branches=self.args.all_branches, + cloud=self.args.cloud, + remote=self.args.remote) return 0 diff --git a/dvc/config.py b/dvc/config.py index 88d0751f0..1bc915605 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -46,12 +46,14 @@ class Config(object): SECTION_CACHE_GS = 'gs' SECTION_CACHE_SSH = 'ssh' SECTION_CACHE_HDFS = 'hdfs' + SECTION_CACHE_AZURE = 'azure' SECTION_CACHE_SCHEMA = { schema.Optional(SECTION_CACHE_LOCAL): str, schema.Optional(SECTION_CACHE_S3): str, schema.Optional(SECTION_CACHE_GS): str, schema.Optional(SECTION_CACHE_HDFS): str, schema.Optional(SECTION_CACHE_SSH): str, + schema.Optional(SECTION_CACHE_AZURE): str, # backward compatibility schema.Optional(SECTION_CACHE_DIR, default='cache'): str, diff --git a/dvc/project.py b/dvc/project.py index 96e031507..7c74e66cc 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -291,21 +291,27 @@ class Project(object): return cache - def gc(self, all_branches=False): + def gc(self, all_branches=False, cloud=False, remote=None): clist = self._used_cache(target=None, all_branches=all_branches) - self.cache.local.gc(clist['local']) + self.cache.local.gc(clist) if self.cache.s3: - self.cache.s3.gc(clist['s3']) + self.cache.s3.gc(clist) if self.cache.gs: - self.cache.gs.gc(clist['gs']) + self.cache.gs.gc(clist) if self.cache.ssh: - self.cache.ssh.gc(clist['ssh']) + self.cache.ssh.gc(clist) if self.cache.hdfs: - self.cache.hdfs.gc(clist['hdfs']) + self.cache.hdfs.gc(clist) + + if self.cache.azure: + self.cache.azure.gc(clist) + + if cloud: + self.cloud._get_cloud(remote).gc(clist) def push(self, target=None, jobs=1, remote=None, all_branches=False): self.cloud.push(self._used_cache(target, all_branches)['local'], jobs, remote=remote) diff --git a/dvc/remote/azure.py b/dvc/remote/azure.py index a38056f57..a8633e5fa 100644 --- a/dvc/remote/azure.py +++ b/dvc/remote/azure.py @@ -17,6 +17,7 @@ from dvc.logger import Logger from dvc.progress import progress from dvc.config import Config from dvc.remote.base import RemoteBase +from dvc.remote.local import RemoteLOCAL class Callback(object): @@ -211,7 +212,8 @@ class RemoteAzure(RemoteBase): progress.finish_target(name) def gc(self, checksum_infos): - used_etags = [info[self.PARAM_ETAG] for info in checksum_infos] + used_etags = [info[self.PARAM_ETAG] for info in checksum_infos['azure']] + used_etags += [info[RemoteLOCAL.PARAM_MD5] for info in checksum_infos['local']] all_blobs = self.blob_service.list_blobs(self.bucket) diff --git a/dvc/remote/gs.py b/dvc/remote/gs.py index 5a0c19206..2b951280b 100644 --- a/dvc/remote/gs.py +++ b/dvc/remote/gs.py @@ -13,6 +13,7 @@ except ImportError: from dvc.logger import Logger from dvc.remote.base import RemoteBase +from dvc.remote.local import RemoteLOCAL from dvc.config import Config from dvc.progress import progress @@ -189,7 +190,8 @@ class RemoteGS(RemoteBase): return [self._path_to_etag(blob.name) for blob in blobs] def gc(self, checksum_infos): - used_etags = [info[self.PARAM_ETAG] for info in checksum_infos] + used_etags = [info[self.PARAM_ETAG] for info in checksum_infos['gs']] + used_etags += [info[RemoteLOCAL.PARAM_MD5] for info in checksum_infos['local']] for etag in self._all_etags(): if etag in used_etags: diff --git a/dvc/remote/hdfs.py b/dvc/remote/hdfs.py index ea1e3cf19..22f67bea6 100644 --- a/dvc/remote/hdfs.py +++ b/dvc/remote/hdfs.py @@ -11,6 +11,7 @@ except ImportError: from dvc.config import Config from dvc.remote.base import RemoteBase +from dvc.remote.local import RemoteLOCAL from dvc.exceptions import DvcException from dvc.logger import Logger @@ -173,7 +174,8 @@ class RemoteHDFS(RemoteBase): return [self._path_to_checksum(path) for path in flist] def gc(self, checksum_infos): - used_checksums = [info[self.PARAM_CHECKSUM] for info in checksum_infos] + used_checksums = [info[self.PARAM_CHECKSUM] for info in checksum_infos['hdfs']] + used_checksums += [info[RemoteLOCAL.PARAM_MD5] for info in checksum_infos['local']] for checksum in self._all_checksums(): if checksum in used_checksums: diff --git a/dvc/remote/local.py b/dvc/remote/local.py index 3b92ff42a..bac674cf1 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -327,7 +327,7 @@ class RemoteLOCAL(RemoteBase): return collected, missing def gc(self, checksum_infos): - used_md5s = [info[self.PARAM_MD5] for info in self._collect(checksum_infos)[0]] + used_md5s = [info[self.PARAM_MD5] for info in self._collect(checksum_infos['local'])[0]] for md5 in self.all(): if md5 in used_md5s: diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index 2dc1159d2..7f3602f07 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -17,6 +17,7 @@ from dvc.logger import Logger from dvc.progress import progress from dvc.config import Config from dvc.remote.base import RemoteBase +from dvc.remote.local import RemoteLOCAL class Callback(object): @@ -229,7 +230,8 @@ class RemoteS3(RemoteBase): return [self._path_to_etag(obj.key) for obj in objects] def gc(self, checksum_infos): - used_etags = [info[self.PARAM_ETAG] for info in checksum_infos] + used_etags = [info[self.PARAM_ETAG] for info in checksum_infos['s3']] + used_etags += [info[RemoteLOCAL.PARAM_MD5] for info in checksum_infos['local']] for etag in self._all(): if etag in used_etags: diff --git a/dvc/remote/ssh.py b/dvc/remote/ssh.py index e0539d206..af1c1684a 100644 --- a/dvc/remote/ssh.py +++ b/dvc/remote/ssh.py @@ -13,6 +13,7 @@ from dvc.logger import Logger from dvc.utils import copyfile, file_md5 from dvc.progress import progress from dvc.remote.base import RemoteBase +from dvc.remote.local import RemoteLOCAL from dvc.config import Config from dvc.exceptions import DvcException @@ -255,7 +256,8 @@ class RemoteSSH(RemoteBase): return [self._path_to_md5(path) for path in flist] def gc(self, checksum_infos): - used_md5s = [info[self.PARAM_MD5] for info in checksum_infos] + used_md5s = [info[self.PARAM_MD5] for info in checksum_infos['ssh']] + used_md5s += [info[RemoteLOCAL.PARAM_MD5] for info in checksum_infos['local']] for md5 in self._all_md5s(): if md5 in used_md5s: