From b15f5ffa3bfd006b3234b79617f1b55f8dc096c3 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Fri, 10 Aug 2018 18:52:35 +0300 Subject: [PATCH] dvc: print a msg when there is no cache to remove Fix #990 Signed-off-by: Ruslan Kuprieiev --- dvc/project.py | 19 ++++++++++++------- dvc/remote/azure.py | 4 ++++ dvc/remote/gs.py | 4 ++++ dvc/remote/hdfs.py | 4 ++++ dvc/remote/local.py | 4 ++++ dvc/remote/s3.py | 4 ++++ dvc/remote/ssh.py | 4 ++++ 7 files changed, 36 insertions(+), 7 deletions(-) diff --git a/dvc/project.py b/dvc/project.py index b2976f31c..d7154d444 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -351,29 +351,34 @@ class Project(object): return cache + def _do_gc(self, typ, func, clist): + removed = func(clist) + if not removed: + self.logger.info("No unused {} cache to remove.".format(typ)) + def gc(self, all_branches=False, cloud=False, remote=None): clist = self._used_cache(target=None, all_branches=all_branches, active=False) - self.cache.local.gc(clist) + self._do_gc('local', self.cache.local.gc, clist) if self.cache.s3: - self.cache.s3.gc(clist) + self._do_gc('s3', self.cache.s3.gc, clist) if self.cache.gs: - self.cache.gs.gc(clist) + self._do_gc('gs', self.cache.gs.gc, clist) if self.cache.ssh: - self.cache.ssh.gc(clist) + self._do_gc('ssh', self.cache.ssh.gc, clist) if self.cache.hdfs: - self.cache.hdfs.gc(clist) + self._do_gc('hdfs', self.cache.hdfs.gc, clist) if self.cache.azure: - self.cache.azure.gc(clist) + self._do_gc('azure', self.cache.azure.gc, clist) if cloud: - self.cloud._get_cloud(remote).gc(clist) + self._do_gc('remote', self.cloud._get_cloud(remote).gc, clist) def push(self, target=None, diff --git a/dvc/remote/azure.py b/dvc/remote/azure.py index 6953b474d..8c23ee408 100644 --- a/dvc/remote/azure.py +++ b/dvc/remote/azure.py @@ -227,6 +227,7 @@ class RemoteAzure(RemoteBase): # # all_blobs = self.blob_service.list_blobs(self.bucket) # +# removed = False # for blob in all_blobs: # etag = blob.properties.etag # if etag in used: @@ -235,3 +236,6 @@ class RemoteAzure(RemoteBase): # 'key': blob.name, # 'bucket': self.bucket} # self.remove(path_info) +# removed = True +# +# return removed diff --git a/dvc/remote/gs.py b/dvc/remote/gs.py index aab3b4bf3..b5a85cc33 100644 --- a/dvc/remote/gs.py +++ b/dvc/remote/gs.py @@ -229,6 +229,7 @@ class RemoteGS(RemoteBase): used = [info[self.PARAM_ETAG] for info in cinfos['gs']] used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']] + removed = False for etag in self._all_etags(): if etag in used: continue @@ -237,3 +238,6 @@ class RemoteGS(RemoteBase): 'key': posixpath.join(self.prefix, etag[0:2], etag[2:])} self.remove(path_info) + removed = True + + return removed diff --git a/dvc/remote/hdfs.py b/dvc/remote/hdfs.py index 3d7fdbbc9..306065f8c 100644 --- a/dvc/remote/hdfs.py +++ b/dvc/remote/hdfs.py @@ -199,6 +199,7 @@ class RemoteHDFS(RemoteBase): used = [info[self.PARAM_CHECKSUM] for info in cinfos['hdfs']] used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']] + removed = False for checksum in self._all_checksums(): if checksum in used: continue @@ -207,3 +208,6 @@ class RemoteHDFS(RemoteBase): 'url': posixpath.join(self.url, checksum[0:2], checksum[2:])} self.remove(path_info) + removed = True + + return removed diff --git a/dvc/remote/local.py b/dvc/remote/local.py index 6c8472e49..392bfa44c 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -464,10 +464,14 @@ class RemoteLOCAL(RemoteBase): checksum_infos = self._collect(checksum_infos['local'])[0] used_md5s = [info[self.PARAM_MD5] for info in checksum_infos] + removed = False for md5 in self.all(): if md5 in used_md5s: continue remove(self.get(md5)) + removed = True + + return removed def status(self, checksum_infos, remote, jobs=1, show_checksums=False): Logger.info("Preparing to pull data from {}".format(remote.url)) diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index c5d5958fd..14a78836c 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -281,6 +281,7 @@ class RemoteS3(RemoteBase): used_etags = [info[self.PARAM_ETAG] for info in cinfos['s3']] used_etags += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']] + removed = False for etag in self._all(): if etag in used_etags: continue @@ -289,3 +290,6 @@ class RemoteS3(RemoteBase): etag[0:2], etag[2:]), 'bucket': self.bucket} self.remove(path_info) + removed = True + + return removed diff --git a/dvc/remote/ssh.py b/dvc/remote/ssh.py index 54f6f54b5..c54a312cb 100644 --- a/dvc/remote/ssh.py +++ b/dvc/remote/ssh.py @@ -289,6 +289,7 @@ class RemoteSSH(RemoteBase): used = [info[self.PARAM_MD5] for info in cinfos['ssh']] used += [info[RemoteLOCAL.PARAM_MD5] for info in cinfos['local']] + removed = False for md5 in self._all_md5s(): if md5 in used: continue @@ -298,3 +299,6 @@ class RemoteSSH(RemoteBase): 'path': posixpath.join(self.prefix, md5[0:2], md5[2:])} self.remove(path_info) + removed = True + + return removed