From d71df35f63aff00f1ab94ef9972f12722de2336d Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Wed, 11 Jul 2018 17:34:08 +0300 Subject: [PATCH] gc: take into account local cache Signed-off-by: Ruslan Kuprieiev --- dvc/remote/azure.py | 2 ++ dvc/remote/gs.py | 2 ++ dvc/remote/hdfs.py | 2 ++ dvc/remote/s3.py | 2 ++ dvc/remote/ssh.py | 2 ++ 5 files changed, 10 insertions(+) diff --git a/dvc/remote/azure.py b/dvc/remote/azure.py index 3254b3e6d..a8633e5fa 100644 --- a/dvc/remote/azure.py +++ b/dvc/remote/azure.py @@ -17,6 +17,7 @@ from dvc.logger import Logger from dvc.progress import progress from dvc.config import Config from dvc.remote.base import RemoteBase +from dvc.remote.local import RemoteLOCAL class Callback(object): @@ -212,6 +213,7 @@ class RemoteAzure(RemoteBase): def gc(self, checksum_infos): used_etags = [info[self.PARAM_ETAG] for info in checksum_infos['azure']] + used_etags += [info[RemoteLOCAL.PARAM_MD5] for info in checksum_infos['local']] all_blobs = self.blob_service.list_blobs(self.bucket) diff --git a/dvc/remote/gs.py b/dvc/remote/gs.py index bb100ad08..2b951280b 100644 --- a/dvc/remote/gs.py +++ b/dvc/remote/gs.py @@ -13,6 +13,7 @@ except ImportError: from dvc.logger import Logger from dvc.remote.base import RemoteBase +from dvc.remote.local import RemoteLOCAL from dvc.config import Config from dvc.progress import progress @@ -190,6 +191,7 @@ class RemoteGS(RemoteBase): def gc(self, checksum_infos): used_etags = [info[self.PARAM_ETAG] for info in checksum_infos['gs']] + used_etags += [info[RemoteLOCAL.PARAM_MD5] for info in checksum_infos['local']] for etag in self._all_etags(): if etag in used_etags: diff --git a/dvc/remote/hdfs.py b/dvc/remote/hdfs.py index 4e000cdf4..22f67bea6 100644 --- a/dvc/remote/hdfs.py +++ b/dvc/remote/hdfs.py @@ -11,6 +11,7 @@ except ImportError: from dvc.config import Config from dvc.remote.base import RemoteBase +from dvc.remote.local import RemoteLOCAL from dvc.exceptions import DvcException from dvc.logger import Logger @@ -174,6 +175,7 @@ class RemoteHDFS(RemoteBase): def gc(self, checksum_infos): used_checksums = [info[self.PARAM_CHECKSUM] for info in checksum_infos['hdfs']] + used_checksums += [info[RemoteLOCAL.PARAM_MD5] for info in checksum_infos['local']] for checksum in self._all_checksums(): if checksum in used_checksums: diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index f39ed8aa3..7f3602f07 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -17,6 +17,7 @@ from dvc.logger import Logger from dvc.progress import progress from dvc.config import Config from dvc.remote.base import RemoteBase +from dvc.remote.local import RemoteLOCAL class Callback(object): @@ -230,6 +231,7 @@ class RemoteS3(RemoteBase): def gc(self, checksum_infos): used_etags = [info[self.PARAM_ETAG] for info in checksum_infos['s3']] + used_etags += [info[RemoteLOCAL.PARAM_MD5] for info in checksum_infos['local']] for etag in self._all(): if etag in used_etags: diff --git a/dvc/remote/ssh.py b/dvc/remote/ssh.py index 72b9e680c..af1c1684a 100644 --- a/dvc/remote/ssh.py +++ b/dvc/remote/ssh.py @@ -13,6 +13,7 @@ from dvc.logger import Logger from dvc.utils import copyfile, file_md5 from dvc.progress import progress from dvc.remote.base import RemoteBase +from dvc.remote.local import RemoteLOCAL from dvc.config import Config from dvc.exceptions import DvcException @@ -256,6 +257,7 @@ class RemoteSSH(RemoteBase): def gc(self, checksum_infos): used_md5s = [info[self.PARAM_MD5] for info in checksum_infos['ssh']] + used_md5s += [info[RemoteLOCAL.PARAM_MD5] for info in checksum_infos['local']] for md5 in self._all_md5s(): if md5 in used_md5s: