From ecb80d98dd9148d997ac829c33f1b692d426e86b Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 20 Dec 2018 19:05:02 +0200 Subject: [PATCH] remote: RFC: unify cache_exists() Signed-off-by: Ruslan Kuprieiev --- dvc/remote/azure.py | 17 ----------------- dvc/remote/base.py | 21 ++++++++++++++++++--- dvc/remote/gs.py | 17 ----------------- dvc/remote/hdfs.py | 18 ++---------------- dvc/remote/s3.py | 17 ----------------- dvc/remote/ssh.py | 15 --------------- 6 files changed, 20 insertions(+), 85 deletions(-) diff --git a/dvc/remote/azure.py b/dvc/remote/azure.py index 6b1c91459..482c254dd 100644 --- a/dvc/remote/azure.py +++ b/dvc/remote/azure.py @@ -101,23 +101,6 @@ class RemoteAzure(RemoteBase): def list_cache_paths(self): return self._list_paths(self.bucket, self.prefix) - def cache_exists(self, md5s): - assert isinstance(md5s, list) - - if len(md5s) == 0: - return [] - - ret = len(md5s) * [False] - paths = [self.checksum_to_path(md5) for md5 in md5s] - for path in self._list_keys(self.bucket, self.prefix): - for i, k in enumerate(paths): - if k == path: - ret[i] = True - - assert len(ret) == len(paths) == len(md5s) - - return ret - def upload(self, from_infos, to_infos, names=None): names = self._verify_path_args(to_infos, from_infos, names) diff --git a/dvc/remote/base.py b/dvc/remote/base.py index 2b63eb9e4..9d67ce099 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -129,9 +129,6 @@ class RemoteBase(object): if e.errno != errno.EEXIST: raise - def cache_exists(self, md5s): - raise NotImplementedError - def exists(self, path_infos): raise NotImplementedError @@ -212,3 +209,21 @@ class RemoteBase(object): return True return False + + def cache_exists(self, checksums): + # NOTE: The reason for such an odd logic is that most of the remotes + # take much shorter time to just retrieve everything they have under + # a certain prefix(e.g. s3, gs, ssh, hdfs). Other remotes that can + # check if particular file exists much quicker, use their own + # implementation of cache_exists(see http, local). + # + # Result of all() might be way too big, so we should walk through + # it in one pass. Also currently, cache_exists() should return + # a list of True/False that matches order in checksums list, + # so we need to use such an ugly logic. + ret = len(checksums) * [False] + for existing in self.all(): + for i, checksum in enumerate(checksums): + if checksum == existing: + ret[i] = True + return ret diff --git a/dvc/remote/gs.py b/dvc/remote/gs.py index 291ae5b8b..387221773 100644 --- a/dvc/remote/gs.py +++ b/dvc/remote/gs.py @@ -170,23 +170,6 @@ class RemoteGS(RemoteBase): paths = self._list_paths(path_info['bucket'], path_info['path']) return any(path_info['path'] == path for path in paths) - def cache_exists(self, md5s): - assert isinstance(md5s, list) - - if len(md5s) == 0: - return [] - - ret = len(md5s) * [False] - paths = [self.checksum_to_path(md5) for md5 in md5s] - for path in self._list_paths(self.bucket, self.prefix): - for i, k in enumerate(paths): - if k == path: - ret[i] = True - - assert len(ret) == len(paths) == len(md5s) - - return ret - def upload(self, from_infos, to_infos, names=None): names = self._verify_path_args(to_infos, from_infos, names) diff --git a/dvc/remote/hdfs.py b/dvc/remote/hdfs.py index 2cdf64a59..5eca0ba1c 100644 --- a/dvc/remote/hdfs.py +++ b/dvc/remote/hdfs.py @@ -21,7 +21,8 @@ class RemoteHDFS(RemoteBase): def __init__(self, project, config): self.project = project - self.prefix = config.get(Config.SECTION_REMOTE_URL, '/') + self.url = config.get(Config.SECTION_REMOTE_URL, '/') + self.prefix = self.url self.user = self.group('user') if not self.user: self.user = config.get(Config.SECTION_REMOTE_USER, @@ -169,21 +170,6 @@ class RemoteHDFS(RemoteBase): except RemoteHDFSCmdError: return False - def cache_exists(self, md5s): - assert isinstance(md5s, list) - - if len(md5s) == 0: - return [] - - existing_md5s = self._all_checksums() - ret = len(md5s) * [False] - for existing_md5 in existing_md5s: - for i, md5 in enumerate(md5s): - if md5 == existing_md5: - ret[i] = True - - return ret - def upload(self, from_infos, to_infos, names=None): names = self._verify_path_args(to_infos, from_infos, names) diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index 849b76a04..3fbacb4ca 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -214,23 +214,6 @@ class RemoteS3(RemoteBase): paths = self._list_paths(path_info['bucket'], path_info['path']) return any(path_info['path'] == path for path in paths) - def cache_exists(self, md5s): - assert isinstance(md5s, list) - - if len(md5s) == 0: - return [] - - ret = len(md5s) * [False] - paths = [self.checksum_to_path(md5) for md5 in md5s] - for path in self._list_paths(self.bucket, self.prefix): - for i, k in enumerate(paths): - if k == path: - ret[i] = True - - assert len(paths) == len(ret) == len(md5s) - - return ret - def upload(self, from_infos, to_infos, names=None): names = self._verify_path_args(to_infos, from_infos, names) diff --git a/dvc/remote/ssh.py b/dvc/remote/ssh.py index 439c6912d..b7da48978 100644 --- a/dvc/remote/ssh.py +++ b/dvc/remote/ssh.py @@ -116,21 +116,6 @@ class RemoteSSH(RemoteBase): return exists - def cache_exists(self, md5s): - assert isinstance(md5s, list) - - if len(md5s) == 0: - return [] - - existing_md5s = self.all() - ret = len(md5s) * [False] - for existing_md5 in existing_md5s: - for i, md5 in enumerate(md5s): - if md5 == existing_md5: - ret[i] = True - - return ret - def _exec(self, ssh, cmd): stdin, stdout, stderr = ssh.exec_command(cmd) channel = stdout.channel