remote: RFC: unify cache_exists()

Signed-off-by: Ruslan Kuprieiev <[email protected]>

dvc/remote/azure.py

@@ -101,23 +101,6 @@ class RemoteAzure(RemoteBase):
     def list_cache_paths(self):
         return self._list_paths(self.bucket, self.prefix)
 
-    def cache_exists(self, md5s):
-        assert isinstance(md5s, list)
-
-        if len(md5s) == 0:
-            return []
-
-        ret = len(md5s) * [False]
-        paths = [self.checksum_to_path(md5) for md5 in md5s]
-        for path in self._list_keys(self.bucket, self.prefix):
-            for i, k in enumerate(paths):
-                if k == path:
-                    ret[i] = True
-
-        assert len(ret) == len(paths) == len(md5s)
-
-        return ret
-
     def upload(self, from_infos, to_infos, names=None):
         names = self._verify_path_args(to_infos, from_infos, names)
 

dvc/remote/base.py

@@ -129,9 +129,6 @@ class RemoteBase(object):
             if e.errno != errno.EEXIST:
                 raise
 
-    def cache_exists(self, md5s):
-        raise NotImplementedError
-
     def exists(self, path_infos):
         raise NotImplementedError
 
@@ -212,3 +209,21 @@ class RemoteBase(object):
             return True
 
         return False
+
+    def cache_exists(self, checksums):
+        # NOTE: The reason for such an odd logic is that most of the remotes
+        # take much shorter time to just retrieve everything they have under
+        # a certain prefix(e.g. s3, gs, ssh, hdfs). Other remotes that can
+        # check if particular file exists much quicker, use their own
+        # implementation of cache_exists(see http, local).
+        #
+        # Result of all() might be way too big, so we should walk through
+        # it in one pass. Also currently, cache_exists() should return
+        # a list of True/False that matches order in checksums list,
+        # so we need to use such an ugly logic.
+        ret = len(checksums) * [False]
+        for existing in self.all():
+            for i, checksum in enumerate(checksums):
+                if checksum == existing:
+                    ret[i] = True
+        return ret

dvc/remote/gs.py

@@ -170,23 +170,6 @@ class RemoteGS(RemoteBase):
         paths = self._list_paths(path_info['bucket'], path_info['path'])
         return any(path_info['path'] == path for path in paths)
 
-    def cache_exists(self, md5s):
-        assert isinstance(md5s, list)
-
-        if len(md5s) == 0:
-            return []
-
-        ret = len(md5s) * [False]
-        paths = [self.checksum_to_path(md5) for md5 in md5s]
-        for path in self._list_paths(self.bucket, self.prefix):
-            for i, k in enumerate(paths):
-                if k == path:
-                    ret[i] = True
-
-        assert len(ret) == len(paths) == len(md5s)
-
-        return ret
-
     def upload(self, from_infos, to_infos, names=None):
         names = self._verify_path_args(to_infos, from_infos, names)
 

dvc/remote/hdfs.py

@@ -21,7 +21,8 @@ class RemoteHDFS(RemoteBase):
 
     def __init__(self, project, config):
         self.project = project
-        self.prefix = config.get(Config.SECTION_REMOTE_URL, '/')
+        self.url = config.get(Config.SECTION_REMOTE_URL, '/')
+        self.prefix = self.url
         self.user = self.group('user')
         if not self.user:
             self.user = config.get(Config.SECTION_REMOTE_USER,
@@ -169,21 +170,6 @@ class RemoteHDFS(RemoteBase):
         except RemoteHDFSCmdError:
             return False
 
-    def cache_exists(self, md5s):
-        assert isinstance(md5s, list)
-
-        if len(md5s) == 0:
-            return []
-
-        existing_md5s = self._all_checksums()
-        ret = len(md5s) * [False]
-        for existing_md5 in existing_md5s:
-            for i, md5 in enumerate(md5s):
-                if md5 == existing_md5:
-                    ret[i] = True
-
-        return ret
-
     def upload(self, from_infos, to_infos, names=None):
         names = self._verify_path_args(to_infos, from_infos, names)
 

dvc/remote/s3.py

@@ -214,23 +214,6 @@ class RemoteS3(RemoteBase):
         paths = self._list_paths(path_info['bucket'], path_info['path'])
         return any(path_info['path'] == path for path in paths)
 
-    def cache_exists(self, md5s):
-        assert isinstance(md5s, list)
-
-        if len(md5s) == 0:
-            return []
-
-        ret = len(md5s) * [False]
-        paths = [self.checksum_to_path(md5) for md5 in md5s]
-        for path in self._list_paths(self.bucket, self.prefix):
-            for i, k in enumerate(paths):
-                if k == path:
-                    ret[i] = True
-
-        assert len(paths) == len(ret) == len(md5s)
-
-        return ret
-
     def upload(self, from_infos, to_infos, names=None):
         names = self._verify_path_args(to_infos, from_infos, names)
 

dvc/remote/ssh.py

@@ -116,21 +116,6 @@ class RemoteSSH(RemoteBase):
 
         return exists
 
-    def cache_exists(self, md5s):
-        assert isinstance(md5s, list)
-
-        if len(md5s) == 0:
-            return []
-
-        existing_md5s = self.all()
-        ret = len(md5s) * [False]
-        for existing_md5 in existing_md5s:
-            for i, md5 in enumerate(md5s):
-                if md5 == existing_md5:
-                    ret[i] = True
-
-        return ret
-
     def _exec(self, ssh, cmd):
         stdin, stdout, stderr = ssh.exec_command(cmd)
         channel = stdout.channel