remote: add comments for s3/azure exists() method

Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
2018-07-09 23:40:56 +03:00
parent abee5f07bb
commit 1f30d29b57
2 changed files with 6 additions and 1 deletions
+2 -1
View File
@@ -137,7 +137,8 @@ class RemoteAzure(RemoteBase):
} for md5 in md5s]
def exists(self, path_infos):
# TODO: why does S3 fetch all blobs first?
# TODO: check whether per-key exists() is faster than listing all
# objects altogether.
ret = []
for path_info in path_infos:
+4
View File
@@ -119,6 +119,10 @@ class RemoteS3(RemoteBase):
'key': posixpath.join(self.prefix, md5[0:2], md5[2:])} for md5 in md5s]
def exists(self, path_infos):
# NOTE: We mostly use exists() method when filtering a bulk of cache
# files to decide if we need to download/upload them and in s3
# list_objects_v2() is much-much faster than trying to check keys
# one-by-one.
ret = []
session = boto3.session.Session()
s3 = session.client('s3')