diff --git a/dvc/remote/azure.py b/dvc/remote/azure.py index 6134716df..22409f2e0 100644 --- a/dvc/remote/azure.py +++ b/dvc/remote/azure.py @@ -137,7 +137,8 @@ class RemoteAzure(RemoteBase): } for md5 in md5s] def exists(self, path_infos): - # TODO: why does S3 fetch all blobs first? + # TODO: check whether per-key exists() is faster than listing all + # objects altogether. ret = [] for path_info in path_infos: diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index 03c699eb2..2dc1159d2 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -119,6 +119,10 @@ class RemoteS3(RemoteBase): 'key': posixpath.join(self.prefix, md5[0:2], md5[2:])} for md5 in md5s] def exists(self, path_infos): + # NOTE: We mostly use exists() method when filtering a bulk of cache + # files to decide if we need to download/upload them and in s3 + # list_objects_v2() is much-much faster than trying to check keys + # one-by-one. ret = [] session = boto3.session.Session() s3 = session.client('s3')