From 1f30d29b57060a8bed8afa1982687ab08d645962 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Mon, 9 Jul 2018 23:40:56 +0300 Subject: [PATCH] remote: add comments for s3/azure exists() method Signed-off-by: Ruslan Kuprieiev --- dvc/remote/azure.py | 3 ++- dvc/remote/s3.py | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/dvc/remote/azure.py b/dvc/remote/azure.py index 6134716df..22409f2e0 100644 --- a/dvc/remote/azure.py +++ b/dvc/remote/azure.py @@ -137,7 +137,8 @@ class RemoteAzure(RemoteBase): } for md5 in md5s] def exists(self, path_infos): - # TODO: why does S3 fetch all blobs first? + # TODO: check whether per-key exists() is faster than listing all + # objects altogether. ret = [] for path_info in path_infos: diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index 03c699eb2..2dc1159d2 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -119,6 +119,10 @@ class RemoteS3(RemoteBase): 'key': posixpath.join(self.prefix, md5[0:2], md5[2:])} for md5 in md5s] def exists(self, path_infos): + # NOTE: We mostly use exists() method when filtering a bulk of cache + # files to decide if we need to download/upload them and in s3 + # list_objects_v2() is much-much faster than trying to check keys + # one-by-one. ret = [] session = boto3.session.Session() s3 = session.client('s3')