Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

S3Tree._list_paths now recognizes the max_items limit (#5239)

Browse Source 
* S3Tree._list_paths now recognizes the max_items limit

* Use modern approach + add a test

* refactor tests

Co-authored-by: Ruslan Kuprieiev <[email protected]>

* lint

Co-authored-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Batuhan Taskaya
2021-01-11 11:54:42 +02:00
committed by GitHub
co-authored by Ruslan Kuprieiev
parent 5374880220
commit 7fd191126e
2 changed files with 14 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/tree/s3.py
+6 -5
View File
@@ -202,12 +202,13 @@ class S3Tree(BaseTree):
return path_info.path in self._list_paths(path_info)
def _list_paths(self, path_info, max_items=None):
kwargs = {"Prefix": path_info.path}
if max_items is not None:
kwargs["MaxKeys"] = max_items
with self._get_bucket(path_info.bucket) as bucket:
for obj_summary in bucket.objects.filter(**kwargs):
obj_summaries = bucket.objects.filter(Prefix=path_info.path)
if max_items is not None:
obj_summaries = obj_summaries.page_size(max_items).limit(
max_items
)
for obj_summary in obj_summaries:
yield obj_summary.key
def walk_files(self, path_info, **kwargs):
tests/func/test_s3.py
+8
View File
@@ -111,3 +111,11 @@ def test_copy_multipart_preserve_etag():
s3.create_bucket(Bucket=from_info.bucket)
_upload_multipart(s3, from_info.bucket, from_info.path)
S3Tree._copy(s3, from_info, to_info, {})
def test_s3_isdir(tmp_dir, dvc, s3):
s3.gen({"data": {"foo": "foo"}})
tree = S3Tree(dvc, s3.config)
assert not tree.isdir(s3 / "data" / "foo")
assert tree.isdir(s3 / "data")