mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-10 23:10:42 +01:00
Mr. Outis
27b63e2674
* s3: list_objects -> list_objects_api * s3: use path_info for _list_paths * s3: separate _list_paths from _list_objects * s3: support max items for listing objects/paths * s3: take into account directories on `exists` * s3: overwrite `isdir` method * s3: custom collect directory entry for s3 * s3: strip checksum when collecting directories * remote: change walk implementation to walk_files * s3: add comment about isdir logic * s3: use `dir_path` correctly on `isdir` * tests: add a few unit tests for S3 * s3: use path_info join instead of posixpath * tests: use parametrize for s3 tests * remote: use posixpath accordingly * tests: remove undeeded AWS variables from S3 test * tests: rearrange parametrize values for readability * tests: add encoding, use a single s3 instance, for loop instead of parameterize * py2: fix encoding
86 lines
2.3 KiB
Python
86 lines
2.3 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
import pytest
|
|
from moto import mock_s3
|
|
|
|
from dvc.remote.s3 import RemoteS3
|
|
|
|
|
|
@pytest.fixture
|
|
def remote():
|
|
"""Returns a RemoteS3 connected to a bucket with the following structure:
|
|
|
|
bucket
|
|
├── data
|
|
│ ├── alice
|
|
│ ├── alpha
|
|
│ └── subdir
|
|
│ ├── 1
|
|
│ ├── 2
|
|
│ └── 3
|
|
├── empty_dir
|
|
├── empty_file
|
|
└── foo
|
|
"""
|
|
with mock_s3():
|
|
remote = RemoteS3(None, {"url": "s3://bucket", "region": "us-east-1"})
|
|
s3 = remote.s3
|
|
|
|
s3.create_bucket(Bucket="bucket")
|
|
s3.put_object(Bucket="bucket", Key="empty_dir/")
|
|
s3.put_object(Bucket="bucket", Key="empty_file", Body=b"")
|
|
s3.put_object(Bucket="bucket", Key="foo", Body=b"foo")
|
|
s3.put_object(Bucket="bucket", Key="data/alice", Body=b"alice")
|
|
s3.put_object(Bucket="bucket", Key="data/alpha", Body=b"alpha")
|
|
s3.put_object(Bucket="bucket", Key="data/subdir/1", Body=b"1")
|
|
s3.put_object(Bucket="bucket", Key="data/subdir/2", Body=b"2")
|
|
s3.put_object(Bucket="bucket", Key="data/subdir/3", Body=b"3")
|
|
|
|
yield remote
|
|
|
|
|
|
def test_isdir(remote):
|
|
test_cases = [
|
|
(True, "data"),
|
|
(True, "data/"),
|
|
(True, "data/subdir"),
|
|
(True, "empty_dir"),
|
|
(False, "foo"),
|
|
(False, "data/alice"),
|
|
(False, "data/al"),
|
|
(False, "data/subdir/1"),
|
|
]
|
|
|
|
for expected, path in test_cases:
|
|
assert remote.isdir(remote.path_info / path) == expected
|
|
|
|
|
|
def test_exists(remote):
|
|
test_cases = [
|
|
(True, "data"),
|
|
(True, "data/"),
|
|
(True, "data/subdir"),
|
|
(True, "empty_dir"),
|
|
(True, "empty_file"),
|
|
(True, "foo"),
|
|
(True, "data/alice"),
|
|
(True, "data/subdir/1"),
|
|
(False, "data/al"),
|
|
(False, "foo/"),
|
|
]
|
|
|
|
for expected, path in test_cases:
|
|
assert remote.exists(remote.path_info / path) == expected
|
|
|
|
|
|
def test_walk_files(remote):
|
|
files = [
|
|
remote.path_info / "data/alice",
|
|
remote.path_info / "data/alpha",
|
|
remote.path_info / "data/subdir/1",
|
|
remote.path_info / "data/subdir/2",
|
|
remote.path_info / "data/subdir/3",
|
|
]
|
|
|
|
assert list(remote.walk_files(remote.path_info / "data")) == files
|