Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

s3: fsspec migration (#5683)

Browse Source 
* s3: migrate to s3fs

* s3: strip etag

* s3: disable instance cache

* tests: use the proper remote:// form in test_checkout

* tests: move/drop some tests

* setup: depend on s3fs

* tests: normalize s3 tests with some cleanups

* Use _download/_upload with old implementation

* fsspec: cache .exists() results

* setup: pin the boto version

* setup: install s3fs[boto3]

* s3: proper exception management

* s3: config improvements

* setup: reference without quotes

* setup: install boto separately

* setup: install moto server

* s3: mocking adjustments for moto_server

* s3: mark some fixtures with scope=session

* s3: s3_fake_creds_file scopes to func

* fsspec: add some docs regarding exists() optimization

* fsspec: exists() fallback

* fsspec: revert exists() optimization

* tests: ssl_verify => verify

* setup: temporarily fetch from master

* s3: add boto as a requirement

* tests: do an extra check regarding race conditions

* tests: use motoserver docker image

* tests: wrap config deletion with try/finally

* tests: disable s3 by default

* setup: upgrade fsspec
This commit is contained in:
Batuhan Taskaya
2021-05-05 11:36:10 +03:00
committed by GitHub
parent 256cb06921
commit d4a3df73df
14 changed files with 314 additions and 688 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/fs/s3.py
+149 -382
View File
@@ -1,75 +1,39 @@
import logging
import functools
import os
import threading
from contextlib import contextmanager
from collections import defaultdict
from funcy import cached_property, wrap_prop
from dvc.config import ConfigError
from dvc.exceptions import DvcException, ETagMismatchError
from dvc.hash_info import HashInfo
from dvc.path_info import CloudURLInfo
from dvc.progress import Tqdm
from dvc.scheme import Schemes
from dvc.utils import conversions, error_link
from .base import BaseFileSystem
logger = logging.getLogger(__name__)
from .fsspec_wrapper import FSSpecWrapper
_AWS_CONFIG_PATH = os.path.join(os.path.expanduser("~"), ".aws", "config")
class S3FileSystem(BaseFileSystem):
class BaseS3FileSystem(FSSpecWrapper):
scheme = Schemes.S3
PATH_CLS = CloudURLInfo
REQUIRES = {"boto3": "boto3"}
REQUIRES = {"s3fs": "s3fs", "boto3": "boto3"}
PARAM_CHECKSUM = "etag"
DETAIL_FIELDS = frozenset(("etag", "size"))
_GRANTS = {
"grant_full_control": "GrantFullControl",
"grant_read": "GrantRead",
"grant_read_acp": "GrantReadACP",
"grant_write_acp": "GrantWriteACP",
}
def __init__(self, repo, config):
super().__init__(repo, config)
url = config.get("url", "s3://")
self.path_info = self.PATH_CLS(url)
self.region = config.get("region")
self.profile = config.get("profile")
self.endpoint_url = config.get("endpointurl")
self.use_ssl = config.get("use_ssl", True)
self.ssl_verify = config.get("ssl_verify", True)
self.extra_args = {}
self.sse = config.get("sse")
if self.sse:
self.extra_args["ServerSideEncryption"] = self.sse
self.sse_kms_key_id = config.get("sse_kms_key_id")
if self.sse_kms_key_id:
self.extra_args["SSEKMSKeyId"] = self.sse_kms_key_id
self.acl = config.get("acl")
if self.acl:
self.extra_args["ACL"] = self.acl
self._append_aws_grants_to_extra_args(config)
self.access_key_id = config.get("access_key_id")
self.secret_access_key = config.get("secret_access_key")
self.session_token = config.get("session_token")
shared_creds = config.get("credentialpath")
if shared_creds:
os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)
config_path = config.get("configpath")
if config_path:
os.environ.setdefault("AWS_CONFIG_FILE", config_path)
self._transfer_config = None
# https://github.com/aws/aws-cli/blob/0376c6262d6b15dc36c82e6da6e1aad10249cc8c/awscli/customizations/s3/transferconfig.py#L107-L113
_TRANSFER_CONFIG_ALIASES = {
"max_queue_size": "max_io_queue",
"max_concurrent_requests": "max_concurrency",
@@ -77,11 +41,15 @@ class S3FileSystem(BaseFileSystem):
"multipart_chunksize": "multipart_chunksize",
}
def _transform_config(self, s3_config):
def _split_s3_config(self, s3_config):
"""Splits the general s3 config into 2 different config
objects, one for transfer.TransferConfig and other is the
general session config"""
from boto3.s3.transfer import TransferConfig
from dvc.utils import conversions
config, transfer_config = {}, {}
for key, value in s3_config.items():
if key in self._TRANSFER_CONFIG_ALIASES:
@@ -94,367 +62,166 @@ class S3FileSystem(BaseFileSystem):
else:
config[key] = value
return config, transfer_config
# pylint: disable=attribute-defined-outside-init
self._transfer_config = TransferConfig(**transfer_config)
return config
def _process_config(self):
from boto3.s3.transfer import TransferConfig
def _load_aws_config_file(self, profile):
from botocore.configloader import load_config
# pylint: disable=attribute-defined-outside-init
self._transfer_config = None
config_path = os.environ.get("AWS_CONFIG_FILE", _AWS_CONFIG_PATH)
if not os.path.exists(config_path):
return None
config = load_config(config_path)
profile = config["profiles"].get(self.profile or "default")
if not profile:
profile_config = config["profiles"].get(profile or "default")
if not profile_config:
return None
s3_config = profile.get("s3", {})
s3_config, transfer_config = self._transform_config(s3_config)
self._transfer_config = TransferConfig(**transfer_config)
return s3_config
s3_config = profile_config.get("s3", {})
return self._split_s3_config(s3_config)
def _prepare_credentials(self, config):
from dvc.config import ConfigError
from dvc.utils.flatten import flatten, unflatten
login_info = defaultdict(dict)
# credentials
login_info["key"] = config.get("access_key_id")
login_info["secret"] = config.get("secret_access_key")
login_info["token"] = config.get("session_token")
# session configuration
login_info["profile"] = config.get("profile")
login_info["use_ssl"] = config.get("use_ssl", True)
# extra client configuration
client = login_info["client_kwargs"]
client["region_name"] = config.get("region")
client["endpoint_url"] = config.get("endpointurl")
client["verify"] = config.get("ssl_verify", True)
# encryptions
additional = login_info["s3_additional_kwargs"]
additional["ServerSideEncryption"] = config.get("sse")
additional["SSEKMSKeyId"] = config.get("sse_kms_key_id")
additional["ACL"] = config.get("acl")
for grant_option, grant_key in self._GRANTS.items():
if config.get(grant_option):
if additional["ACL"]:
raise ConfigError(
"`acl` and `grant_*` AWS S3 config options "
"are mutually exclusive"
)
additional[grant_key] = config[grant_option]
# config kwargs
session_config = login_info["config_kwargs"]
session_config["s3"] = self._load_aws_config_file(
login_info["profile"]
)
return unflatten(
{
key: value
for key, value in flatten(login_info).items()
if value is not None
}
)
def _entry_hook(self, entry):
entry = entry.copy()
if "ETag" in entry:
entry["etag"] = entry["ETag"].strip('"')
return entry
@wrap_prop(threading.Lock())
@cached_property
def fs(self):
from s3fs import S3FileSystem as _S3FileSystem
return _S3FileSystem(**self.fs_args)
def _translate_exceptions(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
try:
func(*args, **kwargs)
except Exception as exc:
from s3fs.errors import translate_boto_error
raise translate_boto_error(exc)
return wrapper
class S3FileSystem(BaseS3FileSystem):
@wrap_prop(threading.Lock())
@cached_property
def s3(self):
import boto3
login_info = self.fs_args
client_kwargs = login_info.get("client_kwargs", {})
session_opts = {
"profile_name": self.profile,
"region_name": self.region,
"profile_name": login_info.get("profile"),
"region_name": client_kwargs.get("region_name"),
}
if self.access_key_id:
session_opts["aws_access_key_id"] = self.access_key_id
if self.secret_access_key:
session_opts["aws_secret_access_key"] = self.secret_access_key
if self.session_token:
session_opts["aws_session_token"] = self.session_token
if "key" in login_info:
session_opts["aws_access_key_id"] = login_info["key"]
if "secret" in login_info:
session_opts["aws_secret_access_key"] = login_info["secret"]
if "token" in login_info:
session_opts["aws_session_token"] = login_info["token"]
session = boto3.session.Session(**session_opts)
s3_config = self._process_config()
return session.resource(
"s3",
endpoint_url=self.endpoint_url,
verify=self.ssl_verify,
use_ssl=self.use_ssl,
config=boto3.session.Config(
signature_version="s3v4", s3=s3_config
),
endpoint_url=client_kwargs.get("endpoint_url"),
use_ssl=login_info["use_ssl"],
)
@contextmanager
def _get_s3(self):
from botocore.exceptions import (
EndpointConnectionError,
NoCredentialsError,
)
try:
yield self.s3
except NoCredentialsError as exc:
link = error_link("no-credentials")
raise DvcException(
f"Unable to find AWS credentials. {link}"
) from exc
except EndpointConnectionError as exc:
link = error_link("connection-error")
name = self.endpoint_url or "AWS S3"
raise DvcException(
f"Unable to connect to '{name}'. {link}"
) from exc
@contextmanager
def _get_bucket(self, bucket):
with self._get_s3() as s3:
try:
yield s3.Bucket(bucket)
except s3.meta.client.exceptions.NoSuchBucket as exc:
link = error_link("no-bucket")
raise DvcException(
f"Bucket '{bucket}' does not exist. {link}"
) from exc
@contextmanager
def _get_obj(self, path_info):
with self._get_bucket(path_info.bucket) as bucket:
try:
yield bucket.Object(path_info.path)
except bucket.meta.client.exceptions.NoSuchKey as exc:
raise DvcException(f"{path_info.url} does not exist") from exc
def _append_aws_grants_to_extra_args(self, config):
# Keys for extra_args can be one of the following list:
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/customizations/s3.html#boto3.s3.transfer.S3Transfer.ALLOWED_UPLOAD_ARGS
"""
ALLOWED_UPLOAD_ARGS = [
'ACL', 'CacheControl', 'ContentDisposition', 'ContentEncoding',
'ContentLanguage', 'ContentType', 'Expires', 'GrantFullControl',
'GrantRead', 'GrantReadACP', 'GrantWriteACP', 'Metadata',
'RequestPayer', 'ServerSideEncryption', 'StorageClass',
'SSECustomerAlgorithm', 'SSECustomerKey', 'SSECustomerKeyMD5',
'SSEKMSKeyId', 'WebsiteRedirectLocation'
]
"""
grants = {
"grant_full_control": "GrantFullControl",
"grant_read": "GrantRead",
"grant_read_acp": "GrantReadACP",
"grant_write_acp": "GrantWriteACP",
}
for grant_option, extra_args_key in grants.items():
if config.get(grant_option):
if self.acl:
raise ConfigError(
"`acl` and `grant_*` AWS S3 config options "
"are mutually exclusive"
)
self.extra_args[extra_args_key] = config.get(grant_option)
def _generate_download_url(self, path_info, expires=3600):
params = {"Bucket": path_info.bucket, "Key": path_info.path}
with self._get_s3() as s3:
return s3.meta.client.generate_presigned_url(
ClientMethod="get_object",
Params=params,
ExpiresIn=int(expires),
)
def exists(self, path_info, use_dvcignore=True):
"""Check if the blob exists. If it does not exist,
it could be a part of a directory path.
eg: if `data/file.txt` exists, check for `data` should return True
"""
return self.isfile(path_info) or self.isdir(path_info)
def isdir(self, path_info):
# S3 doesn't have a concept for directories.
#
# Using `head_object` with a path pointing to a directory
# will throw a 404 error.
#
# A reliable way to know if a given path is a directory is by
# checking if there are more files sharing the same prefix
# with a `list_objects` call.
#
# We need to make sure that the path ends with a forward slash,
# since we can end with false-positives like the following example:
#
# bucket
# └── data
# ├── alice
# └── alpha
#
# Using `data/al` as prefix will return `[data/alice, data/alpha]`,
# While `data/al/` will return nothing.
#
dir_path = path_info / ""
return bool(list(self._list_paths(dir_path, max_items=1)))
def isfile(self, path_info):
if path_info.path.endswith("/"):
return False
return path_info.path in self._list_paths(path_info)
def info(self, path_info):
# temporary workaround, will be removed when migrating to s3fs
if self.isdir(path_info):
return {"type": "directory"}
with self._get_obj(path_info) as obj:
return {
"type": "file",
"etag": obj.e_tag.strip('"'),
"size": obj.content_length,
}
def _list_paths(self, path_info, max_items=None):
with self._get_bucket(path_info.bucket) as bucket:
obj_summaries = bucket.objects.filter(Prefix=path_info.path)
if max_items is not None:
obj_summaries = obj_summaries.page_size(max_items).limit(
max_items
)
for obj_summary in obj_summaries:
yield obj_summary.key
def walk_files(self, path_info, **kwargs):
if not kwargs.pop("prefix", False):
path_info = path_info / ""
for fname in self._list_paths(path_info, **kwargs):
if fname.endswith("/"):
continue
yield path_info.replace(path=fname)
def find(self, path_info, detail=False):
with self._get_bucket(path_info.bucket) as bucket:
for obj_summary in bucket.objects.filter(Prefix=path_info.path):
if detail:
yield {
"type": "file",
"name": obj_summary.key,
"size": obj_summary.size,
"etag": obj_summary.e_tag.strip('"'),
}
else:
yield obj_summary.key
def remove(self, path_info):
if path_info.scheme != "s3":
raise NotImplementedError
logger.debug(f"Removing {path_info}")
with self._get_obj(path_info) as obj:
obj.delete()
def makedirs(self, path_info):
# We need to support creating empty directories, which means
# creating an object with an empty body and a trailing slash `/`.
#
# We are not creating directory objects for every parent prefix,
# as it is not required.
if not path_info.path:
return
dir_path = path_info / ""
with self._get_obj(dir_path) as obj:
obj.put(Body="")
def copy(self, from_info, to_info):
with self._get_s3() as s3:
self._copy(s3.meta.client, from_info, to_info, self.extra_args)
@classmethod
def _copy_multipart(
cls, s3, from_info, to_info, size, n_parts, extra_args
):
mpu = s3.create_multipart_upload(
Bucket=to_info.bucket, Key=to_info.path, **extra_args
)
mpu_id = mpu["UploadId"]
parts = []
byte_position = 0
for i in range(1, n_parts + 1):
obj = s3.head_object(
Bucket=from_info.bucket, Key=from_info.path, PartNumber=i
)
part_size = obj["ContentLength"]
lastbyte = byte_position + part_size - 1
if lastbyte > size:
lastbyte = size - 1
srange = f"bytes={byte_position}-{lastbyte}"
part = s3.upload_part_copy(
Bucket=to_info.bucket,
Key=to_info.path,
PartNumber=i,
UploadId=mpu_id,
CopySourceRange=srange,
CopySource={"Bucket": from_info.bucket, "Key": from_info.path},
)
parts.append(
{"PartNumber": i, "ETag": part["CopyPartResult"]["ETag"]}
)
byte_position += part_size
assert n_parts == len(parts)
s3.complete_multipart_upload(
Bucket=to_info.bucket,
Key=to_info.path,
UploadId=mpu_id,
MultipartUpload={"Parts": parts},
)
@classmethod
def _copy(cls, s3, from_info, to_info, extra_args):
# NOTE: object's etag depends on the way it was uploaded to s3 or the
# way it was copied within the s3. More specifically, it depends on
# the chunk size that was used to transfer it, which would affect
# whether an object would be uploaded as a single part or as a
# multipart.
#
# If an object's etag looks like '8978c98bb5a48c2fb5f2c4c905768afa',
# then it was transferred as a single part, which means that the chunk
# size used to transfer it was greater or equal to the ContentLength
# of that object. So to preserve that tag over the next transfer, we
# could use any value >= ContentLength.
#
# If an object's etag looks like '50d67013a5e1a4070bef1fc8eea4d5f9-13',
# then it was transferred as a multipart, which means that the chunk
# size used to transfer it was less than ContentLength of that object.
# Unfortunately, in general, it doesn't mean that the chunk size was
# the same throughout the transfer, so it means that in order to
# preserve etag, we need to transfer each part separately, so the
# object is transfered in the same chunks as it was originally.
from boto3.s3.transfer import TransferConfig
obj = s3.head_object(Bucket=from_info.bucket, Key=from_info.path)
etag = obj["ETag"].strip('"')
size = obj["ContentLength"]
_, _, parts_suffix = etag.partition("-")
if parts_suffix:
n_parts = int(parts_suffix)
cls._copy_multipart(
s3, from_info, to_info, size, n_parts, extra_args=extra_args
)
else:
source = {"Bucket": from_info.bucket, "Key": from_info.path}
s3.copy(
source,
to_info.bucket,
to_info.path,
ExtraArgs=extra_args,
Config=TransferConfig(multipart_threshold=size + 1),
)
cached_etag = s3.head_object(Bucket=to_info.bucket, Key=to_info.path)[
"ETag"
].strip('"')
if etag != cached_etag:
raise ETagMismatchError(etag, cached_etag)
def etag(self, path_info):
with self._get_obj(path_info) as obj:
return HashInfo("etag", size=obj.content_length,)
def _upload_fobj(self, fobj, to_info):
with self._get_obj(to_info) as obj:
obj.upload_fileobj(fobj, Config=self._transfer_config)
bucket = self.s3.Bucket(path_info.bucket)
return bucket.Object(path_info.path)
@_translate_exceptions
def _upload(
self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
self, from_file, to_info, name=None, no_progress_bar=False, **pbar_args
):
with self._get_obj(to_info) as obj:
total = os.path.getsize(from_file)
with Tqdm(
disable=no_progress_bar, total=total, bytes=True, desc=name
) as pbar:
obj.upload_file(
from_file,
Callback=pbar.update,
ExtraArgs=self.extra_args,
Config=self._transfer_config,
)
total = os.path.getsize(from_file)
with Tqdm(
disable=no_progress_bar,
total=total,
bytes=True,
desc=name,
**pbar_args,
) as pbar:
obj = self._get_obj(to_info)
obj.upload_file(
from_file,
Callback=pbar.update,
ExtraArgs=self.fs_args.get("s3_additional_kwargs"),
Config=self._transfer_config,
)
self.fs.invalidate_cache(self._with_bucket(to_info.parent))
def _download(self, from_info, to_file, name=None, no_progress_bar=False):
with self._get_obj(from_info) as obj:
with Tqdm(
disable=no_progress_bar,
total=obj.content_length,
bytes=True,
desc=name,
) as pbar:
obj.download_file(
to_file, Callback=pbar.update, Config=self._transfer_config
)
@_translate_exceptions
def _download(
self, from_info, to_file, name=None, no_progress_bar=False, **pbar_args
):
obj = self._get_obj(from_info)
with Tqdm(
disable=no_progress_bar,
total=obj.content_length,
bytes=True,
desc=name,
**pbar_args,
) as pbar:
obj.download_file(to_file, Callback=pbar.update)
dvc/logger.py
+1
View File
@@ -196,6 +196,7 @@ def disable_other_loggers():
def setup(level=logging.INFO):
colorama.init()
logging.getLogger("asyncio").setLevel(logging.CRITICAL)
addLoggingLevel("TRACE", logging.DEBUG - 5)
logging.config.dictConfig(
setup.py
+3 -3
View File
@@ -88,16 +88,16 @@ install_requires = [
"python-benedict>=0.21.1",
"pyparsing==2.4.7",
"typing_extensions>=3.7.4",
"fsspec==0.9.0",
"fsspec==2021.4.0",
"diskcache>=5.2.1",
]
# Extra dependencies for remote integrations
gs = ["gcsfs==0.8.0"]
gs = ["gcsfs==2021.4.0"]
gdrive = ["pydrive2>=1.8.1", "six >= 1.13.0"]
s3 = ["boto3>=1.9.201"]
s3 = ["s3fs==2021.4.0", "boto3==1.16.52"]
azure = ["adlfs==0.7.1", "azure-identity>=1.4.0", "knack"]
# https://github.com/Legrandin/pycryptodome/issues/465
oss = ["oss2==2.6.1", "pycryptodome>=3.10"]
test_requirements.txt
-3
View File
@@ -13,9 +13,6 @@ flaky==3.7.0
mock==4.0.3
rangehttpserver==1.2.0
mock-ssh-server==0.9.0
# moto's indirect dependency that is causing problems with flufl.lock's
# dependency (atpublic). See https://github.com/iterative/dvc/pull/4853
moto==2.0.6
beautifulsoup4==4.9.3
wget==3.2
filelock==3.0.12
tests/conftest.py
+1 -1
View File
@@ -19,7 +19,7 @@ REMOTES = {
"hdfs": False,
"http": True,
"oss": False,
"s3": True,
"s3": False,
"ssh": True,
"webdav": True,
}
tests/docker-compose.yml
+4
View File
@@ -6,6 +6,10 @@ services:
command: azurite -L -l /data --blobHost 0.0.0.0 --queueHost 0.0.0.0
ports:
- "10000"
motoserver:
image: motoserver/moto
ports:
- "5000"
oss:
image: rkuprieiev/oss-emulator
ports:
tests/func/test_checkout.py
+1 -1
View File
@@ -856,7 +856,7 @@ def test_checkout_external_modified_file(tmp_dir, dvc, scm, mocker, workspace):
# was attempted without force, dvc checks if it's present in its cache
# before asking user to remove it.
workspace.gen("foo", "foo")
dvc.add(str(workspace / "foo"), external=True)
dvc.add("remote://workspace/foo", external=True)
scm.add(["foo.dvc"])
scm.commit("add foo")
tests/func/test_import_url.py
+1 -1
View File
@@ -369,8 +369,8 @@ empty_xfail = pytest.mark.xfail(
@pytest.mark.parametrize(
"workspace",
[
pytest.lazy_fixture("s3"),
pytest.lazy_fixture("hdfs"),
pytest.param(pytest.lazy_fixture("s3"), marks=empty_xfail),
pytest.param(pytest.lazy_fixture("gs"), marks=empty_xfail),
pytest.param(pytest.lazy_fixture("azure"), marks=empty_xfail),
pytest.param(
tests/func/test_s3.py
-219
View File
@@ -1,219 +0,0 @@
import importlib
import sys
import textwrap
from functools import wraps
import boto3
import moto.s3.models as s3model
import pytest
from moto import mock_s3
from dvc.fs.s3 import S3FileSystem
from dvc.objects.db.base import ObjectDB
from tests.remotes import S3
# from https://github.com/spulec/moto/blob/v1.3.5/tests/test_s3/test_s3.py#L40
REDUCED_PART_SIZE = 256
def reduced_min_part_size(f):
""" speed up tests by temporarily making the multipart minimum part size
small
"""
orig_size = s3model.UPLOAD_PART_MIN_SIZE
@wraps(f)
def wrapped(*args, **kwargs):
try:
s3model.UPLOAD_PART_MIN_SIZE = REDUCED_PART_SIZE
return f(*args, **kwargs)
finally:
s3model.UPLOAD_PART_MIN_SIZE = orig_size
return wrapped
def _get_src_dst():
base_info = S3FileSystem.PATH_CLS(S3.get_url())
return base_info / "from", base_info / "to"
@mock_s3
def test_copy_singlepart_preserve_etag():
from_info, to_info = _get_src_dst()
s3 = boto3.client("s3")
s3.create_bucket(Bucket=from_info.bucket)
s3.put_object(Bucket=from_info.bucket, Key=from_info.path, Body="data")
S3FileSystem._copy(s3, from_info, to_info, {})
@mock_s3
@pytest.mark.parametrize(
"base_info",
[
S3FileSystem.PATH_CLS("s3://bucket/"),
S3FileSystem.PATH_CLS("s3://bucket/ns/"),
],
)
def test_link_created_on_non_nested_path(base_info, tmp_dir, dvc, scm):
from dvc.checkout import _link
fs = S3FileSystem(dvc, {"url": str(base_info.parent)})
odb = ObjectDB(fs)
s3 = odb.fs.s3.meta.client
s3.create_bucket(Bucket=base_info.bucket)
s3.put_object(
Bucket=base_info.bucket, Key=(base_info / "from").path, Body="data"
)
_link(odb, base_info / "from", base_info / "to")
assert odb.fs.exists(base_info / "from")
assert odb.fs.exists(base_info / "to")
@mock_s3
def test_makedirs_doesnot_try_on_top_level_paths(tmp_dir, dvc, scm):
base_info = S3FileSystem.PATH_CLS("s3://bucket/")
fs = S3FileSystem(dvc, {"url": str(base_info)})
fs.makedirs(base_info)
def _upload_multipart(s3, Bucket, Key):
mpu = s3.create_multipart_upload(Bucket=Bucket, Key=Key)
mpu_id = mpu["UploadId"]
parts = []
n_parts = 10
for i in range(1, n_parts + 1):
# NOTE: Generation parts of variable size. Part size should be at
# least 5MB:
# https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadComplete.html
part_size = REDUCED_PART_SIZE + i
body = b"1" * part_size
part = s3.upload_part(
Bucket=Bucket,
Key=Key,
PartNumber=i,
UploadId=mpu_id,
Body=body,
ContentLength=len(body),
)
parts.append({"PartNumber": i, "ETag": part["ETag"]})
s3.complete_multipart_upload(
Bucket=Bucket,
Key=Key,
UploadId=mpu_id,
MultipartUpload={"Parts": parts},
)
@mock_s3
@reduced_min_part_size
def test_copy_multipart_preserve_etag():
from_info, to_info = _get_src_dst()
s3 = boto3.client("s3")
s3.create_bucket(Bucket=from_info.bucket)
_upload_multipart(s3, from_info.bucket, from_info.path)
S3FileSystem._copy(s3, from_info, to_info, {})
def test_s3_isdir(tmp_dir, dvc, s3):
s3.gen({"data": {"foo": "foo"}})
fs = S3FileSystem(dvc, s3.config)
assert not fs.isdir(s3 / "data" / "foo")
assert fs.isdir(s3 / "data")
def test_s3_upload_fobj(tmp_dir, dvc, s3):
s3.gen({"data": {"foo": "foo"}})
fs = S3FileSystem(dvc, s3.config)
to_info = s3 / "data" / "bar"
with fs.open(s3 / "data" / "foo", "rb") as stream:
fs.upload_fobj(stream, to_info, 1)
assert to_info.read_text() == "foo"
KB = 1024
MB = KB ** 2
GB = KB ** 3
def test_s3_aws_config(tmp_dir, dvc, s3, monkeypatch):
config_directory = tmp_dir / ".aws"
config_directory.mkdir()
(config_directory / "config").write_text(
textwrap.dedent(
"""\
[default]
s3 =
max_concurrent_requests = 20000
max_queue_size = 1000
multipart_threshold = 1000KiB
multipart_chunksize = 64MB
use_accelerate_endpoint = true
addressing_style = path
"""
)
)
if sys.platform == "win32":
var = "USERPROFILE"
else:
var = "HOME"
monkeypatch.setenv(var, str(tmp_dir))
# Fresh import to see the effects of changing HOME variable
s3_mod = importlib.reload(sys.modules[S3FileSystem.__module__])
fs = s3_mod.S3FileSystem(dvc, s3.config)
assert fs._transfer_config is None
with fs._get_s3() as s3:
s3_config = s3.meta.client.meta.config.s3
assert s3_config["use_accelerate_endpoint"]
assert s3_config["addressing_style"] == "path"
transfer_config = fs._transfer_config
assert transfer_config.max_io_queue_size == 1000
assert transfer_config.multipart_chunksize == 64 * MB
assert transfer_config.multipart_threshold == 1000 * KB
assert transfer_config.max_request_concurrency == 20000
def test_s3_aws_config_different_profile(tmp_dir, dvc, s3, monkeypatch):
config_file = tmp_dir / "aws_config.ini"
config_file.write_text(
textwrap.dedent(
"""\
[default]
extra = keys
s3 =
addressing_style = auto
use_accelerate_endpoint = true
multipart_threshold = ThisIsNotGoingToBeCasted!
[profile dev]
some_extra = keys
s3 =
addresing_style = virtual
multipart_threshold = 2GiB
"""
)
)
monkeypatch.setenv("AWS_CONFIG_FILE", config_file)
fs = S3FileSystem(dvc, {**s3.config, "profile": "dev"})
assert fs._transfer_config is None
with fs._get_s3() as s3:
s3_config = s3.meta.client.meta.config.s3
assert s3_config["addresing_style"] == "virtual"
assert "use_accelerate_endpoint" not in s3_config
transfer_config = fs._transfer_config
assert transfer_config.multipart_threshold == 2 * GB
tests/remotes/__init__.py
+8 -1
View File
@@ -24,7 +24,14 @@ from .oss import ( # noqa: F401
oss_server,
real_oss,
)
from .s3 import S3, TEST_AWS_REPO_BUCKET, real_s3, s3 # noqa: F401
from .s3 import ( # noqa: F401
S3,
TEST_AWS_REPO_BUCKET,
real_s3,
s3,
s3_fake_creds_file,
s3_server,
)
from .ssh import ( # noqa: F401; noqa: F401
SSHMocked,
ssh,
tests/remotes/s3.py
+57 -7
View File
@@ -4,7 +4,6 @@ import uuid
import pytest
from funcy import cached_property
from moto import mock_s3
from dvc.path_info import CloudURLInfo
from dvc.utils import env2bool
@@ -12,11 +11,17 @@ from dvc.utils import env2bool
from .base import Base
TEST_AWS_REPO_BUCKET = os.environ.get("DVC_TEST_AWS_REPO_BUCKET", "dvc-temp")
TEST_AWS_ENDPOINT_URL = "http://127.0.0.1:{port}/"
class S3(Base, CloudURLInfo):
IS_OBJECT_STORAGE = True
TEST_AWS_ENDPOINT_URL = None
@cached_property
def config(self):
return {"url": self.url, "endpointurl": self.TEST_AWS_ENDPOINT_URL}
@staticmethod
def should_test():
@@ -49,7 +54,7 @@ class S3(Base, CloudURLInfo):
def _s3(self):
import boto3
return boto3.client("s3")
return boto3.client("s3", endpoint_url=self.config["endpointurl"])
def is_file(self):
from botocore.exceptions import ClientError
@@ -95,14 +100,59 @@ class S3(Base, CloudURLInfo):
@pytest.fixture
def s3(test_config):
def s3_fake_creds_file(monkeypatch):
# https://github.com/spulec/moto#other-caveats
import pathlib
aws_dir = pathlib.Path("~").expanduser() / ".aws"
aws_dir.mkdir(exist_ok=True)
aws_creds = aws_dir / "credentials"
initially_exists = aws_creds.exists()
if not initially_exists:
aws_creds.touch()
try:
with monkeypatch.context() as m:
m.setenv("AWS_ACCESS_KEY_ID", "testing")
m.setenv("AWS_SECRET_ACCESS_KEY", "testing")
m.setenv("AWS_SECURITY_TOKEN", "testing")
m.setenv("AWS_SESSION_TOKEN", "testing")
yield
finally:
if aws_creds.exists() and not initially_exists:
aws_creds.unlink()
@pytest.fixture(scope="session")
def s3_server(test_config, docker_compose, docker_services):
import requests
test_config.requires("s3")
with mock_s3():
import boto3
boto3.client("s3").create_bucket(Bucket=TEST_AWS_REPO_BUCKET)
port = docker_services.port_for("motoserver", 5000)
endpoint_url = TEST_AWS_ENDPOINT_URL.format(port=port)
yield S3(S3.get_url())
def _check():
try:
r = requests.get(endpoint_url)
return r.ok
except requests.RequestException:
return False
docker_services.wait_until_responsive(
timeout=60.0, pause=0.1, check=_check
)
S3.TEST_AWS_ENDPOINT_URL = endpoint_url
return endpoint_url
@pytest.fixture
def s3(s3_server, s3_fake_creds_file):
workspace = S3(S3.get_url())
workspace._s3.create_bucket(Bucket=TEST_AWS_REPO_BUCKET)
yield workspace
@pytest.fixture
tests/unit/fs/test_s3.py
+81 -38
View File
@@ -1,7 +1,10 @@
import importlib
import sys
import textwrap
import pytest
from dvc.config import ConfigError
from dvc.exceptions import DvcException
from dvc.fs.s3 import S3FileSystem
bucket_name = "bucket-name"
@@ -35,14 +38,14 @@ def test_verify_ssl_default_param(dvc):
}
fs = S3FileSystem(dvc, config)
assert fs.ssl_verify
assert fs.fs_args["client_kwargs"]["verify"]
def test_ssl_verify_bool_param(dvc):
config = {"url": url, "ssl_verify": False}
fs = S3FileSystem(dvc, config)
assert fs.ssl_verify == config["ssl_verify"]
assert fs.fs_args["client_kwargs"]["verify"] == config["ssl_verify"]
def test_grants(dvc):
@@ -55,13 +58,14 @@ def test_grants(dvc):
}
fs = S3FileSystem(dvc, config)
extra_args = fs.fs_args["s3_additional_kwargs"]
assert (
fs.extra_args["GrantRead"]
extra_args["GrantRead"]
== "id=read-permission-id,id=other-read-permission-id"
)
assert fs.extra_args["GrantReadACP"] == "id=read-acp-permission-id"
assert fs.extra_args["GrantWriteACP"] == "id=write-acp-permission-id"
assert fs.extra_args["GrantFullControl"] == "id=full-control-permission-id"
assert extra_args["GrantReadACP"] == "id=read-acp-permission-id"
assert extra_args["GrantWriteACP"] == "id=write-acp-permission-id"
assert extra_args["GrantFullControl"] == "id=full-control-permission-id"
def test_grants_mutually_exclusive_acl_error(dvc, grants):
@@ -74,7 +78,7 @@ def test_grants_mutually_exclusive_acl_error(dvc, grants):
def test_sse_kms_key_id(dvc):
fs = S3FileSystem(dvc, {"url": url, "sse_kms_key_id": "key"})
assert fs.extra_args["SSEKMSKeyId"] == "key"
assert fs.fs_args["s3_additional_kwargs"]["SSEKMSKeyId"] == "key"
def test_key_id_and_secret(dvc):
@@ -87,44 +91,83 @@ def test_key_id_and_secret(dvc):
"session_token": session_token,
},
)
assert fs.access_key_id == key_id
assert fs.secret_access_key == key_secret
assert fs.session_token == session_token
assert fs.fs_args["key"] == key_id
assert fs.fs_args["secret"] == key_secret
assert fs.fs_args["token"] == session_token
def test_get_s3_no_credentials(mocker):
from botocore.exceptions import NoCredentialsError
fs = S3FileSystem(None, {})
with pytest.raises(DvcException, match="Unable to find AWS credentials"):
with fs._get_s3():
raise NoCredentialsError
KB = 1024
MB = KB ** 2
GB = KB ** 3
def test_get_s3_connection_error(mocker):
from botocore.exceptions import EndpointConnectionError
def test_s3_aws_config(tmp_dir, dvc, s3, monkeypatch):
config_directory = tmp_dir / ".aws"
config_directory.mkdir()
(config_directory / "config").write_text(
textwrap.dedent(
"""\
[default]
s3 =
max_concurrent_requests = 20000
max_queue_size = 1000
multipart_threshold = 1000KiB
multipart_chunksize = 64MB
use_accelerate_endpoint = true
addressing_style = path
"""
)
)
fs = S3FileSystem(None, {})
if sys.platform == "win32":
var = "USERPROFILE"
else:
var = "HOME"
msg = "Unable to connect to 'AWS S3'."
with pytest.raises(DvcException, match=msg):
with fs._get_s3():
raise EndpointConnectionError(endpoint_url="url")
with monkeypatch.context() as m:
m.setenv(var, str(tmp_dir))
# Fresh import to see the effects of changing HOME variable
s3_mod = importlib.reload(sys.modules[S3FileSystem.__module__])
fs = s3_mod.S3FileSystem(dvc, s3.config)
importlib.reload(sys.modules[S3FileSystem.__module__])
s3_config = fs.fs_args["config_kwargs"]["s3"]
assert s3_config["use_accelerate_endpoint"]
assert s3_config["addressing_style"] == "path"
transfer_config = fs._transfer_config
assert transfer_config.max_io_queue_size == 1000
assert transfer_config.multipart_chunksize == 64 * MB
assert transfer_config.multipart_threshold == 1000 * KB
assert transfer_config.max_request_concurrency == 20000
def test_get_s3_connection_error_endpoint(mocker):
from botocore.exceptions import EndpointConnectionError
def test_s3_aws_config_different_profile(tmp_dir, dvc, s3, monkeypatch):
config_file = tmp_dir / "aws_config.ini"
config_file.write_text(
textwrap.dedent(
"""\
[default]
extra = keys
s3 =
addressing_style = auto
use_accelerate_endpoint = true
multipart_threshold = ThisIsNotGoingToBeCasted!
[profile dev]
some_extra = keys
s3 =
addressing_style = virtual
multipart_threshold = 2GiB
"""
)
)
monkeypatch.setenv("AWS_CONFIG_FILE", config_file)
fs = S3FileSystem(None, {"endpointurl": "https://example.com"})
fs = S3FileSystem(dvc, {**s3.config, "profile": "dev"})
msg = "Unable to connect to 'https://example.com'."
with pytest.raises(DvcException, match=msg):
with fs._get_s3():
raise EndpointConnectionError(endpoint_url="url")
s3_config = fs.fs_args["config_kwargs"]["s3"]
assert s3_config["addressing_style"] == "virtual"
def test_get_bucket():
fs = S3FileSystem(None, {"url": "s3://mybucket/path"})
with pytest.raises(DvcException, match="Bucket 'mybucket' does not exist"):
with fs._get_bucket("mybucket") as bucket:
raise bucket.meta.client.exceptions.NoSuchBucket({}, None)
transfer_config = fs._transfer_config
assert transfer_config.multipart_threshold == 2 * GB
tests/unit/remote/test_remote.py
+3 -6
View File
@@ -42,15 +42,12 @@ def test_remote_without_hash_jobs_default(dvc):
assert fs.hash_jobs == fs.HASH_JOBS
@pytest.mark.parametrize(
"fs_name, fs_cls", [("fs", GSFileSystem), ("s3", S3FileSystem)]
)
def test_makedirs_not_create_for_top_level_path(fs_name, fs_cls, dvc, mocker):
@pytest.mark.parametrize("fs_cls", [GSFileSystem, S3FileSystem])
def test_makedirs_not_create_for_top_level_path(fs_cls, dvc, mocker):
url = f"{fs_cls.scheme}://bucket/"
fs = fs_cls(dvc, {"url": url})
mocked_client = mocker.PropertyMock()
# we use remote clients with same name as scheme to interact with remote
mocker.patch.object(fs_cls, fs_name, mocked_client)
mocker.patch.object(fs_cls, "fs", mocked_client)
fs.makedirs(fs.path_info)
assert not mocked_client.called
tests/unit/remote/test_remote_tree.py
+5 -26
View File
@@ -89,10 +89,12 @@ def test_walk_files(remote):
@pytest.mark.parametrize("remote", [pytest.lazy_fixture("s3")], indirect=True)
def test_copy_preserve_etag_across_buckets(remote, dvc):
s3 = remote.fs.s3
s3.Bucket("another").create()
s3 = remote.fs
s3.fs.mkdir("another/")
another = S3FileSystem(dvc, {"url": "s3://another", "region": "us-east-1"})
another = S3FileSystem(
dvc, {**remote.fs.config, "url": "s3://another", "region": "us-east-1"}
)
from_info = remote.fs.path_info / "foo"
to_info = another.path_info / "foo"
@@ -105,29 +107,6 @@ def test_copy_preserve_etag_across_buckets(remote, dvc):
assert from_hash == to_hash
@pytest.mark.parametrize(
"remote",
[
pytest.lazy_fixture("s3"),
pytest.param(
pytest.lazy_fixture("gs"),
marks=pytest.mark.xfail(
reason="https://github.com/iterative/dvc/issues/5521"
),
),
],
indirect=True,
)
def test_makedirs(remote):
fs = remote.fs
empty_dir = remote.fs.path_info / "empty_dir" / ""
fs.remove(empty_dir)
assert not fs.exists(empty_dir)
fs.makedirs(empty_dir)
assert fs.exists(empty_dir)
assert fs.isdir(empty_dir)
@pytest.mark.needs_internet
@pytest.mark.parametrize("remote", remotes, indirect=True)
def test_isfile(remote):