refactor of remote s3 extra_args

This commit is contained in:
dlg227
2019-06-08 17:10:58 -04:00
parent fd3c84d682
commit fa6b764f97
+11 -15
View File
@@ -117,19 +117,16 @@ class RemoteS3(RemoteBASE):
)
return obj
@classmethod
def _copy_multipart(
cls, s3, from_info, to_info, size, n_parts, extra_args={}
):
def _copy_multipart(self, s3, from_info, to_info, size, n_parts):
mpu = s3.create_multipart_upload(
Bucket=to_info.bucket, Key=to_info.path, **extra_args
Bucket=to_info.bucket, Key=to_info.path, **self.extra_args
)
mpu_id = mpu["UploadId"]
parts = []
byte_position = 0
for i in range(1, n_parts + 1):
obj = cls.get_head_object(
obj = self.get_head_object(
s3, from_info.bucket, from_info.path, PartNumber=i
)
part_size = obj["ContentLength"]
@@ -161,8 +158,7 @@ class RemoteS3(RemoteBASE):
MultipartUpload={"Parts": parts},
)
@classmethod
def _copy(cls, s3, from_info, to_info, extra_args={}):
def _copy(self, s3, from_info, to_info):
# NOTE: object's etag depends on the way it was uploaded to s3 or the
# way it was copied within the s3. More specifically, it depends on
# the chunk size that was used to transfer it, which would affect
@@ -183,27 +179,27 @@ class RemoteS3(RemoteBASE):
# preserve etag, we need to transfer each part separately, so the
# object is transfered in the same chunks as it was originally.
obj = cls.get_head_object(s3, from_info.bucket, from_info.path)
obj = self.get_head_object(s3, from_info.bucket, from_info.path)
etag = obj["ETag"].strip('"')
size = obj["ContentLength"]
_, _, parts_suffix = etag.partition("-")
if parts_suffix:
n_parts = int(parts_suffix)
cls._copy_multipart(
s3, from_info, to_info, size, n_parts, extra_args
)
self._copy_multipart(s3, from_info, to_info, size, n_parts)
else:
source = {"Bucket": from_info.bucket, "Key": from_info.path}
s3.copy(source, to_info.bucket, to_info.path, ExtraArgs=extra_args)
s3.copy(
source, to_info.bucket, to_info.path, ExtraArgs=self.extra_args
)
cached_etag = cls.get_etag(s3, to_info.bucket, to_info.path)
cached_etag = self.get_etag(s3, to_info.bucket, to_info.path)
if etag != cached_etag:
raise ETagMismatchError(etag, cached_etag)
def copy(self, from_info, to_info, s3=None):
s3 = s3 if s3 else self.s3
self._copy(s3, from_info, to_info, extra_args=self.extra_args)
self._copy(s3, from_info, to_info)
def remove(self, path_info):
if path_info.scheme != "s3":