Merge pull request #3187 from iterative/pull-3186-restyled

Restyle s3: copy large objects without chunking
This commit is contained in:
Ruslan Kuprieiev
2020-01-17 20:38:36 +02:00
committed by GitHub
+8 -1
View File
@@ -3,6 +3,7 @@
import logging
import os
import threading
from boto3.s3.transfer import TransferConfig
from funcy import cached_property, wrap_prop
@@ -169,7 +170,13 @@ class RemoteS3(RemoteBASE):
)
else:
source = {"Bucket": from_info.bucket, "Key": from_info.path}
s3.copy(source, to_info.bucket, to_info.path, ExtraArgs=extra_args)
s3.copy(
source,
to_info.bucket,
to_info.path,
ExtraArgs=extra_args,
Config=TransferConfig(multipart_threshold=size + 1),
)
cached_etag = cls.get_etag(s3, to_info.bucket, to_info.path)
if etag != cached_etag: