From 0bd433d1539a70c83690ab8602ab3a38cd243d1f Mon Sep 17 00:00:00 2001 From: Andy Sarroff Date: Fri, 17 Jan 2020 09:39:10 -0500 Subject: [PATCH 1/2] s3: copy large objects without chunking When executing `boto3.s3.Client.Copy`, large files are chunked into multiple parts. The default chunk threshold and size is 8MB. This caused an issue when caching an object larger than 8M, where the original object had not been uploaded in multiple parts. The original object and the cached object would have different ETags, causing an exception. This fixes the issue by dynamically setting the chunk threshold to be one byte larger than the size of the original object therefore avoiding multipart copies. Fixes #3174 --- dvc/remote/s3.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index ba043afcd..b6a24c6bb 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -3,6 +3,7 @@ import logging import os import threading +from boto3.s3.transfer import TransferConfig from funcy import cached_property, wrap_prop @@ -169,7 +170,13 @@ class RemoteS3(RemoteBASE): ) else: source = {"Bucket": from_info.bucket, "Key": from_info.path} - s3.copy(source, to_info.bucket, to_info.path, ExtraArgs=extra_args) + s3.copy( + source, + to_info.bucket, + to_info.path, + ExtraArgs=extra_args, + Config=TransferConfig(multipart_threshold=size + 1) + ) cached_etag = cls.get_etag(s3, to_info.bucket, to_info.path) if etag != cached_etag: From c3628c87d5d4fb1ca0995e8e293623a9cbfbc0d0 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Fri, 17 Jan 2020 18:00:19 +0000 Subject: [PATCH 2/2] Restyled by black --- dvc/remote/s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index b6a24c6bb..bcc1cffb8 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -175,7 +175,7 @@ class RemoteS3(RemoteBASE): to_info.bucket, to_info.path, ExtraArgs=extra_args, - Config=TransferConfig(multipart_threshold=size + 1) + Config=TransferConfig(multipart_threshold=size + 1), ) cached_etag = cls.get_etag(s3, to_info.bucket, to_info.path)