diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index 5fce4336c..de271c28a 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -112,16 +112,19 @@ class RemoteS3(RemoteBASE): ) return obj - def _copy_multipart(self, s3, from_info, to_info, size, n_parts): + @classmethod + def _copy_multipart( + cls, s3, from_info, to_info, size, n_parts, extra_args + ): mpu = s3.create_multipart_upload( - Bucket=to_info.bucket, Key=to_info.path, **self.extra_args + Bucket=to_info.bucket, Key=to_info.path, **extra_args ) mpu_id = mpu["UploadId"] parts = [] byte_position = 0 for i in range(1, n_parts + 1): - obj = self.get_head_object( + obj = cls.get_head_object( s3, from_info.bucket, from_info.path, PartNumber=i ) part_size = obj["ContentLength"] @@ -153,7 +156,8 @@ class RemoteS3(RemoteBASE): MultipartUpload={"Parts": parts}, ) - def _copy(self, s3, from_info, to_info): + @classmethod + def _copy(cls, s3, from_info, to_info, extra_args): # NOTE: object's etag depends on the way it was uploaded to s3 or the # way it was copied within the s3. More specifically, it depends on # the chunk size that was used to transfer it, which would affect @@ -174,27 +178,27 @@ class RemoteS3(RemoteBASE): # preserve etag, we need to transfer each part separately, so the # object is transfered in the same chunks as it was originally. - obj = self.get_head_object(s3, from_info.bucket, from_info.path) + obj = cls.get_head_object(s3, from_info.bucket, from_info.path) etag = obj["ETag"].strip('"') size = obj["ContentLength"] _, _, parts_suffix = etag.partition("-") if parts_suffix: n_parts = int(parts_suffix) - self._copy_multipart(s3, from_info, to_info, size, n_parts) + cls._copy_multipart( + s3, from_info, to_info, size, n_parts, extra_args=extra_args + ) else: source = {"Bucket": from_info.bucket, "Key": from_info.path} - s3.copy( - source, to_info.bucket, to_info.path, ExtraArgs=self.extra_args - ) + s3.copy(source, to_info.bucket, to_info.path, ExtraArgs=extra_args) - cached_etag = self.get_etag(s3, to_info.bucket, to_info.path) + cached_etag = cls.get_etag(s3, to_info.bucket, to_info.path) if etag != cached_etag: raise ETagMismatchError(etag, cached_etag) def copy(self, from_info, to_info, s3=None): s3 = s3 if s3 else self.s3 - self._copy(s3, from_info, to_info) + self._copy(s3, from_info, to_info, self.extra_args) def remove(self, path_info): if path_info.scheme != "s3": diff --git a/tests/func/test_s3.py b/tests/func/test_s3.py index 265a8820c..3bcb64ba3 100644 --- a/tests/func/test_s3.py +++ b/tests/func/test_s3.py @@ -19,7 +19,7 @@ def test_copy_singlepart_preserve_etag(): s3 = boto3.client("s3") s3.put_object(Bucket=from_info.bucket, Key=from_info.path, Body="data") - RemoteS3._copy(s3, from_info, to_info) + RemoteS3._copy(s3, from_info, to_info, {}) def _upload_multipart(s3, Bucket, Key): @@ -60,4 +60,4 @@ def test_copy_multipart_preserve_etag(): s3 = boto3.client("s3") _upload_multipart(s3, from_info.bucket, from_info.path) - RemoteS3._copy(s3, from_info, to_info) + RemoteS3._copy(s3, from_info, to_info, {})