Merge pull request #2282 from efiop/moto

tests: s3: use moto for multipart test

scripts/ci/install.sh

@@ -17,6 +17,10 @@ function retry {
 retry pip install --upgrade pip setuptools wheel
 retry pip install .[all,tests]
 
+# NOTE: waiting for https://github.com/spulec/moto/issues/2172
+pip uninstall -y moto
+retry pip install git+https://github.com/efiop/moto.git@move-env-mocking
+
 git config --global user.email "[email protected]"
 git config --global user.name "DVC Tester"
 

setup.py

@@ -102,6 +102,7 @@ tests_requirements = [
     "pydocstyle<4.0",
     "jaraco.windows==3.9.2",
     "mock-ssh-server>=0.5.0",
+    "moto",
 ]
 
 if (sys.version_info) >= (3, 6):

tests/func/test_s3.py

@@ -1,8 +1,32 @@
 import boto3
-import pytest
+
+from moto import mock_s3
+from functools import wraps
+import moto.s3.models as s3model
 
 from dvc.remote.s3 import RemoteS3
-from tests.func.test_data_cloud import _should_test_aws, get_aws_url
+from tests.func.test_data_cloud import get_aws_url
+
+
+# from https://github.com/spulec/moto/blob/v1.3.5/tests/test_s3/test_s3.py#L40
+REDUCED_PART_SIZE = 256
+
+
+def reduced_min_part_size(f):
+    """ speed up tests by temporarily making the multipart minimum part size
+        small
+    """
+    orig_size = s3model.UPLOAD_PART_MIN_SIZE
+
+    @wraps(f)
+    def wrapped(*args, **kwargs):
+        try:
+            s3model.UPLOAD_PART_MIN_SIZE = REDUCED_PART_SIZE
+            return f(*args, **kwargs)
+        finally:
+            s3model.UPLOAD_PART_MIN_SIZE = orig_size
+
+    return wrapped
 
 
 def _get_src_dst():
@@ -10,13 +34,12 @@ def _get_src_dst():
     return base_info / "from", base_info / "to"
 
 
+@mock_s3
 def test_copy_singlepart_preserve_etag():
     from_info, to_info = _get_src_dst()
 
-    if not _should_test_aws():
-        pytest.skip()
-
     s3 = boto3.client("s3")
+    s3.create_bucket(Bucket=from_info.bucket)
     s3.put_object(Bucket=from_info.bucket, Key=from_info.path, Body="data")
 
     RemoteS3._copy(s3, from_info, to_info, {})
@@ -32,8 +55,8 @@ def _upload_multipart(s3, Bucket, Key):
         # NOTE: Generation parts of variable size. Part size should be at
         # least 5MB:
         # https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadComplete.html
-        part_size = 10 * 1024 * 1024 + 2 ** i
-        body = str(i) * part_size
+        part_size = REDUCED_PART_SIZE + i
+        body = b"1" * part_size
         part = s3.upload_part(
             Bucket=Bucket,
             Key=Key,
@@ -52,12 +75,12 @@ def _upload_multipart(s3, Bucket, Key):
     )
 
 
+@mock_s3
+@reduced_min_part_size
 def test_copy_multipart_preserve_etag():
     from_info, to_info = _get_src_dst()
 
-    if not _should_test_aws():
-        pytest.skip()
-
     s3 = boto3.client("s3")
+    s3.create_bucket(Bucket=from_info.bucket)
     _upload_multipart(s3, from_info.bucket, from_info.path)
     RemoteS3._copy(s3, from_info, to_info, {})