s3: use mocked s3 instead of the real one (#4968)

* s3: use mocked s3 instead of the real one

Not removing the fixtures just yet

* Update moto

Tests was failing, because of a change in `responses` package, which
`moto` was mocking. This was fixed on
https://github.com/spulec/moto/pull/3466 which is only available on
`dev122` release.

* Use mocked s3 in checkout tests

setup.py

@@ -133,7 +133,7 @@ tests_requirements = [
     "pydocstyle<4.0",
     "jaraco.windows==3.9.2",
     "mock-ssh-server>=0.8.2",
-    "moto==1.3.16.dev110",
+    "moto==1.3.16.dev122",
     # moto's indirect dependency that is causing problems with flufl.lock's
     # dependency (atpublic). See https://github.com/iterative/dvc/pull/4853
     "aws-sam-translator<1.29.0",

tests/func/test_api.py

@@ -75,7 +75,7 @@ def test_open(tmp_dir, dvc, remote):
     [
         pytest.lazy_fixture(cloud)
         for cloud in [
-            "real_s3",  # NOTE: moto's s3 fails in some tests
+            "s3",
             "gs",
             "azure",
             "gdrive",
@@ -128,7 +128,7 @@ def test_open_granular(tmp_dir, dvc, remote):
     [
         pytest.lazy_fixture(cloud)
         for cloud in [
-            "real_s3",  # NOTE: moto's s3 fails in some tests
+            "s3",
             "gs",
             "azure",
             "gdrive",

tests/func/test_checkout.py

@@ -7,7 +7,6 @@ import shutil
 import pytest
 from mock import patch
 
-from dvc.cache.base import CloudCache
 from dvc.dvcfile import DVC_FILE_SUFFIX, PIPELINE_FILE, Dvcfile
 from dvc.exceptions import (
     CheckoutError,
@@ -17,19 +16,16 @@ from dvc.exceptions import (
     NoOutputOrStageError,
 )
 from dvc.main import main
-from dvc.remote.base import Remote
 from dvc.repo import Repo as DvcRepo
 from dvc.stage import Stage
 from dvc.stage.exceptions import StageFileDoesNotExistError
 from dvc.system import System
 from dvc.tree.local import LocalTree
-from dvc.tree.s3 import S3Tree
 from dvc.utils import relpath
 from dvc.utils.fs import walk_files
 from dvc.utils.serialize import dump_yaml, load_yaml
 from tests.basic_env import TestDvc, TestDvcGit
 from tests.func.test_repro import TestRepro
-from tests.remotes import S3
 
 logger = logging.getLogger("dvc")
 
@@ -757,32 +753,28 @@ def test_checkout_recursive(tmp_dir, dvc):
     }
 
 
-@pytest.mark.skipif(
-    not S3.should_test(), reason="Only run with S3 credentials"
+@pytest.mark.parametrize(
+    "workspace", [pytest.lazy_fixture("s3")], indirect=True
 )
-def test_checkout_for_external_outputs(tmp_dir, dvc):
-    dvc.cache.s3 = CloudCache(S3Tree(dvc, {"url": S3.get_url()}))
+def test_checkout_for_external_outputs(tmp_dir, dvc, workspace):
+    workspace.gen("foo", "foo")
 
-    remote = Remote(S3Tree(dvc, {"url": S3.get_url()}))
-    file_path = remote.tree.path_info / "foo"
-    remote.tree.s3.meta.client.put_object(
-        Bucket=remote.tree.path_info.bucket, Key=file_path.path, Body="foo"
-    )
-
-    dvc.add(str(remote.tree.path_info / "foo"), external=True)
+    file_path = workspace / "foo"
+    dvc.add("remote://workspace/foo")
 
+    remote = dvc.cloud.get_remote("workspace")
     remote.tree.remove(file_path)
+    assert not file_path.exists()
+
     stats = dvc.checkout(force=True)
     assert stats == {**empty_checkout, "added": [str(file_path)]}
-    assert remote.tree.exists(file_path)
+    assert file_path.exists()
+
+    workspace.gen("foo", "foo\nfoo")
 
-    remote.tree.s3.meta.client.put_object(
-        Bucket=remote.tree.path_info.bucket,
-        Key=file_path.path,
-        Body="foo\nfoo",
-    )
     stats = dvc.checkout(force=True)
     assert stats == {**empty_checkout, "modified": [str(file_path)]}
+    assert file_path.read_text() == "foo"
 
 
 def test_checkouts_with_different_addressing(tmp_dir, dvc, run_copy):