tests: launch oss/azure containers from fixtures (#4178)

* tests: launch oss/azure containers from fixtures

Related to #4054

* ci: hdfs: remove unneeded fix

We are using pyarrow 0.17.0 already.

tests/docker-compose.yml

@@ -0,0 +1,11 @@
+---
+version: '3.2'
+services:
+  azurite:
+    image: mcr.microsoft.com/azure-storage/azurite:3.3.0-preview
+    ports:
+      - "10000"
+  oss:
+    image: rkuprieiev/oss-emulator
+    ports:
+      - "8880"

tests/remotes/__init__.py

@@ -1,10 +1,12 @@
+import subprocess
+
 import pytest
 
-from .azure import Azure, azure  # noqa: F401
+from .azure import Azure, azure, azure_server  # noqa: F401
 from .hdfs import HDFS, hdfs  # noqa: F401
 from .http import HTTP, http, http_server  # noqa: F401
 from .local import Local, local_cloud, local_remote  # noqa: F401
-from .oss import OSS, TEST_OSS_REPO_BUCKET, oss  # noqa: F401
+from .oss import OSS, TEST_OSS_REPO_BUCKET, oss, oss_server  # noqa: F401
 from .s3 import S3, TEST_AWS_REPO_BUCKET, real_s3, s3  # noqa: F401
 
 TEST_REMOTE = "upstream"
@@ -35,6 +37,14 @@ from .ssh import (  # noqa: F401; noqa: F401
 )
 
 
+@pytest.fixture(scope="session")
+def docker_compose():
+    try:
+        subprocess.check_output("docker-compose version", shell=True)
+    except (subprocess.CalledProcessError, OSError):
+        pytest.skip("no docker-compose installed")
+
+
 @pytest.fixture
 def remote(tmp_dir, dvc, request):
     cloud = request.param

tests/remotes/azure.py

@@ -1,36 +1,61 @@
 # pylint:disable=abstract-method
 
-import os
 import uuid
 
 import pytest
 
 from dvc.path_info import CloudURLInfo
-from dvc.utils import env2bool
 
 from .base import Base
 
+TEST_AZURE_CONTAINER = "tests"
+TEST_AZURE_CONNECTION_STRING = (
+    "DefaultEndpointsProtocol=http;"
+    "AccountName=devstoreaccount1;"
+    "AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSR"
+    "Z6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;"
+    "BlobEndpoint=http://127.0.0.1:{port}/devstoreaccount1;"
+)
+
 
 class Azure(Base, CloudURLInfo):
-    @staticmethod
-    def should_test():
-        do_test = env2bool("DVC_TEST_AZURE", undefined=None)
-        if do_test is not None:
-            return do_test
+    pass
 
-        return os.getenv("AZURE_STORAGE_CONTAINER_NAME") and os.getenv(
-            "AZURE_STORAGE_CONNECTION_STRING"
-        )
 
-    @staticmethod
-    def get_url():
-        container_name = os.getenv("AZURE_STORAGE_CONTAINER_NAME")
-        assert container_name is not None
-        return "azure://{}/{}".format(container_name, str(uuid.uuid4()))
+@pytest.fixture(scope="session")
+def azure_server(docker_compose, docker_services):
+    from azure.storage.blob import (  # pylint: disable=no-name-in-module
+        BlockBlobService,
+    )
+    from azure.common import (  # pylint: disable=no-name-in-module
+        AzureException,
+    )
+
+    port = docker_services.port_for("azurite", 10000)
+    connection_string = TEST_AZURE_CONNECTION_STRING.format(port=port)
+
+    def _check():
+        try:
+            BlockBlobService(
+                connection_string=connection_string,
+            ).list_containers()
+            return True
+        except AzureException:
+            return False
+
+    docker_services.wait_until_responsive(
+        timeout=60.0, pause=0.1, check=_check
+    )
+
+    return connection_string
 
 
 @pytest.fixture
-def azure():
-    if not Azure.should_test():
-        pytest.skip("no azure running")
-    yield Azure(Azure.get_url())
+def azure(azure_server):
+    url = f"azure://{TEST_AZURE_CONTAINER}/{uuid.uuid4()}"
+    ret = Azure(url)
+    ret.config = {
+        "url": url,
+        "connection_string": azure_server,
+    }
+    return ret

tests/remotes/oss.py

@@ -1,41 +1,54 @@
 # pylint:disable=abstract-method
-import os
 import uuid
 
 import pytest
 
 from dvc.path_info import CloudURLInfo
-from dvc.utils import env2bool
 
 from .base import Base
 
 TEST_OSS_REPO_BUCKET = "dvc-test"
+TEST_OSS_ENDPOINT = "127.0.0.1:{port}"
+TEST_OSS_ACCESS_KEY_ID = "AccessKeyID"
+TEST_OSS_ACCESS_KEY_SECRET = "AccessKeySecret"
 
 
 class OSS(Base, CloudURLInfo):
-    @staticmethod
-    def should_test():
-        do_test = env2bool("DVC_TEST_OSS", undefined=None)
-        if do_test is not None:
-            return do_test
+    pass
 
-        return (
-            os.getenv("OSS_ENDPOINT")
-            and os.getenv("OSS_ACCESS_KEY_ID")
-            and os.getenv("OSS_ACCESS_KEY_SECRET")
-        )
 
-    @staticmethod
-    def _get_storagepath():
-        return f"{TEST_OSS_REPO_BUCKET}/{uuid.uuid4()}"
+@pytest.fixture(scope="session")
+def oss_server(docker_compose, docker_services):
+    import oss2
 
-    @staticmethod
-    def get_url():
-        return f"oss://{OSS._get_storagepath()}"
+    port = docker_services.port_for("oss", 8880)
+    endpoint = TEST_OSS_ENDPOINT.format(port=port)
+
+    def _check():
+        try:
+            auth = oss2.Auth(
+                TEST_OSS_ACCESS_KEY_ID, TEST_OSS_ACCESS_KEY_SECRET
+            )
+            oss2.Bucket(auth, endpoint, "mybucket").get_bucket_info()
+            return True
+        except oss2.exceptions.NoSuchBucket:
+            return True
+        except oss2.exceptions.OssError:
+            return False
+
+    docker_services.wait_until_responsive(timeout=30.0, pause=5, check=_check)
+
+    return endpoint
 
 
 @pytest.fixture
-def oss():
-    if not OSS.should_test():
-        pytest.skip("no oss running")
-    yield OSS(OSS.get_url())
+def oss(oss_server):
+    url = f"oss://{TEST_OSS_REPO_BUCKET}/{uuid.uuid4()}"
+    ret = OSS(url)
+    ret.config = {
+        "url": url,
+        "oss_key_id": TEST_OSS_ACCESS_KEY_ID,
+        "oss_key_secret": TEST_OSS_ACCESS_KEY_SECRET,
+        "oss_endpoint": oss_server,
+    }
+    return ret

tests/unit/remote/test_azure.py

@@ -1,8 +1,5 @@
-import pytest
-
 from dvc.path_info import PathInfo
 from dvc.remote.azure import AzureRemoteTree
-from tests.remotes import Azure
 
 container_name = "container-name"
 connection_string = (
@@ -32,14 +29,11 @@ def test_init(dvc):
     assert tree._conn_kwargs["connection_string"] == connection_string
 
 
-def test_get_file_hash(tmp_dir):
-    if not Azure.should_test():
-        pytest.skip("no azurite running")
-
+def test_get_file_hash(tmp_dir, azure):
     tmp_dir.gen("foo", "foo")
 
-    tree = AzureRemoteTree(None, {})
-    to_info = tree.PATH_CLS(Azure.get_url())
+    tree = AzureRemoteTree(None, azure.config)
+    to_info = azure
     tree.upload(PathInfo("foo"), to_info)
     assert tree.exists(to_info)
     hash_ = tree.get_file_hash(to_info)