tests/remote: adding tests for real oss (#4735)

* tests/remote: adding tests for real oss

the new test suggests #4633 is a bug in the emulator.

* Update tests/func/test_data_cloud.py

Co-authored-by: Ruslan Kuprieiev <[email protected]>

* Update tests/remotes/oss.py

Co-authored-by: Ruslan Kuprieiev <[email protected]>

* tests: correcting small lint issue

Co-authored-by: Ruslan Kuprieiev <[email protected]>

tests/remotes/__init__.py

@@ -6,7 +6,13 @@ from .azure import Azure, azure, azure_server  # noqa: F401
 from .hdfs import HDFS, hadoop, hdfs, hdfs_server  # noqa: F401
 from .http import HTTP, http, http_server  # noqa: F401
 from .local import Local, local_cloud, local_remote  # noqa: F401
-from .oss import OSS, TEST_OSS_REPO_BUCKET, oss, oss_server  # noqa: F401
+from .oss import (  # noqa: F401
+    OSS,
+    TEST_OSS_REPO_BUCKET,
+    oss,
+    oss_server,
+    real_oss,
+)
 from .s3 import S3, TEST_AWS_REPO_BUCKET, real_s3, s3  # noqa: F401
 
 from .gdrive import (  # noqa: F401; noqa: F401

tests/remotes/oss.py

@@ -1,20 +1,37 @@
 # pylint:disable=abstract-method
+import os
 import uuid
 
 import pytest
 
 from dvc.path_info import CloudURLInfo
+from dvc.utils import env2bool
 
 from .base import Base
 
 TEST_OSS_REPO_BUCKET = "dvc-test"
-TEST_OSS_ENDPOINT = "127.0.0.1:{port}"
+EMULATOR_OSS_ENDPOINT = "127.0.0.1:{port}"
-TEST_OSS_ACCESS_KEY_ID = "AccessKeyID"
+EMULATOR_OSS_ACCESS_KEY_ID = "AccessKeyID"
-TEST_OSS_ACCESS_KEY_SECRET = "AccessKeySecret"
+EMULATOR_OSS_ACCESS_KEY_SECRET = "AccessKeySecret"
 
 
 class OSS(Base, CloudURLInfo):
-    pass
+    @staticmethod
+    def get_url():
+        return f"oss://{TEST_OSS_REPO_BUCKET}/{uuid.uuid4()}"
+
+    @staticmethod
+    def should_test():
+        do_test = env2bool("DVC_TEST_OSS", undefined=None)
+        if do_test is not None:
+            return do_test
+
+        if os.getenv("OSS_ACCESS_KEY_ID") and os.getenv(
+            "OSS_ACCESS_KEY_SECRET"
+        ):
+            return True
+
+        return False
 
 
 @pytest.fixture(scope="session")
@@ -22,12 +39,12 @@ def oss_server(docker_compose, docker_services):
     import oss2
 
     port = docker_services.port_for("oss", 8880)
-    endpoint = TEST_OSS_ENDPOINT.format(port=port)
+    endpoint = EMULATOR_OSS_ENDPOINT.format(port=port)
 
     def _check():
         try:
             auth = oss2.Auth(
-                TEST_OSS_ACCESS_KEY_ID, TEST_OSS_ACCESS_KEY_SECRET
+                EMULATOR_OSS_ACCESS_KEY_ID, EMULATOR_OSS_ACCESS_KEY_SECRET
             )
             oss2.Bucket(auth, endpoint, "mybucket").get_bucket_info()
             return True
@@ -43,12 +60,28 @@ def oss_server(docker_compose, docker_services):
 
 @pytest.fixture
 def oss(oss_server):
-    url = f"oss://{TEST_OSS_REPO_BUCKET}/{uuid.uuid4()}"
+    url = OSS.get_url()
     ret = OSS(url)
     ret.config = {
         "url": url,
-        "oss_key_id": TEST_OSS_ACCESS_KEY_ID,
+        "oss_key_id": EMULATOR_OSS_ACCESS_KEY_ID,
-        "oss_key_secret": TEST_OSS_ACCESS_KEY_SECRET,
+        "oss_key_secret": EMULATOR_OSS_ACCESS_KEY_SECRET,
         "oss_endpoint": oss_server,
     }
     return ret
+
+
+@pytest.fixture
+def real_oss():
+    if not OSS.should_test():
+        pytest.skip("no real OSS")
+
+    url = OSS.get_url()
+    ret = OSS(url)
+    ret.config = {
+        "url": url,
+        "oss_key_id": os.environ.get("OSS_ACCESS_KEY_ID"),
+        "oss_key_secret": os.environ.get("OSS_ACCESS_KEY_SECRET"),
+        "oss_endpoint": os.environ.get("OSS_ENDPOINT"),
+    }
+    return ret

tests/remotes_env.sample

@@ -9,3 +9,10 @@
 # Uncomment to test against Microsoft Azure via Azurite
 # export AZURE_STORAGE_CONTAINER_NAME="dvc-test"
 # export AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;"
+
+# Uncomment and fill in to test against Alibaba OSS
+# export OSS_ENDPOINT="...SET-ME.."
+# export OSS_ACCESS_KEY_ID="...SET-ME.."
+# export OSS_ACCESS_KEY_SECRET="...SET-ME.."
+# export DVC_TEST_OSS=1
+# export DVC_TEST_OSS_REPO_BUCKET="...SET-ME..."