diff --git a/tests/remotes/__init__.py b/tests/remotes/__init__.py index e5e4debf0..417bd6899 100644 --- a/tests/remotes/__init__.py +++ b/tests/remotes/__init__.py @@ -6,7 +6,13 @@ from .azure import Azure, azure, azure_server # noqa: F401 from .hdfs import HDFS, hadoop, hdfs, hdfs_server # noqa: F401 from .http import HTTP, http, http_server # noqa: F401 from .local import Local, local_cloud, local_remote # noqa: F401 -from .oss import OSS, TEST_OSS_REPO_BUCKET, oss, oss_server # noqa: F401 +from .oss import ( # noqa: F401 + OSS, + TEST_OSS_REPO_BUCKET, + oss, + oss_server, + real_oss, +) from .s3 import S3, TEST_AWS_REPO_BUCKET, real_s3, s3 # noqa: F401 from .gdrive import ( # noqa: F401; noqa: F401 diff --git a/tests/remotes/oss.py b/tests/remotes/oss.py index 1a163cd3c..8bf7fa14b 100644 --- a/tests/remotes/oss.py +++ b/tests/remotes/oss.py @@ -1,20 +1,37 @@ # pylint:disable=abstract-method +import os import uuid import pytest from dvc.path_info import CloudURLInfo +from dvc.utils import env2bool from .base import Base TEST_OSS_REPO_BUCKET = "dvc-test" -TEST_OSS_ENDPOINT = "127.0.0.1:{port}" -TEST_OSS_ACCESS_KEY_ID = "AccessKeyID" -TEST_OSS_ACCESS_KEY_SECRET = "AccessKeySecret" +EMULATOR_OSS_ENDPOINT = "127.0.0.1:{port}" +EMULATOR_OSS_ACCESS_KEY_ID = "AccessKeyID" +EMULATOR_OSS_ACCESS_KEY_SECRET = "AccessKeySecret" class OSS(Base, CloudURLInfo): - pass + @staticmethod + def get_url(): + return f"oss://{TEST_OSS_REPO_BUCKET}/{uuid.uuid4()}" + + @staticmethod + def should_test(): + do_test = env2bool("DVC_TEST_OSS", undefined=None) + if do_test is not None: + return do_test + + if os.getenv("OSS_ACCESS_KEY_ID") and os.getenv( + "OSS_ACCESS_KEY_SECRET" + ): + return True + + return False @pytest.fixture(scope="session") @@ -22,12 +39,12 @@ def oss_server(docker_compose, docker_services): import oss2 port = docker_services.port_for("oss", 8880) - endpoint = TEST_OSS_ENDPOINT.format(port=port) + endpoint = EMULATOR_OSS_ENDPOINT.format(port=port) def _check(): try: auth = oss2.Auth( - TEST_OSS_ACCESS_KEY_ID, TEST_OSS_ACCESS_KEY_SECRET + EMULATOR_OSS_ACCESS_KEY_ID, EMULATOR_OSS_ACCESS_KEY_SECRET ) oss2.Bucket(auth, endpoint, "mybucket").get_bucket_info() return True @@ -43,12 +60,28 @@ def oss_server(docker_compose, docker_services): @pytest.fixture def oss(oss_server): - url = f"oss://{TEST_OSS_REPO_BUCKET}/{uuid.uuid4()}" + url = OSS.get_url() ret = OSS(url) ret.config = { "url": url, - "oss_key_id": TEST_OSS_ACCESS_KEY_ID, - "oss_key_secret": TEST_OSS_ACCESS_KEY_SECRET, + "oss_key_id": EMULATOR_OSS_ACCESS_KEY_ID, + "oss_key_secret": EMULATOR_OSS_ACCESS_KEY_SECRET, "oss_endpoint": oss_server, } return ret + + +@pytest.fixture +def real_oss(): + if not OSS.should_test(): + pytest.skip("no real OSS") + + url = OSS.get_url() + ret = OSS(url) + ret.config = { + "url": url, + "oss_key_id": os.environ.get("OSS_ACCESS_KEY_ID"), + "oss_key_secret": os.environ.get("OSS_ACCESS_KEY_SECRET"), + "oss_endpoint": os.environ.get("OSS_ENDPOINT"), + } + return ret diff --git a/tests/remotes_env.sample b/tests/remotes_env.sample index 840cf5323..569ef9180 100644 --- a/tests/remotes_env.sample +++ b/tests/remotes_env.sample @@ -9,3 +9,10 @@ # Uncomment to test against Microsoft Azure via Azurite # export AZURE_STORAGE_CONTAINER_NAME="dvc-test" # export AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" + +# Uncomment and fill in to test against Alibaba OSS +# export OSS_ENDPOINT="...SET-ME.." +# export OSS_ACCESS_KEY_ID="...SET-ME.." +# export OSS_ACCESS_KEY_SECRET="...SET-ME.." +# export DVC_TEST_OSS=1 +# export DVC_TEST_OSS_REPO_BUCKET="...SET-ME..."