mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
GaoandRuslan Kuprieiev
93f36d9351
* Migrate to ossfs fix 6177 * Add verification to object stored on oss * Change the oss remote's dependency. pass all of the tests * Performance optimaztion and ossfs version update * Remove the argument to set the resumable get/put 1. Experiments shows resumable get/put has little influnence on performance. 2. Just use resumable ones as default * Use a new api for performance optimization * Without filesize progress bar version * With progress bar version * Bump and add comments 1. Bump ossfs from 2021.6.2 to 2021.7.0 2. add some comments to the verification trigger * Change the name of kwrags following parent class. * Bump to new ossfs and rewrite upload_fobj Because some problem with oss-emulater 1. bump to ossfs 2021.7.1 2. rewrite upload_fobj * Optimization on rm_files * Some optimization follow the old version. * Update dvc/fs/oss.py Co-authored-by: Ruslan Kuprieiev <[email protected]> * Update dvc/objects/db/oss.py Co-authored-by: Ruslan Kuprieiev <[email protected]> * bump into ossfs 2021.7.2 * Bump into ossfs 2021.7.3OC * Switch to real_oss_server * Switch to a real oss server * Use the oss bucket located in US. * Remove some emulator related code * Add envs to ci * remove some oss pytest XFAIL Because we didn't use emulator, we can remove some pytest XFAILs Co-authored-by: Ruslan Kuprieiev <[email protected]>
101 lines
2.4 KiB
Python
101 lines
2.4 KiB
Python
# pylint:disable=abstract-method
|
|
import os
|
|
import uuid
|
|
|
|
import pytest
|
|
|
|
from dvc.path_info import CloudURLInfo
|
|
from dvc.utils import env2bool
|
|
|
|
from .base import Base
|
|
|
|
TEST_OSS_REPO_BUCKET = "dvc-test-github"
|
|
EMULATOR_OSS_ENDPOINT = "127.0.0.1:{port}"
|
|
EMULATOR_OSS_ACCESS_KEY_ID = "AccessKeyID"
|
|
EMULATOR_OSS_ACCESS_KEY_SECRET = "AccessKeySecret"
|
|
|
|
|
|
class OSS(Base, CloudURLInfo):
|
|
|
|
IS_OBJECT_STORAGE = True
|
|
|
|
@staticmethod
|
|
def get_url():
|
|
return f"oss://{TEST_OSS_REPO_BUCKET}/{uuid.uuid4()}"
|
|
|
|
@staticmethod
|
|
def should_test():
|
|
do_test = env2bool("DVC_TEST_OSS", undefined=None)
|
|
if do_test is not None:
|
|
return do_test
|
|
|
|
if os.getenv("OSS_ACCESS_KEY_ID") and os.getenv(
|
|
"OSS_ACCESS_KEY_SECRET"
|
|
):
|
|
return True
|
|
|
|
return False
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def oss_server(test_config, docker_compose, docker_services):
|
|
import oss2
|
|
|
|
test_config.requires("oss")
|
|
port = docker_services.port_for("oss", 8880)
|
|
endpoint = EMULATOR_OSS_ENDPOINT.format(port=port)
|
|
|
|
def _check():
|
|
try:
|
|
auth = oss2.Auth(
|
|
EMULATOR_OSS_ACCESS_KEY_ID, EMULATOR_OSS_ACCESS_KEY_SECRET
|
|
)
|
|
oss2.Bucket(auth, endpoint, "mybucket").get_bucket_info()
|
|
return True
|
|
except oss2.exceptions.NoSuchBucket:
|
|
return True
|
|
except oss2.exceptions.OssError:
|
|
return False
|
|
|
|
docker_services.wait_until_responsive(timeout=30.0, pause=5, check=_check)
|
|
|
|
return endpoint
|
|
|
|
|
|
@pytest.fixture
|
|
def oss(real_oss):
|
|
import oss2
|
|
|
|
ret = real_oss
|
|
|
|
auth = oss2.Auth(ret.config["oss_key_id"], ret.config["oss_key_secret"])
|
|
bucket = oss2.Bucket(
|
|
auth, ret.config["oss_endpoint"], TEST_OSS_REPO_BUCKET
|
|
)
|
|
try:
|
|
bucket.get_bucket_info()
|
|
except oss2.exceptions.NoSuchBucket:
|
|
bucket.create_bucket(
|
|
oss2.BUCKET_ACL_PUBLIC_READ,
|
|
oss2.models.BucketCreateConfig(oss2.BUCKET_STORAGE_CLASS_STANDARD),
|
|
)
|
|
|
|
return ret
|
|
|
|
|
|
@pytest.fixture
|
|
def real_oss(test_config):
|
|
test_config.requires("oss")
|
|
if not OSS.should_test():
|
|
pytest.skip("no real OSS")
|
|
|
|
url = OSS.get_url()
|
|
ret = OSS(url)
|
|
ret.config = {
|
|
"url": url,
|
|
"oss_key_id": os.environ.get("OSS_ACCESS_KEY_ID"),
|
|
"oss_key_secret": os.environ.get("OSS_ACCESS_KEY_SECRET"),
|
|
"oss_endpoint": os.environ.get("OSS_ENDPOINT"),
|
|
}
|
|
return ret
|