mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
Batuhan Taskaya
11b0581536
* import-url: support importing empty directories on object storage services Empty directories are basically files with a trailing slash at the end of their names, with empty contents on object service providers (such as azure, google storage). This patch implements the `is_empty()` method for the `FSSpecWrapper` to determining whether a path is empty (directory or a file) and also tweaks `.isdir()` in order to see empty directories. The base class is only used by azure and gcsfs for now, but if we go on the route of migrating normal providers (like proper filesystems, not just object storages), we might need to do these tweaks on a subclass (like `ObjectStorageFSSpecWrapper` etc), but I don't think it is neccesary as of now. The `is_empty` method is already there, and also used in the importing part to warn users that they are importing something empty (https://github.com/iterative/dvc/blob/e385e59a098cd661ea1ea4257c9a5735c5c32bc6/dvc/output/base.py#L252-L254) so this is not a new method. Just for the reference, as far as I can locate there is no variant of this method in the fsspec. * tests: revert custom mkdir for cloud storages * tests: allign assingments * add some comments
93 lines
2.2 KiB
Python
93 lines
2.2 KiB
Python
# pylint:disable=abstract-method
|
|
import os
|
|
import uuid
|
|
|
|
import pytest
|
|
|
|
from dvc.path_info import CloudURLInfo
|
|
from dvc.utils import env2bool
|
|
|
|
from .base import Base
|
|
|
|
TEST_OSS_REPO_BUCKET = "dvc-test"
|
|
EMULATOR_OSS_ENDPOINT = "127.0.0.1:{port}"
|
|
EMULATOR_OSS_ACCESS_KEY_ID = "AccessKeyID"
|
|
EMULATOR_OSS_ACCESS_KEY_SECRET = "AccessKeySecret"
|
|
|
|
|
|
class OSS(Base, CloudURLInfo):
|
|
|
|
IS_OBJECT_STORAGE = True
|
|
|
|
@staticmethod
|
|
def get_url():
|
|
return f"oss://{TEST_OSS_REPO_BUCKET}/{uuid.uuid4()}"
|
|
|
|
@staticmethod
|
|
def should_test():
|
|
do_test = env2bool("DVC_TEST_OSS", undefined=None)
|
|
if do_test is not None:
|
|
return do_test
|
|
|
|
if os.getenv("OSS_ACCESS_KEY_ID") and os.getenv(
|
|
"OSS_ACCESS_KEY_SECRET"
|
|
):
|
|
return True
|
|
|
|
return False
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def oss_server(test_config, docker_compose, docker_services):
|
|
import oss2
|
|
|
|
test_config.requires("oss")
|
|
port = docker_services.port_for("oss", 8880)
|
|
endpoint = EMULATOR_OSS_ENDPOINT.format(port=port)
|
|
|
|
def _check():
|
|
try:
|
|
auth = oss2.Auth(
|
|
EMULATOR_OSS_ACCESS_KEY_ID, EMULATOR_OSS_ACCESS_KEY_SECRET
|
|
)
|
|
oss2.Bucket(auth, endpoint, "mybucket").get_bucket_info()
|
|
return True
|
|
except oss2.exceptions.NoSuchBucket:
|
|
return True
|
|
except oss2.exceptions.OssError:
|
|
return False
|
|
|
|
docker_services.wait_until_responsive(timeout=30.0, pause=5, check=_check)
|
|
|
|
return endpoint
|
|
|
|
|
|
@pytest.fixture
|
|
def oss(oss_server):
|
|
url = OSS.get_url()
|
|
ret = OSS(url)
|
|
ret.config = {
|
|
"url": url,
|
|
"oss_key_id": EMULATOR_OSS_ACCESS_KEY_ID,
|
|
"oss_key_secret": EMULATOR_OSS_ACCESS_KEY_SECRET,
|
|
"oss_endpoint": oss_server,
|
|
}
|
|
return ret
|
|
|
|
|
|
@pytest.fixture
|
|
def real_oss(test_config):
|
|
test_config.requires("oss")
|
|
if not OSS.should_test():
|
|
pytest.skip("no real OSS")
|
|
|
|
url = OSS.get_url()
|
|
ret = OSS(url)
|
|
ret.config = {
|
|
"url": url,
|
|
"oss_key_id": os.environ.get("OSS_ACCESS_KEY_ID"),
|
|
"oss_key_secret": os.environ.get("OSS_ACCESS_KEY_SECRET"),
|
|
"oss_endpoint": os.environ.get("OSS_ENDPOINT"),
|
|
}
|
|
return ret
|