mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-05 23:09:32 +01:00
Batuhan Taskaya
11b0581536
* import-url: support importing empty directories on object storage services Empty directories are basically files with a trailing slash at the end of their names, with empty contents on object service providers (such as azure, google storage). This patch implements the `is_empty()` method for the `FSSpecWrapper` to determining whether a path is empty (directory or a file) and also tweaks `.isdir()` in order to see empty directories. The base class is only used by azure and gcsfs for now, but if we go on the route of migrating normal providers (like proper filesystems, not just object storages), we might need to do these tweaks on a subclass (like `ObjectStorageFSSpecWrapper` etc), but I don't think it is neccesary as of now. The `is_empty` method is already there, and also used in the importing part to warn users that they are importing something empty (https://github.com/iterative/dvc/blob/e385e59a098cd661ea1ea4257c9a5735c5c32bc6/dvc/output/base.py#L252-L254) so this is not a new method. Just for the reference, as far as I can locate there is no variant of this method in the fsspec. * tests: revert custom mkdir for cloud storages * tests: allign assingments * add some comments
108 lines
2.9 KiB
Python
108 lines
2.9 KiB
Python
# pylint:disable=abstract-method
|
|
|
|
import locale
|
|
import uuid
|
|
|
|
import pytest
|
|
from funcy import cached_property
|
|
|
|
from dvc.path_info import CloudURLInfo
|
|
|
|
from .base import Base
|
|
|
|
TEST_AZURE_CONTAINER = "tests"
|
|
TEST_AZURE_CONNECTION_STRING = (
|
|
"DefaultEndpointsProtocol=http;"
|
|
"AccountName=devstoreaccount1;"
|
|
"AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSR"
|
|
"Z6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;"
|
|
"BlobEndpoint=http://127.0.0.1:{port}/devstoreaccount1;"
|
|
)
|
|
|
|
|
|
class Azure(Base, CloudURLInfo):
|
|
|
|
IS_OBJECT_STORAGE = True
|
|
CONNECTION_STRING = None
|
|
|
|
@cached_property
|
|
def service_client(self):
|
|
# pylint: disable=no-name-in-module
|
|
from azure.core.exceptions import ResourceNotFoundError
|
|
from azure.storage.blob import BlobServiceClient
|
|
|
|
service_client = BlobServiceClient.from_connection_string(
|
|
self.CONNECTION_STRING
|
|
)
|
|
|
|
container_client = service_client.get_container_client(self.bucket)
|
|
try: # verify that container exists
|
|
container_client.get_container_properties()
|
|
except ResourceNotFoundError:
|
|
container_client.create_container()
|
|
|
|
return service_client
|
|
|
|
@property
|
|
def blob_client(self):
|
|
return self.service_client.get_blob_client(self.bucket, self.path)
|
|
|
|
def mkdir(self, mode=0o777, parents=False, exist_ok=False):
|
|
assert mode == 0o777
|
|
assert parents
|
|
|
|
def write_bytes(self, contents):
|
|
self.blob_client.upload_blob(contents, overwrite=True)
|
|
|
|
def read_bytes(self):
|
|
stream = self.blob_client.download_blob()
|
|
return stream.readall()
|
|
|
|
def read_text(self, encoding=None, errors=None):
|
|
if not encoding:
|
|
encoding = locale.getpreferredencoding(False)
|
|
assert errors is None
|
|
return self.read_bytes().decode(encoding)
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def azure_server(test_config, docker_compose, docker_services):
|
|
test_config.requires("azure")
|
|
|
|
from azure.core.exceptions import ( # pylint: disable=no-name-in-module
|
|
AzureError,
|
|
)
|
|
from azure.storage.blob import ( # pylint: disable=no-name-in-module
|
|
BlobServiceClient,
|
|
)
|
|
|
|
port = docker_services.port_for("azurite", 10000)
|
|
connection_string = TEST_AZURE_CONNECTION_STRING.format(port=port)
|
|
|
|
def _check():
|
|
try:
|
|
BlobServiceClient.from_connection_string(
|
|
connection_string
|
|
).list_containers()
|
|
return True
|
|
except AzureError:
|
|
return False
|
|
|
|
docker_services.wait_until_responsive(
|
|
timeout=60.0, pause=0.1, check=_check
|
|
)
|
|
|
|
Azure.CONNECTION_STRING = connection_string
|
|
return connection_string
|
|
|
|
|
|
@pytest.fixture
|
|
def azure(azure_server):
|
|
url = f"azure://{TEST_AZURE_CONTAINER}/{uuid.uuid4()}"
|
|
ret = Azure(url)
|
|
ret.config = {
|
|
"url": url,
|
|
"connection_string": azure_server,
|
|
}
|
|
return ret
|