fs: don't automatically create buckets (#6086)

* fs: azure: don't create bucket

* fs: oss: don't create bucket
This commit is contained in:
Ruslan Kuprieiev
2021-06-01 01:29:17 +03:00
committed by GitHub
parent 1fbf0da3df
commit 488a6f8076
4 changed files with 40 additions and 36 deletions
+1 -7
View File
@@ -84,14 +84,12 @@ class AzureFileSystem(ObjectFSWrapper):
url = config.get("url")
self.path_info = self.PATH_CLS(url)
self.bucket = self.path_info.bucket
if not self.bucket:
if not self.path_info.bucket:
container = self._az_config.get("storage", "container_name", None)
url = f"azure://{container}"
self.path_info = self.PATH_CLS(url)
self.bucket = self.path_info.bucket
def _prepare_credentials(self, **config):
from azure.identity.aio import DefaultAzureCredential
@@ -180,10 +178,6 @@ class AzureFileSystem(ObjectFSWrapper):
try:
with _temp_event_loop():
file_system = AzureBlobFileSystem(**self.fs_args)
if self.bucket not in [
container.rstrip("/") for container in file_system.ls("/")
]:
file_system.mkdir(self.bucket)
except (ValueError, AzureError) as e:
raise AzureAuthError(
f"Authentication to Azure Blob Storage via {self.login_method}"
+15 -22
View File
@@ -45,7 +45,6 @@ class OSSFileSystem(BaseFileSystem): # pylint:disable=abstract-method
url = config.get("url")
self.path_info = self.PATH_CLS(url) if url else None
self.bucket = self.path_info.bucket
self.endpoint = config.get("oss_endpoint") or os.getenv("OSS_ENDPOINT")
self.key_id = (
@@ -68,25 +67,17 @@ class OSSFileSystem(BaseFileSystem): # pylint:disable=abstract-method
logger.debug(f"key id: {self.key_id}")
logger.debug(f"key secret: {self.key_secret}")
auth = oss2.Auth(self.key_id, self.key_secret)
bucket = oss2.Bucket(auth, self.endpoint, self.bucket)
return oss2.Auth(self.key_id, self.key_secret)
# Ensure bucket exists
try:
bucket.get_bucket_info()
except oss2.exceptions.NoSuchBucket:
bucket.create_bucket(
oss2.BUCKET_ACL_PUBLIC_READ,
oss2.models.BucketCreateConfig(
oss2.BUCKET_STORAGE_CLASS_STANDARD
),
)
return bucket
def _get_bucket(self, bucket):
import oss2
return oss2.Bucket(self.oss_service, self.endpoint, bucket)
def _generate_download_url(self, path_info, expires=3600):
assert path_info.bucket == self.bucket
return self.oss_service.sign_url("GET", path_info.path, expires)
return self._get_bucket(path_info.bucket).sign_url(
"GET", path_info.path, expires
)
def exists(self, path_info) -> bool:
paths = self._list_paths(path_info)
@@ -96,7 +87,7 @@ class OSSFileSystem(BaseFileSystem): # pylint:disable=abstract-method
import oss2
for blob in oss2.ObjectIterator(
self.oss_service, prefix=path_info.path
self._get_bucket(path_info.bucket), prefix=path_info.path
):
yield blob.key
@@ -114,16 +105,17 @@ class OSSFileSystem(BaseFileSystem): # pylint:disable=abstract-method
raise NotImplementedError
logger.debug(f"Removing oss://{path_info}")
self.oss_service.delete_object(path_info.path)
self._get_bucket(path_info.bucket).delete_object(path_info.path)
def _upload_fobj(self, fobj, to_info):
self.oss_service.put_object(to_info.path, fobj)
self._get_bucket(to_info.bucket).put_object(to_info.path, fobj)
def _upload(
self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
):
with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar:
self.oss_service.put_object_from_file(
bucket = self._get_bucket(to_info.bucket)
bucket.put_object_from_file(
to_info.path, from_file, progress_callback=pbar.update_to
)
@@ -133,8 +125,9 @@ class OSSFileSystem(BaseFileSystem): # pylint:disable=abstract-method
with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar:
import oss2
bucket = self._get_bucket(from_info.bucket)
oss2.resumable_download(
self.oss_service,
bucket,
from_info.path,
to_file,
progress_callback=pbar.update_to,
+9 -7
View File
@@ -28,19 +28,12 @@ class Azure(Base, CloudURLInfo):
@cached_property
def service_client(self):
# pylint: disable=no-name-in-module
from azure.core.exceptions import ResourceNotFoundError
from azure.storage.blob import BlobServiceClient
service_client = BlobServiceClient.from_connection_string(
self.CONNECTION_STRING
)
container_client = service_client.get_container_client(self.bucket)
try: # verify that container exists
container_client.get_container_properties()
except ResourceNotFoundError:
container_client.create_container()
return service_client
@property
@@ -98,7 +91,16 @@ def azure_server(test_config, docker_compose, docker_services):
@pytest.fixture
def azure(azure_server):
from azure.core.exceptions import ResourceNotFoundError
url = f"azure://{TEST_AZURE_CONTAINER}/{uuid.uuid4()}"
ret = Azure(url)
ret.config = {"url": url, "connection_string": azure_server}
container = ret.service_client.get_container_client(TEST_AZURE_CONTAINER)
try: # verify that container exists
container.get_container_properties()
except ResourceNotFoundError:
container.create_container()
return ret
+15
View File
@@ -64,6 +64,8 @@ def oss_server(test_config, docker_compose, docker_services):
@pytest.fixture
def oss(oss_server):
import oss2
url = OSS.get_url()
ret = OSS(url)
ret.config = {
@@ -72,6 +74,19 @@ def oss(oss_server):
"oss_key_secret": EMULATOR_OSS_ACCESS_KEY_SECRET,
"oss_endpoint": oss_server,
}
auth = oss2.Auth(
EMULATOR_OSS_ACCESS_KEY_ID, EMULATOR_OSS_ACCESS_KEY_SECRET
)
bucket = oss2.Bucket(auth, oss_server, TEST_OSS_REPO_BUCKET)
try:
bucket.get_bucket_info()
except oss2.exceptions.NoSuchBucket:
bucket.create_bucket(
oss2.BUCKET_ACL_PUBLIC_READ,
oss2.models.BucketCreateConfig(oss2.BUCKET_STORAGE_CLASS_STANDARD),
)
return ret