From 488a6f80763903850dc6b42d368cd7421e35165d Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Tue, 1 Jun 2021 01:29:17 +0300 Subject: [PATCH] fs: don't automatically create buckets (#6086) * fs: azure: don't create bucket * fs: oss: don't create bucket --- dvc/fs/azure.py | 8 +------- dvc/fs/oss.py | 37 +++++++++++++++---------------------- tests/remotes/azure.py | 16 +++++++++------- tests/remotes/oss.py | 15 +++++++++++++++ 4 files changed, 40 insertions(+), 36 deletions(-) diff --git a/dvc/fs/azure.py b/dvc/fs/azure.py index 27c9c7d67..eaeffb52e 100644 --- a/dvc/fs/azure.py +++ b/dvc/fs/azure.py @@ -84,14 +84,12 @@ class AzureFileSystem(ObjectFSWrapper): url = config.get("url") self.path_info = self.PATH_CLS(url) - self.bucket = self.path_info.bucket - if not self.bucket: + if not self.path_info.bucket: container = self._az_config.get("storage", "container_name", None) url = f"azure://{container}" self.path_info = self.PATH_CLS(url) - self.bucket = self.path_info.bucket def _prepare_credentials(self, **config): from azure.identity.aio import DefaultAzureCredential @@ -180,10 +178,6 @@ class AzureFileSystem(ObjectFSWrapper): try: with _temp_event_loop(): file_system = AzureBlobFileSystem(**self.fs_args) - if self.bucket not in [ - container.rstrip("/") for container in file_system.ls("/") - ]: - file_system.mkdir(self.bucket) except (ValueError, AzureError) as e: raise AzureAuthError( f"Authentication to Azure Blob Storage via {self.login_method}" diff --git a/dvc/fs/oss.py b/dvc/fs/oss.py index 20f4fde92..b92a87714 100644 --- a/dvc/fs/oss.py +++ b/dvc/fs/oss.py @@ -45,7 +45,6 @@ class OSSFileSystem(BaseFileSystem): # pylint:disable=abstract-method url = config.get("url") self.path_info = self.PATH_CLS(url) if url else None - self.bucket = self.path_info.bucket self.endpoint = config.get("oss_endpoint") or os.getenv("OSS_ENDPOINT") self.key_id = ( @@ -68,25 +67,17 @@ class OSSFileSystem(BaseFileSystem): # pylint:disable=abstract-method logger.debug(f"key id: {self.key_id}") logger.debug(f"key secret: {self.key_secret}") - auth = oss2.Auth(self.key_id, self.key_secret) - bucket = oss2.Bucket(auth, self.endpoint, self.bucket) + return oss2.Auth(self.key_id, self.key_secret) - # Ensure bucket exists - try: - bucket.get_bucket_info() - except oss2.exceptions.NoSuchBucket: - bucket.create_bucket( - oss2.BUCKET_ACL_PUBLIC_READ, - oss2.models.BucketCreateConfig( - oss2.BUCKET_STORAGE_CLASS_STANDARD - ), - ) - return bucket + def _get_bucket(self, bucket): + import oss2 + + return oss2.Bucket(self.oss_service, self.endpoint, bucket) def _generate_download_url(self, path_info, expires=3600): - assert path_info.bucket == self.bucket - - return self.oss_service.sign_url("GET", path_info.path, expires) + return self._get_bucket(path_info.bucket).sign_url( + "GET", path_info.path, expires + ) def exists(self, path_info) -> bool: paths = self._list_paths(path_info) @@ -96,7 +87,7 @@ class OSSFileSystem(BaseFileSystem): # pylint:disable=abstract-method import oss2 for blob in oss2.ObjectIterator( - self.oss_service, prefix=path_info.path + self._get_bucket(path_info.bucket), prefix=path_info.path ): yield blob.key @@ -114,16 +105,17 @@ class OSSFileSystem(BaseFileSystem): # pylint:disable=abstract-method raise NotImplementedError logger.debug(f"Removing oss://{path_info}") - self.oss_service.delete_object(path_info.path) + self._get_bucket(path_info.bucket).delete_object(path_info.path) def _upload_fobj(self, fobj, to_info): - self.oss_service.put_object(to_info.path, fobj) + self._get_bucket(to_info.bucket).put_object(to_info.path, fobj) def _upload( self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs ): with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar: - self.oss_service.put_object_from_file( + bucket = self._get_bucket(to_info.bucket) + bucket.put_object_from_file( to_info.path, from_file, progress_callback=pbar.update_to ) @@ -133,8 +125,9 @@ class OSSFileSystem(BaseFileSystem): # pylint:disable=abstract-method with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar: import oss2 + bucket = self._get_bucket(from_info.bucket) oss2.resumable_download( - self.oss_service, + bucket, from_info.path, to_file, progress_callback=pbar.update_to, diff --git a/tests/remotes/azure.py b/tests/remotes/azure.py index 8a4c4a32d..b2a7996e7 100644 --- a/tests/remotes/azure.py +++ b/tests/remotes/azure.py @@ -28,19 +28,12 @@ class Azure(Base, CloudURLInfo): @cached_property def service_client(self): # pylint: disable=no-name-in-module - from azure.core.exceptions import ResourceNotFoundError from azure.storage.blob import BlobServiceClient service_client = BlobServiceClient.from_connection_string( self.CONNECTION_STRING ) - container_client = service_client.get_container_client(self.bucket) - try: # verify that container exists - container_client.get_container_properties() - except ResourceNotFoundError: - container_client.create_container() - return service_client @property @@ -98,7 +91,16 @@ def azure_server(test_config, docker_compose, docker_services): @pytest.fixture def azure(azure_server): + from azure.core.exceptions import ResourceNotFoundError + url = f"azure://{TEST_AZURE_CONTAINER}/{uuid.uuid4()}" ret = Azure(url) ret.config = {"url": url, "connection_string": azure_server} + + container = ret.service_client.get_container_client(TEST_AZURE_CONTAINER) + try: # verify that container exists + container.get_container_properties() + except ResourceNotFoundError: + container.create_container() + return ret diff --git a/tests/remotes/oss.py b/tests/remotes/oss.py index e84f3948c..42ee82f85 100644 --- a/tests/remotes/oss.py +++ b/tests/remotes/oss.py @@ -64,6 +64,8 @@ def oss_server(test_config, docker_compose, docker_services): @pytest.fixture def oss(oss_server): + import oss2 + url = OSS.get_url() ret = OSS(url) ret.config = { @@ -72,6 +74,19 @@ def oss(oss_server): "oss_key_secret": EMULATOR_OSS_ACCESS_KEY_SECRET, "oss_endpoint": oss_server, } + + auth = oss2.Auth( + EMULATOR_OSS_ACCESS_KEY_ID, EMULATOR_OSS_ACCESS_KEY_SECRET + ) + bucket = oss2.Bucket(auth, oss_server, TEST_OSS_REPO_BUCKET) + try: + bucket.get_bucket_info() + except oss2.exceptions.NoSuchBucket: + bucket.create_bucket( + oss2.BUCKET_ACL_PUBLIC_READ, + oss2.models.BucketCreateConfig(oss2.BUCKET_STORAGE_CLASS_STANDARD), + ) + return ret