From f0c1ff021935ca586e698451544a69a606534b40 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Wed, 16 Aug 2023 05:51:20 +0300 Subject: [PATCH] fs: make _prepare_credentails/fs_args lazy (#9848) * tests: unit: move s3 tests to dvc-s3 * deps: bump dvc-data to 2.14.0 * tests: unit: move azure tests to dvc-azure --- pyproject.toml | 2 +- tests/unit/fs/test_azure.py | 84 ---------------------- tests/unit/fs/test_s3.py | 137 ------------------------------------ 3 files changed, 1 insertion(+), 222 deletions(-) delete mode 100644 tests/unit/fs/test_azure.py delete mode 100644 tests/unit/fs/test_s3.py diff --git a/pyproject.toml b/pyproject.toml index da2769284..d92aa8f62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ "configobj>=5.0.6", "distro>=1.3", "dpath<3,>=2.1.0", - "dvc-data>=2.13.1,<2.14.0", + "dvc-data>=2.14.0,<2.15.0", "dvc-http>=2.29.0", "dvc-render>=0.3.1,<1", "dvc-studio-client>=0.9.2,<1", diff --git a/tests/unit/fs/test_azure.py b/tests/unit/fs/test_azure.py deleted file mode 100644 index 3d8b67255..000000000 --- a/tests/unit/fs/test_azure.py +++ /dev/null @@ -1,84 +0,0 @@ -import pytest -from dvc_azure import AzureAuthError, AzureFileSystem - -container_name = "container-name" -connection_string = ( - "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;" - "AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsu" - "Fq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;" - "BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" -) - - -def test_strip_protocol_env_var(monkeypatch, dvc): - monkeypatch.setenv("AZURE_STORAGE_CONTAINER_NAME", container_name) - monkeypatch.setenv("AZURE_STORAGE_CONNECTION_STRING", connection_string) - - assert AzureFileSystem._strip_protocol("azure://") == container_name - - -def test_strip_protocol(dvc): - assert ( - AzureFileSystem._strip_protocol(f"azure://{container_name}") == container_name - ) - - -def test_init(dvc): - config = {"connection_string": connection_string} - fs = AzureFileSystem(**config) - assert fs.fs_args["connection_string"] == connection_string - - -def test_azure_login_methods(): - def get_login_method(config): - fs = AzureFileSystem(**config) - return fs.login_method - - with pytest.raises(AzureAuthError): - get_login_method({}) - - assert get_login_method({"connection_string": "test"}) == "connection string" - assert get_login_method({"account_name": "test"}).startswith("default credentials") - assert ( - get_login_method({"account_name": "test", "allow_anonymous_login": True}) - == "anonymous login" - ) - - with pytest.raises(AzureAuthError): - get_login_method( - {"tenant_id": "test", "client_id": "test", "client_secret": "test"} - ) - - assert ( - get_login_method( - { - "account_name": "test", - "tenant_id": "test", - "client_id": "test", - "client_secret": "test", - } - ) - == "AD service principal" - ) - - assert ( - get_login_method({"account_name": "test", "account_key": "test"}) - == "account key" - ) - assert ( - get_login_method({"account_name": "test", "sas_token": "test"}) == "SAS token" - ) - assert ( - get_login_method( - { - "connection_string": "test", - "account_name": "test", - "sas_token": "test", - } - ) - == "connection string" - ) - assert ( - get_login_method({"connection_string": "test", "sas_token": "test"}) - == "connection string" - ) diff --git a/tests/unit/fs/test_s3.py b/tests/unit/fs/test_s3.py deleted file mode 100644 index 782dcef91..000000000 --- a/tests/unit/fs/test_s3.py +++ /dev/null @@ -1,137 +0,0 @@ -import os - -import pytest -from dvc_s3 import S3FileSystem - -from dvc.fs import ConfigError - -bucket_name = "bucket-name" -prefix = "some/prefix" -url = f"s3://{bucket_name}/{prefix}" -key_id = "key-id" -key_secret = "key-secret" -session_token = "session-token" - - -@pytest.fixture(autouse=True) -def grants(): - return { - "grant_read": "id=read-permission-id,id=other-read-permission-id", - "grant_read_acp": "id=read-acp-permission-id", - "grant_write_acp": "id=write-acp-permission-id", - "grant_full_control": "id=full-control-permission-id", - } - - -def test_verify_ssl_default_param(dvc): - config = {"url": url} - fs = S3FileSystem(**config) - - assert "client_kwargs" not in fs.fs_args - - config = { - "url": url, - "endpointurl": "https://my.custom.s3:1234", - } - fs = S3FileSystem(**config) - - assert "verify" not in fs.fs_args["client_kwargs"] - - -def test_s3_config_credentialpath(dvc, monkeypatch): - environment = {} - monkeypatch.setattr(os, "environ", environment) - - config = {"url": url, "credentialpath": "somewhere"} - S3FileSystem(**config) - assert environment["AWS_SHARED_CREDENTIALS_FILE"] == "somewhere" - environment.clear() - - config = {"url": url, "configpath": "somewhere"} - S3FileSystem(**config) - assert environment["AWS_CONFIG_FILE"] == "somewhere" - environment.clear() - - config = { - "url": url, - "credentialpath": "somewhere", - "configpath": "elsewhere", - } - S3FileSystem(**config) - assert environment["AWS_SHARED_CREDENTIALS_FILE"] == "somewhere" - assert environment["AWS_CONFIG_FILE"] == "elsewhere" - environment.clear() - - -def test_ssl_verify_bool_param(dvc): - config = {"url": url, "ssl_verify": False} - fs = S3FileSystem(**config) - - assert fs.fs_args["client_kwargs"]["verify"] == config["ssl_verify"] - - -def test_ssl_verify_path_param(dvc): - config = {"url": url, "ssl_verify": "/path/to/custom/cabundle.pem"} - fs = S3FileSystem(**config) - - assert fs.fs_args["client_kwargs"]["verify"] == config["ssl_verify"] - - -def test_ssl_verify_none_param(dvc): - config = {"url": url, "ssl_verify": None} - fs = S3FileSystem(**config) - - assert "client_kwargs" not in fs.fs_args - - config = { - "url": url, - "endpointurl": "https://my.custom.s3:1234", - "ssl_verify": None, - } - fs = S3FileSystem(**config) - - assert "verify" not in fs.fs_args["client_kwargs"] - - -def test_grants(dvc): - config = { - "url": url, - "grant_read": "id=read-permission-id,id=other-read-permission-id", - "grant_read_acp": "id=read-acp-permission-id", - "grant_write_acp": "id=write-acp-permission-id", - "grant_full_control": "id=full-control-permission-id", - } - fs = S3FileSystem(**config) - - extra_args = fs.fs_args["s3_additional_kwargs"] - assert ( - extra_args["GrantRead"] == "id=read-permission-id,id=other-read-permission-id" - ) - assert extra_args["GrantReadACP"] == "id=read-acp-permission-id" - assert extra_args["GrantWriteACP"] == "id=write-acp-permission-id" - assert extra_args["GrantFullControl"] == "id=full-control-permission-id" - - -def test_grants_mutually_exclusive_acl_error(dvc, grants): - for grant_option, grant_value in grants.items(): - config = {"url": url, "acl": "public-read", grant_option: grant_value} - - with pytest.raises(ConfigError): - S3FileSystem(**config) - - -def test_sse_kms_key_id(dvc): - fs = S3FileSystem(url=url, sse_kms_key_id="key") - assert fs.fs_args["s3_additional_kwargs"]["SSEKMSKeyId"] == "key" - - -def test_key_id_and_secret(dvc): - fs = S3FileSystem( - url=url, - access_key_id=key_id, - secret_access_key=key_secret, - session_token=session_token, - ) - assert fs.fs_args["key"] == key_id - assert fs.fs_args["secret"] == key_secret - assert fs.fs_args["token"] == session_token