mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-01 23:08:43 +01:00
fs: make _prepare_credentails/fs_args lazy (#9848)
* tests: unit: move s3 tests to dvc-s3 * deps: bump dvc-data to 2.14.0 * tests: unit: move azure tests to dvc-azure
This commit is contained in:
Ruslan Kuprieiev
+1
-1
@@ -34,7 +34,7 @@ dependencies = [
|
||||
"configobj>=5.0.6",
|
||||
"distro>=1.3",
|
||||
"dpath<3,>=2.1.0",
|
||||
"dvc-data>=2.13.1,<2.14.0",
|
||||
"dvc-data>=2.14.0,<2.15.0",
|
||||
"dvc-http>=2.29.0",
|
||||
"dvc-render>=0.3.1,<1",
|
||||
"dvc-studio-client>=0.9.2,<1",
|
||||
|
||||
@@ -1,84 +0,0 @@
|
||||
import pytest
|
||||
from dvc_azure import AzureAuthError, AzureFileSystem
|
||||
|
||||
container_name = "container-name"
|
||||
connection_string = (
|
||||
"DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;"
|
||||
"AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsu"
|
||||
"Fq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;"
|
||||
"BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;"
|
||||
)
|
||||
|
||||
|
||||
def test_strip_protocol_env_var(monkeypatch, dvc):
|
||||
monkeypatch.setenv("AZURE_STORAGE_CONTAINER_NAME", container_name)
|
||||
monkeypatch.setenv("AZURE_STORAGE_CONNECTION_STRING", connection_string)
|
||||
|
||||
assert AzureFileSystem._strip_protocol("azure://") == container_name
|
||||
|
||||
|
||||
def test_strip_protocol(dvc):
|
||||
assert (
|
||||
AzureFileSystem._strip_protocol(f"azure://{container_name}") == container_name
|
||||
)
|
||||
|
||||
|
||||
def test_init(dvc):
|
||||
config = {"connection_string": connection_string}
|
||||
fs = AzureFileSystem(**config)
|
||||
assert fs.fs_args["connection_string"] == connection_string
|
||||
|
||||
|
||||
def test_azure_login_methods():
|
||||
def get_login_method(config):
|
||||
fs = AzureFileSystem(**config)
|
||||
return fs.login_method
|
||||
|
||||
with pytest.raises(AzureAuthError):
|
||||
get_login_method({})
|
||||
|
||||
assert get_login_method({"connection_string": "test"}) == "connection string"
|
||||
assert get_login_method({"account_name": "test"}).startswith("default credentials")
|
||||
assert (
|
||||
get_login_method({"account_name": "test", "allow_anonymous_login": True})
|
||||
== "anonymous login"
|
||||
)
|
||||
|
||||
with pytest.raises(AzureAuthError):
|
||||
get_login_method(
|
||||
{"tenant_id": "test", "client_id": "test", "client_secret": "test"}
|
||||
)
|
||||
|
||||
assert (
|
||||
get_login_method(
|
||||
{
|
||||
"account_name": "test",
|
||||
"tenant_id": "test",
|
||||
"client_id": "test",
|
||||
"client_secret": "test",
|
||||
}
|
||||
)
|
||||
== "AD service principal"
|
||||
)
|
||||
|
||||
assert (
|
||||
get_login_method({"account_name": "test", "account_key": "test"})
|
||||
== "account key"
|
||||
)
|
||||
assert (
|
||||
get_login_method({"account_name": "test", "sas_token": "test"}) == "SAS token"
|
||||
)
|
||||
assert (
|
||||
get_login_method(
|
||||
{
|
||||
"connection_string": "test",
|
||||
"account_name": "test",
|
||||
"sas_token": "test",
|
||||
}
|
||||
)
|
||||
== "connection string"
|
||||
)
|
||||
assert (
|
||||
get_login_method({"connection_string": "test", "sas_token": "test"})
|
||||
== "connection string"
|
||||
)
|
||||
@@ -1,137 +0,0 @@
|
||||
import os
|
||||
|
||||
import pytest
|
||||
from dvc_s3 import S3FileSystem
|
||||
|
||||
from dvc.fs import ConfigError
|
||||
|
||||
bucket_name = "bucket-name"
|
||||
prefix = "some/prefix"
|
||||
url = f"s3://{bucket_name}/{prefix}"
|
||||
key_id = "key-id"
|
||||
key_secret = "key-secret"
|
||||
session_token = "session-token"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def grants():
|
||||
return {
|
||||
"grant_read": "id=read-permission-id,id=other-read-permission-id",
|
||||
"grant_read_acp": "id=read-acp-permission-id",
|
||||
"grant_write_acp": "id=write-acp-permission-id",
|
||||
"grant_full_control": "id=full-control-permission-id",
|
||||
}
|
||||
|
||||
|
||||
def test_verify_ssl_default_param(dvc):
|
||||
config = {"url": url}
|
||||
fs = S3FileSystem(**config)
|
||||
|
||||
assert "client_kwargs" not in fs.fs_args
|
||||
|
||||
config = {
|
||||
"url": url,
|
||||
"endpointurl": "https://my.custom.s3:1234",
|
||||
}
|
||||
fs = S3FileSystem(**config)
|
||||
|
||||
assert "verify" not in fs.fs_args["client_kwargs"]
|
||||
|
||||
|
||||
def test_s3_config_credentialpath(dvc, monkeypatch):
|
||||
environment = {}
|
||||
monkeypatch.setattr(os, "environ", environment)
|
||||
|
||||
config = {"url": url, "credentialpath": "somewhere"}
|
||||
S3FileSystem(**config)
|
||||
assert environment["AWS_SHARED_CREDENTIALS_FILE"] == "somewhere"
|
||||
environment.clear()
|
||||
|
||||
config = {"url": url, "configpath": "somewhere"}
|
||||
S3FileSystem(**config)
|
||||
assert environment["AWS_CONFIG_FILE"] == "somewhere"
|
||||
environment.clear()
|
||||
|
||||
config = {
|
||||
"url": url,
|
||||
"credentialpath": "somewhere",
|
||||
"configpath": "elsewhere",
|
||||
}
|
||||
S3FileSystem(**config)
|
||||
assert environment["AWS_SHARED_CREDENTIALS_FILE"] == "somewhere"
|
||||
assert environment["AWS_CONFIG_FILE"] == "elsewhere"
|
||||
environment.clear()
|
||||
|
||||
|
||||
def test_ssl_verify_bool_param(dvc):
|
||||
config = {"url": url, "ssl_verify": False}
|
||||
fs = S3FileSystem(**config)
|
||||
|
||||
assert fs.fs_args["client_kwargs"]["verify"] == config["ssl_verify"]
|
||||
|
||||
|
||||
def test_ssl_verify_path_param(dvc):
|
||||
config = {"url": url, "ssl_verify": "/path/to/custom/cabundle.pem"}
|
||||
fs = S3FileSystem(**config)
|
||||
|
||||
assert fs.fs_args["client_kwargs"]["verify"] == config["ssl_verify"]
|
||||
|
||||
|
||||
def test_ssl_verify_none_param(dvc):
|
||||
config = {"url": url, "ssl_verify": None}
|
||||
fs = S3FileSystem(**config)
|
||||
|
||||
assert "client_kwargs" not in fs.fs_args
|
||||
|
||||
config = {
|
||||
"url": url,
|
||||
"endpointurl": "https://my.custom.s3:1234",
|
||||
"ssl_verify": None,
|
||||
}
|
||||
fs = S3FileSystem(**config)
|
||||
|
||||
assert "verify" not in fs.fs_args["client_kwargs"]
|
||||
|
||||
|
||||
def test_grants(dvc):
|
||||
config = {
|
||||
"url": url,
|
||||
"grant_read": "id=read-permission-id,id=other-read-permission-id",
|
||||
"grant_read_acp": "id=read-acp-permission-id",
|
||||
"grant_write_acp": "id=write-acp-permission-id",
|
||||
"grant_full_control": "id=full-control-permission-id",
|
||||
}
|
||||
fs = S3FileSystem(**config)
|
||||
|
||||
extra_args = fs.fs_args["s3_additional_kwargs"]
|
||||
assert (
|
||||
extra_args["GrantRead"] == "id=read-permission-id,id=other-read-permission-id"
|
||||
)
|
||||
assert extra_args["GrantReadACP"] == "id=read-acp-permission-id"
|
||||
assert extra_args["GrantWriteACP"] == "id=write-acp-permission-id"
|
||||
assert extra_args["GrantFullControl"] == "id=full-control-permission-id"
|
||||
|
||||
|
||||
def test_grants_mutually_exclusive_acl_error(dvc, grants):
|
||||
for grant_option, grant_value in grants.items():
|
||||
config = {"url": url, "acl": "public-read", grant_option: grant_value}
|
||||
|
||||
with pytest.raises(ConfigError):
|
||||
S3FileSystem(**config)
|
||||
|
||||
|
||||
def test_sse_kms_key_id(dvc):
|
||||
fs = S3FileSystem(url=url, sse_kms_key_id="key")
|
||||
assert fs.fs_args["s3_additional_kwargs"]["SSEKMSKeyId"] == "key"
|
||||
|
||||
|
||||
def test_key_id_and_secret(dvc):
|
||||
fs = S3FileSystem(
|
||||
url=url,
|
||||
access_key_id=key_id,
|
||||
secret_access_key=key_secret,
|
||||
session_token=session_token,
|
||||
)
|
||||
assert fs.fs_args["key"] == key_id
|
||||
assert fs.fs_args["secret"] == key_secret
|
||||
assert fs.fs_args["token"] == session_token
|
||||
Reference in New Issue
Block a user