fs: test and slightly refactor get_fs_config()

This commit is contained in:
Ronan Lamy
2022-09-28 22:06:40 +03:00
committed by Ruslan Kuprieiev
parent 13b2ac3bd1
commit 04af944e22
3 changed files with 52 additions and 12 deletions
+13 -9
View File
@@ -2,6 +2,9 @@ from urllib.parse import urlparse
from dvc_http import HTTPFileSystem, HTTPSFileSystem # noqa: F401
from dvc.config import ConfigError as RepoConfigError
from dvc.config_schema import SCHEMA, Invalid
# pylint: disable=unused-import
from dvc_objects.fs import utils # noqa: F401
from dvc_objects.fs import ( # noqa: F401
@@ -44,7 +47,7 @@ known_implementations.update(
# pylint: enable=unused-import
def get_fs_config(repo, config, **kwargs):
def get_fs_config(config, **kwargs):
name = kwargs.get("name")
if name:
try:
@@ -55,10 +58,10 @@ def get_fs_config(repo, config, **kwargs):
raise RemoteNotFoundError(f"remote '{name}' doesn't exist")
else:
remote_conf = kwargs
return _resolve_remote_refs(repo, config, remote_conf)
return _resolve_remote_refs(config, remote_conf)
def _resolve_remote_refs(repo, config, remote_conf):
def _resolve_remote_refs(config, remote_conf):
# Support for cross referenced remotes.
# This will merge the settings, shadowing base ref with remote_conf.
# For example, having:
@@ -84,21 +87,22 @@ def _resolve_remote_refs(repo, config, remote_conf):
if parsed.scheme != "remote":
return remote_conf
base = get_fs_config(repo, config, name=parsed.netloc)
cls, _, _ = get_cloud_fs(repo, **base)
base = get_fs_config(config, name=parsed.netloc)
cls, _, _ = _get_cloud_fs(config, **base)
relpath = parsed.path.lstrip("/").replace("/", cls.sep)
url = cls.sep.join((base["url"], relpath))
return {**base, **remote_conf, "url": url}
def get_cloud_fs(repo, **kwargs):
from dvc.config import ConfigError as RepoConfigError
from dvc.config_schema import SCHEMA, Invalid
repo_config = repo.config if repo else {}
return _get_cloud_fs(repo_config, **kwargs)
def _get_cloud_fs(repo_config, **kwargs):
core_config = repo_config.get("core", {})
remote_conf = get_fs_config(repo, repo_config, **kwargs)
remote_conf = get_fs_config(repo_config, **kwargs)
try:
remote_conf = SCHEMA["remote"][str](remote_conf)
except Invalid as exc:
+1 -1
View File
@@ -76,7 +76,7 @@ def _get_caches(cache):
def _get_remotes(config):
schemes = (
get_fs_cls(get_fs_config(None, config, name=remote)).protocol
get_fs_cls(get_fs_config(config, name=remote)).protocol
for remote in config["remote"]
)
+38 -2
View File
@@ -1,10 +1,12 @@
from dataclasses import dataclass
import pytest
from dvc_http import HTTPFileSystem, HTTPSFileSystem
from dvc_s3 import S3FileSystem
from dvc_ssh import SSHFileSystem
from dvc.config import RemoteNotFoundError
from dvc.fs import LocalFileSystem, get_fs_cls, get_fs_config
from dvc.fs import LocalFileSystem, get_cloud_fs, get_fs_cls, get_fs_config
url_cls_pairs = [
("s3://bucket/path", S3FileSystem),
@@ -36,5 +38,39 @@ def test_get_fs_cls(url, cls):
def test_get_fs_config():
result = get_fs_config({}, url="ssh://example.com:/dir/path")
assert result == {"url": "ssh://example.com:/dir/path"}
def test_get_fs_config_error():
with pytest.raises(RemoteNotFoundError):
get_fs_config(None, {"remote": {}}, name="myremote")
get_fs_config({"remote": {}}, name="myremote")
def test_remote_url():
config = {
"remote": {
"base": {"url": "http://example.com"},
"r1": {"url": "remote://base/r1", "user": "user"},
"r2": {"url": "remote://r1/r2", "password": "123"},
}
}
result = get_fs_config(config, url="remote://r2/foo")
assert result == {
"password": "123",
"user": "user",
"url": "http://example.com/r1/r2/foo",
}
@dataclass
class FakeRepo:
config: dict
def test_get_cloud_fs():
repo = FakeRepo({})
cls, config, path = get_cloud_fs(repo, url="ssh://example.com:/dir/path")
assert cls is SSHFileSystem
assert config == {"host": "example.com", "verify": False}
assert path == "/dir/path"