mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-11 07:18:28 +01:00
fs: test and slightly refactor get_fs_config()
This commit is contained in:
committed by
Ruslan Kuprieiev
parent
13b2ac3bd1
commit
04af944e22
+13
-9
@@ -2,6 +2,9 @@ from urllib.parse import urlparse
|
||||
|
||||
from dvc_http import HTTPFileSystem, HTTPSFileSystem # noqa: F401
|
||||
|
||||
from dvc.config import ConfigError as RepoConfigError
|
||||
from dvc.config_schema import SCHEMA, Invalid
|
||||
|
||||
# pylint: disable=unused-import
|
||||
from dvc_objects.fs import utils # noqa: F401
|
||||
from dvc_objects.fs import ( # noqa: F401
|
||||
@@ -44,7 +47,7 @@ known_implementations.update(
|
||||
# pylint: enable=unused-import
|
||||
|
||||
|
||||
def get_fs_config(repo, config, **kwargs):
|
||||
def get_fs_config(config, **kwargs):
|
||||
name = kwargs.get("name")
|
||||
if name:
|
||||
try:
|
||||
@@ -55,10 +58,10 @@ def get_fs_config(repo, config, **kwargs):
|
||||
raise RemoteNotFoundError(f"remote '{name}' doesn't exist")
|
||||
else:
|
||||
remote_conf = kwargs
|
||||
return _resolve_remote_refs(repo, config, remote_conf)
|
||||
return _resolve_remote_refs(config, remote_conf)
|
||||
|
||||
|
||||
def _resolve_remote_refs(repo, config, remote_conf):
|
||||
def _resolve_remote_refs(config, remote_conf):
|
||||
# Support for cross referenced remotes.
|
||||
# This will merge the settings, shadowing base ref with remote_conf.
|
||||
# For example, having:
|
||||
@@ -84,21 +87,22 @@ def _resolve_remote_refs(repo, config, remote_conf):
|
||||
if parsed.scheme != "remote":
|
||||
return remote_conf
|
||||
|
||||
base = get_fs_config(repo, config, name=parsed.netloc)
|
||||
cls, _, _ = get_cloud_fs(repo, **base)
|
||||
base = get_fs_config(config, name=parsed.netloc)
|
||||
cls, _, _ = _get_cloud_fs(config, **base)
|
||||
relpath = parsed.path.lstrip("/").replace("/", cls.sep)
|
||||
url = cls.sep.join((base["url"], relpath))
|
||||
return {**base, **remote_conf, "url": url}
|
||||
|
||||
|
||||
def get_cloud_fs(repo, **kwargs):
|
||||
from dvc.config import ConfigError as RepoConfigError
|
||||
from dvc.config_schema import SCHEMA, Invalid
|
||||
|
||||
repo_config = repo.config if repo else {}
|
||||
return _get_cloud_fs(repo_config, **kwargs)
|
||||
|
||||
|
||||
def _get_cloud_fs(repo_config, **kwargs):
|
||||
core_config = repo_config.get("core", {})
|
||||
|
||||
remote_conf = get_fs_config(repo, repo_config, **kwargs)
|
||||
remote_conf = get_fs_config(repo_config, **kwargs)
|
||||
try:
|
||||
remote_conf = SCHEMA["remote"][str](remote_conf)
|
||||
except Invalid as exc:
|
||||
|
||||
+1
-1
@@ -76,7 +76,7 @@ def _get_caches(cache):
|
||||
|
||||
def _get_remotes(config):
|
||||
schemes = (
|
||||
get_fs_cls(get_fs_config(None, config, name=remote)).protocol
|
||||
get_fs_cls(get_fs_config(config, name=remote)).protocol
|
||||
for remote in config["remote"]
|
||||
)
|
||||
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
from dvc_http import HTTPFileSystem, HTTPSFileSystem
|
||||
from dvc_s3 import S3FileSystem
|
||||
from dvc_ssh import SSHFileSystem
|
||||
|
||||
from dvc.config import RemoteNotFoundError
|
||||
from dvc.fs import LocalFileSystem, get_fs_cls, get_fs_config
|
||||
from dvc.fs import LocalFileSystem, get_cloud_fs, get_fs_cls, get_fs_config
|
||||
|
||||
url_cls_pairs = [
|
||||
("s3://bucket/path", S3FileSystem),
|
||||
@@ -36,5 +38,39 @@ def test_get_fs_cls(url, cls):
|
||||
|
||||
|
||||
def test_get_fs_config():
|
||||
result = get_fs_config({}, url="ssh://example.com:/dir/path")
|
||||
assert result == {"url": "ssh://example.com:/dir/path"}
|
||||
|
||||
|
||||
def test_get_fs_config_error():
|
||||
with pytest.raises(RemoteNotFoundError):
|
||||
get_fs_config(None, {"remote": {}}, name="myremote")
|
||||
get_fs_config({"remote": {}}, name="myremote")
|
||||
|
||||
|
||||
def test_remote_url():
|
||||
config = {
|
||||
"remote": {
|
||||
"base": {"url": "http://example.com"},
|
||||
"r1": {"url": "remote://base/r1", "user": "user"},
|
||||
"r2": {"url": "remote://r1/r2", "password": "123"},
|
||||
}
|
||||
}
|
||||
result = get_fs_config(config, url="remote://r2/foo")
|
||||
assert result == {
|
||||
"password": "123",
|
||||
"user": "user",
|
||||
"url": "http://example.com/r1/r2/foo",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class FakeRepo:
|
||||
config: dict
|
||||
|
||||
|
||||
def test_get_cloud_fs():
|
||||
repo = FakeRepo({})
|
||||
cls, config, path = get_cloud_fs(repo, url="ssh://example.com:/dir/path")
|
||||
assert cls is SSHFileSystem
|
||||
assert config == {"host": "example.com", "verify": False}
|
||||
assert path == "/dir/path"
|
||||
|
||||
Reference in New Issue
Block a user