mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
92 lines
2.6 KiB
Python
92 lines
2.6 KiB
Python
import posixpath
|
|
from urllib.parse import urlparse
|
|
|
|
from .azure import AzureFileSystem
|
|
from .gdrive import GDriveFileSystem
|
|
from .gs import GSFileSystem
|
|
from .hdfs import HDFSFileSystem
|
|
from .http import HTTPFileSystem
|
|
from .https import HTTPSFileSystem
|
|
from .local import LocalFileSystem
|
|
from .oss import OSSFileSystem
|
|
from .s3 import S3FileSystem
|
|
from .ssh import SSHFileSystem
|
|
from .webdav import WebDAVFileSystem
|
|
from .webdavs import WebDAVSFileSystem
|
|
from .webhdfs import WebHDFSFileSystem
|
|
|
|
ALL_FS = [
|
|
AzureFileSystem,
|
|
GDriveFileSystem,
|
|
GSFileSystem,
|
|
HDFSFileSystem,
|
|
HTTPFileSystem,
|
|
HTTPSFileSystem,
|
|
S3FileSystem,
|
|
SSHFileSystem,
|
|
OSSFileSystem,
|
|
WebDAVFileSystem,
|
|
WebDAVSFileSystem,
|
|
WebHDFSFileSystem
|
|
# NOTE: LocalFileSystem is the default
|
|
]
|
|
|
|
|
|
def get_fs_cls(remote_conf):
|
|
for fs_cls in ALL_FS:
|
|
if fs_cls.supported(remote_conf):
|
|
return fs_cls
|
|
return LocalFileSystem
|
|
|
|
|
|
def get_fs_config(config, **kwargs):
|
|
name = kwargs.get("name")
|
|
if name:
|
|
remote_conf = config["remote"][name.lower()]
|
|
else:
|
|
remote_conf = kwargs
|
|
return _resolve_remote_refs(config, remote_conf)
|
|
|
|
|
|
def _resolve_remote_refs(config, remote_conf):
|
|
# Support for cross referenced remotes.
|
|
# This will merge the settings, shadowing base ref with remote_conf.
|
|
# For example, having:
|
|
#
|
|
# dvc remote add server ssh://localhost
|
|
# dvc remote modify server user root
|
|
# dvc remote modify server ask_password true
|
|
#
|
|
# dvc remote add images remote://server/tmp/pictures
|
|
# dvc remote modify images user alice
|
|
# dvc remote modify images ask_password false
|
|
# dvc remote modify images password asdf1234
|
|
#
|
|
# Results on a config dictionary like:
|
|
#
|
|
# {
|
|
# "url": "ssh://localhost/tmp/pictures",
|
|
# "user": "alice",
|
|
# "password": "asdf1234",
|
|
# "ask_password": False,
|
|
# }
|
|
parsed = urlparse(remote_conf["url"])
|
|
if parsed.scheme != "remote":
|
|
return remote_conf
|
|
|
|
base = get_fs_config(config, name=parsed.netloc)
|
|
url = posixpath.join(base["url"], parsed.path.lstrip("/"))
|
|
return {**base, **remote_conf, "url": url}
|
|
|
|
|
|
def get_cloud_fs(repo, **kwargs):
|
|
from dvc.config import ConfigError
|
|
from dvc.config_schema import SCHEMA, Invalid
|
|
|
|
remote_conf = get_fs_config(repo.config, **kwargs)
|
|
try:
|
|
remote_conf = SCHEMA["remote"][str](remote_conf)
|
|
except Invalid as exc:
|
|
raise ConfigError(str(exc)) from None
|
|
return get_fs_cls(remote_conf)(repo, remote_conf)
|