diff --git a/dvc/checkout.py b/dvc/checkout.py index 54b3ec900..89d7c422a 100644 --- a/dvc/checkout.py +++ b/dvc/checkout.py @@ -134,7 +134,7 @@ def _cache_is_copy(cache, path_info): return True workspace_file = path_info.with_name("." + uuid()) - test_cache_file = cache.fs.path_info / ".cache_type_test_file" + test_cache_file = cache.path_info / ".cache_type_test_file" if not cache.fs.exists(test_cache_file): cache.makedirs(test_cache_file.parent) with cache.fs.open(test_cache_file, "wb") as fobj: diff --git a/dvc/fs/__init__.py b/dvc/fs/__init__.py index 6e3696e30..fe38415e2 100644 --- a/dvc/fs/__init__.py +++ b/dvc/fs/__init__.py @@ -106,4 +106,11 @@ def get_cloud_fs(repo, **kwargs): if cls == GDriveFileSystem and repo: remote_conf["gdrive_credentials_tmp_dir"] = repo.tmp_dir - return cls, remote_conf + url = remote_conf.pop("url") + path_info = cls.PATH_CLS( + cls._strip_protocol(url) # pylint:disable=protected-access + ) + + extras = cls._get_kwargs_from_urls(url) # pylint:disable=protected-access + + return cls, dict(**extras, **remote_conf), path_info diff --git a/dvc/fs/azure.py b/dvc/fs/azure.py index eaeffb52e..acdf06fe1 100644 --- a/dvc/fs/azure.py +++ b/dvc/fs/azure.py @@ -5,7 +5,8 @@ import sys import threading from contextlib import contextmanager -from funcy import cached_property, wrap_prop +from fsspec.utils import infer_storage_options +from funcy import cached_property, memoize, wrap_prop from dvc.exceptions import DvcException from dvc.path_info import CloudURLInfo @@ -67,6 +68,19 @@ class AzureAuthError(DvcException): pass +@memoize +def _az_config(): + # NOTE: ideally we would've used get_default_cli().config from + # azure.cli.core, but azure-cli-core has a lot of conflicts with other + # dependencies. So instead we are just use knack directly + from knack.config import CLIConfig + + config_dir = os.getenv( + "AZURE_CONFIG_DIR", os.path.expanduser(os.path.join("~", ".azure")) + ) + return CLIConfig(config_dir=config_dir, config_env_var_prefix="AZURE") + + # pylint:disable=abstract-method class AzureFileSystem(ObjectFSWrapper): scheme = Schemes.AZURE @@ -79,17 +93,21 @@ class AzureFileSystem(ObjectFSWrapper): "azure-identity": "azure.identity", } - def __init__(self, **config): - super().__init__(**config) + @classmethod + def _strip_protocol(cls, path: str): + bucket = infer_storage_options(path).get("host") + if bucket: + return path - url = config.get("url") - self.path_info = self.PATH_CLS(url) + bucket = _az_config().get("storage", "container_name", None) + return f"azure://{bucket}" - if not self.path_info.bucket: - container = self._az_config.get("storage", "container_name", None) - url = f"azure://{container}" - - self.path_info = self.PATH_CLS(url) + @staticmethod + def _get_kwargs_from_urls(urlpath): + ops = infer_storage_options(urlpath) + if "host" in ops: + return {"bucket": ops["host"]} + return {} def _prepare_credentials(self, **config): from azure.identity.aio import DefaultAzureCredential @@ -100,16 +118,16 @@ class AzureFileSystem(ObjectFSWrapper): login_info = {} login_info["connection_string"] = config.get( "connection_string", - self._az_config.get("storage", "connection_string", None), + _az_config().get("storage", "connection_string", None), ) login_info["account_name"] = config.get( - "account_name", self._az_config.get("storage", "account", None) + "account_name", _az_config().get("storage", "account", None) ) login_info["account_key"] = config.get( - "account_key", self._az_config.get("storage", "key", None) + "account_key", _az_config().get("storage", "key", None) ) login_info["sas_token"] = config.get( - "sas_token", self._az_config.get("storage", "sas_token", None) + "sas_token", _az_config().get("storage", "sas_token", None) ) login_info["tenant_id"] = config.get("tenant_id") login_info["client_id"] = config.get("client_id") @@ -157,18 +175,6 @@ class AzureFileSystem(ObjectFSWrapper): self.login_method = login_method return login_info - @cached_property - def _az_config(self): - # NOTE: ideally we would've used get_default_cli().config from - # azure.cli.core, but azure-cli-core has a lot of conflicts with other - # dependencies. So instead we are just use knack directly - from knack.config import CLIConfig - - config_dir = os.getenv( - "AZURE_CONFIG_DIR", os.path.expanduser(os.path.join("~", ".azure")) - ) - return CLIConfig(config_dir=config_dir, config_env_var_prefix="AZURE") - @wrap_prop(threading.Lock()) @cached_property def fs(self): diff --git a/dvc/fs/base.py b/dvc/fs/base.py index 392ec5407..e4c4513f5 100644 --- a/dvc/fs/base.py +++ b/dvc/fs/base.py @@ -55,11 +55,17 @@ class BaseFileSystem: def __init__(self, **kwargs): self._check_requires(**kwargs) - self.path_info = None - self.jobs = kwargs.get("jobs") or self._JOBS self.hash_jobs = kwargs.get("checksum_jobs") or self.HASH_JOBS + @classmethod + def _strip_protocol(cls, path: str): + return path + + @staticmethod + def _get_kwargs_from_urls(urlpath): # pylint:disable=unused-argument + return {} + @classmethod def get_missing_deps(cls): import importlib diff --git a/dvc/fs/gdrive.py b/dvc/fs/gdrive.py index 5a65d9997..3cceee896 100644 --- a/dvc/fs/gdrive.py +++ b/dvc/fs/gdrive.py @@ -142,6 +142,10 @@ class GDriveFileSystem(BaseFileSystem): # pylint:disable=abstract-method ) ) + @staticmethod + def _get_kwargs_from_urls(urlpath): + return {"url": urlpath} + def _validate_config(self): # Validate Service Account configuration if ( diff --git a/dvc/fs/gs.py b/dvc/fs/gs.py index a375ac50c..5a585ea58 100644 --- a/dvc/fs/gs.py +++ b/dvc/fs/gs.py @@ -17,12 +17,6 @@ class GSFileSystem(ObjectFSWrapper): PARAM_CHECKSUM = "etag" DETAIL_FIELDS = frozenset(("etag", "size")) - def __init__(self, **config): - super().__init__(**config) - - url = config.get("url", "gs://") - self.path_info = self.PATH_CLS(url) - def _prepare_credentials(self, **config): login_info = {"consistency": None} login_info["project"] = config.get("projectname") diff --git a/dvc/fs/hdfs.py b/dvc/fs/hdfs.py index 635cf3e38..91e4fc203 100644 --- a/dvc/fs/hdfs.py +++ b/dvc/fs/hdfs.py @@ -6,7 +6,6 @@ import shutil import subprocess from collections import deque from contextlib import closing, contextmanager -from urllib.parse import urlparse from dvc.hash_info import HashInfo from dvc.progress import Tqdm @@ -68,24 +67,19 @@ class HDFSFileSystem(BaseFileSystem): def __init__(self, **config): super().__init__(**config) - self.path_info = None - url = config.get("url") - if not url: - return - - parsed = urlparse(url) - user = parsed.username or config.get("user") - - self.path_info = self.PATH_CLS.from_parts( - scheme=self.scheme, - host=parsed.hostname, - user=user, - port=parsed.port, - path=parsed.path, - ) + self.host = config["host"] + self.user = config.get("user") + self.port = config.get("port") self.kerb_ticket = config.get("kerb_ticket") + @staticmethod + def _get_kwargs_from_urls(urlpath): + from fsspec.implementations.hdfs import PyArrowHDFS + + # pylint:disable=protected-access + return PyArrowHDFS._get_kwargs_from_urls(urlpath) + def hdfs(self, path_info): import pyarrow.fs diff --git a/dvc/fs/http.py b/dvc/fs/http.py index 455f2ca33..022890db9 100644 --- a/dvc/fs/http.py +++ b/dvc/fs/http.py @@ -2,6 +2,7 @@ import logging import os.path import threading from typing import Optional +from urllib.parse import urlparse from funcy import cached_property, memoize, wrap_prop, wrap_with @@ -39,13 +40,7 @@ class HTTPFileSystem(BaseFileSystem): # pylint:disable=abstract-method def __init__(self, **config): super().__init__(**config) - url = config.get("url") - if url: - self.path_info = self.PATH_CLS(url) - self.user = config.get("user", None) - self.host = self.path_info.host - else: - self.path_info = None + self.user = config.get("user", None) self.auth = config.get("auth", None) self.custom_auth_header = config.get("custom_auth_header", None) @@ -55,12 +50,12 @@ class HTTPFileSystem(BaseFileSystem): # pylint:disable=abstract-method self.ssl_verify = config.get("ssl_verify", True) self.method = config.get("method", "POST") - def _auth_method(self): + def _auth_method(self, url): from requests.auth import HTTPBasicAuth, HTTPDigestAuth if self.auth: if self.ask_password and self.password is None: - self.password = ask_password(self.host, self.user) + self.password = ask_password(urlparse(url).hostname, self.user) if self.auth == "basic": return HTTPBasicAuth(self.user, self.password) if self.auth == "digest": @@ -103,7 +98,7 @@ class HTTPFileSystem(BaseFileSystem): # pylint:disable=abstract-method res = self._session.request( method, url, - auth=self._auth_method(), + auth=self._auth_method(url), headers=self.headers, **kwargs, ) diff --git a/dvc/fs/local.py b/dvc/fs/local.py index 0a2120356..9cb406c40 100644 --- a/dvc/fs/local.py +++ b/dvc/fs/local.py @@ -25,8 +25,6 @@ class LocalFileSystem(BaseFileSystem): super().__init__(**config) self.fs = LocalFS() - url = config.get("url") - self.path_info = self.PATH_CLS(url) if url else None @staticmethod def open(path_info, mode="r", encoding=None, **kwargs): diff --git a/dvc/fs/oss.py b/dvc/fs/oss.py index b92a87714..19ed1795f 100644 --- a/dvc/fs/oss.py +++ b/dvc/fs/oss.py @@ -42,9 +42,6 @@ class OSSFileSystem(BaseFileSystem): # pylint:disable=abstract-method def __init__(self, **config): super().__init__(**config) - url = config.get("url") - self.path_info = self.PATH_CLS(url) if url else None - self.endpoint = config.get("oss_endpoint") or os.getenv("OSS_ENDPOINT") self.key_id = ( diff --git a/dvc/fs/s3.py b/dvc/fs/s3.py index 40bcb1756..c7bdba2a7 100644 --- a/dvc/fs/s3.py +++ b/dvc/fs/s3.py @@ -29,12 +29,6 @@ class BaseS3FileSystem(ObjectFSWrapper): "grant_write_acp": "GrantWriteACP", } - def __init__(self, **config): - super().__init__(**config) - - url = config.get("url", "s3://") - self.path_info = self.PATH_CLS(url) - _TRANSFER_CONFIG_ALIASES = { "max_queue_size": "max_io_queue", "max_concurrent_requests": "max_concurrency", diff --git a/dvc/fs/ssh/__init__.py b/dvc/fs/ssh/__init__.py index 1df4fcb3d..93a9e4021 100644 --- a/dvc/fs/ssh/__init__.py +++ b/dvc/fs/ssh/__init__.py @@ -5,7 +5,6 @@ import os import shutil import threading from contextlib import closing, contextmanager -from urllib.parse import urlparse from funcy import first, memoize, silent, wrap_with @@ -48,34 +47,19 @@ class SSHFileSystem(BaseFileSystem): # pylint:disable=abstract-method def __init__(self, **config): super().__init__(**config) - url = config.get("url") - if url: - parsed = urlparse(url) - user_ssh_config = self._load_user_ssh_config(parsed.hostname) + user_ssh_config = self._load_user_ssh_config(config["host"]) - host = user_ssh_config.get("hostname", parsed.hostname) - user = ( - config.get("user") - or parsed.username - or user_ssh_config.get("user") - or getpass.getuser() - ) - port = ( - config.get("port") - or parsed.port - or self._try_get_ssh_config_port(user_ssh_config) - or self.DEFAULT_PORT - ) - self.path_info = self.PATH_CLS.from_parts( - scheme=self.scheme, - host=host, - user=user, - port=port, - path=parsed.path, - ) - else: - self.path_info = None - user_ssh_config = {} + self.host = user_ssh_config.get("hostname", config["host"]) + self.user = ( + config.get("user") + or user_ssh_config.get("user") + or getpass.getuser() + ) + self.port = ( + config.get("port") + or self._try_get_ssh_config_port(user_ssh_config) + or self.DEFAULT_PORT + ) self.keyfile = config.get( "keyfile" @@ -93,6 +77,16 @@ class SSHFileSystem(BaseFileSystem): # pylint:disable=abstract-method self.sock = None self.allow_agent = config.get("allow_agent", True) + @staticmethod + def _get_kwargs_from_urls(urlpath): + from fsspec.implementations.sftp import SFTPFileSystem + + # pylint:disable=protected-access + kwargs = SFTPFileSystem._get_kwargs_from_urls(urlpath) + if "username" in kwargs: + kwargs["user"] = kwargs.pop("username") + return kwargs + @staticmethod def ssh_config_filename(): return os.path.expanduser(os.path.join("~", ".ssh", "config")) @@ -120,17 +114,13 @@ class SSHFileSystem(BaseFileSystem): # pylint:disable=abstract-method def _try_get_ssh_config_keyfile(user_ssh_config): return first(user_ssh_config.get("identityfile") or ()) - def ensure_credentials(self, path_info=None): - if path_info is None: - path_info = self.path_info - + def ensure_credentials(self): # NOTE: we use the same password regardless of the server :( if self.ask_password and self.password is None: - host, user, port = path_info.host, path_info.user, path_info.port - self.password = ask_password(host, user, port) + self.password = ask_password(self.host, self.user, self.port) def ssh(self, path_info): - self.ensure_credentials(path_info) + self.ensure_credentials() from .connection import SSHConnection diff --git a/dvc/fs/webdav.py b/dvc/fs/webdav.py index 1e393c602..4c17a9392 100644 --- a/dvc/fs/webdav.py +++ b/dvc/fs/webdav.py @@ -71,20 +71,18 @@ class WebDAVFileSystem(BaseFileSystem): # pylint:disable=abstract-method # Connection timeout self.timeout = config.get("timeout", 30) - # Get URL from configuration - self.url = config.get("url", None) + self.hostname = config["host"] + self.prefix = config.get("prefix") + self.user = config.get("user") - # If URL in config parse path_info - if self.url: - self.path_info = self.PATH_CLS(self.url) - - # If username not specified try to use from URL - if self.user is None and self.path_info.user is not None: - self.user = self.path_info.user - - # Construct hostname from path_info by stripping path - http_info = HTTPURLInfo(self.path_info.url) - self.hostname = http_info.replace(path="").url + @staticmethod + def _get_kwargs_from_urls(urlpath): + path_info = WebDAVURLInfo(urlpath) + http_info = HTTPURLInfo(path_info.url) + return { + "prefix": path_info.path, + "host": http_info.replace(path="").url, + } # Webdav client @wrap_prop(threading.Lock()) @@ -116,10 +114,6 @@ class WebDAVFileSystem(BaseFileSystem): # pylint:disable=abstract-method f"Configuration for WebDAV {self.hostname} is invalid." ) - # Check whether connection is valid (root should always exist) - if not client.check(self.path_info.path): - raise WebDAVConnectionError(self.hostname) - return client def open(self, path_info, mode="r", encoding=None, **kwargs): @@ -202,7 +196,7 @@ class WebDAVFileSystem(BaseFileSystem): # pylint:disable=abstract-method # Creates directories def makedirs(self, path_info): # Terminate recursion - if path_info.path == self.path_info.path or self.exists(path_info): + if path_info.path == self.prefix or self.exists(path_info): return # Recursively descent to root diff --git a/dvc/fs/webhdfs.py b/dvc/fs/webhdfs.py index 1dc8b3c34..2d14fd737 100644 --- a/dvc/fs/webhdfs.py +++ b/dvc/fs/webhdfs.py @@ -4,7 +4,6 @@ import posixpath import shutil import threading from contextlib import contextmanager -from urllib.parse import urlparse from funcy import cached_property, wrap_prop @@ -40,23 +39,24 @@ class WebHDFSFileSystem(BaseFileSystem): # pylint:disable=abstract-method def __init__(self, **config): super().__init__(**config) - self.path_info = None - url = config.get("url") - if not url: - return - - self.path_info = self.PATH_CLS(url) - - parsed = urlparse(url) - - self.host = parsed.hostname - self.user = parsed.username or config.get("user") - self.port = parsed.port + self.host = config["host"] + self.user = config.get("user") + self.port = config.get("port") self.hdfscli_config = config.get("hdfscli_config") self.token = config.get("webhdfs_token") self.alias = config.get("webhdfs_alias") + @staticmethod + def _get_kwargs_from_urls(urlpath): + from fsspec.implementations.webhdfs import WebHDFS + + return ( + WebHDFS._get_kwargs_from_urls( # pylint:disable=protected-access + urlpath + ) + ) + @wrap_prop(threading.Lock()) @cached_property def hdfs_client(self): diff --git a/dvc/objects/db/__init__.py b/dvc/objects/db/__init__.py index 41f45f0cf..507184a8e 100644 --- a/dvc/objects/db/__init__.py +++ b/dvc/objects/db/__init__.py @@ -3,22 +3,22 @@ from collections import defaultdict from dvc.scheme import Schemes -def get_odb(fs, **config): +def get_odb(fs, path_info, **config): from .base import ObjectDB from .gdrive import GDriveObjectDB from .local import LocalObjectDB from .ssh import SSHObjectDB if fs.scheme == Schemes.LOCAL: - return LocalObjectDB(fs, **config) + return LocalObjectDB(fs, path_info, **config) if fs.scheme == Schemes.SSH: - return SSHObjectDB(fs, **config) + return SSHObjectDB(fs, path_info, **config) if fs.scheme == Schemes.GDRIVE: - return GDriveObjectDB(fs, **config) + return GDriveObjectDB(fs, path_info, **config) - return ObjectDB(fs, **config) + return ObjectDB(fs, path_info, **config) def _get_odb(repo, settings): @@ -27,8 +27,8 @@ def _get_odb(repo, settings): if not settings: return None - cls, config = get_cloud_fs(repo, **settings) - return get_odb(cls(**config), state=repo.state, **config) + cls, config, path_info = get_cloud_fs(repo, **settings) + return get_odb(cls(**config), path_info, state=repo.state, **config) class ODBManager: diff --git a/dvc/objects/db/base.py b/dvc/objects/db/base.py index 9707df9c7..edb3bb970 100644 --- a/dvc/objects/db/base.py +++ b/dvc/objects/db/base.py @@ -18,10 +18,11 @@ class ObjectDB: DEFAULT_CACHE_TYPES = ["copy"] CACHE_MODE: Optional[int] = None - def __init__(self, fs, **config): + def __init__(self, fs, path_info, **config): from dvc.state import StateNoop self.fs = fs + self.path_info = path_info self.state = config.get("state", StateNoop()) self.verify = config.get("verify", self.DEFAULT_VERIFY) self.cache_types = config.get("type") or copy(self.DEFAULT_CACHE_TYPES) @@ -87,7 +88,7 @@ class ObjectDB: callback(1) def hash_to_path_info(self, hash_): - return self.fs.path_info / hash_[0:2] / hash_[2:] + return self.path_info / hash_[0:2] / hash_[2:] # Override to return path as a string instead of PathInfo for clouds # which support string paths (see local) @@ -143,12 +144,12 @@ class ObjectDB: def _list_paths(self, prefix=None, progress_callback=None): if prefix: if len(prefix) > 2: - path_info = self.fs.path_info / prefix[:2] / prefix[2:] + path_info = self.path_info / prefix[:2] / prefix[2:] else: - path_info = self.fs.path_info / prefix[:2] + path_info = self.path_info / prefix[:2] prefix = True else: - path_info = self.fs.path_info + path_info = self.path_info prefix = False if progress_callback: for file_info in self.fs.walk_files(path_info, prefix=prefix): @@ -319,7 +320,7 @@ class ObjectDB: removed = False # hashes must be sorted to ensure we always remove .dir files first for hash_ in sorted( - self.all(jobs, str(self.fs.path_info)), + self.all(jobs, str(self.path_info)), key=self.fs.is_dir_hash, reverse=True, ): diff --git a/dvc/objects/db/local.py b/dvc/objects/db/local.py index 65c101990..4af4366fc 100644 --- a/dvc/objects/db/local.py +++ b/dvc/objects/db/local.py @@ -22,9 +22,9 @@ class LocalObjectDB(ObjectDB): CACHE_MODE = 0o444 UNPACKED_DIR_SUFFIX = ".unpacked" - def __init__(self, fs, **config): - super().__init__(fs, **config) - self.cache_dir = config.get("url") + def __init__(self, fs, path_info, **config): + super().__init__(fs, path_info, **config) + self.cache_dir = path_info.fspath shared = config.get("shared") if shared: @@ -36,11 +36,11 @@ class LocalObjectDB(ObjectDB): @property def cache_dir(self): - return self.fs.path_info.fspath if self.fs.path_info else None + return self.path_info.fspath if self.path_info else None @cache_dir.setter def cache_dir(self, value): - self.fs.path_info = PathInfo(value) if value else None + self.path_info = PathInfo(value) if value else None @cached_property def cache_path(self): @@ -81,13 +81,13 @@ class LocalObjectDB(ObjectDB): return ret def _list_paths(self, prefix=None, progress_callback=None): - assert self.fs.path_info is not None + assert self.path_info is not None if prefix: - path_info = self.fs.path_info / prefix[:2] + path_info = self.path_info / prefix[:2] if not self.fs.exists(path_info): return else: - path_info = self.fs.path_info + path_info = self.path_info # NOTE: use utils.fs walk_files since fs.walk_files will not follow # symlinks if progress_callback: diff --git a/dvc/objects/db/ssh.py b/dvc/objects/db/ssh.py index 5be814871..ded0d41db 100644 --- a/dvc/objects/db/ssh.py +++ b/dvc/objects/db/ssh.py @@ -75,10 +75,10 @@ class SSHObjectDB(ObjectDB): def _list_paths(self, prefix=None, progress_callback=None): if prefix: - root = posixpath.join(self.fs.path_info.path, prefix[:2]) + root = posixpath.join(self.path_info.path, prefix[:2]) else: - root = self.fs.path_info.path - with self.fs.ssh(self.fs.path_info) as ssh: + root = self.path_info.path + with self.fs.ssh(self.path_info) as ssh: if prefix and not ssh.exists(root): return # If we simply return an iterator then with above closes instantly diff --git a/dvc/objects/stage.py b/dvc/objects/stage.py index 142d67cf3..e66d426e8 100644 --- a/dvc/objects/stage.py +++ b/dvc/objects/stage.py @@ -15,7 +15,7 @@ def _upload_file(path_info, fs, odb): from dvc.utils import tmp_fname from dvc.utils.stream import HashedStreamReader - tmp_info = odb.fs.path_info / tmp_fname() + tmp_info = odb.path_info / tmp_fname() with fs.open(path_info, mode="rb", chunk_size=fs.CHUNK_SIZE) as stream: stream = HashedStreamReader(stream) odb.fs.upload_fobj( diff --git a/dvc/output.py b/dvc/output.py index 608a03bda..0c21bd071 100644 --- a/dvc/output.py +++ b/dvc/output.py @@ -279,9 +279,8 @@ class Output: ): self.repo = stage.repo if stage else None - fs_cls, fs_config = get_cloud_fs(self.repo, url=path) + fs_cls, fs_config, path_info = get_cloud_fs(self.repo, url=path) self.fs = fs_cls(**fs_config) - path_info = self.fs.path_info if ( self.fs.scheme == "local" @@ -741,9 +740,8 @@ class Output: if odb is None: odb = self.odb - cls, config = get_cloud_fs(self.repo, url=source) + cls, config, from_info = get_cloud_fs(self.repo, url=source) from_fs = cls(**config) - from_info = from_fs.path_info # When running import-url --to-remote / add --to-remote/-o ... we # assume that it is unlikely that the odb will contain majority of the diff --git a/dvc/remote/__init__.py b/dvc/remote/__init__.py index 4f82cfdc2..3e695c86b 100644 --- a/dvc/remote/__init__.py +++ b/dvc/remote/__init__.py @@ -4,8 +4,8 @@ from .local import LocalRemote def get_remote(repo, **kwargs): - cls, config = get_cloud_fs(repo, **kwargs) + cls, config, path_info = get_cloud_fs(repo, **kwargs) fs = cls(**config) if fs.scheme == "local": - return LocalRemote(fs, repo.tmp_dir, **config) - return Remote(fs, repo.tmp_dir, **config) + return LocalRemote(fs, path_info, repo.tmp_dir, **config) + return Remote(fs, path_info, repo.tmp_dir, **config) diff --git a/dvc/remote/base.py b/dvc/remote/base.py index 8d2003b48..e878196c3 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -10,7 +10,7 @@ from dvc.exceptions import DownloadError, UploadError from dvc.hash_info import HashInfo from ..progress import Tqdm -from .index import RemoteIndex, RemoteIndexNoop +from .index import RemoteIndex logger = logging.getLogger(__name__) @@ -67,25 +67,22 @@ class Remote: INDEX_CLS = RemoteIndex - def __init__(self, fs, tmp_dir, **config): + def __init__(self, fs, path_info, tmp_dir, **config): from dvc.objects.db import get_odb self.fs = fs - self.odb = get_odb(self.fs, **config) + self.odb = get_odb(self.fs, path_info, **config) - url = config.get("url") - if url: - index_name = hashlib.sha256(url.encode("utf-8")).hexdigest() - self.index = self.INDEX_CLS( - tmp_dir, index_name, dir_suffix=self.fs.CHECKSUM_DIR_SUFFIX - ) - else: - self.index = RemoteIndexNoop() + url = getattr(path_info, "fspath", path_info.url) + index_name = hashlib.sha256(url.encode("utf-8")).hexdigest() + self.index = self.INDEX_CLS( + tmp_dir, index_name, dir_suffix=self.fs.CHECKSUM_DIR_SUFFIX + ) def __repr__(self): return "{class_name}: '{path_info}'".format( class_name=type(self).__name__, - path_info=self.fs.path_info or "No path", + path_info=self.odb.path_info or "No path", ) @index_locked @@ -145,7 +142,7 @@ class Remote: {dir_hash: set(file_hash, ...)} which can be used to map a .dir file to its file contents. """ - logger.debug(f"Preparing to collect status from {self.fs.path_info}") + logger.debug(f"Preparing to collect status from {self.odb.path_info}") md5s = set(named_cache.scheme_keys(cache.fs.scheme)) logger.debug("Collecting information from local cache...") @@ -171,7 +168,7 @@ class Remote: if md5s: remote_exists.update( self.hashes_exist( - md5s, jobs=jobs, name=str(self.fs.path_info) + md5s, jobs=jobs, name=str(self.odb.path_info) ) ) return self._make_status( @@ -302,7 +299,7 @@ class Remote: logger.debug( "Preparing to {} '{}'".format( "download data from" if download else "upload data to", - self.fs.path_info, + self.odb.path_info, ) ) diff --git a/dvc/stage/cache.py b/dvc/stage/cache.py index 18bc5d809..7b9df6f77 100644 --- a/dvc/stage/cache.py +++ b/dvc/stage/cache.py @@ -212,15 +212,17 @@ class StageCache: ret = [] runs = from_remote.path_info / "runs" - if not from_remote.exists(runs): + if not from_remote.fs.exists(runs): return [] - for src in from_remote.walk_files(runs): + for src in from_remote.fs.walk_files(runs): rel = src.relative_to(from_remote.path_info) dst = to_remote.path_info / rel key = dst.parent # check if any build cache already exists for this key - if to_remote.exists(key) and first(to_remote.walk_files(key)): + if to_remote.fs.exists(key) and first( + to_remote.fs.walk_files(key) + ): continue func(src, dst) ret.append((src.parent.name, src.name)) @@ -233,8 +235,8 @@ class StageCache: remote = self.repo.cloud.get_remote(remote) return self._transfer( _log_exceptions(remote.fs.upload, "upload"), - self.repo.odb.local.fs, - remote.fs, + self.repo.odb.local, + remote.odb, ) def pull(self, remote): @@ -243,8 +245,8 @@ class StageCache: remote = self.repo.cloud.get_remote(remote) return self._transfer( _log_exceptions(remote.fs.download, "download"), - remote.fs, - self.repo.odb.local.fs, + remote.odb, + self.repo.odb.local, ) def get_used_cache(self, used_run_cache, *args, **kwargs): diff --git a/tests/func/test_fs.py b/tests/func/test_fs.py index 5403ff696..bbfe95bf3 100644 --- a/tests/func/test_fs.py +++ b/tests/func/test_fs.py @@ -244,9 +244,8 @@ def test_walk_dont_ignore_subrepos(tmp_dir, scm, dvc): ) def test_fs_getsize(dvc, cloud): cloud.gen({"data": {"foo": "foo"}, "baz": "baz baz"}) - cls, config = get_cloud_fs(dvc, **cloud.config) + cls, config, path_info = get_cloud_fs(dvc, **cloud.config) fs = cls(**config) - path_info = fs.path_info assert fs.getsize(path_info / "baz") == 7 assert fs.getsize(path_info / "data" / "foo") == 3 @@ -270,11 +269,11 @@ def test_fs_getsize(dvc, cloud): ) def test_fs_upload_fobj(dvc, tmp_dir, cloud): tmp_dir.gen("foo", "foo") - cls, config = get_cloud_fs(dvc, **cloud.config) + cls, config, path_info = get_cloud_fs(dvc, **cloud.config) fs = cls(**config) from_info = tmp_dir / "foo" - to_info = fs.path_info / "foo" + to_info = path_info / "foo" with open(from_info, "rb") as stream: fs.upload_fobj(stream, to_info) @@ -297,9 +296,9 @@ def test_fs_ls(dvc, cloud): } } ) - cls, config = get_cloud_fs(dvc, **cloud.config) + cls, config, path_info = get_cloud_fs(dvc, **cloud.config) fs = cls(**config) - path_info = cloud / "directory" + path_info /= "directory" assert {os.path.basename(file_key) for file_key in fs.ls(path_info)} == { "foo", @@ -327,9 +326,8 @@ def test_fs_ls(dvc, cloud): ) def test_fs_find_recursive(dvc, cloud): cloud.gen({"data": {"foo": "foo", "bar": {"baz": "baz"}, "quux": "quux"}}) - cls, config = get_cloud_fs(dvc, **cloud.config) + cls, config, path_info = get_cloud_fs(dvc, **cloud.config) fs = cls(**config) - path_info = fs.path_info assert { os.path.basename(file_key) for file_key in fs.find(path_info / "data") @@ -349,9 +347,8 @@ def test_fs_find_recursive(dvc, cloud): ) def test_fs_find_with_etag(dvc, cloud): cloud.gen({"data": {"foo": "foo", "bar": {"baz": "baz"}, "quux": "quux"}}) - cls, config = get_cloud_fs(dvc, **cloud.config) + cls, config, path_info = get_cloud_fs(dvc, **cloud.config) fs = cls(**config) - path_info = fs.path_info for details in fs.find(path_info / "data", detail=True): assert ( @@ -371,7 +368,7 @@ def test_fs_find_with_etag(dvc, cloud): ) def test_fs_fsspec_path_management(dvc, cloud): cloud.gen({"foo": "foo", "data": {"bar": "bar", "baz": {"foo": "foo"}}}) - cls, config = get_cloud_fs(dvc, **cloud.config) + cls, config, _ = get_cloud_fs(dvc, **cloud.config) fs = cls(**config) root = cloud.parents[len(cloud.parents) - 1] diff --git a/tests/func/test_odb.py b/tests/func/test_odb.py index 464314a40..a04c1d4cd 100644 --- a/tests/func/test_odb.py +++ b/tests/func/test_odb.py @@ -88,7 +88,7 @@ class TestExternalCacheDir(TestDvc): self.dvc.__init__() - assert self.dvc.odb.ssh.fs.path_info == ssh_url + "/tmp" + assert self.dvc.odb.ssh.path_info == ssh_url + "/tmp" class TestSharedCacheDir(TestDir): diff --git a/tests/func/test_remote.py b/tests/func/test_remote.py index 719a17c76..e7cd7eb4f 100644 --- a/tests/func/test_remote.py +++ b/tests/func/test_remote.py @@ -451,7 +451,7 @@ def test_upload_exists(tmp_dir, dvc, local_remote): remote.fs.CACHE_MODE = 0o644 from_info = PathInfo(tmp_dir / "foo") - to_info = remote.fs.path_info / "foo" + to_info = remote.odb.path_info / "foo" remote.fs.upload(from_info, to_info) assert remote.fs.exists(to_info) diff --git a/tests/unit/fs/test_azure.py b/tests/unit/fs/test_azure.py index cf74a32d9..01d0e5889 100644 --- a/tests/unit/fs/test_azure.py +++ b/tests/unit/fs/test_azure.py @@ -15,21 +15,27 @@ connection_string = ( ) -def test_init_env_var(monkeypatch, dvc): +def test_strip_protocol_env_var(monkeypatch, dvc): monkeypatch.setenv("AZURE_STORAGE_CONTAINER_NAME", container_name) monkeypatch.setenv("AZURE_STORAGE_CONNECTION_STRING", connection_string) - config = {"url": "azure://"} - fs = AzureFileSystem(**config) - assert fs.path_info == "azure://" + container_name + assert ( + AzureFileSystem._strip_protocol("azure://") + == f"azure://{container_name}" + ) + + +def test_strip_protocol(dvc): + assert ( + AzureFileSystem._strip_protocol(f"azure://{container_name}") + == f"azure://{container_name}" + ) def test_init(dvc): - prefix = "some/prefix" - url = f"azure://{container_name}/{prefix}" - config = {"url": url, "connection_string": connection_string} + config = {"connection_string": connection_string} fs = AzureFileSystem(**config) - assert fs.path_info == url + assert fs.fs_args["connection_string"] == connection_string def test_info(tmp_dir, azure): diff --git a/tests/unit/fs/test_s3.py b/tests/unit/fs/test_s3.py index ba3f0aaec..d5195848d 100644 --- a/tests/unit/fs/test_s3.py +++ b/tests/unit/fs/test_s3.py @@ -26,13 +26,6 @@ def grants(): } -def test_init(dvc): - config = {"url": url} - fs = S3FileSystem(**config) - - assert fs.path_info == url - - def test_verify_ssl_default_param(dvc): config = {"url": url} fs = S3FileSystem(**config) diff --git a/tests/unit/fs/test_tree.py b/tests/unit/fs/test_tree.py index 14fcfd0c4..c5f8769cb 100644 --- a/tests/unit/fs/test_tree.py +++ b/tests/unit/fs/test_tree.py @@ -16,12 +16,12 @@ def test_get_cloud_fs(tmp_dir, dvc): first = base / "first" second = first / "second" - cls, config = get_cloud_fs(dvc, name="base") - assert cls(**config).path_info == base - cls, config = get_cloud_fs(dvc, name="first") - assert cls(**config).path_info == first - cls, config = get_cloud_fs(dvc, name="second") - assert cls(**config).path_info == second + _, _, path_info = get_cloud_fs(dvc, name="base") + assert path_info == base + _, _, path_info = get_cloud_fs(dvc, name="first") + assert path_info == first + _, _, path_info = get_cloud_fs(dvc, name="second") + assert path_info == second def test_get_cloud_fs_validate(tmp_dir, dvc): @@ -39,11 +39,9 @@ def test_get_cloud_fs_validate(tmp_dir, dvc): default=False, ) - assert get_cloud_fs(dvc, name="base")[1] == { - "url": "ssh://example.com/path" - } + assert get_cloud_fs(dvc, name="base")[1] == {"host": "example.com"} assert get_cloud_fs(dvc, name="first")[1] == { - "url": "ssh://example.com/path/first", + "host": "example.com", "type": ["symlink"], } diff --git a/tests/unit/objects/db/test_local.py b/tests/unit/objects/db/test_local.py index 1653f138b..6448b60cc 100644 --- a/tests/unit/objects/db/test_local.py +++ b/tests/unit/objects/db/test_local.py @@ -15,7 +15,7 @@ def test_status_download_optimization(mocker, dvc): And the desired files to fetch are already on the local cache, Don't check the existence of the desired files on the remote cache """ - odb = LocalObjectDB(LocalFileSystem()) + odb = LocalObjectDB(LocalFileSystem(), PathInfo(".")) infos = NamedCache() infos.add("local", "acbd18db4cc2f85cedef654fccc4a4d8", "foo") diff --git a/tests/unit/remote/ssh/test_ssh.py b/tests/unit/remote/ssh/test_ssh.py index f853986ea..bd2570187 100644 --- a/tests/unit/remote/ssh/test_ssh.py +++ b/tests/unit/remote/ssh/test_ssh.py @@ -9,7 +9,7 @@ from dvc.fs.ssh import SSHFileSystem from dvc.system import System -def test_url(dvc): +def test_get_kwargs_from_urls(): user = "test" host = "123.45.67.89" port = 1234 @@ -17,23 +17,27 @@ def test_url(dvc): # URL ssh://[user@]host.xz[:port]/path url = f"ssh://{user}@{host}:{port}{path}" - config = {"url": url} - fs = SSHFileSystem(**config) - assert fs.path_info == url + assert SSHFileSystem._get_kwargs_from_urls(url) == { + "user": user, + "host": host, + "port": port, + } # SCP-like URL ssh://[user@]host.xz:/absolute/path url = f"ssh://{user}@{host}:{path}" - config = {"url": url} - fs = SSHFileSystem(**config) - assert fs.path_info == url + assert SSHFileSystem._get_kwargs_from_urls(url) == { + "user": user, + "host": host, + } -def test_no_path(dvc): - config = {"url": "ssh://127.0.0.1"} - fs = SSHFileSystem(**config) - assert fs.path_info.path == "" +def test_init(): + fs = SSHFileSystem(user="test", host="12.34.56.78", port="1234") + assert fs.user == "test" + assert fs.host == "12.34.56.78" + assert fs.port == "1234" mock_ssh_config = """ @@ -53,8 +57,8 @@ else: @pytest.mark.parametrize( "config,expected_host", [ - ({"url": "ssh://example.com"}, "1.2.3.4"), - ({"url": "ssh://not_in_ssh_config.com"}, "not_in_ssh_config.com"), + ({"host": "example.com"}, "1.2.3.4"), + ({"host": "not_in_ssh_config.com"}, "not_in_ssh_config.com"), ], ) @patch("os.path.exists", return_value=True) @@ -64,27 +68,22 @@ else: read_data=mock_ssh_config, ) def test_ssh_host_override_from_config( - mock_file, mock_exists, dvc, config, expected_host + mock_file, mock_exists, config, expected_host ): fs = SSHFileSystem(**config) mock_exists.assert_called_with(SSHFileSystem.ssh_config_filename()) mock_file.assert_called_with(SSHFileSystem.ssh_config_filename()) - assert fs.path_info.host == expected_host + assert fs.host == expected_host @pytest.mark.parametrize( "config,expected_user", [ - ({"url": "ssh://test1@example.com"}, "test1"), - ({"url": "ssh://example.com", "user": "test2"}, "test2"), - ({"url": "ssh://example.com"}, "ubuntu"), - ({"url": "ssh://test1@not_in_ssh_config.com"}, "test1"), - ( - {"url": "ssh://test1@not_in_ssh_config.com", "user": "test2"}, - "test2", - ), - ({"url": "ssh://not_in_ssh_config.com"}, getpass.getuser()), + ({"host": "example.com", "user": "test1"}, "test1"), + ({"host": "example.com"}, "ubuntu"), + ({"host": "not_in_ssh_config.com", "user": "test1"}, "test1"), + ({"host": "not_in_ssh_config.com"}, getpass.getuser()), ], ) @patch("os.path.exists", return_value=True) @@ -93,23 +92,21 @@ def test_ssh_host_override_from_config( new_callable=mock_open, read_data=mock_ssh_config, ) -def test_ssh_user(mock_file, mock_exists, dvc, config, expected_user): +def test_ssh_user(mock_file, mock_exists, config, expected_user): fs = SSHFileSystem(**config) mock_exists.assert_called_with(SSHFileSystem.ssh_config_filename()) mock_file.assert_called_with(SSHFileSystem.ssh_config_filename()) - assert fs.path_info.user == expected_user + assert fs.user == expected_user @pytest.mark.parametrize( "config,expected_port", [ - ({"url": "ssh://example.com:2222"}, 2222), - ({"url": "ssh://example.com"}, 1234), - ({"url": "ssh://example.com", "port": 4321}, 4321), - ({"url": "ssh://not_in_ssh_config.com"}, SSHFileSystem.DEFAULT_PORT), - ({"url": "ssh://not_in_ssh_config.com:2222"}, 2222), - ({"url": "ssh://not_in_ssh_config.com:2222", "port": 4321}, 4321), + ({"host": "example.com"}, 1234), + ({"host": "example.com", "port": 4321}, 4321), + ({"host": "not_in_ssh_config.com"}, SSHFileSystem.DEFAULT_PORT), + ({"host": "not_in_ssh_config.com", "port": 2222}, 2222), ], ) @patch("os.path.exists", return_value=True) @@ -118,33 +115,33 @@ def test_ssh_user(mock_file, mock_exists, dvc, config, expected_user): new_callable=mock_open, read_data=mock_ssh_config, ) -def test_ssh_port(mock_file, mock_exists, dvc, config, expected_port): +def test_ssh_port(mock_file, mock_exists, config, expected_port): fs = SSHFileSystem(**config) mock_exists.assert_called_with(SSHFileSystem.ssh_config_filename()) mock_file.assert_called_with(SSHFileSystem.ssh_config_filename()) - assert fs.path_info.port == expected_port + assert fs.port == expected_port @pytest.mark.parametrize( "config,expected_keyfile", [ ( - {"url": "ssh://example.com", "keyfile": "dvc_config.key"}, + {"host": "example.com", "keyfile": "dvc_config.key"}, "dvc_config.key", ), ( - {"url": "ssh://example.com"}, + {"host": "example.com"}, os.path.expanduser("~/.ssh/not_default.key"), ), ( { - "url": "ssh://not_in_ssh_config.com", + "host": "not_in_ssh_config.com", "keyfile": "dvc_config.key", }, "dvc_config.key", ), - ({"url": "ssh://not_in_ssh_config.com"}, None), + ({"host": "not_in_ssh_config.com"}, None), ], ) @patch("os.path.exists", return_value=True) @@ -153,7 +150,7 @@ def test_ssh_port(mock_file, mock_exists, dvc, config, expected_port): new_callable=mock_open, read_data=mock_ssh_config, ) -def test_ssh_keyfile(mock_file, mock_exists, dvc, config, expected_keyfile): +def test_ssh_keyfile(mock_file, mock_exists, config, expected_keyfile): fs = SSHFileSystem(**config) mock_exists.assert_called_with(SSHFileSystem.ssh_config_filename()) @@ -164,9 +161,9 @@ def test_ssh_keyfile(mock_file, mock_exists, dvc, config, expected_keyfile): @pytest.mark.parametrize( "config,expected_gss_auth", [ - ({"url": "ssh://example.com", "gss_auth": True}, True), - ({"url": "ssh://example.com", "gss_auth": False}, False), - ({"url": "ssh://not_in_ssh_config.com"}, False), + ({"host": "example.com", "gss_auth": True}, True), + ({"host": "example.com", "gss_auth": False}, False), + ({"host": "not_in_ssh_config.com"}, False), ], ) @patch("os.path.exists", return_value=True) @@ -175,7 +172,7 @@ def test_ssh_keyfile(mock_file, mock_exists, dvc, config, expected_keyfile): new_callable=mock_open, read_data=mock_ssh_config, ) -def test_ssh_gss_auth(mock_file, mock_exists, dvc, config, expected_gss_auth): +def test_ssh_gss_auth(mock_file, mock_exists, config, expected_gss_auth): fs = SSHFileSystem(**config) mock_exists.assert_called_with(SSHFileSystem.ssh_config_filename()) @@ -186,10 +183,10 @@ def test_ssh_gss_auth(mock_file, mock_exists, dvc, config, expected_gss_auth): @pytest.mark.parametrize( "config,expected_allow_agent", [ - ({"url": "ssh://example.com"}, True), - ({"url": "ssh://not_in_ssh_config.com"}, True), - ({"url": "ssh://example.com", "allow_agent": True}, True), - ({"url": "ssh://example.com", "allow_agent": False}, False), + ({"host": "example.com"}, True), + ({"host": "not_in_ssh_config.com"}, True), + ({"host": "example.com", "allow_agent": True}, True), + ({"host": "example.com", "allow_agent": False}, False), ], ) @patch("os.path.exists", return_value=True) @@ -198,9 +195,7 @@ def test_ssh_gss_auth(mock_file, mock_exists, dvc, config, expected_gss_auth): new_callable=mock_open, read_data=mock_ssh_config, ) -def test_ssh_allow_agent( - mock_file, mock_exists, dvc, config, expected_allow_agent -): +def test_ssh_allow_agent(mock_file, mock_exists, config, expected_allow_agent): fs = SSHFileSystem(**config) mock_exists.assert_called_with(SSHFileSystem.ssh_config_filename()) @@ -208,11 +203,15 @@ def test_ssh_allow_agent( assert fs.allow_agent == expected_allow_agent -def test_hardlink_optimization(dvc, tmp_dir, ssh): - fs = SSHFileSystem(**ssh.config) +def test_hardlink_optimization(tmp_dir, dvc, ssh): + from dvc.fs import get_cloud_fs - from_info = fs.path_info / "empty" - to_info = fs.path_info / "link" + cls, config, path_info = get_cloud_fs(dvc, **ssh.config) + fs = cls(**config) + assert isinstance(fs, SSHFileSystem) + + from_info = path_info / "empty" + to_info = path_info / "link" with fs.open(from_info, "wb"): pass diff --git a/tests/unit/remote/test_base.py b/tests/unit/remote/test_base.py index bc0064adb..3a9a44c84 100644 --- a/tests/unit/remote/test_base.py +++ b/tests/unit/remote/test_base.py @@ -37,7 +37,7 @@ def test_cmd_error(dvc): @mock.patch.object(ObjectDB, "list_hashes_traverse") @mock.patch.object(ObjectDB, "list_hashes_exists") def test_hashes_exist(object_exists, traverse, dvc): - odb = ObjectDB(BaseFileSystem()) + odb = ObjectDB(BaseFileSystem(), None) # remote does not support traverse odb.fs.CAN_TRAVERSE = False @@ -86,8 +86,8 @@ def test_hashes_exist(object_exists, traverse, dvc): @mock.patch.object(ObjectDB, "list_hashes", return_value=[]) @mock.patch.object(ObjectDB, "_path_to_hash", side_effect=lambda x: x) def test_list_hashes_traverse(_path_to_hash, list_hashes, dvc): - odb = ObjectDB(BaseFileSystem()) - odb.fs.path_info = PathInfo("foo") + odb = ObjectDB(BaseFileSystem(), None) + odb.path_info = PathInfo("foo") # parallel traverse size = 256 / odb.fs._JOBS * odb.fs.LIST_OBJECT_PAGE_SIZE @@ -111,8 +111,8 @@ def test_list_hashes_traverse(_path_to_hash, list_hashes, dvc): def test_list_hashes(dvc): - odb = ObjectDB(BaseFileSystem()) - odb.fs.path_info = PathInfo("foo") + odb = ObjectDB(BaseFileSystem(), None) + odb.path_info = PathInfo("foo") with mock.patch.object( odb, "_list_paths", return_value=["12/3456", "bar"] @@ -122,19 +122,17 @@ def test_list_hashes(dvc): def test_list_paths(dvc): - odb = ObjectDB(BaseFileSystem()) - odb.fs.path_info = PathInfo("foo") + path_info = PathInfo("foo") + odb = ObjectDB(BaseFileSystem(), path_info) with mock.patch.object(odb.fs, "walk_files", return_value=[]) as walk_mock: for _ in odb._list_paths(): pass - walk_mock.assert_called_with(odb.fs.path_info, prefix=False) + walk_mock.assert_called_with(path_info, prefix=False) for _ in odb._list_paths(prefix="000"): pass - walk_mock.assert_called_with( - odb.fs.path_info / "00" / "0", prefix=True - ) + walk_mock.assert_called_with(path_info / "00" / "0", prefix=True) @pytest.mark.parametrize( diff --git a/tests/unit/remote/test_http.py b/tests/unit/remote/test_http.py index 3b5bda1e0..81324edb6 100644 --- a/tests/unit/remote/test_http.py +++ b/tests/unit/remote/test_http.py @@ -25,7 +25,7 @@ def test_public_auth_method(dvc): fs = HTTPFileSystem(**config) - assert fs._auth_method() is None + assert fs._auth_method(config["url"]) is None def test_basic_auth_method(dvc): @@ -44,8 +44,8 @@ def test_basic_auth_method(dvc): fs = HTTPFileSystem(**config) - assert fs._auth_method() == auth - assert isinstance(fs._auth_method(), HTTPBasicAuth) + assert fs._auth_method(config["url"]) == auth + assert isinstance(fs._auth_method(config["url"]), HTTPBasicAuth) def test_digest_auth_method(dvc): @@ -64,8 +64,8 @@ def test_digest_auth_method(dvc): fs = HTTPFileSystem(**config) - assert fs._auth_method() == auth - assert isinstance(fs._auth_method(), HTTPDigestAuth) + assert fs._auth_method(config["url"]) == auth + assert isinstance(fs._auth_method(config["url"]), HTTPDigestAuth) def test_custom_auth_method(dvc): @@ -81,7 +81,7 @@ def test_custom_auth_method(dvc): fs = HTTPFileSystem(**config) - assert fs._auth_method() is None + assert fs._auth_method(config["url"]) is None assert header in fs.headers assert fs.headers[header] == password @@ -123,9 +123,9 @@ def test_http_method(dvc): fs = HTTPFileSystem(**config) - assert fs._auth_method() == auth + assert fs._auth_method(config["url"]) == auth assert fs.method == "PUT" - assert isinstance(fs._auth_method(), HTTPBasicAuth) + assert isinstance(fs._auth_method(config["url"]), HTTPBasicAuth) def test_exists(mocker): diff --git a/tests/unit/remote/test_oss.py b/tests/unit/remote/test_oss.py index 558fab00f..da3db3fec 100644 --- a/tests/unit/remote/test_oss.py +++ b/tests/unit/remote/test_oss.py @@ -16,7 +16,6 @@ def test_init(dvc): "oss_endpoint": endpoint, } fs = OSSFileSystem(**config) - assert fs.path_info == url assert fs.endpoint == endpoint assert fs.key_id == key_id assert fs.key_secret == key_secret diff --git a/tests/unit/remote/test_remote.py b/tests/unit/remote/test_remote.py index 881aca6d8..5106761a1 100644 --- a/tests/unit/remote/test_remote.py +++ b/tests/unit/remote/test_remote.py @@ -12,7 +12,7 @@ def test_remote_with_hash_jobs(dvc): } dvc.config["core"]["checksum_jobs"] = 200 - cls, config = get_cloud_fs(dvc, name="with_hash_jobs") + cls, config, _ = get_cloud_fs(dvc, name="with_hash_jobs") fs = cls(**config) assert fs.hash_jobs == 100 @@ -24,7 +24,7 @@ def test_remote_with_jobs(dvc): } dvc.config["core"]["jobs"] = 200 - cls, config = get_cloud_fs(dvc, name="with_jobs") + cls, config, _ = get_cloud_fs(dvc, name="with_jobs") fs = cls(**config) assert fs.jobs == 100 @@ -33,7 +33,7 @@ def test_remote_without_hash_jobs(dvc): dvc.config["remote"]["without_hash_jobs"] = {"url": "s3://bucket/name"} dvc.config["core"]["checksum_jobs"] = 200 - cls, config = get_cloud_fs(dvc, name="without_hash_jobs") + cls, config, _ = get_cloud_fs(dvc, name="without_hash_jobs") fs = cls(**config) assert fs.hash_jobs == 200 @@ -41,7 +41,7 @@ def test_remote_without_hash_jobs(dvc): def test_remote_without_hash_jobs_default(dvc): dvc.config["remote"]["without_hash_jobs"] = {"url": "s3://bucket/name"} - cls, config = get_cloud_fs(dvc, name="without_hash_jobs") + cls, config, _ = get_cloud_fs(dvc, name="without_hash_jobs") fs = cls(**config) assert fs.hash_jobs == fs.HASH_JOBS @@ -53,5 +53,5 @@ def test_makedirs_not_create_for_top_level_path(fs_cls, dvc, mocker): mocked_client = mocker.PropertyMock() mocker.patch.object(fs_cls, "fs", mocked_client) - fs.makedirs(fs.path_info) + fs.makedirs(fs.PATH_CLS(url)) assert not mocked_client.called diff --git a/tests/unit/remote/test_remote_tree.py b/tests/unit/remote/test_remote_tree.py index 502594f88..e0e7eefd3 100644 --- a/tests/unit/remote/test_remote_tree.py +++ b/tests/unit/remote/test_remote_tree.py @@ -47,7 +47,7 @@ def test_isdir(remote): ] for expected, path in test_cases: - assert remote.fs.isdir(remote.fs.path_info / path) == expected + assert remote.fs.isdir(remote.odb.path_info / path) == expected @pytest.mark.needs_internet @@ -68,23 +68,23 @@ def test_exists(remote): ] for expected, path in test_cases: - assert remote.fs.exists(remote.fs.path_info / path) == expected + assert remote.fs.exists(remote.odb.path_info / path) == expected @pytest.mark.needs_internet @pytest.mark.parametrize("remote", remotes, indirect=True) def test_walk_files(remote): files = [ - remote.fs.path_info / "data/alice", - remote.fs.path_info / "data/alpha", - remote.fs.path_info / "data/subdir-file.txt", - remote.fs.path_info / "data/subdir/1", - remote.fs.path_info / "data/subdir/2", - remote.fs.path_info / "data/subdir/3", - remote.fs.path_info / "data/subdir/empty_file", + remote.odb.path_info / "data/alice", + remote.odb.path_info / "data/alpha", + remote.odb.path_info / "data/subdir-file.txt", + remote.odb.path_info / "data/subdir/1", + remote.odb.path_info / "data/subdir/2", + remote.odb.path_info / "data/subdir/3", + remote.odb.path_info / "data/subdir/empty_file", ] - assert list(remote.fs.walk_files(remote.fs.path_info / "data")) == files + assert list(remote.fs.walk_files(remote.odb.path_info / "data")) == files @pytest.mark.parametrize("cloud", [pytest.lazy_fixture("s3")]) @@ -100,8 +100,8 @@ def test_copy_preserve_etag_across_buckets(cloud, dvc): another = S3FileSystem(**config) - from_info = rem.fs.path_info / "foo" - to_info = another.path_info / "foo" + from_info = rem.odb.path_info / "foo" + to_info = another.PATH_CLS("s3://another/foo") rem.fs.copy(from_info, to_info) @@ -131,7 +131,7 @@ def test_isfile(remote): ] for expected, path in test_cases: - assert remote.fs.isfile(remote.fs.path_info / path) == expected + assert remote.fs.isfile(remote.odb.path_info / path) == expected @pytest.mark.needs_internet @@ -139,7 +139,7 @@ def test_isfile(remote): def test_download_dir(remote, tmpdir): path = str(tmpdir / "data") to_info = PathInfo(path) - remote.fs.download(remote.fs.path_info / "data", to_info) + remote.fs.download(remote.odb.path_info / "data", to_info) assert os.path.isdir(path) data_dir = tmpdir / "data" assert len(list(walk_files(path))) == 7 diff --git a/tests/unit/remote/test_webdav.py b/tests/unit/remote/test_webdav.py index bf9d3c400..a8c620736 100644 --- a/tests/unit/remote/test_webdav.py +++ b/tests/unit/remote/test_webdav.py @@ -1,5 +1,3 @@ -import pytest - from dvc.fs.webdav import WebDAVFileSystem # Test configuration @@ -9,37 +7,14 @@ userurl = f"webdavs://{user}@example.com/public.php/webdav" password = "password" -# Test minimum requiered configuration (url) -def test_init(dvc): - config = {"url": url} - fs = WebDAVFileSystem(**config) - - assert fs.path_info == url - - -# Test username from configuration -@pytest.mark.parametrize( - "config", [{"url": url, "user": user}, {"url": userurl}] -) -def test_user(dvc, config): - fs = WebDAVFileSystem(**config) +def test_user(dvc): + fs = WebDAVFileSystem(host=url, user=user) assert fs.user == user -# Test username extraction from url -def test_userurl(dvc): - config = {"url": userurl} - fs = WebDAVFileSystem(**config) - - assert fs.path_info == userurl - assert fs.user == user - assert fs.path_info.user == user - - -# test password from config def test_password(dvc): - config = {"url": url, "user": user, "password": password} + config = {"host": url, "user": user, "password": password} fs = WebDAVFileSystem(**config) assert fs.password == password diff --git a/tests/unit/remote/test_webhdfs.py b/tests/unit/remote/test_webhdfs.py index bef48f609..0e2bbca48 100644 --- a/tests/unit/remote/test_webhdfs.py +++ b/tests/unit/remote/test_webhdfs.py @@ -9,7 +9,7 @@ hdfscli_config = "path/to/cli/config" def test_init(dvc): url = "webhdfs://test@127.0.0.1:50070" config = { - "url": url, + "host": url, "webhdfs_token": webhdfs_token, "webhdfs_alias": webhdfs_alias, "hdfscli_config": hdfscli_config, @@ -17,8 +17,7 @@ def test_init(dvc): } fs = WebHDFSFileSystem(**config) - assert fs.path_info == url assert fs.token == webhdfs_token assert fs.alias == webhdfs_alias - assert fs.path_info.user == user + assert fs.user == user assert fs.hdfscli_config == hdfscli_config