Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

odb: don't use fs.path_info (#6059)

Browse Source 
* odb: don't use fs.path_info

* fs: get rid of fs.path_info
This commit is contained in:
Ruslan Kuprieiev
2021-06-02 20:43:51 +03:00
committed by GitHub
parent 0e3db0d323
commit 5fb181fcf8
38 changed files with 297 additions and 356 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/checkout.py
+1 -1
View File
@@ -134,7 +134,7 @@ def _cache_is_copy(cache, path_info):
return True
workspace_file = path_info.with_name("." + uuid())
test_cache_file = cache.fs.path_info / ".cache_type_test_file"
test_cache_file = cache.path_info / ".cache_type_test_file"
if not cache.fs.exists(test_cache_file):
cache.makedirs(test_cache_file.parent)
with cache.fs.open(test_cache_file, "wb") as fobj:
dvc/fs/__init__.py
+8 -1
View File
@@ -106,4 +106,11 @@ def get_cloud_fs(repo, **kwargs):
if cls == GDriveFileSystem and repo:
remote_conf["gdrive_credentials_tmp_dir"] = repo.tmp_dir
return cls, remote_conf
url = remote_conf.pop("url")
path_info = cls.PATH_CLS(
cls._strip_protocol(url) # pylint:disable=protected-access
)
extras = cls._get_kwargs_from_urls(url) # pylint:disable=protected-access
return cls, dict(**extras, **remote_conf), path_info
dvc/fs/azure.py
+32 -26
View File
@@ -5,7 +5,8 @@ import sys
import threading
from contextlib import contextmanager
from funcy import cached_property, wrap_prop
from fsspec.utils import infer_storage_options
from funcy import cached_property, memoize, wrap_prop
from dvc.exceptions import DvcException
from dvc.path_info import CloudURLInfo
@@ -67,6 +68,19 @@ class AzureAuthError(DvcException):
pass
@memoize
def _az_config():
# NOTE: ideally we would've used get_default_cli().config from
# azure.cli.core, but azure-cli-core has a lot of conflicts with other
# dependencies. So instead we are just use knack directly
from knack.config import CLIConfig
config_dir = os.getenv(
"AZURE_CONFIG_DIR", os.path.expanduser(os.path.join("~", ".azure"))
)
return CLIConfig(config_dir=config_dir, config_env_var_prefix="AZURE")
# pylint:disable=abstract-method
class AzureFileSystem(ObjectFSWrapper):
scheme = Schemes.AZURE
@@ -79,17 +93,21 @@ class AzureFileSystem(ObjectFSWrapper):
"azure-identity": "azure.identity",
}
def __init__(self, **config):
super().__init__(**config)
@classmethod
def _strip_protocol(cls, path: str):
bucket = infer_storage_options(path).get("host")
if bucket:
return path
url = config.get("url")
self.path_info = self.PATH_CLS(url)
bucket = _az_config().get("storage", "container_name", None)
return f"azure://{bucket}"
if not self.path_info.bucket:
container = self._az_config.get("storage", "container_name", None)
url = f"azure://{container}"
self.path_info = self.PATH_CLS(url)
@staticmethod
def _get_kwargs_from_urls(urlpath):
ops = infer_storage_options(urlpath)
if "host" in ops:
return {"bucket": ops["host"]}
return {}
def _prepare_credentials(self, **config):
from azure.identity.aio import DefaultAzureCredential
@@ -100,16 +118,16 @@ class AzureFileSystem(ObjectFSWrapper):
login_info = {}
login_info["connection_string"] = config.get(
"connection_string",
self._az_config.get("storage", "connection_string", None),
_az_config().get("storage", "connection_string", None),
)
login_info["account_name"] = config.get(
"account_name", self._az_config.get("storage", "account", None)
"account_name", _az_config().get("storage", "account", None)
)
login_info["account_key"] = config.get(
"account_key", self._az_config.get("storage", "key", None)
"account_key", _az_config().get("storage", "key", None)
)
login_info["sas_token"] = config.get(
"sas_token", self._az_config.get("storage", "sas_token", None)
"sas_token", _az_config().get("storage", "sas_token", None)
)
login_info["tenant_id"] = config.get("tenant_id")
login_info["client_id"] = config.get("client_id")
@@ -157,18 +175,6 @@ class AzureFileSystem(ObjectFSWrapper):
self.login_method = login_method
return login_info
@cached_property
def _az_config(self):
# NOTE: ideally we would've used get_default_cli().config from
# azure.cli.core, but azure-cli-core has a lot of conflicts with other
# dependencies. So instead we are just use knack directly
from knack.config import CLIConfig
config_dir = os.getenv(
"AZURE_CONFIG_DIR", os.path.expanduser(os.path.join("~", ".azure"))
)
return CLIConfig(config_dir=config_dir, config_env_var_prefix="AZURE")
@wrap_prop(threading.Lock())
@cached_property
def fs(self):
dvc/fs/base.py
+8 -2
View File
@@ -55,11 +55,17 @@ class BaseFileSystem:
def __init__(self, **kwargs):
self._check_requires(**kwargs)
self.path_info = None
self.jobs = kwargs.get("jobs") or self._JOBS
self.hash_jobs = kwargs.get("checksum_jobs") or self.HASH_JOBS
@classmethod
def _strip_protocol(cls, path: str):
return path
@staticmethod
def _get_kwargs_from_urls(urlpath): # pylint:disable=unused-argument
return {}
@classmethod
def get_missing_deps(cls):
import importlib
dvc/fs/gdrive.py
+4
View File
@@ -142,6 +142,10 @@ class GDriveFileSystem(BaseFileSystem): # pylint:disable=abstract-method
)
)
@staticmethod
def _get_kwargs_from_urls(urlpath):
return {"url": urlpath}
def _validate_config(self):
# Validate Service Account configuration
if (
dvc/fs/gs.py
-6
View File
@@ -17,12 +17,6 @@ class GSFileSystem(ObjectFSWrapper):
PARAM_CHECKSUM = "etag"
DETAIL_FIELDS = frozenset(("etag", "size"))
def __init__(self, **config):
super().__init__(**config)
url = config.get("url", "gs://")
self.path_info = self.PATH_CLS(url)
def _prepare_credentials(self, **config):
login_info = {"consistency": None}
login_info["project"] = config.get("projectname")
dvc/fs/hdfs.py
+10 -16
View File
@@ -6,7 +6,6 @@ import shutil
import subprocess
from collections import deque
from contextlib import closing, contextmanager
from urllib.parse import urlparse
from dvc.hash_info import HashInfo
from dvc.progress import Tqdm
@@ -68,24 +67,19 @@ class HDFSFileSystem(BaseFileSystem):
def __init__(self, **config):
super().__init__(**config)
self.path_info = None
url = config.get("url")
if not url:
return
parsed = urlparse(url)
user = parsed.username or config.get("user")
self.path_info = self.PATH_CLS.from_parts(
scheme=self.scheme,
host=parsed.hostname,
user=user,
port=parsed.port,
path=parsed.path,
)
self.host = config["host"]
self.user = config.get("user")
self.port = config.get("port")
self.kerb_ticket = config.get("kerb_ticket")
@staticmethod
def _get_kwargs_from_urls(urlpath):
from fsspec.implementations.hdfs import PyArrowHDFS
# pylint:disable=protected-access
return PyArrowHDFS._get_kwargs_from_urls(urlpath)
def hdfs(self, path_info):
import pyarrow.fs
dvc/fs/http.py
+5 -10
View File
@@ -2,6 +2,7 @@ import logging
import os.path
import threading
from typing import Optional
from urllib.parse import urlparse
from funcy import cached_property, memoize, wrap_prop, wrap_with
@@ -39,13 +40,7 @@ class HTTPFileSystem(BaseFileSystem): # pylint:disable=abstract-method
def __init__(self, **config):
super().__init__(**config)
url = config.get("url")
if url:
self.path_info = self.PATH_CLS(url)
self.user = config.get("user", None)
self.host = self.path_info.host
else:
self.path_info = None
self.user = config.get("user", None)
self.auth = config.get("auth", None)
self.custom_auth_header = config.get("custom_auth_header", None)
@@ -55,12 +50,12 @@ class HTTPFileSystem(BaseFileSystem): # pylint:disable=abstract-method
self.ssl_verify = config.get("ssl_verify", True)
self.method = config.get("method", "POST")
def _auth_method(self):
def _auth_method(self, url):
from requests.auth import HTTPBasicAuth, HTTPDigestAuth
if self.auth:
if self.ask_password and self.password is None:
self.password = ask_password(self.host, self.user)
self.password = ask_password(urlparse(url).hostname, self.user)
if self.auth == "basic":
return HTTPBasicAuth(self.user, self.password)
if self.auth == "digest":
@@ -103,7 +98,7 @@ class HTTPFileSystem(BaseFileSystem): # pylint:disable=abstract-method
res = self._session.request(
method,
url,
auth=self._auth_method(),
auth=self._auth_method(url),
headers=self.headers,
**kwargs,
)
dvc/fs/local.py
-2
View File
@@ -25,8 +25,6 @@ class LocalFileSystem(BaseFileSystem):
super().__init__(**config)
self.fs = LocalFS()
url = config.get("url")
self.path_info = self.PATH_CLS(url) if url else None
@staticmethod
def open(path_info, mode="r", encoding=None, **kwargs):
dvc/fs/oss.py
-3
View File
@@ -42,9 +42,6 @@ class OSSFileSystem(BaseFileSystem): # pylint:disable=abstract-method
def __init__(self, **config):
super().__init__(**config)
url = config.get("url")
self.path_info = self.PATH_CLS(url) if url else None
self.endpoint = config.get("oss_endpoint") or os.getenv("OSS_ENDPOINT")
self.key_id = (
dvc/fs/s3.py
-6
View File
@@ -29,12 +29,6 @@ class BaseS3FileSystem(ObjectFSWrapper):
"grant_write_acp": "GrantWriteACP",
}
def __init__(self, **config):
super().__init__(**config)
url = config.get("url", "s3://")
self.path_info = self.PATH_CLS(url)
_TRANSFER_CONFIG_ALIASES = {
"max_queue_size": "max_io_queue",
"max_concurrent_requests": "max_concurrency",
dvc/fs/ssh/__init__.py
+25 -35
View File
@@ -5,7 +5,6 @@ import os
import shutil
import threading
from contextlib import closing, contextmanager
from urllib.parse import urlparse
from funcy import first, memoize, silent, wrap_with
@@ -48,34 +47,19 @@ class SSHFileSystem(BaseFileSystem): # pylint:disable=abstract-method
def __init__(self, **config):
super().__init__(**config)
url = config.get("url")
if url:
parsed = urlparse(url)
user_ssh_config = self._load_user_ssh_config(parsed.hostname)
user_ssh_config = self._load_user_ssh_config(config["host"])
host = user_ssh_config.get("hostname", parsed.hostname)
user = (
config.get("user")
or parsed.username
or user_ssh_config.get("user")
or getpass.getuser()
)
port = (
config.get("port")
or parsed.port
or self._try_get_ssh_config_port(user_ssh_config)
or self.DEFAULT_PORT
)
self.path_info = self.PATH_CLS.from_parts(
scheme=self.scheme,
host=host,
user=user,
port=port,
path=parsed.path,
)
else:
self.path_info = None
user_ssh_config = {}
self.host = user_ssh_config.get("hostname", config["host"])
self.user = (
config.get("user")
or user_ssh_config.get("user")
or getpass.getuser()
)
self.port = (
config.get("port")
or self._try_get_ssh_config_port(user_ssh_config)
or self.DEFAULT_PORT
)
self.keyfile = config.get(
"keyfile"
@@ -93,6 +77,16 @@ class SSHFileSystem(BaseFileSystem): # pylint:disable=abstract-method
self.sock = None
self.allow_agent = config.get("allow_agent", True)
@staticmethod
def _get_kwargs_from_urls(urlpath):
from fsspec.implementations.sftp import SFTPFileSystem
# pylint:disable=protected-access
kwargs = SFTPFileSystem._get_kwargs_from_urls(urlpath)
if "username" in kwargs:
kwargs["user"] = kwargs.pop("username")
return kwargs
@staticmethod
def ssh_config_filename():
return os.path.expanduser(os.path.join("~", ".ssh", "config"))
@@ -120,17 +114,13 @@ class SSHFileSystem(BaseFileSystem): # pylint:disable=abstract-method
def _try_get_ssh_config_keyfile(user_ssh_config):
return first(user_ssh_config.get("identityfile") or ())
def ensure_credentials(self, path_info=None):
if path_info is None:
path_info = self.path_info
def ensure_credentials(self):
# NOTE: we use the same password regardless of the server :(
if self.ask_password and self.password is None:
host, user, port = path_info.host, path_info.user, path_info.port
self.password = ask_password(host, user, port)
self.password = ask_password(self.host, self.user, self.port)
def ssh(self, path_info):
self.ensure_credentials(path_info)
self.ensure_credentials()
from .connection import SSHConnection
dvc/fs/webdav.py
+12 -18
View File
@@ -71,20 +71,18 @@ class WebDAVFileSystem(BaseFileSystem): # pylint:disable=abstract-method
# Connection timeout
self.timeout = config.get("timeout", 30)
# Get URL from configuration
self.url = config.get("url", None)
self.hostname = config["host"]
self.prefix = config.get("prefix")
self.user = config.get("user")
# If URL in config parse path_info
if self.url:
self.path_info = self.PATH_CLS(self.url)
# If username not specified try to use from URL
if self.user is None and self.path_info.user is not None:
self.user = self.path_info.user
# Construct hostname from path_info by stripping path
http_info = HTTPURLInfo(self.path_info.url)
self.hostname = http_info.replace(path="").url
@staticmethod
def _get_kwargs_from_urls(urlpath):
path_info = WebDAVURLInfo(urlpath)
http_info = HTTPURLInfo(path_info.url)
return {
"prefix": path_info.path,
"host": http_info.replace(path="").url,
}
# Webdav client
@wrap_prop(threading.Lock())
@@ -116,10 +114,6 @@ class WebDAVFileSystem(BaseFileSystem): # pylint:disable=abstract-method
f"Configuration for WebDAV {self.hostname} is invalid."
)
# Check whether connection is valid (root should always exist)
if not client.check(self.path_info.path):
raise WebDAVConnectionError(self.hostname)
return client
def open(self, path_info, mode="r", encoding=None, **kwargs):
@@ -202,7 +196,7 @@ class WebDAVFileSystem(BaseFileSystem): # pylint:disable=abstract-method
# Creates directories
def makedirs(self, path_info):
# Terminate recursion
if path_info.path == self.path_info.path or self.exists(path_info):
if path_info.path == self.prefix or self.exists(path_info):
return
# Recursively descent to root
dvc/fs/webhdfs.py
+13 -13
View File
@@ -4,7 +4,6 @@ import posixpath
import shutil
import threading
from contextlib import contextmanager
from urllib.parse import urlparse
from funcy import cached_property, wrap_prop
@@ -40,23 +39,24 @@ class WebHDFSFileSystem(BaseFileSystem): # pylint:disable=abstract-method
def __init__(self, **config):
super().__init__(**config)
self.path_info = None
url = config.get("url")
if not url:
return
self.path_info = self.PATH_CLS(url)
parsed = urlparse(url)
self.host = parsed.hostname
self.user = parsed.username or config.get("user")
self.port = parsed.port
self.host = config["host"]
self.user = config.get("user")
self.port = config.get("port")
self.hdfscli_config = config.get("hdfscli_config")
self.token = config.get("webhdfs_token")
self.alias = config.get("webhdfs_alias")
@staticmethod
def _get_kwargs_from_urls(urlpath):
from fsspec.implementations.webhdfs import WebHDFS
return (
WebHDFS._get_kwargs_from_urls( # pylint:disable=protected-access
urlpath
)
)
@wrap_prop(threading.Lock())
@cached_property
def hdfs_client(self):
dvc/objects/db/__init__.py
+7 -7
View File
@@ -3,22 +3,22 @@ from collections import defaultdict
from dvc.scheme import Schemes
def get_odb(fs, **config):
def get_odb(fs, path_info, **config):
from .base import ObjectDB
from .gdrive import GDriveObjectDB
from .local import LocalObjectDB
from .ssh import SSHObjectDB
if fs.scheme == Schemes.LOCAL:
return LocalObjectDB(fs, **config)
return LocalObjectDB(fs, path_info, **config)
if fs.scheme == Schemes.SSH:
return SSHObjectDB(fs, **config)
return SSHObjectDB(fs, path_info, **config)
if fs.scheme == Schemes.GDRIVE:
return GDriveObjectDB(fs, **config)
return GDriveObjectDB(fs, path_info, **config)
return ObjectDB(fs, **config)
return ObjectDB(fs, path_info, **config)
def _get_odb(repo, settings):
@@ -27,8 +27,8 @@ def _get_odb(repo, settings):
if not settings:
return None
cls, config = get_cloud_fs(repo, **settings)
return get_odb(cls(**config), state=repo.state, **config)
cls, config, path_info = get_cloud_fs(repo, **settings)
return get_odb(cls(**config), path_info, state=repo.state, **config)
class ODBManager:
dvc/objects/db/base.py
+7 -6
View File
@@ -18,10 +18,11 @@ class ObjectDB:
DEFAULT_CACHE_TYPES = ["copy"]
CACHE_MODE: Optional[int] = None
def __init__(self, fs, **config):
def __init__(self, fs, path_info, **config):
from dvc.state import StateNoop
self.fs = fs
self.path_info = path_info
self.state = config.get("state", StateNoop())
self.verify = config.get("verify", self.DEFAULT_VERIFY)
self.cache_types = config.get("type") or copy(self.DEFAULT_CACHE_TYPES)
@@ -87,7 +88,7 @@ class ObjectDB:
callback(1)
def hash_to_path_info(self, hash_):
return self.fs.path_info / hash_[0:2] / hash_[2:]
return self.path_info / hash_[0:2] / hash_[2:]
# Override to return path as a string instead of PathInfo for clouds
# which support string paths (see local)
@@ -143,12 +144,12 @@ class ObjectDB:
def _list_paths(self, prefix=None, progress_callback=None):
if prefix:
if len(prefix) > 2:
path_info = self.fs.path_info / prefix[:2] / prefix[2:]
path_info = self.path_info / prefix[:2] / prefix[2:]
else:
path_info = self.fs.path_info / prefix[:2]
path_info = self.path_info / prefix[:2]
prefix = True
else:
path_info = self.fs.path_info
path_info = self.path_info
prefix = False
if progress_callback:
for file_info in self.fs.walk_files(path_info, prefix=prefix):
@@ -319,7 +320,7 @@ class ObjectDB:
removed = False
# hashes must be sorted to ensure we always remove .dir files first
for hash_ in sorted(
self.all(jobs, str(self.fs.path_info)),
self.all(jobs, str(self.path_info)),
key=self.fs.is_dir_hash,
reverse=True,
):
dvc/objects/db/local.py
+8 -8
View File
@@ -22,9 +22,9 @@ class LocalObjectDB(ObjectDB):
CACHE_MODE = 0o444
UNPACKED_DIR_SUFFIX = ".unpacked"
def __init__(self, fs, **config):
super().__init__(fs, **config)
self.cache_dir = config.get("url")
def __init__(self, fs, path_info, **config):
super().__init__(fs, path_info, **config)
self.cache_dir = path_info.fspath
shared = config.get("shared")
if shared:
@@ -36,11 +36,11 @@ class LocalObjectDB(ObjectDB):
@property
def cache_dir(self):
return self.fs.path_info.fspath if self.fs.path_info else None
return self.path_info.fspath if self.path_info else None
@cache_dir.setter
def cache_dir(self, value):
self.fs.path_info = PathInfo(value) if value else None
self.path_info = PathInfo(value) if value else None
@cached_property
def cache_path(self):
@@ -81,13 +81,13 @@ class LocalObjectDB(ObjectDB):
return ret
def _list_paths(self, prefix=None, progress_callback=None):
assert self.fs.path_info is not None
assert self.path_info is not None
if prefix:
path_info = self.fs.path_info / prefix[:2]
path_info = self.path_info / prefix[:2]
if not self.fs.exists(path_info):
return
else:
path_info = self.fs.path_info
path_info = self.path_info
# NOTE: use utils.fs walk_files since fs.walk_files will not follow
# symlinks
if progress_callback:
dvc/objects/db/ssh.py
+3 -3
View File
@@ -75,10 +75,10 @@ class SSHObjectDB(ObjectDB):
def _list_paths(self, prefix=None, progress_callback=None):
if prefix:
root = posixpath.join(self.fs.path_info.path, prefix[:2])
root = posixpath.join(self.path_info.path, prefix[:2])
else:
root = self.fs.path_info.path
with self.fs.ssh(self.fs.path_info) as ssh:
root = self.path_info.path
with self.fs.ssh(self.path_info) as ssh:
if prefix and not ssh.exists(root):
return
# If we simply return an iterator then with above closes instantly
dvc/objects/stage.py
+1 -1
View File
@@ -15,7 +15,7 @@ def _upload_file(path_info, fs, odb):
from dvc.utils import tmp_fname
from dvc.utils.stream import HashedStreamReader
tmp_info = odb.fs.path_info / tmp_fname()
tmp_info = odb.path_info / tmp_fname()
with fs.open(path_info, mode="rb", chunk_size=fs.CHUNK_SIZE) as stream:
stream = HashedStreamReader(stream)
odb.fs.upload_fobj(
dvc/output.py
+2 -4
View File
@@ -279,9 +279,8 @@ class Output:
):
self.repo = stage.repo if stage else None
fs_cls, fs_config = get_cloud_fs(self.repo, url=path)
fs_cls, fs_config, path_info = get_cloud_fs(self.repo, url=path)
self.fs = fs_cls(**fs_config)
path_info = self.fs.path_info
if (
self.fs.scheme == "local"
@@ -741,9 +740,8 @@ class Output:
if odb is None:
odb = self.odb
cls, config = get_cloud_fs(self.repo, url=source)
cls, config, from_info = get_cloud_fs(self.repo, url=source)
from_fs = cls(**config)
from_info = from_fs.path_info
# When running import-url --to-remote / add --to-remote/-o ... we
# assume that it is unlikely that the odb will contain majority of the
dvc/remote/__init__.py
+3 -3
View File
@@ -4,8 +4,8 @@ from .local import LocalRemote
def get_remote(repo, **kwargs):
cls, config = get_cloud_fs(repo, **kwargs)
cls, config, path_info = get_cloud_fs(repo, **kwargs)
fs = cls(**config)
if fs.scheme == "local":
return LocalRemote(fs, repo.tmp_dir, **config)
return Remote(fs, repo.tmp_dir, **config)
return LocalRemote(fs, path_info, repo.tmp_dir, **config)
return Remote(fs, path_info, repo.tmp_dir, **config)
dvc/remote/base.py
+12 -15
View File
@@ -10,7 +10,7 @@ from dvc.exceptions import DownloadError, UploadError
from dvc.hash_info import HashInfo
from ..progress import Tqdm
from .index import RemoteIndex, RemoteIndexNoop
from .index import RemoteIndex
logger = logging.getLogger(__name__)
@@ -67,25 +67,22 @@ class Remote:
INDEX_CLS = RemoteIndex
def __init__(self, fs, tmp_dir, **config):
def __init__(self, fs, path_info, tmp_dir, **config):
from dvc.objects.db import get_odb
self.fs = fs
self.odb = get_odb(self.fs, **config)
self.odb = get_odb(self.fs, path_info, **config)
url = config.get("url")
if url:
index_name = hashlib.sha256(url.encode("utf-8")).hexdigest()
self.index = self.INDEX_CLS(
tmp_dir, index_name, dir_suffix=self.fs.CHECKSUM_DIR_SUFFIX
)
else:
self.index = RemoteIndexNoop()
url = getattr(path_info, "fspath", path_info.url)
index_name = hashlib.sha256(url.encode("utf-8")).hexdigest()
self.index = self.INDEX_CLS(
tmp_dir, index_name, dir_suffix=self.fs.CHECKSUM_DIR_SUFFIX
)
def __repr__(self):
return "{class_name}: '{path_info}'".format(
class_name=type(self).__name__,
path_info=self.fs.path_info or "No path",
path_info=self.odb.path_info or "No path",
)
@index_locked
@@ -145,7 +142,7 @@ class Remote:
{dir_hash: set(file_hash, ...)} which can be used to map
a .dir file to its file contents.
"""
logger.debug(f"Preparing to collect status from {self.fs.path_info}")
logger.debug(f"Preparing to collect status from {self.odb.path_info}")
md5s = set(named_cache.scheme_keys(cache.fs.scheme))
logger.debug("Collecting information from local cache...")
@@ -171,7 +168,7 @@ class Remote:
if md5s:
remote_exists.update(
self.hashes_exist(
md5s, jobs=jobs, name=str(self.fs.path_info)
md5s, jobs=jobs, name=str(self.odb.path_info)
)
)
return self._make_status(
@@ -302,7 +299,7 @@ class Remote:
logger.debug(
"Preparing to {} '{}'".format(
"download data from" if download else "upload data to",
self.fs.path_info,
self.odb.path_info,
)
)
dvc/stage/cache.py
+9 -7
View File
@@ -212,15 +212,17 @@ class StageCache:
ret = []
runs = from_remote.path_info / "runs"
if not from_remote.exists(runs):
if not from_remote.fs.exists(runs):
return []
for src in from_remote.walk_files(runs):
for src in from_remote.fs.walk_files(runs):
rel = src.relative_to(from_remote.path_info)
dst = to_remote.path_info / rel
key = dst.parent
# check if any build cache already exists for this key
if to_remote.exists(key) and first(to_remote.walk_files(key)):
if to_remote.fs.exists(key) and first(
to_remote.fs.walk_files(key)
):
continue
func(src, dst)
ret.append((src.parent.name, src.name))
@@ -233,8 +235,8 @@ class StageCache:
remote = self.repo.cloud.get_remote(remote)
return self._transfer(
_log_exceptions(remote.fs.upload, "upload"),
self.repo.odb.local.fs,
remote.fs,
self.repo.odb.local,
remote.odb,
)
def pull(self, remote):
@@ -243,8 +245,8 @@ class StageCache:
remote = self.repo.cloud.get_remote(remote)
return self._transfer(
_log_exceptions(remote.fs.download, "download"),
remote.fs,
self.repo.odb.local.fs,
remote.odb,
self.repo.odb.local,
)
def get_used_cache(self, used_run_cache, *args, **kwargs):
tests/func/test_fs.py
+8 -11
View File
@@ -244,9 +244,8 @@ def test_walk_dont_ignore_subrepos(tmp_dir, scm, dvc):
)
def test_fs_getsize(dvc, cloud):
cloud.gen({"data": {"foo": "foo"}, "baz": "baz baz"})
cls, config = get_cloud_fs(dvc, **cloud.config)
cls, config, path_info = get_cloud_fs(dvc, **cloud.config)
fs = cls(**config)
path_info = fs.path_info
assert fs.getsize(path_info / "baz") == 7
assert fs.getsize(path_info / "data" / "foo") == 3
@@ -270,11 +269,11 @@ def test_fs_getsize(dvc, cloud):
)
def test_fs_upload_fobj(dvc, tmp_dir, cloud):
tmp_dir.gen("foo", "foo")
cls, config = get_cloud_fs(dvc, **cloud.config)
cls, config, path_info = get_cloud_fs(dvc, **cloud.config)
fs = cls(**config)
from_info = tmp_dir / "foo"
to_info = fs.path_info / "foo"
to_info = path_info / "foo"
with open(from_info, "rb") as stream:
fs.upload_fobj(stream, to_info)
@@ -297,9 +296,9 @@ def test_fs_ls(dvc, cloud):
}
}
)
cls, config = get_cloud_fs(dvc, **cloud.config)
cls, config, path_info = get_cloud_fs(dvc, **cloud.config)
fs = cls(**config)
path_info = cloud / "directory"
path_info /= "directory"
assert {os.path.basename(file_key) for file_key in fs.ls(path_info)} == {
"foo",
@@ -327,9 +326,8 @@ def test_fs_ls(dvc, cloud):
)
def test_fs_find_recursive(dvc, cloud):
cloud.gen({"data": {"foo": "foo", "bar": {"baz": "baz"}, "quux": "quux"}})
cls, config = get_cloud_fs(dvc, **cloud.config)
cls, config, path_info = get_cloud_fs(dvc, **cloud.config)
fs = cls(**config)
path_info = fs.path_info
assert {
os.path.basename(file_key) for file_key in fs.find(path_info / "data")
@@ -349,9 +347,8 @@ def test_fs_find_recursive(dvc, cloud):
)
def test_fs_find_with_etag(dvc, cloud):
cloud.gen({"data": {"foo": "foo", "bar": {"baz": "baz"}, "quux": "quux"}})
cls, config = get_cloud_fs(dvc, **cloud.config)
cls, config, path_info = get_cloud_fs(dvc, **cloud.config)
fs = cls(**config)
path_info = fs.path_info
for details in fs.find(path_info / "data", detail=True):
assert (
@@ -371,7 +368,7 @@ def test_fs_find_with_etag(dvc, cloud):
)
def test_fs_fsspec_path_management(dvc, cloud):
cloud.gen({"foo": "foo", "data": {"bar": "bar", "baz": {"foo": "foo"}}})
cls, config = get_cloud_fs(dvc, **cloud.config)
cls, config, _ = get_cloud_fs(dvc, **cloud.config)
fs = cls(**config)
root = cloud.parents[len(cloud.parents) - 1]
tests/func/test_odb.py
+1 -1
View File
@@ -88,7 +88,7 @@ class TestExternalCacheDir(TestDvc):
self.dvc.__init__()
assert self.dvc.odb.ssh.fs.path_info == ssh_url + "/tmp"
assert self.dvc.odb.ssh.path_info == ssh_url + "/tmp"
class TestSharedCacheDir(TestDir):
tests/func/test_remote.py
+1 -1
View File
@@ -451,7 +451,7 @@ def test_upload_exists(tmp_dir, dvc, local_remote):
remote.fs.CACHE_MODE = 0o644
from_info = PathInfo(tmp_dir / "foo")
to_info = remote.fs.path_info / "foo"
to_info = remote.odb.path_info / "foo"
remote.fs.upload(from_info, to_info)
assert remote.fs.exists(to_info)
tests/unit/fs/test_azure.py
+14 -8
View File
@@ -15,21 +15,27 @@ connection_string = (
)
def test_init_env_var(monkeypatch, dvc):
def test_strip_protocol_env_var(monkeypatch, dvc):
monkeypatch.setenv("AZURE_STORAGE_CONTAINER_NAME", container_name)
monkeypatch.setenv("AZURE_STORAGE_CONNECTION_STRING", connection_string)
config = {"url": "azure://"}
fs = AzureFileSystem(**config)
assert fs.path_info == "azure://" + container_name
assert (
AzureFileSystem._strip_protocol("azure://")
== f"azure://{container_name}"
)
def test_strip_protocol(dvc):
assert (
AzureFileSystem._strip_protocol(f"azure://{container_name}")
== f"azure://{container_name}"
)
def test_init(dvc):
prefix = "some/prefix"
url = f"azure://{container_name}/{prefix}"
config = {"url": url, "connection_string": connection_string}
config = {"connection_string": connection_string}
fs = AzureFileSystem(**config)
assert fs.path_info == url
assert fs.fs_args["connection_string"] == connection_string
def test_info(tmp_dir, azure):
tests/unit/fs/test_s3.py
-7
View File
@@ -26,13 +26,6 @@ def grants():
}
def test_init(dvc):
config = {"url": url}
fs = S3FileSystem(**config)
assert fs.path_info == url
def test_verify_ssl_default_param(dvc):
config = {"url": url}
fs = S3FileSystem(**config)
tests/unit/fs/test_tree.py
+8 -10
View File
@@ -16,12 +16,12 @@ def test_get_cloud_fs(tmp_dir, dvc):
first = base / "first"
second = first / "second"
cls, config = get_cloud_fs(dvc, name="base")
assert cls(**config).path_info == base
cls, config = get_cloud_fs(dvc, name="first")
assert cls(**config).path_info == first
cls, config = get_cloud_fs(dvc, name="second")
assert cls(**config).path_info == second
_, _, path_info = get_cloud_fs(dvc, name="base")
assert path_info == base
_, _, path_info = get_cloud_fs(dvc, name="first")
assert path_info == first
_, _, path_info = get_cloud_fs(dvc, name="second")
assert path_info == second
def test_get_cloud_fs_validate(tmp_dir, dvc):
@@ -39,11 +39,9 @@ def test_get_cloud_fs_validate(tmp_dir, dvc):
default=False,
)
assert get_cloud_fs(dvc, name="base")[1] == {
"url": "ssh://example.com/path"
}
assert get_cloud_fs(dvc, name="base")[1] == {"host": "example.com"}
assert get_cloud_fs(dvc, name="first")[1] == {
"url": "ssh://example.com/path/first",
"host": "example.com",
"type": ["symlink"],
}
tests/unit/objects/db/test_local.py
+1 -1
View File
@@ -15,7 +15,7 @@ def test_status_download_optimization(mocker, dvc):
And the desired files to fetch are already on the local cache,
Don't check the existence of the desired files on the remote cache
"""
odb = LocalObjectDB(LocalFileSystem())
odb = LocalObjectDB(LocalFileSystem(), PathInfo("."))
infos = NamedCache()
infos.add("local", "acbd18db4cc2f85cedef654fccc4a4d8", "foo")
tests/unit/remote/ssh/test_ssh.py
+53 -54
View File
@@ -9,7 +9,7 @@ from dvc.fs.ssh import SSHFileSystem
from dvc.system import System
def test_url(dvc):
def test_get_kwargs_from_urls():
user = "test"
host = "123.45.67.89"
port = 1234
@@ -17,23 +17,27 @@ def test_url(dvc):
# URL ssh://[user@]host.xz[:port]/path
url = f"ssh://{user}@{host}:{port}{path}"
config = {"url": url}
fs = SSHFileSystem(**config)
assert fs.path_info == url
assert SSHFileSystem._get_kwargs_from_urls(url) == {
"user": user,
"host": host,
"port": port,
}
# SCP-like URL ssh://[user@]host.xz:/absolute/path
url = f"ssh://{user}@{host}:{path}"
config = {"url": url}
fs = SSHFileSystem(**config)
assert fs.path_info == url
assert SSHFileSystem._get_kwargs_from_urls(url) == {
"user": user,
"host": host,
}
def test_no_path(dvc):
config = {"url": "ssh://127.0.0.1"}
fs = SSHFileSystem(**config)
assert fs.path_info.path == ""
def test_init():
fs = SSHFileSystem(user="test", host="12.34.56.78", port="1234")
assert fs.user == "test"
assert fs.host == "12.34.56.78"
assert fs.port == "1234"
mock_ssh_config = """
@@ -53,8 +57,8 @@ else:
@pytest.mark.parametrize(
"config,expected_host",
[
({"url": "ssh://example.com"}, "1.2.3.4"),
({"url": "ssh://not_in_ssh_config.com"}, "not_in_ssh_config.com"),
({"host": "example.com"}, "1.2.3.4"),
({"host": "not_in_ssh_config.com"}, "not_in_ssh_config.com"),
],
)
@patch("os.path.exists", return_value=True)
@@ -64,27 +68,22 @@ else:
read_data=mock_ssh_config,
)
def test_ssh_host_override_from_config(
mock_file, mock_exists, dvc, config, expected_host
mock_file, mock_exists, config, expected_host
):
fs = SSHFileSystem(**config)
mock_exists.assert_called_with(SSHFileSystem.ssh_config_filename())
mock_file.assert_called_with(SSHFileSystem.ssh_config_filename())
assert fs.path_info.host == expected_host
assert fs.host == expected_host
@pytest.mark.parametrize(
"config,expected_user",
[
({"url": "ssh://[email protected]"}, "test1"),
({"url": "ssh://example.com", "user": "test2"}, "test2"),
({"url": "ssh://example.com"}, "ubuntu"),
({"url": "ssh://test1@not_in_ssh_config.com"}, "test1"),
(
{"url": "ssh://test1@not_in_ssh_config.com", "user": "test2"},
"test2",
),
({"url": "ssh://not_in_ssh_config.com"}, getpass.getuser()),
({"host": "example.com", "user": "test1"}, "test1"),
({"host": "example.com"}, "ubuntu"),
({"host": "not_in_ssh_config.com", "user": "test1"}, "test1"),
({"host": "not_in_ssh_config.com"}, getpass.getuser()),
],
)
@patch("os.path.exists", return_value=True)
@@ -93,23 +92,21 @@ def test_ssh_host_override_from_config(
new_callable=mock_open,
read_data=mock_ssh_config,
)
def test_ssh_user(mock_file, mock_exists, dvc, config, expected_user):
def test_ssh_user(mock_file, mock_exists, config, expected_user):
fs = SSHFileSystem(**config)
mock_exists.assert_called_with(SSHFileSystem.ssh_config_filename())
mock_file.assert_called_with(SSHFileSystem.ssh_config_filename())
assert fs.path_info.user == expected_user
assert fs.user == expected_user
@pytest.mark.parametrize(
"config,expected_port",
[
({"url": "ssh://example.com:2222"}, 2222),
({"url": "ssh://example.com"}, 1234),
({"url": "ssh://example.com", "port": 4321}, 4321),
({"url": "ssh://not_in_ssh_config.com"}, SSHFileSystem.DEFAULT_PORT),
({"url": "ssh://not_in_ssh_config.com:2222"}, 2222),
({"url": "ssh://not_in_ssh_config.com:2222", "port": 4321}, 4321),
({"host": "example.com"}, 1234),
({"host": "example.com", "port": 4321}, 4321),
({"host": "not_in_ssh_config.com"}, SSHFileSystem.DEFAULT_PORT),
({"host": "not_in_ssh_config.com", "port": 2222}, 2222),
],
)
@patch("os.path.exists", return_value=True)
@@ -118,33 +115,33 @@ def test_ssh_user(mock_file, mock_exists, dvc, config, expected_user):
new_callable=mock_open,
read_data=mock_ssh_config,
)
def test_ssh_port(mock_file, mock_exists, dvc, config, expected_port):
def test_ssh_port(mock_file, mock_exists, config, expected_port):
fs = SSHFileSystem(**config)
mock_exists.assert_called_with(SSHFileSystem.ssh_config_filename())
mock_file.assert_called_with(SSHFileSystem.ssh_config_filename())
assert fs.path_info.port == expected_port
assert fs.port == expected_port
@pytest.mark.parametrize(
"config,expected_keyfile",
[
(
{"url": "ssh://example.com", "keyfile": "dvc_config.key"},
{"host": "example.com", "keyfile": "dvc_config.key"},
"dvc_config.key",
),
(
{"url": "ssh://example.com"},
{"host": "example.com"},
os.path.expanduser("~/.ssh/not_default.key"),
),
(
{
"url": "ssh://not_in_ssh_config.com",
"host": "not_in_ssh_config.com",
"keyfile": "dvc_config.key",
},
"dvc_config.key",
),
({"url": "ssh://not_in_ssh_config.com"}, None),
({"host": "not_in_ssh_config.com"}, None),
],
)
@patch("os.path.exists", return_value=True)
@@ -153,7 +150,7 @@ def test_ssh_port(mock_file, mock_exists, dvc, config, expected_port):
new_callable=mock_open,
read_data=mock_ssh_config,
)
def test_ssh_keyfile(mock_file, mock_exists, dvc, config, expected_keyfile):
def test_ssh_keyfile(mock_file, mock_exists, config, expected_keyfile):
fs = SSHFileSystem(**config)
mock_exists.assert_called_with(SSHFileSystem.ssh_config_filename())
@@ -164,9 +161,9 @@ def test_ssh_keyfile(mock_file, mock_exists, dvc, config, expected_keyfile):
@pytest.mark.parametrize(
"config,expected_gss_auth",
[
({"url": "ssh://example.com", "gss_auth": True}, True),
({"url": "ssh://example.com", "gss_auth": False}, False),
({"url": "ssh://not_in_ssh_config.com"}, False),
({"host": "example.com", "gss_auth": True}, True),
({"host": "example.com", "gss_auth": False}, False),
({"host": "not_in_ssh_config.com"}, False),
],
)
@patch("os.path.exists", return_value=True)
@@ -175,7 +172,7 @@ def test_ssh_keyfile(mock_file, mock_exists, dvc, config, expected_keyfile):
new_callable=mock_open,
read_data=mock_ssh_config,
)
def test_ssh_gss_auth(mock_file, mock_exists, dvc, config, expected_gss_auth):
def test_ssh_gss_auth(mock_file, mock_exists, config, expected_gss_auth):
fs = SSHFileSystem(**config)
mock_exists.assert_called_with(SSHFileSystem.ssh_config_filename())
@@ -186,10 +183,10 @@ def test_ssh_gss_auth(mock_file, mock_exists, dvc, config, expected_gss_auth):
@pytest.mark.parametrize(
"config,expected_allow_agent",
[
({"url": "ssh://example.com"}, True),
({"url": "ssh://not_in_ssh_config.com"}, True),
({"url": "ssh://example.com", "allow_agent": True}, True),
({"url": "ssh://example.com", "allow_agent": False}, False),
({"host": "example.com"}, True),
({"host": "not_in_ssh_config.com"}, True),
({"host": "example.com", "allow_agent": True}, True),
({"host": "example.com", "allow_agent": False}, False),
],
)
@patch("os.path.exists", return_value=True)
@@ -198,9 +195,7 @@ def test_ssh_gss_auth(mock_file, mock_exists, dvc, config, expected_gss_auth):
new_callable=mock_open,
read_data=mock_ssh_config,
)
def test_ssh_allow_agent(
mock_file, mock_exists, dvc, config, expected_allow_agent
):
def test_ssh_allow_agent(mock_file, mock_exists, config, expected_allow_agent):
fs = SSHFileSystem(**config)
mock_exists.assert_called_with(SSHFileSystem.ssh_config_filename())
@@ -208,11 +203,15 @@ def test_ssh_allow_agent(
assert fs.allow_agent == expected_allow_agent
def test_hardlink_optimization(dvc, tmp_dir, ssh):
fs = SSHFileSystem(**ssh.config)
def test_hardlink_optimization(tmp_dir, dvc, ssh):
from dvc.fs import get_cloud_fs
from_info = fs.path_info / "empty"
to_info = fs.path_info / "link"
cls, config, path_info = get_cloud_fs(dvc, **ssh.config)
fs = cls(**config)
assert isinstance(fs, SSHFileSystem)
from_info = path_info / "empty"
to_info = path_info / "link"
with fs.open(from_info, "wb"):
pass
tests/unit/remote/test_base.py
+9 -11
View File
@@ -37,7 +37,7 @@ def test_cmd_error(dvc):
@mock.patch.object(ObjectDB, "list_hashes_traverse")
@mock.patch.object(ObjectDB, "list_hashes_exists")
def test_hashes_exist(object_exists, traverse, dvc):
odb = ObjectDB(BaseFileSystem())
odb = ObjectDB(BaseFileSystem(), None)
# remote does not support traverse
odb.fs.CAN_TRAVERSE = False
@@ -86,8 +86,8 @@ def test_hashes_exist(object_exists, traverse, dvc):
@mock.patch.object(ObjectDB, "list_hashes", return_value=[])
@mock.patch.object(ObjectDB, "_path_to_hash", side_effect=lambda x: x)
def test_list_hashes_traverse(_path_to_hash, list_hashes, dvc):
odb = ObjectDB(BaseFileSystem())
odb.fs.path_info = PathInfo("foo")
odb = ObjectDB(BaseFileSystem(), None)
odb.path_info = PathInfo("foo")
# parallel traverse
size = 256 / odb.fs._JOBS * odb.fs.LIST_OBJECT_PAGE_SIZE
@@ -111,8 +111,8 @@ def test_list_hashes_traverse(_path_to_hash, list_hashes, dvc):
def test_list_hashes(dvc):
odb = ObjectDB(BaseFileSystem())
odb.fs.path_info = PathInfo("foo")
odb = ObjectDB(BaseFileSystem(), None)
odb.path_info = PathInfo("foo")
with mock.patch.object(
odb, "_list_paths", return_value=["12/3456", "bar"]
@@ -122,19 +122,17 @@ def test_list_hashes(dvc):
def test_list_paths(dvc):
odb = ObjectDB(BaseFileSystem())
odb.fs.path_info = PathInfo("foo")
path_info = PathInfo("foo")
odb = ObjectDB(BaseFileSystem(), path_info)
with mock.patch.object(odb.fs, "walk_files", return_value=[]) as walk_mock:
for _ in odb._list_paths():
pass
walk_mock.assert_called_with(odb.fs.path_info, prefix=False)
walk_mock.assert_called_with(path_info, prefix=False)
for _ in odb._list_paths(prefix="000"):
pass
walk_mock.assert_called_with(
odb.fs.path_info / "00" / "0", prefix=True
)
walk_mock.assert_called_with(path_info / "00" / "0", prefix=True)
@pytest.mark.parametrize(
tests/unit/remote/test_http.py
+8 -8
View File
@@ -25,7 +25,7 @@ def test_public_auth_method(dvc):
fs = HTTPFileSystem(**config)
assert fs._auth_method() is None
assert fs._auth_method(config["url"]) is None
def test_basic_auth_method(dvc):
@@ -44,8 +44,8 @@ def test_basic_auth_method(dvc):
fs = HTTPFileSystem(**config)
assert fs._auth_method() == auth
assert isinstance(fs._auth_method(), HTTPBasicAuth)
assert fs._auth_method(config["url"]) == auth
assert isinstance(fs._auth_method(config["url"]), HTTPBasicAuth)
def test_digest_auth_method(dvc):
@@ -64,8 +64,8 @@ def test_digest_auth_method(dvc):
fs = HTTPFileSystem(**config)
assert fs._auth_method() == auth
assert isinstance(fs._auth_method(), HTTPDigestAuth)
assert fs._auth_method(config["url"]) == auth
assert isinstance(fs._auth_method(config["url"]), HTTPDigestAuth)
def test_custom_auth_method(dvc):
@@ -81,7 +81,7 @@ def test_custom_auth_method(dvc):
fs = HTTPFileSystem(**config)
assert fs._auth_method() is None
assert fs._auth_method(config["url"]) is None
assert header in fs.headers
assert fs.headers[header] == password
@@ -123,9 +123,9 @@ def test_http_method(dvc):
fs = HTTPFileSystem(**config)
assert fs._auth_method() == auth
assert fs._auth_method(config["url"]) == auth
assert fs.method == "PUT"
assert isinstance(fs._auth_method(), HTTPBasicAuth)
assert isinstance(fs._auth_method(config["url"]), HTTPBasicAuth)
def test_exists(mocker):
tests/unit/remote/test_oss.py
-1
View File
@@ -16,7 +16,6 @@ def test_init(dvc):
"oss_endpoint": endpoint,
}
fs = OSSFileSystem(**config)
assert fs.path_info == url
assert fs.endpoint == endpoint
assert fs.key_id == key_id
assert fs.key_secret == key_secret
tests/unit/remote/test_remote.py
+5 -5
View File
@@ -12,7 +12,7 @@ def test_remote_with_hash_jobs(dvc):
}
dvc.config["core"]["checksum_jobs"] = 200
cls, config = get_cloud_fs(dvc, name="with_hash_jobs")
cls, config, _ = get_cloud_fs(dvc, name="with_hash_jobs")
fs = cls(**config)
assert fs.hash_jobs == 100
@@ -24,7 +24,7 @@ def test_remote_with_jobs(dvc):
}
dvc.config["core"]["jobs"] = 200
cls, config = get_cloud_fs(dvc, name="with_jobs")
cls, config, _ = get_cloud_fs(dvc, name="with_jobs")
fs = cls(**config)
assert fs.jobs == 100
@@ -33,7 +33,7 @@ def test_remote_without_hash_jobs(dvc):
dvc.config["remote"]["without_hash_jobs"] = {"url": "s3://bucket/name"}
dvc.config["core"]["checksum_jobs"] = 200
cls, config = get_cloud_fs(dvc, name="without_hash_jobs")
cls, config, _ = get_cloud_fs(dvc, name="without_hash_jobs")
fs = cls(**config)
assert fs.hash_jobs == 200
@@ -41,7 +41,7 @@ def test_remote_without_hash_jobs(dvc):
def test_remote_without_hash_jobs_default(dvc):
dvc.config["remote"]["without_hash_jobs"] = {"url": "s3://bucket/name"}
cls, config = get_cloud_fs(dvc, name="without_hash_jobs")
cls, config, _ = get_cloud_fs(dvc, name="without_hash_jobs")
fs = cls(**config)
assert fs.hash_jobs == fs.HASH_JOBS
@@ -53,5 +53,5 @@ def test_makedirs_not_create_for_top_level_path(fs_cls, dvc, mocker):
mocked_client = mocker.PropertyMock()
mocker.patch.object(fs_cls, "fs", mocked_client)
fs.makedirs(fs.path_info)
fs.makedirs(fs.PATH_CLS(url))
assert not mocked_client.called
tests/unit/remote/test_remote_tree.py
+14 -14
View File
@@ -47,7 +47,7 @@ def test_isdir(remote):
]
for expected, path in test_cases:
assert remote.fs.isdir(remote.fs.path_info / path) == expected
assert remote.fs.isdir(remote.odb.path_info / path) == expected
@pytest.mark.needs_internet
@@ -68,23 +68,23 @@ def test_exists(remote):
]
for expected, path in test_cases:
assert remote.fs.exists(remote.fs.path_info / path) == expected
assert remote.fs.exists(remote.odb.path_info / path) == expected
@pytest.mark.needs_internet
@pytest.mark.parametrize("remote", remotes, indirect=True)
def test_walk_files(remote):
files = [
remote.fs.path_info / "data/alice",
remote.fs.path_info / "data/alpha",
remote.fs.path_info / "data/subdir-file.txt",
remote.fs.path_info / "data/subdir/1",
remote.fs.path_info / "data/subdir/2",
remote.fs.path_info / "data/subdir/3",
remote.fs.path_info / "data/subdir/empty_file",
remote.odb.path_info / "data/alice",
remote.odb.path_info / "data/alpha",
remote.odb.path_info / "data/subdir-file.txt",
remote.odb.path_info / "data/subdir/1",
remote.odb.path_info / "data/subdir/2",
remote.odb.path_info / "data/subdir/3",
remote.odb.path_info / "data/subdir/empty_file",
]
assert list(remote.fs.walk_files(remote.fs.path_info / "data")) == files
assert list(remote.fs.walk_files(remote.odb.path_info / "data")) == files
@pytest.mark.parametrize("cloud", [pytest.lazy_fixture("s3")])
@@ -100,8 +100,8 @@ def test_copy_preserve_etag_across_buckets(cloud, dvc):
another = S3FileSystem(**config)
from_info = rem.fs.path_info / "foo"
to_info = another.path_info / "foo"
from_info = rem.odb.path_info / "foo"
to_info = another.PATH_CLS("s3://another/foo")
rem.fs.copy(from_info, to_info)
@@ -131,7 +131,7 @@ def test_isfile(remote):
]
for expected, path in test_cases:
assert remote.fs.isfile(remote.fs.path_info / path) == expected
assert remote.fs.isfile(remote.odb.path_info / path) == expected
@pytest.mark.needs_internet
@@ -139,7 +139,7 @@ def test_isfile(remote):
def test_download_dir(remote, tmpdir):
path = str(tmpdir / "data")
to_info = PathInfo(path)
remote.fs.download(remote.fs.path_info / "data", to_info)
remote.fs.download(remote.odb.path_info / "data", to_info)
assert os.path.isdir(path)
data_dir = tmpdir / "data"
assert len(list(walk_files(path))) == 7
tests/unit/remote/test_webdav.py
+3 -28
View File
@@ -1,5 +1,3 @@
import pytest
from dvc.fs.webdav import WebDAVFileSystem
# Test configuration
@@ -9,37 +7,14 @@ userurl = f"webdavs://{user}@example.com/public.php/webdav"
password = "password"
# Test minimum requiered configuration (url)
def test_init(dvc):
config = {"url": url}
fs = WebDAVFileSystem(**config)
assert fs.path_info == url
# Test username from configuration
@pytest.mark.parametrize(
"config", [{"url": url, "user": user}, {"url": userurl}]
)
def test_user(dvc, config):
fs = WebDAVFileSystem(**config)
def test_user(dvc):
fs = WebDAVFileSystem(host=url, user=user)
assert fs.user == user
# Test username extraction from url
def test_userurl(dvc):
config = {"url": userurl}
fs = WebDAVFileSystem(**config)
assert fs.path_info == userurl
assert fs.user == user
assert fs.path_info.user == user
# test password from config
def test_password(dvc):
config = {"url": url, "user": user, "password": password}
config = {"host": url, "user": user, "password": password}
fs = WebDAVFileSystem(**config)
assert fs.password == password
tests/unit/remote/test_webhdfs.py
+2 -3
View File
@@ -9,7 +9,7 @@ hdfscli_config = "path/to/cli/config"
def test_init(dvc):
url = "webhdfs://[email protected]:50070"
config = {
"url": url,
"host": url,
"webhdfs_token": webhdfs_token,
"webhdfs_alias": webhdfs_alias,
"hdfscli_config": hdfscli_config,
@@ -17,8 +17,7 @@ def test_init(dvc):
}
fs = WebHDFSFileSystem(**config)
assert fs.path_info == url
assert fs.token == webhdfs_token
assert fs.alias == webhdfs_alias
assert fs.path_info.user == user
assert fs.user == user
assert fs.hdfscli_config == hdfscli_config