mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 15:28:28 +01:00
fs: add basic type annotations to FsspecWrapper (#6779)
This commit is contained in:
@@ -175,12 +175,6 @@ class BaseFileSystem:
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def ls(self, path_info, detail=False):
|
||||
raise RemoteActionNotImplemented("ls", self.scheme)
|
||||
|
||||
def find(self, path_info, detail=False, prefix=None):
|
||||
raise RemoteActionNotImplemented("find", self.scheme)
|
||||
|
||||
def is_empty(self, path_info):
|
||||
return False
|
||||
|
||||
|
||||
+121
-41
@@ -1,14 +1,38 @@
|
||||
import os
|
||||
import shutil
|
||||
from functools import lru_cache
|
||||
from typing import (
|
||||
IO,
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Dict,
|
||||
Iterator,
|
||||
Optional,
|
||||
Union,
|
||||
overload,
|
||||
)
|
||||
|
||||
from funcy import cached_property
|
||||
from tqdm.utils import CallbackIOWrapper
|
||||
|
||||
from dvc.path_info import URLInfo
|
||||
from dvc.progress import DEFAULT_CALLBACK
|
||||
|
||||
from .base import BaseFileSystem
|
||||
|
||||
FSPath = URLInfo
|
||||
AnyFSPath = Union[FSPath, str]
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from typing import Iterable
|
||||
|
||||
from typing_extensions import Literal
|
||||
|
||||
|
||||
# An info() entry, might evolve to a TypedDict
|
||||
# in the future (e.g for properly type 'size' etc).
|
||||
Entry = Dict[str, Any]
|
||||
|
||||
|
||||
# pylint: disable=no-member
|
||||
class FSSpecWrapper(BaseFileSystem):
|
||||
@@ -19,18 +43,8 @@ class FSSpecWrapper(BaseFileSystem):
|
||||
self.fs_args = {"skip_instance_cache": True}
|
||||
self.fs_args.update(self._prepare_credentials(**kwargs))
|
||||
|
||||
@cached_property
|
||||
def fs(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def _with_bucket(self, path):
|
||||
return str(path)
|
||||
|
||||
def _strip_bucket(self, entry):
|
||||
return entry
|
||||
|
||||
@staticmethod
|
||||
def _get_kwargs_from_urls(urlpath):
|
||||
def _get_kwargs_from_urls(urlpath: str) -> "Dict[str, Any]":
|
||||
from fsspec.utils import infer_storage_options
|
||||
|
||||
options = infer_storage_options(urlpath)
|
||||
@@ -38,6 +52,28 @@ class FSSpecWrapper(BaseFileSystem):
|
||||
options.pop("protocol", None)
|
||||
return options
|
||||
|
||||
@cached_property
|
||||
def fs(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def _with_bucket(self, path: AnyFSPath) -> str:
|
||||
return str(path)
|
||||
|
||||
def _strip_bucket(self, entry: str) -> str:
|
||||
return entry
|
||||
|
||||
@overload
|
||||
def _strip_buckets(
|
||||
self, entries: "Iterable[Entry]", detail: "Literal[True]"
|
||||
) -> "Iterator[Entry]":
|
||||
...
|
||||
|
||||
@overload
|
||||
def _strip_buckets(
|
||||
self, entries: "Iterable[str]", detail: "Literal[False]" = False
|
||||
) -> "Iterator[str]":
|
||||
...
|
||||
|
||||
def _strip_buckets(self, entries, detail=False):
|
||||
for entry in entries:
|
||||
if detail:
|
||||
@@ -47,59 +83,91 @@ class FSSpecWrapper(BaseFileSystem):
|
||||
entry = self._strip_bucket(entry)
|
||||
yield entry
|
||||
|
||||
def _entry_hook(self, entry):
|
||||
def _entry_hook(self, entry: "Entry") -> "Entry":
|
||||
"""Simple hook method to be overridden when wanted to process
|
||||
entries within info() and ls(detail=True) calls"""
|
||||
return entry
|
||||
|
||||
def _prepare_credentials(
|
||||
self, **config
|
||||
): # pylint: disable=unused-argument
|
||||
self, **config: Dict[str, Any] # pylint: disable=unused-argument
|
||||
) -> Dict[str, Any]:
|
||||
"""Prepare the arguments for authentication to the
|
||||
host filesystem"""
|
||||
return {}
|
||||
|
||||
def _isdir(self, path_info):
|
||||
def _isdir(self, path_info: FSPath) -> bool:
|
||||
return self.fs.isdir(self._with_bucket(path_info))
|
||||
|
||||
def isdir(self, path_info):
|
||||
def isdir(self, path_info: FSPath) -> bool:
|
||||
try:
|
||||
return self._isdir(path_info)
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
def isfile(self, path_info):
|
||||
def isfile(self, path_info: FSPath) -> bool:
|
||||
try:
|
||||
return not self._isdir(path_info)
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
def is_empty(self, path_info):
|
||||
def is_empty(self, path_info: FSPath) -> bool:
|
||||
entry = self.info(path_info)
|
||||
if entry["type"] == "directory":
|
||||
return not self.fs.ls(self._with_bucket(path_info))
|
||||
return entry["size"] == 0
|
||||
|
||||
def open(
|
||||
self, path_info, mode="r", **kwargs
|
||||
): # pylint: disable=arguments-differ
|
||||
self,
|
||||
path_info: FSPath,
|
||||
mode: str = "r",
|
||||
encoding: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> "IO": # pylint: disable=arguments-differ
|
||||
return self.fs.open(self._with_bucket(path_info), mode=mode)
|
||||
|
||||
def checksum(self, path_info):
|
||||
def checksum(self, path_info: FSPath) -> str:
|
||||
return self.fs.checksum(self._with_bucket(path_info))
|
||||
|
||||
def copy(self, from_info, to_info):
|
||||
def copy(self, from_info: FSPath, to_info: FSPath) -> None:
|
||||
self.makedirs(to_info.parent)
|
||||
self.fs.copy(self._with_bucket(from_info), self._with_bucket(to_info))
|
||||
|
||||
def exists(self, path_info) -> bool:
|
||||
def exists(self, path_info: FSPath) -> bool:
|
||||
return self.fs.exists(self._with_bucket(path_info))
|
||||
|
||||
@overload
|
||||
def ls(
|
||||
self, path_info: FSPath, detail: "Literal[True]"
|
||||
) -> "Iterator[Entry]":
|
||||
...
|
||||
|
||||
@overload
|
||||
def ls(self, path_info: FSPath, detail: "Literal[False]") -> Iterator[str]:
|
||||
...
|
||||
|
||||
def ls(self, path_info, detail=False):
|
||||
path = self._with_bucket(path_info)
|
||||
files = self.fs.ls(path, detail=detail)
|
||||
yield from self._strip_buckets(files, detail=detail)
|
||||
|
||||
@overload
|
||||
def find(
|
||||
self,
|
||||
path_info: FSPath,
|
||||
detail: "Literal[True]",
|
||||
prefix: Optional[bool],
|
||||
) -> "Iterator[Entry]":
|
||||
...
|
||||
|
||||
@overload
|
||||
def find(
|
||||
self,
|
||||
path_info: FSPath,
|
||||
detail: "Literal[False]",
|
||||
prefix: Optional[bool],
|
||||
) -> Iterator[str]:
|
||||
...
|
||||
|
||||
# pylint: disable=unused-argument
|
||||
def find(self, path_info, detail=False, prefix=None):
|
||||
path = self._with_bucket(path_info)
|
||||
@@ -109,46 +177,56 @@ class FSSpecWrapper(BaseFileSystem):
|
||||
|
||||
yield from self._strip_buckets(files, detail=detail)
|
||||
|
||||
def walk_files(self, path_info, **kwargs):
|
||||
def walk_files(self, path_info: FSPath, **kwargs) -> Iterator[FSPath]:
|
||||
for file in self.find(path_info, **kwargs):
|
||||
yield path_info.replace(path=file)
|
||||
|
||||
def move(self, from_info, to_info):
|
||||
def move(self, from_info: FSPath, to_info: FSPath) -> None:
|
||||
self.fs.move(self._with_bucket(from_info), self._with_bucket(to_info))
|
||||
|
||||
def remove(self, path_info):
|
||||
def remove(self, path_info: FSPath) -> None:
|
||||
self.fs.rm_file(self._with_bucket(path_info))
|
||||
|
||||
def info(self, path_info):
|
||||
def info(self, path_info: FSPath) -> "Entry":
|
||||
info = self.fs.info(self._with_bucket(path_info)).copy()
|
||||
info = self._entry_hook(info)
|
||||
info["name"] = self._strip_bucket(info["name"])
|
||||
return info
|
||||
|
||||
def makedirs(self, path_info, **kwargs):
|
||||
def makedirs(self, path_info: FSPath, **kwargs) -> None:
|
||||
self.fs.makedirs(
|
||||
self._with_bucket(path_info), exist_ok=kwargs.pop("exist_ok", True)
|
||||
)
|
||||
|
||||
def put_file(
|
||||
self, from_file, to_info, callback=DEFAULT_CALLBACK, **kwargs
|
||||
):
|
||||
self,
|
||||
from_file: FSPath,
|
||||
to_info: FSPath,
|
||||
callback: Any = DEFAULT_CALLBACK,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
self.fs.put_file(
|
||||
from_file, self._with_bucket(to_info), callback=callback, **kwargs
|
||||
)
|
||||
self.fs.invalidate_cache(self._with_bucket(to_info.parent))
|
||||
|
||||
def get_file(
|
||||
self, from_info, to_info, callback=DEFAULT_CALLBACK, **kwargs
|
||||
):
|
||||
self,
|
||||
from_info: FSPath,
|
||||
to_info: FSPath,
|
||||
callback: Any = DEFAULT_CALLBACK,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
self.fs.get_file(
|
||||
self._with_bucket(from_info), to_info, callback=callback, **kwargs
|
||||
)
|
||||
|
||||
def upload_fobj(self, fobj, to_info, **kwargs):
|
||||
def upload_fobj(self, fobj: IO, to_info: FSPath, **kwargs) -> None:
|
||||
self.makedirs(to_info.parent)
|
||||
with self.open(to_info, "wb") as fdest:
|
||||
shutil.copyfileobj(fobj, fdest, length=fdest.blocksize)
|
||||
shutil.copyfileobj(
|
||||
fobj, fdest, length=getattr(fdest, "blocksize", None)
|
||||
)
|
||||
|
||||
|
||||
# pylint: disable=abstract-method
|
||||
@@ -156,21 +234,21 @@ class ObjectFSWrapper(FSSpecWrapper):
|
||||
TRAVERSE_PREFIX_LEN = 3
|
||||
|
||||
@lru_cache(512)
|
||||
def _with_bucket(self, path):
|
||||
def _with_bucket(self, path: AnyFSPath) -> str:
|
||||
if isinstance(path, self.PATH_CLS):
|
||||
return f"{path.bucket}/{path.path}"
|
||||
return path
|
||||
return str(path)
|
||||
|
||||
def _strip_bucket(self, entry):
|
||||
def _strip_bucket(self, entry: str) -> str:
|
||||
try:
|
||||
bucket, path = entry.split("/", 1)
|
||||
except ValueError:
|
||||
# If there is no path attached, only returns
|
||||
# the bucket (top-level).
|
||||
bucket, path = entry, None
|
||||
bucket, path = entry, ""
|
||||
return path or bucket
|
||||
|
||||
def makedirs(self, path_info, **kwargs):
|
||||
def makedirs(self, path_info: FSPath, **kwargs) -> None:
|
||||
# For object storages make this method a no-op. The original
|
||||
# fs.makedirs() method will only check if the bucket exists
|
||||
# and create if it doesn't though we don't want to support
|
||||
@@ -178,7 +256,7 @@ class ObjectFSWrapper(FSSpecWrapper):
|
||||
# simply ignore all mkdir()/makedirs() calls.
|
||||
return None
|
||||
|
||||
def _isdir(self, path_info):
|
||||
def _isdir(self, path_info: FSPath) -> bool:
|
||||
# Directory in object storages are interpreted differently
|
||||
# among different fsspec providers, so this logic is a temporary
|
||||
# measure for us to adapt as of now. It checks whether it is a
|
||||
@@ -213,7 +291,9 @@ class ObjectFSWrapper(FSSpecWrapper):
|
||||
if len(files) == 1 and files[0] == path and self.isdir(path_info):
|
||||
return None
|
||||
|
||||
yield from self._strip_buckets(files, detail=detail)
|
||||
yield from self._strip_buckets( # pylint: disable=not-an-iterable
|
||||
files, detail=detail
|
||||
)
|
||||
|
||||
|
||||
# pylint: disable=arguments-differ
|
||||
|
||||
+2
-4
@@ -36,10 +36,8 @@ class SSHFileSystem(FSSpecWrapper):
|
||||
|
||||
def _with_bucket(self, path):
|
||||
if isinstance(path, self.PATH_CLS):
|
||||
path_str = path.path
|
||||
else:
|
||||
path_str = super()._with_bucket(path)
|
||||
return path_str
|
||||
return path.path
|
||||
return super()._with_bucket(path)
|
||||
|
||||
def _prepare_credentials(self, **config):
|
||||
self.CAN_TRAVERSE = True
|
||||
|
||||
+1
-1
@@ -84,7 +84,7 @@ class WebDAVFileSystem(FSSpecWrapper): # pylint:disable=abstract-method
|
||||
return path.path[len(self.prefix) :].lstrip("/")
|
||||
return path
|
||||
|
||||
_with_bucket = translate_path_info
|
||||
_with_bucket = translate_path_info # type: ignore
|
||||
|
||||
|
||||
class WebDAVSFileSystem(WebDAVFileSystem): # pylint:disable=abstract-method
|
||||
|
||||
Reference in New Issue
Block a user