fs: add basic type annotations to FsspecWrapper (#6779)

This commit is contained in:
Batuhan Taskaya
2021-10-11 15:15:56 +03:00
committed by GitHub
parent d6ecef7169
commit be0c827b95
4 changed files with 124 additions and 52 deletions
-6
View File
@@ -175,12 +175,6 @@ class BaseFileSystem:
"""
raise NotImplementedError
def ls(self, path_info, detail=False):
raise RemoteActionNotImplemented("ls", self.scheme)
def find(self, path_info, detail=False, prefix=None):
raise RemoteActionNotImplemented("find", self.scheme)
def is_empty(self, path_info):
return False
+121 -41
View File
@@ -1,14 +1,38 @@
import os
import shutil
from functools import lru_cache
from typing import (
IO,
TYPE_CHECKING,
Any,
Dict,
Iterator,
Optional,
Union,
overload,
)
from funcy import cached_property
from tqdm.utils import CallbackIOWrapper
from dvc.path_info import URLInfo
from dvc.progress import DEFAULT_CALLBACK
from .base import BaseFileSystem
FSPath = URLInfo
AnyFSPath = Union[FSPath, str]
if TYPE_CHECKING:
from typing import Iterable
from typing_extensions import Literal
# An info() entry, might evolve to a TypedDict
# in the future (e.g for properly type 'size' etc).
Entry = Dict[str, Any]
# pylint: disable=no-member
class FSSpecWrapper(BaseFileSystem):
@@ -19,18 +43,8 @@ class FSSpecWrapper(BaseFileSystem):
self.fs_args = {"skip_instance_cache": True}
self.fs_args.update(self._prepare_credentials(**kwargs))
@cached_property
def fs(self):
raise NotImplementedError
def _with_bucket(self, path):
return str(path)
def _strip_bucket(self, entry):
return entry
@staticmethod
def _get_kwargs_from_urls(urlpath):
def _get_kwargs_from_urls(urlpath: str) -> "Dict[str, Any]":
from fsspec.utils import infer_storage_options
options = infer_storage_options(urlpath)
@@ -38,6 +52,28 @@ class FSSpecWrapper(BaseFileSystem):
options.pop("protocol", None)
return options
@cached_property
def fs(self):
raise NotImplementedError
def _with_bucket(self, path: AnyFSPath) -> str:
return str(path)
def _strip_bucket(self, entry: str) -> str:
return entry
@overload
def _strip_buckets(
self, entries: "Iterable[Entry]", detail: "Literal[True]"
) -> "Iterator[Entry]":
...
@overload
def _strip_buckets(
self, entries: "Iterable[str]", detail: "Literal[False]" = False
) -> "Iterator[str]":
...
def _strip_buckets(self, entries, detail=False):
for entry in entries:
if detail:
@@ -47,59 +83,91 @@ class FSSpecWrapper(BaseFileSystem):
entry = self._strip_bucket(entry)
yield entry
def _entry_hook(self, entry):
def _entry_hook(self, entry: "Entry") -> "Entry":
"""Simple hook method to be overridden when wanted to process
entries within info() and ls(detail=True) calls"""
return entry
def _prepare_credentials(
self, **config
): # pylint: disable=unused-argument
self, **config: Dict[str, Any] # pylint: disable=unused-argument
) -> Dict[str, Any]:
"""Prepare the arguments for authentication to the
host filesystem"""
return {}
def _isdir(self, path_info):
def _isdir(self, path_info: FSPath) -> bool:
return self.fs.isdir(self._with_bucket(path_info))
def isdir(self, path_info):
def isdir(self, path_info: FSPath) -> bool:
try:
return self._isdir(path_info)
except FileNotFoundError:
return False
def isfile(self, path_info):
def isfile(self, path_info: FSPath) -> bool:
try:
return not self._isdir(path_info)
except FileNotFoundError:
return False
def is_empty(self, path_info):
def is_empty(self, path_info: FSPath) -> bool:
entry = self.info(path_info)
if entry["type"] == "directory":
return not self.fs.ls(self._with_bucket(path_info))
return entry["size"] == 0
def open(
self, path_info, mode="r", **kwargs
): # pylint: disable=arguments-differ
self,
path_info: FSPath,
mode: str = "r",
encoding: Optional[str] = None,
**kwargs,
) -> "IO": # pylint: disable=arguments-differ
return self.fs.open(self._with_bucket(path_info), mode=mode)
def checksum(self, path_info):
def checksum(self, path_info: FSPath) -> str:
return self.fs.checksum(self._with_bucket(path_info))
def copy(self, from_info, to_info):
def copy(self, from_info: FSPath, to_info: FSPath) -> None:
self.makedirs(to_info.parent)
self.fs.copy(self._with_bucket(from_info), self._with_bucket(to_info))
def exists(self, path_info) -> bool:
def exists(self, path_info: FSPath) -> bool:
return self.fs.exists(self._with_bucket(path_info))
@overload
def ls(
self, path_info: FSPath, detail: "Literal[True]"
) -> "Iterator[Entry]":
...
@overload
def ls(self, path_info: FSPath, detail: "Literal[False]") -> Iterator[str]:
...
def ls(self, path_info, detail=False):
path = self._with_bucket(path_info)
files = self.fs.ls(path, detail=detail)
yield from self._strip_buckets(files, detail=detail)
@overload
def find(
self,
path_info: FSPath,
detail: "Literal[True]",
prefix: Optional[bool],
) -> "Iterator[Entry]":
...
@overload
def find(
self,
path_info: FSPath,
detail: "Literal[False]",
prefix: Optional[bool],
) -> Iterator[str]:
...
# pylint: disable=unused-argument
def find(self, path_info, detail=False, prefix=None):
path = self._with_bucket(path_info)
@@ -109,46 +177,56 @@ class FSSpecWrapper(BaseFileSystem):
yield from self._strip_buckets(files, detail=detail)
def walk_files(self, path_info, **kwargs):
def walk_files(self, path_info: FSPath, **kwargs) -> Iterator[FSPath]:
for file in self.find(path_info, **kwargs):
yield path_info.replace(path=file)
def move(self, from_info, to_info):
def move(self, from_info: FSPath, to_info: FSPath) -> None:
self.fs.move(self._with_bucket(from_info), self._with_bucket(to_info))
def remove(self, path_info):
def remove(self, path_info: FSPath) -> None:
self.fs.rm_file(self._with_bucket(path_info))
def info(self, path_info):
def info(self, path_info: FSPath) -> "Entry":
info = self.fs.info(self._with_bucket(path_info)).copy()
info = self._entry_hook(info)
info["name"] = self._strip_bucket(info["name"])
return info
def makedirs(self, path_info, **kwargs):
def makedirs(self, path_info: FSPath, **kwargs) -> None:
self.fs.makedirs(
self._with_bucket(path_info), exist_ok=kwargs.pop("exist_ok", True)
)
def put_file(
self, from_file, to_info, callback=DEFAULT_CALLBACK, **kwargs
):
self,
from_file: FSPath,
to_info: FSPath,
callback: Any = DEFAULT_CALLBACK,
**kwargs,
) -> None:
self.fs.put_file(
from_file, self._with_bucket(to_info), callback=callback, **kwargs
)
self.fs.invalidate_cache(self._with_bucket(to_info.parent))
def get_file(
self, from_info, to_info, callback=DEFAULT_CALLBACK, **kwargs
):
self,
from_info: FSPath,
to_info: FSPath,
callback: Any = DEFAULT_CALLBACK,
**kwargs,
) -> None:
self.fs.get_file(
self._with_bucket(from_info), to_info, callback=callback, **kwargs
)
def upload_fobj(self, fobj, to_info, **kwargs):
def upload_fobj(self, fobj: IO, to_info: FSPath, **kwargs) -> None:
self.makedirs(to_info.parent)
with self.open(to_info, "wb") as fdest:
shutil.copyfileobj(fobj, fdest, length=fdest.blocksize)
shutil.copyfileobj(
fobj, fdest, length=getattr(fdest, "blocksize", None)
)
# pylint: disable=abstract-method
@@ -156,21 +234,21 @@ class ObjectFSWrapper(FSSpecWrapper):
TRAVERSE_PREFIX_LEN = 3
@lru_cache(512)
def _with_bucket(self, path):
def _with_bucket(self, path: AnyFSPath) -> str:
if isinstance(path, self.PATH_CLS):
return f"{path.bucket}/{path.path}"
return path
return str(path)
def _strip_bucket(self, entry):
def _strip_bucket(self, entry: str) -> str:
try:
bucket, path = entry.split("/", 1)
except ValueError:
# If there is no path attached, only returns
# the bucket (top-level).
bucket, path = entry, None
bucket, path = entry, ""
return path or bucket
def makedirs(self, path_info, **kwargs):
def makedirs(self, path_info: FSPath, **kwargs) -> None:
# For object storages make this method a no-op. The original
# fs.makedirs() method will only check if the bucket exists
# and create if it doesn't though we don't want to support
@@ -178,7 +256,7 @@ class ObjectFSWrapper(FSSpecWrapper):
# simply ignore all mkdir()/makedirs() calls.
return None
def _isdir(self, path_info):
def _isdir(self, path_info: FSPath) -> bool:
# Directory in object storages are interpreted differently
# among different fsspec providers, so this logic is a temporary
# measure for us to adapt as of now. It checks whether it is a
@@ -213,7 +291,9 @@ class ObjectFSWrapper(FSSpecWrapper):
if len(files) == 1 and files[0] == path and self.isdir(path_info):
return None
yield from self._strip_buckets(files, detail=detail)
yield from self._strip_buckets( # pylint: disable=not-an-iterable
files, detail=detail
)
# pylint: disable=arguments-differ
+2 -4
View File
@@ -36,10 +36,8 @@ class SSHFileSystem(FSSpecWrapper):
def _with_bucket(self, path):
if isinstance(path, self.PATH_CLS):
path_str = path.path
else:
path_str = super()._with_bucket(path)
return path_str
return path.path
return super()._with_bucket(path)
def _prepare_credentials(self, **config):
self.CAN_TRAVERSE = True
+1 -1
View File
@@ -84,7 +84,7 @@ class WebDAVFileSystem(FSSpecWrapper): # pylint:disable=abstract-method
return path.path[len(self.prefix) :].lstrip("/")
return path
_with_bucket = translate_path_info
_with_bucket = translate_path_info # type: ignore
class WebDAVSFileSystem(WebDAVFileSystem): # pylint:disable=abstract-method