From be0c827b95dd2a3f7a60c61857e6d5e96f3816a9 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Mon, 11 Oct 2021 14:15:56 +0200 Subject: [PATCH] fs: add basic type annotations to FsspecWrapper (#6779) --- dvc/fs/base.py | 6 -- dvc/fs/fsspec_wrapper.py | 162 +++++++++++++++++++++++++++++---------- dvc/fs/ssh.py | 6 +- dvc/fs/webdav.py | 2 +- 4 files changed, 124 insertions(+), 52 deletions(-) diff --git a/dvc/fs/base.py b/dvc/fs/base.py index 5df14dc13..beefcb18e 100644 --- a/dvc/fs/base.py +++ b/dvc/fs/base.py @@ -175,12 +175,6 @@ class BaseFileSystem: """ raise NotImplementedError - def ls(self, path_info, detail=False): - raise RemoteActionNotImplemented("ls", self.scheme) - - def find(self, path_info, detail=False, prefix=None): - raise RemoteActionNotImplemented("find", self.scheme) - def is_empty(self, path_info): return False diff --git a/dvc/fs/fsspec_wrapper.py b/dvc/fs/fsspec_wrapper.py index 117175e1f..19c813538 100644 --- a/dvc/fs/fsspec_wrapper.py +++ b/dvc/fs/fsspec_wrapper.py @@ -1,14 +1,38 @@ import os import shutil from functools import lru_cache +from typing import ( + IO, + TYPE_CHECKING, + Any, + Dict, + Iterator, + Optional, + Union, + overload, +) from funcy import cached_property from tqdm.utils import CallbackIOWrapper +from dvc.path_info import URLInfo from dvc.progress import DEFAULT_CALLBACK from .base import BaseFileSystem +FSPath = URLInfo +AnyFSPath = Union[FSPath, str] + +if TYPE_CHECKING: + from typing import Iterable + + from typing_extensions import Literal + + +# An info() entry, might evolve to a TypedDict +# in the future (e.g for properly type 'size' etc). +Entry = Dict[str, Any] + # pylint: disable=no-member class FSSpecWrapper(BaseFileSystem): @@ -19,18 +43,8 @@ class FSSpecWrapper(BaseFileSystem): self.fs_args = {"skip_instance_cache": True} self.fs_args.update(self._prepare_credentials(**kwargs)) - @cached_property - def fs(self): - raise NotImplementedError - - def _with_bucket(self, path): - return str(path) - - def _strip_bucket(self, entry): - return entry - @staticmethod - def _get_kwargs_from_urls(urlpath): + def _get_kwargs_from_urls(urlpath: str) -> "Dict[str, Any]": from fsspec.utils import infer_storage_options options = infer_storage_options(urlpath) @@ -38,6 +52,28 @@ class FSSpecWrapper(BaseFileSystem): options.pop("protocol", None) return options + @cached_property + def fs(self): + raise NotImplementedError + + def _with_bucket(self, path: AnyFSPath) -> str: + return str(path) + + def _strip_bucket(self, entry: str) -> str: + return entry + + @overload + def _strip_buckets( + self, entries: "Iterable[Entry]", detail: "Literal[True]" + ) -> "Iterator[Entry]": + ... + + @overload + def _strip_buckets( + self, entries: "Iterable[str]", detail: "Literal[False]" = False + ) -> "Iterator[str]": + ... + def _strip_buckets(self, entries, detail=False): for entry in entries: if detail: @@ -47,59 +83,91 @@ class FSSpecWrapper(BaseFileSystem): entry = self._strip_bucket(entry) yield entry - def _entry_hook(self, entry): + def _entry_hook(self, entry: "Entry") -> "Entry": """Simple hook method to be overridden when wanted to process entries within info() and ls(detail=True) calls""" return entry def _prepare_credentials( - self, **config - ): # pylint: disable=unused-argument + self, **config: Dict[str, Any] # pylint: disable=unused-argument + ) -> Dict[str, Any]: """Prepare the arguments for authentication to the host filesystem""" return {} - def _isdir(self, path_info): + def _isdir(self, path_info: FSPath) -> bool: return self.fs.isdir(self._with_bucket(path_info)) - def isdir(self, path_info): + def isdir(self, path_info: FSPath) -> bool: try: return self._isdir(path_info) except FileNotFoundError: return False - def isfile(self, path_info): + def isfile(self, path_info: FSPath) -> bool: try: return not self._isdir(path_info) except FileNotFoundError: return False - def is_empty(self, path_info): + def is_empty(self, path_info: FSPath) -> bool: entry = self.info(path_info) if entry["type"] == "directory": return not self.fs.ls(self._with_bucket(path_info)) return entry["size"] == 0 def open( - self, path_info, mode="r", **kwargs - ): # pylint: disable=arguments-differ + self, + path_info: FSPath, + mode: str = "r", + encoding: Optional[str] = None, + **kwargs, + ) -> "IO": # pylint: disable=arguments-differ return self.fs.open(self._with_bucket(path_info), mode=mode) - def checksum(self, path_info): + def checksum(self, path_info: FSPath) -> str: return self.fs.checksum(self._with_bucket(path_info)) - def copy(self, from_info, to_info): + def copy(self, from_info: FSPath, to_info: FSPath) -> None: self.makedirs(to_info.parent) self.fs.copy(self._with_bucket(from_info), self._with_bucket(to_info)) - def exists(self, path_info) -> bool: + def exists(self, path_info: FSPath) -> bool: return self.fs.exists(self._with_bucket(path_info)) + @overload + def ls( + self, path_info: FSPath, detail: "Literal[True]" + ) -> "Iterator[Entry]": + ... + + @overload + def ls(self, path_info: FSPath, detail: "Literal[False]") -> Iterator[str]: + ... + def ls(self, path_info, detail=False): path = self._with_bucket(path_info) files = self.fs.ls(path, detail=detail) yield from self._strip_buckets(files, detail=detail) + @overload + def find( + self, + path_info: FSPath, + detail: "Literal[True]", + prefix: Optional[bool], + ) -> "Iterator[Entry]": + ... + + @overload + def find( + self, + path_info: FSPath, + detail: "Literal[False]", + prefix: Optional[bool], + ) -> Iterator[str]: + ... + # pylint: disable=unused-argument def find(self, path_info, detail=False, prefix=None): path = self._with_bucket(path_info) @@ -109,46 +177,56 @@ class FSSpecWrapper(BaseFileSystem): yield from self._strip_buckets(files, detail=detail) - def walk_files(self, path_info, **kwargs): + def walk_files(self, path_info: FSPath, **kwargs) -> Iterator[FSPath]: for file in self.find(path_info, **kwargs): yield path_info.replace(path=file) - def move(self, from_info, to_info): + def move(self, from_info: FSPath, to_info: FSPath) -> None: self.fs.move(self._with_bucket(from_info), self._with_bucket(to_info)) - def remove(self, path_info): + def remove(self, path_info: FSPath) -> None: self.fs.rm_file(self._with_bucket(path_info)) - def info(self, path_info): + def info(self, path_info: FSPath) -> "Entry": info = self.fs.info(self._with_bucket(path_info)).copy() info = self._entry_hook(info) info["name"] = self._strip_bucket(info["name"]) return info - def makedirs(self, path_info, **kwargs): + def makedirs(self, path_info: FSPath, **kwargs) -> None: self.fs.makedirs( self._with_bucket(path_info), exist_ok=kwargs.pop("exist_ok", True) ) def put_file( - self, from_file, to_info, callback=DEFAULT_CALLBACK, **kwargs - ): + self, + from_file: FSPath, + to_info: FSPath, + callback: Any = DEFAULT_CALLBACK, + **kwargs, + ) -> None: self.fs.put_file( from_file, self._with_bucket(to_info), callback=callback, **kwargs ) self.fs.invalidate_cache(self._with_bucket(to_info.parent)) def get_file( - self, from_info, to_info, callback=DEFAULT_CALLBACK, **kwargs - ): + self, + from_info: FSPath, + to_info: FSPath, + callback: Any = DEFAULT_CALLBACK, + **kwargs, + ) -> None: self.fs.get_file( self._with_bucket(from_info), to_info, callback=callback, **kwargs ) - def upload_fobj(self, fobj, to_info, **kwargs): + def upload_fobj(self, fobj: IO, to_info: FSPath, **kwargs) -> None: self.makedirs(to_info.parent) with self.open(to_info, "wb") as fdest: - shutil.copyfileobj(fobj, fdest, length=fdest.blocksize) + shutil.copyfileobj( + fobj, fdest, length=getattr(fdest, "blocksize", None) + ) # pylint: disable=abstract-method @@ -156,21 +234,21 @@ class ObjectFSWrapper(FSSpecWrapper): TRAVERSE_PREFIX_LEN = 3 @lru_cache(512) - def _with_bucket(self, path): + def _with_bucket(self, path: AnyFSPath) -> str: if isinstance(path, self.PATH_CLS): return f"{path.bucket}/{path.path}" - return path + return str(path) - def _strip_bucket(self, entry): + def _strip_bucket(self, entry: str) -> str: try: bucket, path = entry.split("/", 1) except ValueError: # If there is no path attached, only returns # the bucket (top-level). - bucket, path = entry, None + bucket, path = entry, "" return path or bucket - def makedirs(self, path_info, **kwargs): + def makedirs(self, path_info: FSPath, **kwargs) -> None: # For object storages make this method a no-op. The original # fs.makedirs() method will only check if the bucket exists # and create if it doesn't though we don't want to support @@ -178,7 +256,7 @@ class ObjectFSWrapper(FSSpecWrapper): # simply ignore all mkdir()/makedirs() calls. return None - def _isdir(self, path_info): + def _isdir(self, path_info: FSPath) -> bool: # Directory in object storages are interpreted differently # among different fsspec providers, so this logic is a temporary # measure for us to adapt as of now. It checks whether it is a @@ -213,7 +291,9 @@ class ObjectFSWrapper(FSSpecWrapper): if len(files) == 1 and files[0] == path and self.isdir(path_info): return None - yield from self._strip_buckets(files, detail=detail) + yield from self._strip_buckets( # pylint: disable=not-an-iterable + files, detail=detail + ) # pylint: disable=arguments-differ diff --git a/dvc/fs/ssh.py b/dvc/fs/ssh.py index b2e4cc7fe..745d86264 100644 --- a/dvc/fs/ssh.py +++ b/dvc/fs/ssh.py @@ -36,10 +36,8 @@ class SSHFileSystem(FSSpecWrapper): def _with_bucket(self, path): if isinstance(path, self.PATH_CLS): - path_str = path.path - else: - path_str = super()._with_bucket(path) - return path_str + return path.path + return super()._with_bucket(path) def _prepare_credentials(self, **config): self.CAN_TRAVERSE = True diff --git a/dvc/fs/webdav.py b/dvc/fs/webdav.py index da4d451ae..c1ab08aa5 100644 --- a/dvc/fs/webdav.py +++ b/dvc/fs/webdav.py @@ -84,7 +84,7 @@ class WebDAVFileSystem(FSSpecWrapper): # pylint:disable=abstract-method return path.path[len(self.prefix) :].lstrip("/") return path - _with_bucket = translate_path_info + _with_bucket = translate_path_info # type: ignore class WebDAVSFileSystem(WebDAVFileSystem): # pylint:disable=abstract-method