fs: add more fsspec methods

This commit is contained in:
Saugat Pachhai (सौगात)
2022-05-05 08:16:34 +05:45
committed by skshetry
parent 9d476aab8b
commit c64db917f5
2 changed files with 84 additions and 3 deletions
+75 -3
View File
@@ -1,3 +1,4 @@
import datetime
import logging
import os
import shutil
@@ -184,6 +185,15 @@ class FileSystem:
) -> "IO": # pylint: disable=arguments-differ
return self.fs.open(path, mode=mode, encoding=encoding, **kwargs)
def read_block(
self,
path: AnyFSPath,
offset: int,
length: int,
delimiter: bytes = None,
) -> bytes:
return self.fs.read_block(path, offset, length, delimiter=delimiter)
def cat(
self,
path: Union[AnyFSPath, List[AnyFSPath]],
@@ -216,6 +226,12 @@ class FileSystem:
) -> bytes:
return self.fs.cat_file(path, start=start, end=end, **kwargs)
def head(self, path: AnyFSPath, size: int = 1024) -> bytes:
return self.fs.head(path, size=size)
def tail(self, path: AnyFSPath, size: int = 1024) -> bytes:
return self.fs.tail(path, size=size)
def pipe_file(self, path: AnyFSPath, value: bytes, **kwargs: Any) -> None:
return self.fs.pipe_file(path, value, **kwargs)
@@ -239,6 +255,11 @@ class FileSystem:
self.makedirs(self.path.parent(to_info))
self.fs.copy(from_info, to_info)
def cp_file(
self, from_info: AnyFSPath, to_info: AnyFSPath, **kwargs: Any
) -> None:
self.fs.cp_file(from_info, to_info, **kwargs)
def exists(self, path: AnyFSPath) -> bool:
return self.fs.exists(path)
@@ -298,15 +319,32 @@ class FileSystem:
) -> Iterator[str]:
yield from self.fs.find(path)
def move(self, from_info: AnyFSPath, to_info: AnyFSPath) -> None:
self.fs.move(from_info, to_info)
def mv(
self, from_info: AnyFSPath, to_info: AnyFSPath, **kwargs: Any
) -> None:
self.fs.mv(from_info, to_info)
def remove(self, path: AnyFSPath) -> None:
move = mv
def rmdir(self, path: AnyFSPath) -> None:
self.fs.rmdir(path)
def rm_file(self, path: AnyFSPath) -> None:
self.fs.rm_file(path)
def rm(self, path: AnyFSPath) -> None:
self.fs.rm(path, recursive=True)
remove = rm
def info(self, path: AnyFSPath) -> "Entry":
return self.fs.info(path)
def mkdir(
self, path: AnyFSPath, create_parents: bool = True, **kwargs: Any
) -> None:
self.fs.mkdir(path, create_parents=create_parents, **kwargs)
def makedirs(self, path: AnyFSPath, **kwargs: Any) -> None:
self.fs.makedirs(path, exist_ok=kwargs.pop("exist_ok", True))
@@ -356,9 +394,24 @@ class FileSystem:
):
return self.fs.walk(path, topdown=topdown, **kwargs)
def glob(self, path: AnyFSPath, **kwargs: Any):
return self.fs.glob(path, **kwargs)
def size(self, path: AnyFSPath) -> Optional[int]:
return self.fs.size(path)
def sizes(self, paths: List[AnyFSPath]) -> List[Optional[int]]:
return self.fs.sizes(paths)
def du(
self,
path: AnyFSPath,
total: bool = True,
maxdepth: int = None,
**kwargs: Any,
) -> Union[int, Dict[AnyFSPath, int]]:
return self.fs.du(path, total=total, maxdepth=maxdepth, **kwargs)
def get(
self,
from_info: AnyFSPath,
@@ -404,6 +457,20 @@ class FileSystem:
)
)
def ukey(self, path: AnyFSPath) -> str:
return self.fs.ukey(path)
def created(self, path: AnyFSPath) -> datetime.datetime:
return self.fs.created(path)
def modified(self, path: AnyFSPath) -> datetime.datetime:
return self.fs.modified(path)
def sign(
self, path: AnyFSPath, expiration: int = 100, **kwargs: Any
) -> str:
return self.fs.sign(path, expiration=expiration, **kwargs)
class ObjectFileSystem(FileSystem): # pylint: disable=abstract-method
TRAVERSE_PREFIX_LEN = 3
@@ -416,6 +483,11 @@ class ObjectFileSystem(FileSystem): # pylint: disable=abstract-method
# simply ignore all mkdir()/makedirs() calls.
return None
def mkdir(
self, path: AnyFSPath, create_parents: bool = True, **kwargs: Any
) -> None:
return None
def _isdir(self, path: AnyFSPath) -> bool:
# Directory in object storages are interpreted differently
# among different fsspec providers, so this logic is a temporary
+9
View File
@@ -107,6 +107,9 @@ class FsspecLocalFileSystem(AbstractFileSystem):
def rm(self, path, recursive=False, maxdepth=None):
remove(path)
def cp_file(self, path1, path2, **kwargs):
return self.copy(path1, path2, **kwargs)
def copy(self, path1, path2, recursive=False, on_error=None, **kwargs):
tmp_info = os.path.join(self._parent(path2), tmp_fname(""))
try:
@@ -155,6 +158,12 @@ class FsspecLocalFileSystem(AbstractFileSystem):
def reflink(self, path1, path2):
return System.reflink(path1, path2)
def created(self, path):
return self.fs.created(path)
def modified(self, path):
return self.fs.modified(path)
class LocalFileSystem(FileSystem):
sep = os.sep