s3, gs, azure: re-enable prefix-based search optimization (#6151)

This commit is contained in:
Batuhan Taskaya
2021-06-13 16:55:36 +03:00
committed by GitHub
parent df67dc2db8
commit b9df4451b1
3 changed files with 17 additions and 7 deletions
+1 -1
View File
@@ -170,7 +170,7 @@ class BaseFileSystem:
def ls(self, path_info, detail=False):
raise RemoteActionNotImplemented("ls", self.scheme)
def find(self, path_info, detail=False):
def find(self, path_info, detail=False, prefix=None):
raise RemoteActionNotImplemented("find", self.scheme)
def is_empty(self, path_info):
+15 -5
View File
@@ -96,7 +96,8 @@ class FSSpecWrapper(BaseFileSystem):
files = self.fs.ls(path, detail=detail)
yield from self._strip_buckets(files, detail=detail)
def find(self, path_info, detail=False):
# pylint: disable=unused-argument
def find(self, path_info, detail=False, prefix=None):
path = self._with_bucket(path_info)
files = self.fs.find(path, detail=detail)
if detail:
@@ -105,7 +106,7 @@ class FSSpecWrapper(BaseFileSystem):
yield from self._strip_buckets(files, detail=detail)
def walk_files(self, path_info, **kwargs):
for file in self.find(path_info):
for file in self.find(path_info, **kwargs):
yield path_info.replace(path=file)
def remove(self, path_info):
@@ -155,6 +156,8 @@ class FSSpecWrapper(BaseFileSystem):
# pylint: disable=abstract-method
class ObjectFSWrapper(FSSpecWrapper):
TRAVERSE_PREFIX_LEN = 3
def _isdir(self, path_info):
# Directory in object storages are interpreted differently
# among different fsspec providers, so this logic is a temporary
@@ -169,9 +172,16 @@ class ObjectFSWrapper(FSSpecWrapper):
and entry["name"].endswith("/")
)
def find(self, path_info, detail=False):
path = self._with_bucket(path_info)
files = self.fs.find(path, detail=detail)
def find(self, path_info, detail=False, prefix=None):
if prefix is not None:
path = self._with_bucket(path_info.parent)
files = self.fs.find(
path, detail=detail, prefix=path_info.parts[-1]
)
else:
path = self._with_bucket(path_info)
files = self.fs.find(path, detail=detail)
if detail:
files = files.values()
+1 -1
View File
@@ -538,7 +538,7 @@ class GDriveFileSystem(BaseFileSystem): # pylint:disable=abstract-method
query = f"({query}) and trashed=false"
return self._gdrive_list(query)
def find(self, path_info, detail=False):
def find(self, path_info, detail=False, prefix=None):
root_path = path_info.path
seen_paths = set()