fsspec: normalize checksum() to return string (#6767)

This commit is contained in:
Batuhan Taskaya
2021-10-08 16:57:37 +03:00
committed by GitHub
parent f628f670c9
commit 50cdf4f213
3 changed files with 11 additions and 22 deletions
+1 -6
View File
@@ -5,7 +5,6 @@ import threading
from funcy import cached_property, wrap_prop
from dvc.hash_info import HashInfo
from dvc.scheme import Schemes
from dvc.utils import fix_env
@@ -43,11 +42,7 @@ class HDFSFileSystem(CallbackMixin, FSSpecWrapper):
return HadoopFileSystem(**self.fs_args)
def checksum(self, path_info):
return HashInfo(
"checksum",
self._checksum(path_info),
size=self.getsize(path_info),
)
return self._checksum(path_info)
def _checksum(self, path_info, **kwargs):
# PyArrow doesn't natively support retrieving the
+1 -5
View File
@@ -2,7 +2,6 @@ import threading
from funcy import cached_property, wrap_prop
from dvc.hash_info import HashInfo
from dvc.path_info import CloudURLInfo
from dvc.scheme import Schemes
@@ -47,7 +46,4 @@ class WebHDFSFileSystem(CallbackMixin, FSSpecWrapper):
def checksum(self, path_info):
path = self._with_bucket(path_info)
ukey = self.fs.ukey(path)
return HashInfo(
"checksum", ukey["bytes"], size=self.getsize(path_info)
)
return ukey["bytes"]
+9 -11
View File
@@ -45,18 +45,16 @@ def _get_file_hash(path_info, fs, name):
info = fs.info(path_info)
if name in info:
assert not info[name].endswith(".dir")
return HashInfo(name, info[name], size=info["size"])
hash_value = info[name]
elif hasattr(fs, name):
func = getattr(fs, name)
hash_value = func(path_info)
elif name == "md5":
hash_value = file_md5(path_info, fs)
else:
raise NotImplementedError
func = getattr(fs, name, None)
if func:
return func(path_info)
if name == "md5":
return HashInfo(
name, file_md5(path_info, fs), size=fs.getsize(path_info)
)
raise NotImplementedError
return HashInfo(name, hash_value, size=info["size"])
def get_file_hash(path_info, fs, name, state=None):