From 50cdf4f213c4f03493abe43783b63655c457df2e Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Fri, 8 Oct 2021 15:57:37 +0200 Subject: [PATCH] fsspec: normalize checksum() to return string (#6767) --- dvc/fs/hdfs.py | 7 +------ dvc/fs/webhdfs.py | 6 +----- dvc/objects/stage.py | 20 +++++++++----------- 3 files changed, 11 insertions(+), 22 deletions(-) diff --git a/dvc/fs/hdfs.py b/dvc/fs/hdfs.py index d56f79ef3..6acac63cd 100644 --- a/dvc/fs/hdfs.py +++ b/dvc/fs/hdfs.py @@ -5,7 +5,6 @@ import threading from funcy import cached_property, wrap_prop -from dvc.hash_info import HashInfo from dvc.scheme import Schemes from dvc.utils import fix_env @@ -43,11 +42,7 @@ class HDFSFileSystem(CallbackMixin, FSSpecWrapper): return HadoopFileSystem(**self.fs_args) def checksum(self, path_info): - return HashInfo( - "checksum", - self._checksum(path_info), - size=self.getsize(path_info), - ) + return self._checksum(path_info) def _checksum(self, path_info, **kwargs): # PyArrow doesn't natively support retrieving the diff --git a/dvc/fs/webhdfs.py b/dvc/fs/webhdfs.py index 224732cf7..964b5b3fb 100644 --- a/dvc/fs/webhdfs.py +++ b/dvc/fs/webhdfs.py @@ -2,7 +2,6 @@ import threading from funcy import cached_property, wrap_prop -from dvc.hash_info import HashInfo from dvc.path_info import CloudURLInfo from dvc.scheme import Schemes @@ -47,7 +46,4 @@ class WebHDFSFileSystem(CallbackMixin, FSSpecWrapper): def checksum(self, path_info): path = self._with_bucket(path_info) ukey = self.fs.ukey(path) - - return HashInfo( - "checksum", ukey["bytes"], size=self.getsize(path_info) - ) + return ukey["bytes"] diff --git a/dvc/objects/stage.py b/dvc/objects/stage.py index 4550782d4..99d99675c 100644 --- a/dvc/objects/stage.py +++ b/dvc/objects/stage.py @@ -45,18 +45,16 @@ def _get_file_hash(path_info, fs, name): info = fs.info(path_info) if name in info: assert not info[name].endswith(".dir") - return HashInfo(name, info[name], size=info["size"]) + hash_value = info[name] + elif hasattr(fs, name): + func = getattr(fs, name) + hash_value = func(path_info) + elif name == "md5": + hash_value = file_md5(path_info, fs) + else: + raise NotImplementedError - func = getattr(fs, name, None) - if func: - return func(path_info) - - if name == "md5": - return HashInfo( - name, file_md5(path_info, fs), size=fs.getsize(path_info) - ) - - raise NotImplementedError + return HashInfo(name, hash_value, size=info["size"]) def get_file_hash(path_info, fs, name, state=None):