diff --git a/dvc/fs/azure.py b/dvc/fs/azure.py index 41ee4554b..f9eeb0640 100644 --- a/dvc/fs/azure.py +++ b/dvc/fs/azure.py @@ -10,7 +10,7 @@ from dvc.exceptions import DvcException from dvc.scheme import Schemes from dvc.utils import format_link -from .fsspec_wrapper import CallbackMixin, ObjectFSWrapper +from .fsspec_wrapper import ObjectFSWrapper logger = logging.getLogger(__name__) _DEFAULT_CREDS_STEPS = ( @@ -38,7 +38,7 @@ def _az_config(): # pylint:disable=abstract-method -class AzureFileSystem(CallbackMixin, ObjectFSWrapper): +class AzureFileSystem(ObjectFSWrapper): scheme = Schemes.AZURE PARAM_CHECKSUM = "etag" REQUIRES = { diff --git a/dvc/fs/fsspec_wrapper.py b/dvc/fs/fsspec_wrapper.py index 89a3e5ac6..cb1793d71 100644 --- a/dvc/fs/fsspec_wrapper.py +++ b/dvc/fs/fsspec_wrapper.py @@ -1,9 +1,7 @@ -import os import shutil from typing import IO, TYPE_CHECKING, Any, Dict, Iterator, Optional, overload from funcy import cached_property -from tqdm.utils import CallbackIOWrapper from ._callback import DEFAULT_CALLBACK from .base import FileSystem @@ -218,40 +216,3 @@ class NoDirectoriesMixin: def ls(self, *args, **kwargs): raise NotImplementedError - - -class CallbackMixin: - """Provides callback support for the filesystem that don't support yet.""" - - def put_file( - self, - from_file, - to_info, - callback=DEFAULT_CALLBACK, - **kwargs, - ): - """Add compatibility support for Callback.""" - # pylint: disable=protected-access - self.makedirs(self.path.parent(to_info)) - size = os.path.getsize(from_file) - with open(from_file, "rb") as fobj: - callback.set_size(size) - wrapped = CallbackIOWrapper(callback.relative_update, fobj) - self.upload_fobj(wrapped, to_info) - self.fs.invalidate_cache(self.path.parent(to_info)) - - def get_file( - self, - from_info, - to_info, - callback=DEFAULT_CALLBACK, - **kwargs, - ): - # pylint: disable=protected-access - total: int = self.getsize(from_info) - if total: - callback.set_size(total) - - with self.open(from_info, "rb") as fobj, open(to_info, "wb") as fdest: - wrapped = CallbackIOWrapper(callback.relative_update, fobj) - shutil.copyfileobj(wrapped, fdest, length=fobj.blocksize) diff --git a/dvc/fs/gs.py b/dvc/fs/gs.py index 41ef7bfb0..56f17340f 100644 --- a/dvc/fs/gs.py +++ b/dvc/fs/gs.py @@ -5,10 +5,10 @@ from funcy import cached_property, wrap_prop from dvc.scheme import Schemes # pylint:disable=abstract-method -from .fsspec_wrapper import CallbackMixin, ObjectFSWrapper +from .fsspec_wrapper import ObjectFSWrapper -class GSFileSystem(CallbackMixin, ObjectFSWrapper): +class GSFileSystem(ObjectFSWrapper): scheme = Schemes.GS REQUIRES = {"gcsfs": "gcsfs"} PARAM_CHECKSUM = "etag" diff --git a/dvc/fs/hdfs.py b/dvc/fs/hdfs.py index e77fdf9dd..c2abc1ae4 100644 --- a/dvc/fs/hdfs.py +++ b/dvc/fs/hdfs.py @@ -8,13 +8,13 @@ from funcy import cached_property, wrap_prop from dvc.scheme import Schemes from dvc.utils import fix_env -from .fsspec_wrapper import CallbackMixin, FSSpecWrapper +from .fsspec_wrapper import FSSpecWrapper CHECKSUM_REGEX = re.compile(r".*\t.*\t(?P.*)") # pylint: disable=abstract-method -class HDFSFileSystem(CallbackMixin, FSSpecWrapper): +class HDFSFileSystem(FSSpecWrapper): scheme = Schemes.HDFS REQUIRES = {"fsspec": "fsspec", "pyarrow": "pyarrow"} PARAM_CHECKSUM = "checksum" diff --git a/dvc/fs/webhdfs.py b/dvc/fs/webhdfs.py index 29d055af9..a862867b3 100644 --- a/dvc/fs/webhdfs.py +++ b/dvc/fs/webhdfs.py @@ -5,10 +5,10 @@ from funcy import cached_property, wrap_prop from dvc.scheme import Schemes # pylint:disable=abstract-method -from .fsspec_wrapper import CallbackMixin, FSSpecWrapper +from .fsspec_wrapper import FSSpecWrapper -class WebHDFSFileSystem(CallbackMixin, FSSpecWrapper): +class WebHDFSFileSystem(FSSpecWrapper): scheme = Schemes.WEBHDFS REQUIRES = {"fsspec": "fsspec"} PARAM_CHECKSUM = "checksum"