diff --git a/dvc/tree/hdfs.py b/dvc/tree/hdfs.py index 60c0da7fe..a57493c77 100644 --- a/dvc/tree/hdfs.py +++ b/dvc/tree/hdfs.py @@ -8,6 +8,7 @@ from contextlib import closing, contextmanager from urllib.parse import urlparse from dvc.hash_info import HashInfo +from dvc.progress import Tqdm from dvc.scheme import Schemes from dvc.utils import fix_env, tmp_fname @@ -180,14 +181,34 @@ class HDFSTree(BaseTree): self.PARAM_CHECKSUM, self._group(regex, stdout, "checksum") ) - def _upload(self, from_file, to_info, **_kwargs): + def _upload( + self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs + ): with self.hdfs(to_info) as hdfs: tmp_file = tmp_fname(to_info.path) + total = os.path.getsize(from_file) with open(from_file, "rb") as fobj: - hdfs.upload(tmp_file, fobj) + with Tqdm.wrapattr( + fobj, + "read", + desc=name, + total=total, + disable=no_progress_bar, + ) as wrapped: + hdfs.upload(tmp_file, wrapped) hdfs.rename(tmp_file, to_info.path) - def _download(self, from_info, to_file, **_kwargs): + def _download( + self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs + ): with self.hdfs(from_info) as hdfs: + total = hdfs.info(from_info.path)["size"] with open(to_file, "wb+") as fobj: - hdfs.download(from_info.path, fobj) + with Tqdm.wrapattr( + fobj, + "write", + desc=name, + total=total, + disable=no_progress_bar, + ) as wrapped: + hdfs.download(from_info.path, wrapped) diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index c474935fd..9619b146f 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -21,7 +21,7 @@ from dvc.utils.serialize import dump_yaml, load_yaml all_clouds = [ pytest.lazy_fixture(cloud) - for cloud in ["s3", "gs", "azure", "gdrive", "ssh", "http"] + for cloud in ["s3", "gs", "azure", "gdrive", "ssh", "http", "hdfs"] ] + [ pytest.param( pytest.lazy_fixture("oss"),