diff --git a/dvc/progress.py b/dvc/progress.py index 7f0ab98a3..17085a48a 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -8,14 +8,16 @@ class Tqdm(tqdm): """ maximum-compatibility tqdm-based progressbars """ + def __init__( - self, - iterable=None, - disable=logging.getLogger( - __name__).getEffectiveLevel() >= logging.CRITICAL, - bytes=False, # pylint: disable=W0622 - desc_truncate=None, - **kwargs): + self, + iterable=None, + disable=logging.getLogger(__name__).getEffectiveLevel() + >= logging.CRITICAL, + bytes=False, # pylint: disable=W0622 + desc_truncate=None, + **kwargs + ): """ bytes : shortcut for `unit='B', unit_scale=True, unit_divisor=1024, miniters=1` @@ -24,15 +26,15 @@ class Tqdm(tqdm): """ # kwargs = deepcopy(kwargs) if bytes: - for k, v in dict(unit='B', unit_scale=True, unit_divisor=1024, - miniters=1): + for k, v in dict( + unit="B", unit_scale=True, unit_divisor=1024, miniters=1 + ): kwargs.setdefault(k, v) if desc_truncate is not None: - kwargs.setdefault('desc', self.truncate(desc_truncate)) + kwargs.setdefault("desc", self.truncate(desc_truncate)) super(Tqdm, self).__init__( - iterable=iterable, - disable=disable, - **kwargs) + iterable=iterable, disable=disable, **kwargs + ) # self.set_lock(Lock()) def update_desc(self, desc, n=1, truncate=True): @@ -46,7 +48,7 @@ class Tqdm(tqdm): def update_to(self, current, total=None): if total: - self.total = total # pylint: disable=W0613 + self.total = total # pylint: disable=W0613,W0201 self.update(current - self.n) @classmethod diff --git a/dvc/remote/azure.py b/dvc/remote/azure.py index 5af5c31bc..5f8839d1e 100644 --- a/dvc/remote/azure.py +++ b/dvc/remote/azure.py @@ -118,8 +118,10 @@ class RemoteAZURE(RemoteBASE): ): with Tqdm(desc_truncate=name, disable=no_progress_bar) as pbar: self.blob_service.create_blob_from_path( - to_info.bucket, to_info.path, from_file, - progress_callback=pbar.update_to + to_info.bucket, + to_info.path, + from_file, + progress_callback=pbar.update_to, ) def _download( @@ -127,8 +129,10 @@ class RemoteAZURE(RemoteBASE): ): with Tqdm(desc_truncate=name, disable=no_progress_bar) as pbar: self.blob_service.get_blob_to_path( - from_info.bucket, from_info.path, to_file, - progress_callback=pbar.update_to + from_info.bucket, + from_info.path, + to_file, + progress_callback=pbar.update_to, ) def open(self, path_info, mode="r", encoding=None): diff --git a/dvc/remote/http.py b/dvc/remote/http.py index 64648bdd1..68fa9b4fd 100644 --- a/dvc/remote/http.py +++ b/dvc/remote/http.py @@ -36,9 +36,13 @@ class RemoteHTTP(RemoteBASE): # if total: # leave = total > self.CHUNK_SIZE * 100 - with Tqdm(total=total, leave=leave, bytes=True, - desc_truncate=name, - disable=no_progress_bar) as pbar: + with Tqdm( + total=total, + leave=leave, + bytes=True, + desc_truncate=name, + disable=no_progress_bar, + ) as pbar: with open(to_file, "wb") as fd: for chunk in request.iter_content(chunk_size=self.CHUNK_SIZE): fd.write(chunk) diff --git a/dvc/remote/local/__init__.py b/dvc/remote/local/__init__.py index 9de7c445b..22ec5377d 100644 --- a/dvc/remote/local/__init__.py +++ b/dvc/remote/local/__init__.py @@ -255,7 +255,7 @@ class RemoteLOCAL(RemoteBASE): def cache_exists(self, checksums, jobs=None): return [ checksum - for checksum in Tqdm(checksums, unit='md5') + for checksum in Tqdm(checksums, unit="md5") if not self.changed_cache_file(checksum) ] diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index 382aba9ae..72eaf41d1 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -214,8 +214,12 @@ class RemoteS3(RemoteBASE): def _upload(self, from_file, to_info, name=None, no_progress_bar=False): total = os.path.getsize(from_file) - with Tqdm(disable=no_progress_bar, total=total, bytes=True, - desc_truncate=name) as pbar: + with Tqdm( + disable=no_progress_bar, + total=total, + bytes=True, + desc_truncate=name, + ) as pbar: self.s3.upload_file( from_file, to_info.bucket, @@ -228,8 +232,12 @@ class RemoteS3(RemoteBASE): total = self.s3.head_object( Bucket=from_info.bucket, Key=from_info.path )["ContentLength"] - with Tqdm(disable=no_progress_bar, total=total, bytes=True, - desc_truncate=name) as pbar: + with Tqdm( + disable=no_progress_bar, + total=total, + bytes=True, + desc_truncate=name, + ) as pbar: self.s3.download_file( from_info.bucket, from_info.path, to_file, Callback=pbar.update ) diff --git a/dvc/remote/ssh/connection.py b/dvc/remote/ssh/connection.py index 1a0e9fb2e..d30e84194 100644 --- a/dvc/remote/ssh/connection.py +++ b/dvc/remote/ssh/connection.py @@ -172,8 +172,7 @@ class SSHConnection: progress_title = os.path.basename(src) with Tqdm( - desc_truncate=progress_title, - disable=no_progress_bar + desc_truncate=progress_title, disable=no_progress_bar ) as pbar: self.sftp.get(src, dest, callback=pbar.update_to) @@ -188,8 +187,7 @@ class SSHConnection: progress_title = posixpath.basename(dest) with Tqdm( - desc_truncate=progress_title, - disable=no_progress_bar + desc_truncate=progress_title, disable=no_progress_bar ) as pbar: self.sftp.put(src, tmp_file, callback=pbar.update_to) diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index 3a247be82..8254ee11d 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -59,8 +59,11 @@ def file_md5(fname): name = relpath(fname) with Tqdm( - desc_truncate=name, disable=not bar, total=size, bytes=True, - leave=False + desc_truncate=name, + disable=not bar, + total=size, + bytes=True, + leave=False, ) as pbar: with open(fname, "rb") as fobj: while True: @@ -129,7 +132,9 @@ def copyfile(src, dest, no_progress_bar=False, name=None): System.reflink(src, dest) except DvcException: with Tqdm( - desc_truncate=name, disable=no_progress_bar, total=total, + desc_truncate=name, + disable=no_progress_bar, + total=total, bytes=True, ) as pbar: with open(src, "rb") as fsrc, open(dest, "wb+") as fdest: