tqdm remote.s3, flake8

dvc/progress.py

@@ -22,7 +22,7 @@ class Tqdm(tqdm):
         kwargs  : anything accepted by `tqdm.tqdm()`
         """
         if bytes:
-            #kwargs = deepcopy(kwargs)
+            # kwargs = deepcopy(kwargs)
             for k, v in dict(unit='B', unit_scale=True, unit_divisor=1024,
                              miniters=1):
                 kwargs.setdefault(k, v)
@@ -54,6 +54,7 @@ class Tqdm(tqdm):
         i = max_len - len(fill)
         return (fill + s[-i:]) if end else (s[:i] + fill)
 
+
 class Progress(tqdm):
     """
     Simple multi-target progress bar.

dvc/remote/http.py

@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 from dvc.scheme import Schemes
 from dvc.utils.compat import open
 
-import threading
 import requests
 import logging
 
@@ -31,9 +30,9 @@ class RemoteHTTP(RemoteBASE):
         request = self._request("GET", from_info.url, stream=True)
         total = self._content_length(from_info.url)
         leave = False
-        #TODO: persistent progress only for "large" files?
-        #if total:
-        #    leave = total > self.CHUNK_SIZE * 100
+        # TODO: persistent progress only for "large" files?
+        # if total:
+        #     leave = total > self.CHUNK_SIZE * 100
 
         with Tqdm(total=total, leave=leave, bytes=True,
                   desc=Tqdm.truncate(to_file, 10),

dvc/remote/s3.py

@@ -1,7 +1,6 @@
 from __future__ import unicode_literals
 
 import os
-import threading
 import logging
 import itertools
 from funcy import cached_property
@@ -11,7 +10,7 @@ try:
 except ImportError:
     boto3 = None
 
-from dvc.progress import progress
+from dvc.progress import Tqdm
 from dvc.config import Config
 from dvc.remote.base import RemoteBASE
 from dvc.exceptions import DvcException, ETagMismatchError
@@ -21,19 +20,6 @@ from dvc.scheme import Schemes
 logger = logging.getLogger(__name__)
 
 
-class Callback(object):
-    def __init__(self, name, total):
-        self.name = name
-        self.total = total
-        self.current = 0
-        self.lock = threading.Lock()
-
-    def __call__(self, byts):
-        with self.lock:
-            self.current += byts
-            progress.update_target(self.name, self.current, self.total)
-
-
 class RemoteS3(RemoteBASE):
     scheme = Schemes.S3
     path_cls = CloudURLInfo
@@ -228,24 +214,24 @@ class RemoteS3(RemoteBASE):
 
     def _upload(self, from_file, to_info, name=None, no_progress_bar=False):
         total = os.path.getsize(from_file)
-        cb = None if no_progress_bar else Callback(name, total)
-        self.s3.upload_file(
-            from_file,
-            to_info.bucket,
-            to_info.path,
-            Callback=cb,
-            ExtraArgs=self.extra_args,
-        )
+        desc = Tqdm.truncate(name) if name else None
+        with Tqdm(disable=no_progress_bar, total=total, bytes=True,
+                  desc=desc) as pbar:
+            self.s3.upload_file(
+                from_file,
+                to_info.bucket,
+                to_info.path,
+                Callback=pbar.update,
+                ExtraArgs=self.extra_args,
+            )
 
     def _download(self, from_info, to_file, name=None, no_progress_bar=False):
-        if no_progress_bar:
-            cb = None
-        else:
-            total = self.s3.head_object(
-                Bucket=from_info.bucket, Key=from_info.path
-            )["ContentLength"]
-            cb = Callback(name, total)
-
-        self.s3.download_file(
-            from_info.bucket, from_info.path, to_file, Callback=cb
-        )
+        total = self.s3.head_object(
+            Bucket=from_info.bucket, Key=from_info.path
+        )["ContentLength"]
+        desc = Tqdm.truncate(name) if name else None
+        with Tqdm(disable=no_progress_bar, total=total, bytes=True,
+                  desc=desc) as pbar:
+            self.s3.download_file(
+                from_info.bucket, from_info.path, to_file, Callback=pbar.update
+            )