Merge pull request #1229 from efiop/master

progress reporting fixes

dvc/logger.py

@@ -106,25 +106,35 @@ class Logger(object):
     def debug_prefix():
         return Logger._prefix('Debug', 'debug')
 
+    @staticmethod
+    def _with_progress(func, msg):
+        from dvc.progress import progress
+        with progress:
+            func(msg)
+
     @staticmethod
     def error(msg, exc=None):
         prefix = Logger.error_prefix()
         str_exc, str_tb = Logger.parse_exc(exc)
         if Logger.logger().getEffectiveLevel() == logging.DEBUG and exc:
             str_tb = str_tb if str_tb else traceback.format_exc()
-            Logger.logger().error(prefix + str_tb)
+            Logger._with_progress(Logger.logger().error,
+                                  prefix + str_tb)
         chat = "\n\nHaving any troubles? Hit us up at dvc.org/support, " \
                "we are always happy to help!"
-        return Logger.logger().error(prefix + msg + str_exc + chat)
+        Logger._with_progress(Logger.logger().error,
+                              prefix + msg + str_exc + chat)
 
     @staticmethod
     def warn(msg):
-        return Logger.logger().warn(Logger.warning_prefix() + msg)
+        Logger._with_progress(Logger.logger().warn,
+                              Logger.warning_prefix() + msg)
 
     @staticmethod
     def debug(msg):
-        return Logger.logger().debug(Logger.debug_prefix() + msg)
+        Logger._with_progress(Logger.logger().debug,
+                              Logger.debug_prefix() + msg)
 
     @staticmethod
     def info(msg):
-        return Logger.logger().info(msg)
+        Logger._with_progress(Logger.logger().info, msg)

dvc/progress.py

@@ -11,6 +11,7 @@ class Progress(object):
         self._n_total = 0
         self._n_finished = 0
         self._lock = threading.Lock()
+        self._line = None
 
     def set_n_total(self, total):
         self._n_total = total
@@ -28,18 +29,23 @@ class Progress(object):
         print(line, end='')
         sys.stdout.flush()
 
-    def update_target(self, name, current, total):
+    def refresh(self, line=None):
         # Just go away if it is locked. Will update next time
         if not self._lock.acquire(False):
             return
 
-        bar = self._bar(name, current, total)
+        if line is None:
+            line = self._line
 
-        if sys.stdout.isatty():
-            self._writeln(bar)
+        if sys.stdout.isatty() and line is not None:
+            self._writeln(line)
+            self._line = line
 
         self._lock.release()
 
+    def update_target(self, name, current, total):
+        self.refresh(self._bar(name, current, total))
+
     def finish_target(self, name):
         # We have to write a msg about finished target
         with self._lock:
@@ -51,6 +57,7 @@ class Progress(object):
             print(bar)
 
             self._n_finished += 1
+            self._line = None
 
     def _bar(self, target_name, current, total):
         """
@@ -78,5 +85,15 @@ class Progress(object):
 
         return num + bar + percent + target_name
 
+    def __enter__(self):
+        self._lock.acquire(True)
+        if self._line is not None:
+            self._clearln()
+
+    def __exit__(self, type, value, tb):
+        if self._line is not None:
+            self.refresh()
+        self._lock.release()
+
 
 progress = Progress()

dvc/remote/local.py

@@ -158,14 +158,15 @@ class RemoteLOCAL(RemoteBase):
     def collect_dir_cache(self, dname):
         dir_info = []
 
-        bar = False
         for root, dirs, files in os.walk(dname):
+            bar = False
+
             if len(files) > LARGE_DIR_SIZE:
                 msg = "Computing md5 for a large directory {}. " \
                       "This is only done once."
-                Logger.info(msg.format(os.path.relpath(dname)))
+                Logger.info(msg.format(os.path.relpath(root)))
                 bar = True
-                title = os.path.relpath(dname)
+                title = os.path.relpath(root)
                 processed = 0
                 total = len(files)
                 progress.update_target(title, 0, total)
@@ -182,8 +183,8 @@ class RemoteLOCAL(RemoteBase):
                 dir_info.append({self.PARAM_RELPATH: relpath,
                                  self.PARAM_MD5: md5})
 
-        if bar:
-            progress.finish_target(title)
+            if bar:
+                progress.finish_target(title)
 
         # NOTE: sorting the list by path to ensure reproducibility
         dir_info = sorted(dir_info, key=itemgetter(self.PARAM_RELPATH))