diff --git a/dvc/logger.py b/dvc/logger.py index 378015227..e21c0f460 100644 --- a/dvc/logger.py +++ b/dvc/logger.py @@ -106,25 +106,35 @@ class Logger(object): def debug_prefix(): return Logger._prefix('Debug', 'debug') + @staticmethod + def _with_progress(func, msg): + from dvc.progress import progress + with progress: + func(msg) + @staticmethod def error(msg, exc=None): prefix = Logger.error_prefix() str_exc, str_tb = Logger.parse_exc(exc) if Logger.logger().getEffectiveLevel() == logging.DEBUG and exc: str_tb = str_tb if str_tb else traceback.format_exc() - Logger.logger().error(prefix + str_tb) + Logger._with_progress(Logger.logger().error, + prefix + str_tb) chat = "\n\nHaving any troubles? Hit us up at dvc.org/support, " \ "we are always happy to help!" - return Logger.logger().error(prefix + msg + str_exc + chat) + Logger._with_progress(Logger.logger().error, + prefix + msg + str_exc + chat) @staticmethod def warn(msg): - return Logger.logger().warn(Logger.warning_prefix() + msg) + Logger._with_progress(Logger.logger().warn, + Logger.warning_prefix() + msg) @staticmethod def debug(msg): - return Logger.logger().debug(Logger.debug_prefix() + msg) + Logger._with_progress(Logger.logger().debug, + Logger.debug_prefix() + msg) @staticmethod def info(msg): - return Logger.logger().info(msg) + Logger._with_progress(Logger.logger().info, msg) diff --git a/dvc/progress.py b/dvc/progress.py index 27b73fc4a..f26cced78 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -11,6 +11,7 @@ class Progress(object): self._n_total = 0 self._n_finished = 0 self._lock = threading.Lock() + self._line = None def set_n_total(self, total): self._n_total = total @@ -28,18 +29,23 @@ class Progress(object): print(line, end='') sys.stdout.flush() - def update_target(self, name, current, total): + def refresh(self, line=None): # Just go away if it is locked. Will update next time if not self._lock.acquire(False): return - bar = self._bar(name, current, total) + if line is None: + line = self._line - if sys.stdout.isatty(): - self._writeln(bar) + if sys.stdout.isatty() and line is not None: + self._writeln(line) + self._line = line self._lock.release() + def update_target(self, name, current, total): + self.refresh(self._bar(name, current, total)) + def finish_target(self, name): # We have to write a msg about finished target with self._lock: @@ -51,6 +57,7 @@ class Progress(object): print(bar) self._n_finished += 1 + self._line = None def _bar(self, target_name, current, total): """ @@ -78,5 +85,15 @@ class Progress(object): return num + bar + percent + target_name + def __enter__(self): + self._lock.acquire(True) + if self._line is not None: + self._clearln() + + def __exit__(self, type, value, tb): + if self._line is not None: + self.refresh() + self._lock.release() + progress = Progress() diff --git a/dvc/remote/local.py b/dvc/remote/local.py index 2effd5b2b..81bb0cac1 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -158,14 +158,15 @@ class RemoteLOCAL(RemoteBase): def collect_dir_cache(self, dname): dir_info = [] - bar = False for root, dirs, files in os.walk(dname): + bar = False + if len(files) > LARGE_DIR_SIZE: msg = "Computing md5 for a large directory {}. " \ "This is only done once." - Logger.info(msg.format(os.path.relpath(dname))) + Logger.info(msg.format(os.path.relpath(root))) bar = True - title = os.path.relpath(dname) + title = os.path.relpath(root) processed = 0 total = len(files) progress.update_target(title, 0, total) @@ -182,8 +183,8 @@ class RemoteLOCAL(RemoteBase): dir_info.append({self.PARAM_RELPATH: relpath, self.PARAM_MD5: md5}) - if bar: - progress.finish_target(title) + if bar: + progress.finish_target(title) # NOTE: sorting the list by path to ensure reproducibility dir_info = sorted(dir_info, key=itemgetter(self.PARAM_RELPATH))