From 1c36ef3f4345466f8f818e8f7f0fb1bc700ec418 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Fri, 19 Oct 2018 02:55:03 +0300 Subject: [PATCH] remote: use warning if upload/download failed Signed-off-by: Ruslan Kuprieiev --- dvc/logger.py | 43 ++++++++++++++++++++++++++++--------------- dvc/remote/azure.py | 8 ++++---- dvc/remote/gs.py | 12 ++++++------ dvc/remote/local.py | 8 ++++---- dvc/remote/s3.py | 6 +++--- dvc/remote/ssh.py | 12 ++++++------ 6 files changed, 51 insertions(+), 38 deletions(-) diff --git a/dvc/logger.py b/dvc/logger.py index e21c0f460..e1f361ee4 100644 --- a/dvc/logger.py +++ b/dvc/logger.py @@ -113,27 +113,40 @@ class Logger(object): func(msg) @staticmethod - def error(msg, exc=None): + def _error_exc(exc): + if exc is None: + return + + if Logger.logger().getEffectiveLevel() != logging.DEBUG: + return + prefix = Logger.error_prefix() str_exc, str_tb = Logger.parse_exc(exc) - if Logger.logger().getEffectiveLevel() == logging.DEBUG and exc: - str_tb = str_tb if str_tb else traceback.format_exc() - Logger._with_progress(Logger.logger().error, - prefix + str_tb) + str_tb = str_tb if str_tb else traceback.format_exc() + Logger._with_progress(Logger.logger().error, prefix + str_tb) + + @staticmethod + def _with_exc(func, msg, suffix="", exc=None): + Logger._error_exc(exc) + msg = msg + Logger.parse_exc(exc)[0] + suffix + Logger._with_progress(func, msg) + + @staticmethod + def error(msg, exc=None): chat = "\n\nHaving any troubles? Hit us up at dvc.org/support, " \ "we are always happy to help!" - Logger._with_progress(Logger.logger().error, - prefix + msg + str_exc + chat) + Logger._with_exc(Logger.logger().error, + Logger.error_prefix() + msg, + suffix=chat, + exc=exc) - @staticmethod - def warn(msg): - Logger._with_progress(Logger.logger().warn, - Logger.warning_prefix() + msg) + @classmethod + def warn(cls, msg, exc=None): + cls._with_exc(cls.logger().warn, cls.warning_prefix() + msg, exc=exc) - @staticmethod - def debug(msg): - Logger._with_progress(Logger.logger().debug, - Logger.debug_prefix() + msg) + @classmethod + def debug(cls, msg, exc=None): + cls._with_exc(cls.logger().debug, cls.debug_prefix() + msg, exc=exc) @staticmethod def info(msg): diff --git a/dvc/remote/azure.py b/dvc/remote/azure.py index 4c6692a31..0aaf665d1 100644 --- a/dvc/remote/azure.py +++ b/dvc/remote/azure.py @@ -176,8 +176,8 @@ class RemoteAzure(RemoteBase): self.blob_service.create_blob_from_path( bucket, key, from_info['path'], progress_callback=cb) except Exception as ex: - Logger.error("Failed to upload '{}'".format(from_info['path']), - ex) + msg = "Failed to upload '{}'".format(from_info['path']) + Logger.warn(msg, ex) else: progress.finish_target(name) @@ -213,8 +213,8 @@ class RemoteAzure(RemoteBase): self.blob_service.get_blob_to_path( bucket, key, tmp_file, progress_callback=cb) except Exception as exc: - Logger.error("Failed to download '{}/{}'".format( - bucket, key), exc) + msg = "Failed to download '{}/{}'".format(bucket, key) + Logger.warn(msg, exc) else: os.rename(tmp_file, to_info['path']) diff --git a/dvc/remote/gs.py b/dvc/remote/gs.py index d44eaa343..922b9caa9 100644 --- a/dvc/remote/gs.py +++ b/dvc/remote/gs.py @@ -214,9 +214,9 @@ class RemoteGS(RemoteBase): blob.upload_from_filename(from_info['path']) except Exception as exc: msg = "Failed to upload '{}' to '{}/{}'" - Logger.error(msg.format(from_info['path'], - to_info['bucket'], - to_info['key']), exc) + Logger.warn(msg.format(from_info['path'], + to_info['bucket'], + to_info['key']), exc) continue progress.finish_target(name) @@ -263,9 +263,9 @@ class RemoteGS(RemoteBase): blob.download_to_filename(tmp_file) except Exception as exc: msg = "Failed to download '{}/{}' to '{}'" - Logger.error(msg.format(from_info['bucket'], - from_info['key'], - to_info['path']), exc) + Logger.warn(msg.format(from_info['bucket'], + from_info['key'], + to_info['path']), exc) continue os.rename(tmp_file, to_info['path']) diff --git a/dvc/remote/local.py b/dvc/remote/local.py index a6fbe3fd6..3406af86a 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -417,8 +417,8 @@ class RemoteLOCAL(RemoteBase): copyfile(from_info['path'], to_info['path'], name=name) except Exception as exc: msg = "Failed to upload '{}' tp '{}'" - Logger.error(msg.format(from_info['path'], - to_info['path']), exc) + Logger.warn(msg.format(from_info['path'], + to_info['path']), exc) def download(self, from_infos, @@ -449,8 +449,8 @@ class RemoteLOCAL(RemoteBase): name=name) except Exception as exc: msg = "Failed to download '{}' to '{}'" - Logger.error(msg.format(from_info['path'], - to_info['path']), exc) + Logger.warn(msg.format(from_info['path'], + to_info['path']), exc) continue os.rename(tmp_file, to_info['path']) diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index a3dce4b24..be170da54 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -279,7 +279,7 @@ class RemoteS3(RemoteBase): Callback=cb) except Exception as exc: msg = "Failed to upload '{}'".format(from_info['path']) - Logger.error(msg, exc) + Logger.warn(msg, exc) continue progress.finish_target(name) @@ -331,8 +331,8 @@ class RemoteS3(RemoteBase): except Exception as exc: msg = "Failed to download '{}/{}'".format(from_info['bucket'], from_info['key']) - Logger.error(msg, exc) - return + Logger.warn(msg, exc) + continue os.rename(tmp_file, to_info['path']) diff --git a/dvc/remote/ssh.py b/dvc/remote/ssh.py index ea144c768..121a5fcf4 100644 --- a/dvc/remote/ssh.py +++ b/dvc/remote/ssh.py @@ -349,9 +349,9 @@ class RemoteSSH(RemoteBase): callback=create_cb(name)) except Exception as exc: msg = "Failed to download '{}/{}' to '{}'" - Logger.error(msg.format(from_info['host'], - from_info['path'], - to_info['path']), exc) + Logger.warn(msg.format(from_info['host'], + from_info['path'], + to_info['path']), exc) continue os.rename(tmp_file, to_info['path']) @@ -390,9 +390,9 @@ class RemoteSSH(RemoteBase): callback=create_cb(name)) except Exception as exc: msg = "Failed to upload '{}' to '{}/{}'" - Logger.error(msg.format(from_info['path'], - to_info['host'], - to_info['path'], exc)) + Logger.warn(msg.format(from_info['path'], + to_info['host'], + to_info['path'], exc)) continue progress.finish_target(name)