diff --git a/dvc/analytics.py b/dvc/analytics.py index de1a5c513..584233dae 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -2,8 +2,8 @@ import os import json import errno +import dvc.logger as logger from dvc import VERSION -from dvc.logger import logger class Analytics(object): diff --git a/dvc/cli.py b/dvc/cli.py index a503bdc65..08d726fc0 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -4,6 +4,7 @@ import os import sys import argparse +import dvc.logger as logger from dvc.command.init import CmdInit from dvc.command.destroy import CmdDestroy from dvc.command.remove import CmdRemove @@ -28,7 +29,6 @@ from dvc.command.lock import CmdLock, CmdUnlock from dvc.command.pipeline import CmdPipelineShow, CmdPipelineList from dvc.command.daemon import CmdDaemonUpdater, CmdDaemonAnalytics from dvc.exceptions import DvcParserError -from dvc.logger import logger from dvc import VERSION @@ -44,7 +44,7 @@ def _fix_subparsers(subparsers): class DvcParser(argparse.ArgumentParser): def error(self, message): - sys.stderr.write('{}: {}\n'.format(logger.error_prefix(), message)) + logger.error(message) self.print_help() raise DvcParserError() @@ -86,7 +86,7 @@ def parse_args(argv=None): '--version', action=VersionAction, nargs=0, - help='Show program\'s version.') + help="Show program's version.") # Sub commands subparsers = parser.add_subparsers( @@ -462,7 +462,7 @@ def parse_args(argv=None): status_parser.set_defaults(func=CmdDataStatus) # Repro - REPRO_HELP = 'Reproduce DVC file. Default file name - \'Dvcfile\'.' + REPRO_HELP = "Reproduce DVC file. Default file name - 'Dvcfile'." repro_parser = subparsers.add_parser( 'repro', parents=[parent_parser], @@ -529,7 +529,7 @@ def parse_args(argv=None): action='store_true', default=False, help='Reproduce all descendants of a changed stage' - 'even if their direct dependencies didn\'t ' + "even if their direct dependencies didn't " 'change.') repro_parser.set_defaults(func=CmdRepro) @@ -897,18 +897,16 @@ def parse_args(argv=None): help='JSON path.') metrics_show_group.add_argument( '--tsv-path', - help='TSV path \'row,column\'(e.g. \'1,2\').') + help="TSV path 'row,column' (e.g. '1,2').") metrics_show_group.add_argument( '--htsv-path', - help='Headed TSV path \'row,column\'' - '(e.g. \'Name,3\').') + help="Headed TSV path 'row,column (e.g. 'Name,3').") metrics_show_group.add_argument( '--csv-path', - help='CSV path \'row,column\'(e.g. \'1,2\').') + help="CSV path 'row,column' (e.g. '1,2').") metrics_show_group.add_argument( '--hcsv-path', - help='Headed CSV path \'row,column\'' - '(e.g. \'Name,3\').') + help="Headed CSV path 'row,column' (e.g. 'Name,3').") metrics_show_parser.add_argument( '-a', '--all-branches', @@ -982,7 +980,7 @@ def parse_args(argv=None): install_parser.set_defaults(func=CmdInstall) # Root - ROOT_HELP = 'Relative path to project\'s directory.' + ROOT_HELP = "Relative path to project's directory." root_parser = subparsers.add_parser( 'root', parents=[parent_parser], diff --git a/dvc/command/add.py b/dvc/command/add.py index 1b81361e6..33677828b 100644 --- a/dvc/command/add.py +++ b/dvc/command/add.py @@ -1,6 +1,6 @@ +import dvc.logger as logger from dvc.exceptions import DvcException from dvc.command.base import CmdBase -from dvc.logger import logger class CmdAdd(CmdBase): @@ -8,7 +8,7 @@ class CmdAdd(CmdBase): for target in self.args.targets: try: self.project.add(target, recursive=self.args.recursive) - except DvcException as ex: - logger.error('Failed to add \'{}\''.format(target), ex) + except DvcException: + logger.error('failed to add file') return 1 return 0 diff --git a/dvc/command/base.py b/dvc/command/base.py index 1eadfd541..81e05d90a 100644 --- a/dvc/command/base.py +++ b/dvc/command/base.py @@ -1,4 +1,4 @@ -from dvc.logger import logger +import dvc.logger as logger class CmdBase(object): @@ -24,8 +24,8 @@ class CmdBase(object): try: with self.project.lock: return self.run() - except LockError as ex: - logger.error('Failed to lock before running a command', ex) + except LockError: + logger.error('failed to lock before running a command') return 1 # Abstract methods that have to be implemented by any inheritance class diff --git a/dvc/command/config.py b/dvc/command/config.py index ea33654ae..9e0ce946a 100644 --- a/dvc/command/config.py +++ b/dvc/command/config.py @@ -1,7 +1,7 @@ import os +import dvc.logger as logger from dvc.command.base import CmdBase -from dvc.logger import logger from dvc.config import Config from dvc.exceptions import DvcException @@ -43,16 +43,16 @@ class CmdConfig(CmdBase): try: self.config.unset(configobj, section, opt) self.config.save(configobj) - except DvcException as exc: - logger.error("Failed to unset '{}'".format(self.args.name), exc) + except DvcException: + logger.error("failed to unset '{}'".format(self.args.name)) return 1 return 0 def _show(self, section, opt): try: self.config.show(self.configobj, section, opt) - except DvcException as exc: - logger.error("Failed to show '{}'".format(self.args.name), exc) + except DvcException: + logger.error("failed to show '{}'".format(self.args.name)) return 1 return 0 @@ -60,11 +60,9 @@ class CmdConfig(CmdBase): try: self.config.set(self.configobj, section, opt, value) self.config.save(self.configobj) - except DvcException as exc: - logger.error("Failed to set '{}.{}' to '{}'".format(section, - opt, - value), - exc) + except DvcException: + logger.error("failed to set '{}.{}' to '{}'" + .format(section, opt, value)) return 1 return 0 diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py index 62586f90a..0fa44e162 100644 --- a/dvc/command/data_sync.py +++ b/dvc/command/data_sync.py @@ -1,4 +1,5 @@ from dvc.command.base import CmdBase +import dvc.logger as logger class CmdDataBase(CmdBase): @@ -27,9 +28,8 @@ class CmdDataPull(CmdDataBase): all_tags=self.args.all_tags, with_deps=self.args.with_deps, force=self.args.force) - except Exception as exc: - self.project.logger.error('Failed to pull data from the cloud', - exc) + except Exception: + logger.error('failed to pull data from the cloud') return 1 return 0 @@ -44,8 +44,8 @@ class CmdDataPush(CmdDataBase): all_branches=self.args.all_branches, all_tags=self.args.all_tags, with_deps=self.args.with_deps) - except Exception as exc: - self.project.logger.error('Failed to push data to the cloud', exc) + except Exception: + logger.error('failed to push data to the cloud') return 1 return 0 @@ -60,8 +60,7 @@ class CmdDataFetch(CmdDataBase): all_branches=self.args.all_branches, all_tags=self.args.all_tags, with_deps=self.args.with_deps) - except Exception as exc: - self.project.logger.error('Failed to fetch data from the cloud', - exc) + except Exception: + logger.error('failed to fetch data from the cloud') return 1 return 0 diff --git a/dvc/command/destroy.py b/dvc/command/destroy.py index c0305b950..afd7a0449 100644 --- a/dvc/command/destroy.py +++ b/dvc/command/destroy.py @@ -1,4 +1,5 @@ import dvc.prompt as prompt +import dvc.logger as logger from dvc.command.base import CmdBase from dvc.exceptions import DvcException @@ -6,17 +7,21 @@ from dvc.exceptions import DvcException class CmdDestroy(CmdBase): def run_cmd(self): try: - msg = u'This will destroy all information about your pipelines, ' \ - u'all data files, as well as cache in .dvc/cache.\n' \ - u'Are you sure you want to continue?' + statement = ( + 'This will destroy all information about your pipelines,' + ' all data files, as well as cache in .dvc/cache.' + '\n' + 'Are you sure you want to continue?' + ) - if not self.args.force and not prompt.confirm(msg): - msg = u'Cannot destroy without a confirmation from the ' \ - u'user. Use \'-f\' to force.' - raise DvcException(msg) + if not self.args.force and not prompt.confirm(statement): + raise DvcException( + "cannot destroy without a confirmation from the user." + " Use '-f' to force." + ) self.project.destroy() - except Exception as exc: - self.project.logger.error('Failed to destroy DVC', exc) + except Exception: + logger.error('failed to destroy DVC') return 1 return 0 diff --git a/dvc/command/gc.py b/dvc/command/gc.py index 19fd24b43..84c6cd4e5 100644 --- a/dvc/command/gc.py +++ b/dvc/command/gc.py @@ -1,12 +1,13 @@ import os import dvc.prompt as prompt +import dvc.logger as logger from dvc.command.base import CmdBase class CmdGC(CmdBase): def run(self): - msg = 'This will remove all cache except the cache that is used in ' + msg = 'this will remove all cache except the cache that is used in ' if not self.args.all_branches and not self.args.all_tags: msg += 'the current git branch' elif self.args.all_branches and not self.args.all_tags: @@ -24,7 +25,7 @@ class CmdGC(CmdBase): else: msg += ' of the current project.' - self.project.logger.warn(msg) + logger.warning(msg) msg = 'Are you sure you want to proceed?' if not self.args.force and not prompt.confirm(msg): diff --git a/dvc/command/imp.py b/dvc/command/imp.py index a8316f3de..7f9a2dfa6 100644 --- a/dvc/command/imp.py +++ b/dvc/command/imp.py @@ -5,9 +5,9 @@ try: except ImportError: from urllib.parse import urlparse +import dvc.logger as logger from dvc.exceptions import DvcException from dvc.command.base import CmdBase -from dvc.logger import logger class CmdImport(CmdBase): @@ -20,7 +20,7 @@ class CmdImport(CmdBase): out = self.args.out or default_out self.project.imp(self.args.url, out) - except DvcException as ex: - logger.error('Failed to import {}'.format(self.args.url), ex) + except DvcException: + logger.error('failed to import {}'.format(self.args.url)) return 1 return 0 diff --git a/dvc/command/init.py b/dvc/command/init.py index a8bcc3376..932d27916 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -1,4 +1,4 @@ -from dvc.logger import logger +import dvc.logger as logger class CmdInit(object): @@ -13,7 +13,7 @@ class CmdInit(object): no_scm=self.args.no_scm, force=self.args.force) self.config = self.project.config - except InitError as e: - logger.error('Failed to initiate dvc', e) + except InitError: + logger.error('failed to initiate dvc') return 1 return 0 diff --git a/dvc/command/install.py b/dvc/command/install.py index 0ac3c2bd8..1a2e0dda1 100644 --- a/dvc/command/install.py +++ b/dvc/command/install.py @@ -1,4 +1,4 @@ -from dvc.logger import logger +import dvc.logger as logger from dvc.command.base import CmdBase @@ -6,7 +6,7 @@ class CmdInstall(CmdBase): def run_cmd(self): try: self.project.install() - except Exception as e: - logger.error('Failed to install dvc hooks', e) + except Exception: + logger.error('failed to install dvc hooks') return 1 return 0 diff --git a/dvc/command/lock.py b/dvc/command/lock.py index 00f51b25c..faaa23e42 100644 --- a/dvc/command/lock.py +++ b/dvc/command/lock.py @@ -1,3 +1,4 @@ +import dvc.logger as logger from dvc.exceptions import DvcException from dvc.command.base import CmdBase @@ -7,10 +8,12 @@ class CmdLockBase(CmdBase): for target in self.args.targets: try: self.project.lock_stage(target, unlock=unlock) - except DvcException as ex: - msg = 'Failed to {}lock \'{}\'' - msg = msg.format('un' if unlock else '', target) - self.project.logger.error(msg, ex) + except DvcException: + logger.error( + "failed to {}lock '{}'" + .format('un' if unlock else '', target) + ) + return 1 return 0 diff --git a/dvc/command/metrics.py b/dvc/command/metrics.py index e0da68767..98012eeeb 100644 --- a/dvc/command/metrics.py +++ b/dvc/command/metrics.py @@ -1,3 +1,4 @@ +import dvc.logger as logger from dvc.command.base import CmdBase from dvc.exceptions import DvcException @@ -30,8 +31,8 @@ class CmdMetricsShow(CmdBase): xpath=xpath, all_branches=self.args.all_branches, all_tags=self.args.all_tags) - except DvcException as exc: - self.project.logger.error('Failed to show metrics', exc) + except DvcException: + logger.error('failed to show metrics') return 1 return 0 @@ -43,8 +44,8 @@ class CmdMetricsModify(CmdBase): self.project.metrics_modify(self.args.path, typ=self.args.type, xpath=self.args.xpath) - except DvcException as exc: - self.project.logger.error('Failed to modify metrics', exc) + except DvcException: + logger.error('failed to modify metrics') return 1 return 0 @@ -56,9 +57,9 @@ class CmdMetricsAdd(CmdBase): self.project.metrics_add(self.args.path, self.args.type, self.args.xpath) - except DvcException as exc: - msg = 'Failed to add metric file \'{}\''.format(self.args.path) - self.project.logger.error(msg, exc) + except DvcException: + msg = "failed to add metric file '{}'".format(self.args.path) + logger.error(msg) return 1 return 0 @@ -68,9 +69,9 @@ class CmdMetricsRemove(CmdBase): def run(self): try: self.project.metrics_remove(self.args.path) - except DvcException as exc: - msg = 'Failed to remove metric file \'{}\''.format(self.args.path) - self.project.logger.error(msg, exc) + except DvcException: + msg = "failed to remove metric file '{}'".format(self.args.path) + logger.error(msg) return 1 return 0 diff --git a/dvc/command/move.py b/dvc/command/move.py index 87c6a8f2c..a76706fe5 100644 --- a/dvc/command/move.py +++ b/dvc/command/move.py @@ -1,3 +1,4 @@ +import dvc.logger as logger from dvc.exceptions import DvcException from dvc.command.base import CmdBase @@ -6,9 +7,9 @@ class CmdMove(CmdBase): def run(self): try: self.project.move(self.args.src, self.args.dst) - except DvcException as ex: - msg = 'Failed to move \'{}\' -> \'{}\''.format(self.args.src, - self.args.dst) - self.project.logger.error(msg, ex) + except DvcException: + msg = "failed to move '{}' -> '{}'".format(self.args.src, + self.args.dst) + logger.error(msg) return 1 return 0 diff --git a/dvc/command/pipeline.py b/dvc/command/pipeline.py index c17b02357..37ceeb172 100644 --- a/dvc/command/pipeline.py +++ b/dvc/command/pipeline.py @@ -1,5 +1,6 @@ import os +import dvc.logger as logger from dvc.exceptions import DvcException from dvc.command.base import CmdBase @@ -16,12 +17,12 @@ class CmdPipelineShow(CmdBase): for n in networkx.dfs_postorder_nodes(G, node): if commands: - self.project.logger.info(stages[n].cmd) + logger.info(stages[n].cmd) elif outs: for out in stages[n].outs: - self.project.logger.info(str(out)) + logger.info(str(out)) else: - self.project.logger.info(n) + logger.info(n) def __build_graph(self, target, commands, outs): import networkx @@ -106,9 +107,9 @@ class CmdPipelineShow(CmdBase): self._show(target, self.args.commands, self.args.outs) - except DvcException as ex: - msg = 'Failed to show pipeline for \'{}\''.format(target) - self.project.logger.error(msg, ex) + except DvcException: + msg = "failed to show pipeline for '{}'".format(target) + logger.error(msg) return 1 return 0 @@ -121,9 +122,9 @@ class CmdPipelineList(CmdBase): for p in pipelines: stages = networkx.get_node_attributes(p, 'stage') for stage in stages: - self.project.logger.info(stage) + logger.info(stage) if len(stages) != 0: - self.project.logger.info("="*80) - self.project.logger.info("{} pipeline(s) total".format(len(pipelines))) + logger.info('=' * 80) + logger.info('{} pipeline(s) total'.format(len(pipelines))) return 0 diff --git a/dvc/command/remote.py b/dvc/command/remote.py index 607ce2f71..682aef333 100644 --- a/dvc/command/remote.py +++ b/dvc/command/remote.py @@ -1,8 +1,8 @@ import re +import dvc.logger as logger from dvc.config import Config from dvc.command.config import CmdConfig -from dvc.logger import logger class CmdRemoteAdd(CmdConfig): @@ -13,7 +13,7 @@ class CmdRemoteAdd(CmdConfig): return ret if self.args.default: - msg = 'Setting \'{}\' as a default remote.'.format(self.args.name) + msg = "Setting '{}' as a default remote.".format(self.args.name) logger.info(msg) ret = self._set(Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, diff --git a/dvc/command/remove.py b/dvc/command/remove.py index b3d491c15..3a2396ba1 100644 --- a/dvc/command/remove.py +++ b/dvc/command/remove.py @@ -1,7 +1,7 @@ import dvc.prompt as prompt +import dvc.logger as logger from dvc.exceptions import DvcException from dvc.command.base import CmdBase -from dvc.logger import logger class CmdRemove(CmdBase): @@ -30,7 +30,7 @@ class CmdRemove(CmdBase): try: outs_only = self._is_outs_only(target) self.project.remove(target, outs_only=outs_only) - except DvcException as ex: - logger.error('Failed to remove {}'.format(target), ex) + except DvcException: + logger.error('failed to remove {}'.format(target)) return 1 return 0 diff --git a/dvc/command/repro.py b/dvc/command/repro.py index d51254833..41679c338 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -1,5 +1,6 @@ import os +import dvc.logger as logger from dvc.command.base import CmdBase from dvc.command.status import CmdDataStatus from dvc.exceptions import DvcException @@ -30,13 +31,12 @@ class CmdRepro(CmdBase): ignore_build_cache=self.args.ignore_build_cache) if len(stages) == 0: - self.project.logger.info(CmdDataStatus.UP_TO_DATE_MSG) + logger.info(CmdDataStatus.UP_TO_DATE_MSG) if self.args.metrics: self.project.metrics_show() - except DvcException as ex: - msg = 'Failed to reproduce \'{}\''.format(target) - self.project.logger.error(msg, ex) + except DvcException: + logger.error() ret = 1 break diff --git a/dvc/command/root.py b/dvc/command/root.py index feff9cb01..c4cac440e 100644 --- a/dvc/command/root.py +++ b/dvc/command/root.py @@ -1,5 +1,6 @@ import os +import dvc.logger as logger from dvc.command.base import CmdBase @@ -8,5 +9,5 @@ class CmdRoot(CmdBase): return self.run() def run(self): - self.project.logger.info(os.path.relpath(self.project.root_dir)) + logger.info(os.path.relpath(self.project.root_dir)) return 0 diff --git a/dvc/command/run.py b/dvc/command/run.py index 79857919d..f72ac71ab 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -1,3 +1,4 @@ +import dvc.logger as logger from dvc.command.base import CmdBase from dvc.exceptions import DvcException @@ -33,8 +34,8 @@ class CmdRun(CmdBase): overwrite=overwrite, ignore_build_cache=self.args.ignore_build_cache, remove_outs=self.args.remove_outs) - except DvcException as ex: - self.project.logger.error('Failed to run command', ex) + except DvcException: + logger.error('failed to run command') return 1 return 0 diff --git a/dvc/command/status.py b/dvc/command/status.py index aeae8652f..fb8cce33f 100644 --- a/dvc/command/status.py +++ b/dvc/command/status.py @@ -1,3 +1,4 @@ +import dvc.logger as logger from dvc.command.data_sync import CmdDataBase @@ -16,11 +17,11 @@ class CmdDataStatus(CmdDataBase): for key, value in status.items(): if isinstance(value, dict): - self.project.logger.info('{}{}'.format(ind, key)) + logger.info('{}{}'.format(ind, key)) self._show(value, indent+1) else: msg = '{}{}{}'.format(ind, self._normalize(value), key) - self.project.logger.info(msg) + logger.info(msg) def do_run(self, target=None): indent = 1 if self.args.cloud else 0 @@ -36,9 +37,9 @@ class CmdDataStatus(CmdDataBase): if st: self._show(st, indent) else: - self.project.logger.info(self.UP_TO_DATE_MSG) + logger.info(self.UP_TO_DATE_MSG) - except Exception as exc: - self.project.logger.error('Failed to obtain data status', exc) + except Exception: + logger.error('failed to obtain data status') return 1 return 0 diff --git a/dvc/command/unprotect.py b/dvc/command/unprotect.py index 4b8210df9..cb1efcb01 100644 --- a/dvc/command/unprotect.py +++ b/dvc/command/unprotect.py @@ -1,3 +1,4 @@ +import dvc.logger as logger from dvc.exceptions import DvcException from dvc.command.base import CmdBase @@ -7,8 +8,8 @@ class CmdUnprotect(CmdBase): for target in self.args.targets: try: self.project.unprotect(target) - except DvcException as ex: - msg = "Failed to unprotect '{}'".format(target) - self.project.logger.error(msg, ex) + except DvcException: + msg = "failed to unprotect '{}'".format(target) + logger.error(msg) return 1 return 0 diff --git a/dvc/config.py b/dvc/config.py index aaa34a879..bdc6c08ac 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -6,14 +6,14 @@ import errno import configobj from schema import Schema, Optional, And, Use, Regex -from dvc.logger import logger +import dvc.logger as logger from dvc.exceptions import DvcException class ConfigError(DvcException): """ DVC config exception """ def __init__(self, msg, ex=None): - super(ConfigError, self).__init__('Config file error: {}'.format(msg), + super(ConfigError, self).__init__('config file error: {}'.format(msg), ex) @@ -312,19 +312,19 @@ class Config(object): raise conf.write() except Exception as exc: - msg = "Failed to write config '{}'".format(conf.filename) + msg = "failed to write config '{}'".format(conf.filename) raise ConfigError(msg, exc) def unset(self, config, section, opt=None): if section not in config.keys(): - raise ConfigError("Section '{}' doesn't exist".format(section)) + raise ConfigError("section '{}' doesn't exist".format(section)) if opt is None: del config[section] return if opt not in config[section].keys(): - raise ConfigError("Option '{}.{}' doesn't exist".format(section, + raise ConfigError("option '{}.{}' doesn't exist".format(section, opt)) del config[section][opt] @@ -339,10 +339,10 @@ class Config(object): def show(self, config, section, opt): if section not in config.keys(): - raise ConfigError("Section '{}' doesn't exist".format(section)) + raise ConfigError("section '{}' doesn't exist".format(section)) if opt not in config[section].keys(): - raise ConfigError("Option '{}.{}' doesn't exist".format(section, + raise ConfigError("option '{}.{}' doesn't exist".format(section, opt)) logger.info(config[section][opt]) diff --git a/dvc/daemon.py b/dvc/daemon.py index 46cd7adf7..e093bd497 100644 --- a/dvc/daemon.py +++ b/dvc/daemon.py @@ -2,7 +2,7 @@ import os import sys from subprocess import Popen -from dvc.logger import logger +import dvc.logger as logger from dvc.utils import is_binary, fix_env @@ -34,8 +34,8 @@ class Daemon(object): # pragma: no cover pid = os.fork() if pid > 0: return - except OSError as exc: - logger.error("Failed at first fork", exc) + except OSError: + logger.error("failed at first fork") os._exit(1) os.setsid() @@ -45,8 +45,8 @@ class Daemon(object): # pragma: no cover pid = os.fork() if pid > 0: os._exit(0) - except OSError as exc: - logger.error("Failed at second fork", exc) + except OSError: + logger.error("failed at second fork") os._exit(1) sys.stdin.close() diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index fd55f665b..9461599d8 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -1,4 +1,4 @@ -from dvc.logger import logger +import dvc.logger as logger from dvc.config import Config, ConfigError from dvc.remote.s3 import RemoteS3 @@ -35,8 +35,8 @@ class DataCloud(object): if self._core.get(Config.SECTION_CORE_CLOUD, None): # backward compatibility - msg = 'Using obsoleted config format. Consider updating.' - logger.warn(msg) + msg = 'using obsoleted config format. Consider updating.' + logger.warning(msg) return self._init_compat() return None @@ -52,12 +52,12 @@ class DataCloud(object): section = Config.SECTION_REMOTE_FMT.format(remote) cloud_config = self._config.get(section, None) if not cloud_config: - msg = "Can't find remote section '{}' in config" + msg = "can't find remote section '{}' in config" raise ConfigError(msg.format(section)) cloud_type = self.supported(cloud_config) if not cloud_type: - raise ConfigError("Unsupported cloud '{}'".format(cloud_config)) + raise ConfigError("unsupported cloud '{}'".format(cloud_config)) return self._init_cloud(cloud_config, cloud_type) @@ -69,25 +69,25 @@ class DataCloud(object): cloud_type = self.CLOUD_MAP.get(name, None) if not cloud_type: - msg = "Wrong cloud type {} specified" - raise ConfigError(msg.format(name)) + msg = "wrong cloud type '{}' specified".format(name) + raise ConfigError(msg) cloud_config = self._config.get(name, None) if not cloud_config: - msg = "Can't find cloud section '{}' in config" - raise ConfigError(msg.format(name)) + msg = "can't find cloud section '{}' in config".format(name) + raise ConfigError(msg) # NOTE: check if the class itself has everything needed for operation. # E.g. all the imported packages. if not cloud_type.supported(cloud_type.compat_config(cloud_config)): - raise ConfigError("Unsupported cloud '{}'".format(name)) + raise ConfigError("unsupported cloud '{}'".format(name)) return self._init_cloud(cloud_config, cloud_type) def _init_cloud(self, cloud_config, cloud_type): global_storage_path = self._core.get(Config.SECTION_CORE_STORAGEPATH) if global_storage_path: - logger.warn('Using obsoleted config format. Consider updating.') + logger.warning('using obsoleted config format. Consider updating.') cloud = cloud_type(self.project, cloud_config) return cloud @@ -99,17 +99,16 @@ class DataCloud(object): if self._cloud: return self._cloud - msg = "No remote repository specified. Setup default " \ - "repository with\n"\ - " dvc config core.remote \n" \ - "or use:\n" \ - " dvc {} -r ".format(cmd) - raise ConfigError(msg) + raise ConfigError( + 'No remote repository specified. Setup default repository with\n' + ' dvc config core.remote \n' + 'or use:\n' + ' dvc {} -r \n' + .format(cmd) + ) def push(self, targets, jobs=1, remote=None, show_checksums=False): - """ - Push data items in a cloud-agnostic way. - """ + """Push data items in a cloud-agnostic way.""" return self.project.cache.local.push(targets, jobs=jobs, remote=self._get_cloud(remote, @@ -117,9 +116,7 @@ class DataCloud(object): show_checksums=show_checksums) def pull(self, targets, jobs=1, remote=None, show_checksums=False): - """ - Pull data items in a cloud-agnostic way. - """ + """Pull data items in a cloud-agnostic way.""" return self.project.cache.local.pull(targets, jobs=jobs, remote=self._get_cloud(remote, @@ -127,9 +124,7 @@ class DataCloud(object): show_checksums=show_checksums) def status(self, targets, jobs=1, remote=None, show_checksums=False): - """ - Check status of data items in a cloud-agnostic way. - """ + """Check status of data items in a cloud-agnostic way.""" cloud = self._get_cloud(remote, 'status') return self.project.cache.local.status(targets, jobs=jobs, diff --git a/dvc/dependency/base.py b/dvc/dependency/base.py index 4700e8fc8..7a519f9cf 100644 --- a/dvc/dependency/base.py +++ b/dvc/dependency/base.py @@ -3,22 +3,16 @@ import re from dvc.exceptions import DvcException -class DependencyError(DvcException): - def __init__(self, path, msg): - msg = 'Dependency \'{}\' error: {}'.format(path, msg) - super(DependencyError, self).__init__(msg) - - -class DependencyDoesNotExistError(DependencyError): +class DependencyDoesNotExistError(DvcException): def __init__(self, path): - msg = 'does not exist' - super(DependencyDoesNotExistError, self).__init__(path, msg) + msg = "dependency '{}' does not exist".format(path) + super(DependencyDoesNotExistError, self).__init__(msg) -class DependencyIsNotFileOrDirError(DependencyError): +class DependencyIsNotFileOrDirError(DvcException): def __init__(self, path): - msg = 'not a file or directory' - super(DependencyIsNotFileOrDirError, self).__init__(path, msg) + msg = "dependency '{}' is not a file or directory".format(path) + super(DependencyIsNotFileOrDirError, self).__init__(msg) class DependencyBase(object): diff --git a/dvc/dependency/local.py b/dvc/dependency/local.py index a9ffb6efc..f944e5e6a 100644 --- a/dvc/dependency/local.py +++ b/dvc/dependency/local.py @@ -5,6 +5,7 @@ try: except ImportError: from urllib.parse import urlparse +import dvc.logger as logger from dvc.dependency.base import DependencyBase from dvc.dependency.base import DependencyDoesNotExistError from dvc.dependency.base import DependencyIsNotFileOrDirError @@ -72,8 +73,8 @@ class DependencyLOCAL(DependencyBase): if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \ (os.path.isdir(self.path) and len(os.listdir(self.path)) == 0): - msg = "File/directory '{}' is empty.".format(self.rel_path) - self.project.logger.warn(msg) + msg = "file/directory '{}' is empty.".format(self.rel_path) + logger.warning(msg) self.info = self.remote.save_info(self.path_info) diff --git a/dvc/exceptions.py b/dvc/exceptions.py index 462970807..d3da24075 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -15,7 +15,7 @@ class DvcException(Exception): class UnsupportedRemoteError(DvcException): def __init__(self, config): - msg = "Remote '{}' is not supported.".format(config) + msg = "remote '{}' is not supported.".format(config) super(UnsupportedRemoteError, self).__init__(msg) @@ -24,7 +24,7 @@ class OutputDuplicationError(DvcException): assert isinstance(output, str) assert isinstance(stages, list) assert all(isinstance(stage, str) for stage in stages) - msg = "File '{}' is specified as an output in more than one stage:{}" + msg = "file '{}' is specified as an output in more than one stage:{}" s = "" for stage in stages: s += "\n " + stage @@ -36,7 +36,7 @@ class WorkingDirectoryAsOutputError(DvcException): assert isinstance(cwd, str) assert isinstance(fname, str) msg = ( - "Current working directory '{cwd}' is specified as an output in" + "current working directory '{cwd}' is specified as an output in" " '{fname}'. Use another CWD to prevent any data removal." .format(cwd=cwd, fname=fname) ) @@ -46,20 +46,20 @@ class WorkingDirectoryAsOutputError(DvcException): class CircularDependencyError(DvcException): def __init__(self, dependency): assert isinstance(dependency, str) - msg = "File '{}' is specified as an output and as a dependency." + msg = "file '{}' is specified as an output and as a dependency." super(CircularDependencyError, self).__init__(msg.format(dependency)) class ArgumentDuplicationError(DvcException): def __init__(self, path): assert isinstance(path, str) - msg = "File '{}' is specified more than once." + msg = "file '{}' is specified more than once." super(ArgumentDuplicationError, self).__init__(msg.format(path)) class MoveNotDataSourceError(DvcException): def __init__(self, path): - msg = "Move is not permitted for stages that are not data sources. " \ + msg = "move is not permitted for stages that are not data sources. " \ "You need to either move '{path}' to a new location and edit " \ "it by hand, or remove '{path}' and create a new one at the " \ "desired location." @@ -68,10 +68,10 @@ class MoveNotDataSourceError(DvcException): class NotDvcProjectError(DvcException): def __init__(self, root): - msg = "Not a dvc repository (checked up to mount point {})" + msg = "not a dvc repository (checked up to mount point '{}')" super(NotDvcProjectError, self).__init__(msg.format(root)) class DvcParserError(DvcException): def __init__(self): - super(DvcException, self).__init__("Parser error") + super(DvcException, self).__init__("parser error") diff --git a/dvc/lock.py b/dvc/lock.py index 35a73dab6..a8109b21d 100644 --- a/dvc/lock.py +++ b/dvc/lock.py @@ -21,7 +21,7 @@ class Lock(object): try: self._lock = zc.lockfile.LockFile(self.lock_file) except zc.lockfile.LockError: - raise LockError('Cannot perform the cmd since DVC is busy and ' + raise LockError('cannot perform the cmd since DVC is busy and ' 'locked. Please retry the cmd later.') def lock(self): diff --git a/dvc/logger.py b/dvc/logger.py index b750743df..4c075f268 100644 --- a/dvc/logger.py +++ b/dvc/logger.py @@ -4,239 +4,240 @@ import colorama import traceback import re - -colorama.init() +from contextlib import contextmanager -def visual_width(line): - """ Get the the number of columns required to display a string """ - - return len(re.sub(colorama.ansitowin32.AnsiToWin32.ANSI_CSI_RE, '', line)) +def info(message): + logger.info(message) -def visual_center(line, width): - """ Center align string according to it's visual width """ +def debug(message): + prefix = colorize('Debug', color='blue') - spaces = max(width - visual_width(line), 0) - left_padding = int(spaces / 2) - right_padding = (spaces - left_padding) + out = '{prefix}: {message}'.format(prefix=prefix, message=message) - return (left_padding * ' ') + line + (right_padding * ' ') + logger.debug(out) -class Logger(object): - FMT = '%(message)s' - DEFAULT_LEVEL = logging.INFO - DEFAULT_LEVEL_NAME = 'info' +def warning(message): + prefix = colorize('Warning', color='yellow') - LEVEL_MAP = { - 'debug': logging.DEBUG, - 'info': logging.INFO, - 'warn': logging.WARNING, - 'error': logging.ERROR + out = '{prefix}: {message}'.format(prefix=prefix, message=message) + + logger.warning(out) + + +def error(message=None): + prefix = colorize('Error', color='red') + + out = ( + '{prefix}: {description}' + '\n' + '{stack_trace}' + '\n' + '{footer}'.format( + prefix=prefix, + description=_description(message), + stack_trace=_stack_trace(), + footer=_footer(), + ) + ) + + logger.error(out) + + +def box(message, border_color=None): + lines = message.split('\n') + max_width = max(_visual_width(line) for line in lines) + + padding_horizontal = 5 + padding_vertical = 1 + + box_size_horizontal = (max_width + (padding_horizontal * 2)) + + chars = { + 'corner': '+', + 'horizontal': '-', + 'vertical': '|', + 'empty': ' ', } - COLOR_MAP = { + margin = "{corner}{line}{corner}\n".format( + corner=chars['corner'], + line=chars['horizontal'] * box_size_horizontal, + ) + + padding_lines = [ + "{border}{space}{border}\n".format( + border=colorize(chars['vertical'], color=border_color), + space=chars['empty'] * box_size_horizontal, + ) * padding_vertical + ] + + content_lines = [ + "{border}{space}{content}{space}{border}\n".format( + border=colorize(chars['vertical'], color=border_color), + space=chars['empty'] * padding_horizontal, + content=_visual_center(line, max_width), + ) for line in lines + ] + + box = "{margin}{padding}{content}{padding}{margin}".format( + margin=colorize(margin, color=border_color), + padding=''.join(padding_lines), + content=''.join(content_lines), + ) + + logger.info(box) + + +def level(): + return logger.getEffectiveLevel() + + +def set_level(level_name): + if not level_name: + return + + levels = { + 'info': logging.INFO, + 'debug': logging.DEBUG, + 'warning': logging.WARNING, + 'error': logging.ERROR, + 'critical': logging.CRITICAL, + } + + logger.setLevel(levels.get(level_name)) + + +def be_quiet(): + logger.setLevel(logging.CRITICAL) + + +def be_verbose(): + logger.setLevel(logging.DEBUG) + + +@contextmanager +def verbose(): + previous_level = level() + be_verbose() + yield + logger.setLevel(previous_level) + + +@contextmanager +def quiet(): + previous_level = level() + be_quiet() + yield + logger.setLevel(previous_level) + + +def is_quiet(): + return level() == logging.CRITICAL + + +def is_verbose(): + return level() == logging.DEBUG + + +def colorize(message, color=None): + if not color: + return message + + colors = { 'green': colorama.Fore.GREEN, 'yellow': colorama.Fore.YELLOW, 'blue': colorama.Fore.BLUE, 'red': colorama.Fore.RED, } - LEVEL_COLOR_MAP = { - 'debug': 'blue', - 'warn': 'yellow', - 'error': 'red', - } - - def __init__(self, loglevel=None, force=False): - self.logger = logging.getLogger('dvc') - if force or not self._already_initialized(): - class LogLevelFilter(logging.Filter): - def filter(self, record): - return record.levelno <= logging.WARNING - - sh_out = logging.StreamHandler(sys.stdout) - sh_out.setFormatter(logging.Formatter(self.FMT)) - sh_out.setLevel(logging.DEBUG) - sh_out.addFilter(LogLevelFilter()) - - sh_err = logging.StreamHandler(sys.stderr) - sh_err.setFormatter(logging.Formatter(self.FMT)) - sh_err.setLevel(logging.ERROR) - - self.logger.addHandler(sh_out) - self.logger.addHandler(sh_err) - - self.set_level(loglevel) - - def set_level(self, level=None): - if not level: - lvl = self.DEFAULT_LEVEL - else: - level = level.lower() - lvl = self.LEVEL_MAP.get(level, self.DEFAULT_LEVEL) - self.logger.setLevel(lvl) - self.lvl = lvl - - def be_quiet(self): - self.lvl = logging.CRITICAL - self.logger.setLevel(logging.CRITICAL) - - def be_verbose(self): - self.lvl = logging.DEBUG - self.logger.setLevel(logging.DEBUG) - - def colorize(self, msg, color): - header = '' - footer = '' - - if sys.stdout.isatty(): # pragma: no cover - header = self.COLOR_MAP.get(color.lower(), '') - footer = colorama.Style.RESET_ALL - - return u'{}{}{}'.format(header, msg, footer) - - def parse_exc(self, exc, tb=None): - str_tb = tb if tb else None - str_exc = str(exc) if exc else "" - l_str_exc = [] - - if len(str_exc) != 0: - l_str_exc.append(str_exc) - - if exc and hasattr(exc, 'cause') and exc.cause: - cause_tb = exc.cause_tb if hasattr(exc, 'cause_tb') else None - l_cause_str_exc, cause_str_tb = self.parse_exc(exc.cause, cause_tb) - - str_tb = cause_str_tb - l_str_exc += l_cause_str_exc - - return (l_str_exc, str_tb) - - def _prefix(self, msg, typ): - color = self.LEVEL_COLOR_MAP.get(typ.lower(), '') - return self.colorize('{}'.format(msg), color) - - def error_prefix(self): - return self._prefix('Error', 'error') - - def warning_prefix(self): - return self._prefix('Warning', 'warn') - - def debug_prefix(self): - return self._prefix('Debug', 'debug') - - def _with_progress(self, func, msg): - from dvc.progress import progress - with progress: - func(msg) - - def _with_exc(self, func, prefix, msg, suffix="", exc=None): - l_str_exc, str_tb = self.parse_exc(exc) - - if exc is not None and self.is_verbose(): - str_tb = str_tb if str_tb else traceback.format_exc() - self._with_progress(self.logger.error, str_tb) - - l_msg = [prefix] - if msg is not None and len(msg) != 0: - l_msg.append(msg) - l_msg += l_str_exc - - self._with_progress(func, ': '.join(l_msg) + suffix) - - def error(self, msg, exc=None): - if self.is_quiet(): - return - - chat = "\n\nHaving any troubles? Hit us up at dvc.org/support, " \ - "we are always happy to help!" - self._with_exc(self.logger.error, - self.error_prefix(), - msg, - suffix=chat, - exc=exc) - - def warn(self, msg, exc=None): - if self.is_quiet(): - return - - self._with_exc(self.logger.warning, - self.warning_prefix(), - msg, - exc=exc) - - def debug(self, msg, exc=None): - if not self.is_verbose(): - return - - self._with_exc(self.logger.debug, - self.debug_prefix(), - msg, - exc=exc) - - def info(self, msg): - if self.is_quiet(): - return - - self._with_progress(self.logger.info, msg) - - def is_quiet(self): - return self.lvl == logging.CRITICAL - - def is_verbose(self): - return self.lvl == logging.DEBUG - - def box(self, msg, border_color=''): - if self.is_quiet(): - return - - lines = msg.split('\n') - max_width = max(visual_width(line) for line in lines) - - padding_horizontal = 5 - padding_vertical = 1 - - box_size_horizontal = (max_width + (padding_horizontal * 2)) - - chars = { - 'corner': '+', - 'horizontal': '-', - 'vertical': '|', - 'empty': ' ', - } - - margin = "{corner}{line}{corner}\n".format( - corner=chars['corner'], - line=chars['horizontal'] * box_size_horizontal, - ) - - padding_lines = [ - "{border}{space}{border}\n".format( - border=self.colorize(chars['vertical'], border_color), - space=chars['empty'] * box_size_horizontal, - ) * padding_vertical - ] - - content_lines = [ - "{border}{space}{content}{space}{border}\n".format( - border=self.colorize(chars['vertical'], border_color), - space=chars['empty'] * padding_horizontal, - content=visual_center(line, max_width), - ) for line in lines - ] - - box = "{margin}{padding}{content}{padding}{margin}".format( - margin=self.colorize(margin, border_color), - padding=''.join(padding_lines), - content=''.join(content_lines), - ) - - print(box) - - def _already_initialized(self): - return bool(self.logger.handlers) + return u'{color}{message}{nc}'.format( + color=colors.get(color, ''), + message=message, + nc=colorama.Fore.RESET, + ) -logger = Logger() +def _init_colorama(): + colorama.init() + + +def _set_default_level(): + logger.setLevel(logging.INFO) + + +def _add_handlers(): + formatter = '%(message)s' + + class LogLevelFilter(logging.Filter): + def filter(self, record): + return record.levelno <= logging.WARNING + + sh_out = logging.StreamHandler(sys.stdout) + sh_out.setFormatter(logging.Formatter(formatter)) + sh_out.setLevel(logging.DEBUG) + sh_out.addFilter(LogLevelFilter()) + + sh_err = logging.StreamHandler(sys.stderr) + sh_err.setFormatter(logging.Formatter(formatter)) + sh_err.setLevel(logging.ERROR) + + logger.addHandler(sh_out) + logger.addHandler(sh_err) + + +def _stack_trace(): + trace = sys.exc_info()[2] + + if not is_verbose() or not trace: + return '' + + return '{line}\n{stack_trace}{line}\n'.format( + line=colorize('-' * 60, color='red'), + stack_trace=traceback.format_exc(), + ) + + +def _description(message): + exception = sys.exc_info()[1] + + if exception and message: + description = '{message} - {exception}' + elif exception: + description = '{exception}' + elif message: + description = '{message}' + + return description.format(message=message, exception=str(exception)) + + +def _footer(): + return '{phrase} Hit us up at {url}, we are always happy to help!'.format( + phrase=colorize('Having any troubles?', 'yellow'), + url=colorize('https://dvc.org/support', 'blue'), + ) + + +def _visual_width(line): + """Get the the number of columns required to display a string""" + + return len(re.sub(colorama.ansitowin32.AnsiToWin32.ANSI_CSI_RE, '', line)) + + +def _visual_center(line, width): + """Center align string according to it's visual width""" + + spaces = max(width - _visual_width(line), 0) + left_padding = int(spaces / 2) + right_padding = (spaces - left_padding) + + return (left_padding * ' ') + line + (right_padding * ' ') + + +logger = logging.getLogger('dvc') + +_set_default_level() +_add_handlers() +_init_colorama() diff --git a/dvc/main.py b/dvc/main.py index 61ee3562e..c9a136405 100644 --- a/dvc/main.py +++ b/dvc/main.py @@ -1,4 +1,4 @@ -from dvc.logger import logger +import dvc.logger as logger from dvc.cli import parse_args from dvc.command.base import CmdBase from dvc.analytics import Analytics @@ -18,16 +18,16 @@ def main(argv=None): cmd = args.func(args) ret = cmd.run_cmd() - except KeyboardInterrupt as ex: - logger.error("Interrupted by the user", ex) + except KeyboardInterrupt: + logger.error("interrupted by the user") ret = 252 - except NotDvcProjectError as ex: - logger.error("", ex) + except NotDvcProjectError: + logger.error() ret = 253 except DvcParserError: ret = 254 - except Exception as ex: - logger.error('Unexpected error', ex) + except Exception: + logger.error('unexpected error') ret = 255 Analytics().send_cmd(cmd, args, ret) diff --git a/dvc/output/base.py b/dvc/output/base.py index 9ca2a17e1..2153a8d95 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -1,24 +1,19 @@ from dvc.exceptions import DvcException -class OutputError(DvcException): - def __init__(self, path, msg): - msg = 'Output \'{}\' error: {}'.format(path, msg) - super(OutputError, self).__init__(msg) - - -class OutputDoesNotExistError(OutputError): +class OutputDoesNotExistError(DvcException): def __init__(self, path): - super(OutputDoesNotExistError, self).__init__(path, 'does not exist') + msg = "output '{}' does not exists".format(path) + super(OutputDoesNotExistError, self).__init__(msg) -class OutputIsNotFileOrDirError(OutputError): +class OutputIsNotFileOrDirError(DvcException): def __init__(self, path): - msg = 'not a file or directory' - super(OutputIsNotFileOrDirError, self).__init__(path, msg) + msg = "output '{}' is not a file or directory".format(path) + super(OutputIsNotFileOrDirError, self).__init__(msg) -class OutputAlreadyTrackedError(OutputError): +class OutputAlreadyTrackedError(DvcException): def __init__(self, path): - msg = 'already tracked by scm(e.g. git)' - super(OutputAlreadyTrackedError, self).__init__(path, msg) + msg = "output '{}' is already tracked by scm (e.g. git)".format(path) + super(OutputAlreadyTrackedError, self).__init__(msg) diff --git a/dvc/output/gs.py b/dvc/output/gs.py index 405e5a8f8..2e483377d 100644 --- a/dvc/output/gs.py +++ b/dvc/output/gs.py @@ -17,7 +17,7 @@ class OutputGS(DependencyGS): self.use_cache = cache self.metric = metric if cache and self.project.cache.gs is None: - raise DvcException("No cache location setup for \'gs\' outputs.") + raise DvcException("no cache location setup for 'gs' outputs.") def dumpd(self): ret = super(OutputGS, self).dumpd() diff --git a/dvc/output/hdfs.py b/dvc/output/hdfs.py index 85b435c4f..e3e6bce9d 100644 --- a/dvc/output/hdfs.py +++ b/dvc/output/hdfs.py @@ -17,7 +17,7 @@ class OutputHDFS(DependencyHDFS): self.use_cache = cache self.metric = metric if cache and self.project.cache.hdfs is None: - raise DvcException("No cache location setup for \'hdfs\' outputs.") + raise DvcException("no cache location setup for 'hdfs' outputs.") def dumpd(self): ret = super(OutputHDFS, self).dumpd() diff --git a/dvc/output/local.py b/dvc/output/local.py index 6c1cf6541..eda2aa6b0 100644 --- a/dvc/output/local.py +++ b/dvc/output/local.py @@ -1,6 +1,7 @@ import os from schema import Optional, Or +import dvc.logger as logger from dvc.output.base import OutputDoesNotExistError, OutputIsNotFileOrDirError from dvc.output.base import OutputAlreadyTrackedError from dvc.dependency.local import DependencyLOCAL @@ -76,36 +77,35 @@ class OutputLOCAL(DependencyLOCAL): return if os.path.isdir(self.path): - msg = 'Directory \'{}\' cannot be used as metrics.' + msg = "directory '{}' cannot be used as metrics." raise DvcException(msg.format(self.rel_path)) if not istextfile(self.path): - msg = 'Binary file \'{}\' cannot be used as metrics.' + msg = "binary file '{}' cannot be used as metrics." raise DvcException(msg.format(self.rel_path)) def save(self): if not self.use_cache: super(OutputLOCAL, self).save() self._verify_metric() - msg = 'Output \'{}\' doesn\'t use cache. Skipping saving.' - self.project.logger.info(msg.format(self.rel_path)) + msg = "Output '{}' doesn't use cache. Skipping saving." + logger.info(msg.format(self.rel_path)) return if not os.path.exists(self.path): raise self.DoesNotExistError(self.rel_path) - if not os.path.isfile(self.path) \ - and not os.path.isdir(self.path): # pragma: no cover + if (not os.path.isfile(self.path) and not os.path.isdir(self.path)): raise self.IsNotFileOrDirError(self.rel_path) if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \ (os.path.isdir(self.path) and len(os.listdir(self.path)) == 0): - msg = "File/directory '{}' is empty.".format(self.rel_path) - self.project.logger.warn(msg) + msg = "file/directory '{}' is empty.".format(self.rel_path) + logger.warning(msg) if not self.changed(): - msg = 'Output \'{}\' didn\'t change. Skipping saving.' - self.project.logger.info(msg.format(self.rel_path)) + msg = "Output '{}' didn't change. Skipping saving." + logger.info(msg.format(self.rel_path)) return if self.is_local: diff --git a/dvc/output/s3.py b/dvc/output/s3.py index 1bbba03bd..25c3be04d 100644 --- a/dvc/output/s3.py +++ b/dvc/output/s3.py @@ -17,7 +17,7 @@ class OutputS3(DependencyS3): self.use_cache = cache self.metric = metric if cache and self.project.cache.s3 is None: - raise DvcException("No cache location setup for \'s3\' outputs.") + raise DvcException("no cache location setup for 's3' outputs.") def dumpd(self): ret = super(OutputS3, self).dumpd() diff --git a/dvc/output/ssh.py b/dvc/output/ssh.py index 4238c1f3d..90af3b964 100644 --- a/dvc/output/ssh.py +++ b/dvc/output/ssh.py @@ -17,7 +17,7 @@ class OutputSSH(DependencySSH): self.use_cache = cache self.metric = metric if cache and self.project.cache.ssh is None: - raise DvcException("No cache location setup for \'ssh\' outputs.") + raise DvcException("no cache location setup for 'ssh' outputs.") def dumpd(self): ret = super(OutputSSH, self).dumpd() diff --git a/dvc/progress.py b/dvc/progress.py index ad8c9867c..a4d10ab93 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -1,6 +1,7 @@ from __future__ import print_function import sys import threading +import dvc.logger as logger class Progress(object): @@ -86,8 +87,6 @@ class Progress(object): return num + bar + percent + target_name def _print(self, *args, **kwargs): - from dvc.logger import logger - if logger.is_quiet(): return diff --git a/dvc/project.py b/dvc/project.py index dcd307487..0db6f9a1b 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -1,6 +1,7 @@ import collections import os import dvc.prompt as prompt +import dvc.logger as logger from dvc.exceptions import DvcException, MoveNotDataSourceError from dvc.exceptions import NotDvcProjectError @@ -14,7 +15,7 @@ class InitError(DvcException): class ReproductionError(DvcException): def __init__(self, dvc_file_name, ex): self.path = dvc_file_name - msg = 'Failed to reproduce \'{}\''.format(dvc_file_name) + msg = "failed to reproduce '{}'".format(dvc_file_name) super(ReproductionError, self).__init__(msg, cause=ex) @@ -22,7 +23,6 @@ class Project(object): DVC_DIR = '.dvc' def __init__(self, root_dir=None): - from dvc.logger import logger from dvc.config import Config from dvc.state import State from dvc.lock import Lock @@ -45,8 +45,8 @@ class Project(object): self.state = State(self, self.config._config) core = self.config._config[Config.SECTION_CORE] - self.logger = logger - self.logger.set_level(core.get(Config.SECTION_CORE_LOGLEVEL, None)) + + logger.set_level(core.get(Config.SECTION_CORE_LOGLEVEL)) self.cache = Cache(self) self.cloud = DataCloud(self, config=self.config._config) @@ -89,7 +89,7 @@ class Project(object): msg = '\nTo track the changes with git run:\n\n' msg += '\tgit add ' + " ".join(self._files_to_git_add) - self.logger.info(msg) + logger.info(msg) @staticmethod def init(root_dir=os.curdir, no_scm=False, force=False): @@ -108,7 +108,6 @@ class Project(object): import shutil from dvc.scm import SCM, Base from dvc.config import Config - from dvc.logger import logger root_dir = os.path.abspath(root_dir) dvc_dir = os.path.join(root_dir, Project.DVC_DIR) @@ -302,7 +301,7 @@ class Project(object): self._remind_to_git_add() if not found: - msg = 'Unable to find dvcfile with output \'{}\'' + msg = "unable to find dvcfile with output '{}'" raise DvcException(msg.format(from_path)) def _unprotect_file(self, path): @@ -312,7 +311,7 @@ class Project(object): from dvc.utils import copyfile, move, remove if System.is_symlink(path) or System.is_hardlink(path): - self.logger.debug("Unprotecting '{}'".format(path)) + logger.debug("Unprotecting '{}'".format(path)) tmp = os.path.join(os.path.dirname(path), '.' + str(uuid.uuid4())) move(path, tmp) @@ -321,8 +320,8 @@ class Project(object): remove(tmp) else: - self.logger.debug("Skipping copying for '{}', since it is not " - "a symlink or a hardlink.".format(path)) + logger.debug("Skipping copying for '{}', since it is not " + "a symlink or a hardlink.".format(path)) os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE) @@ -334,8 +333,10 @@ class Project(object): def unprotect(self, path): if not os.path.exists(path): - raise DvcException("Can't unprotect non-existing " - "data '{}'".format(path)) + raise DvcException( + "can't unprotect non-existing data '{}'" + .format(path) + ) if os.path.isdir(path): self._unprotect_dir(path) @@ -415,9 +416,11 @@ class Project(object): stage = stages[node] if stage.locked: - msg = 'DVC file \'{}\' is locked. Its dependencies are not ' \ - 'going to be reproduced.' - self.logger.warn(msg.format(stage.relpath)) + logger.warning( + "DVC file '{path}' is locked. Its dependencies are" + " not going to be reproduced." + .format(path=stage.relpath) + ) stage = stage.reproduce(force=force, dry=dry, interactive=interactive) if not stage: @@ -563,9 +566,11 @@ class Project(object): for stage in stages: if stage.locked: - msg = 'DVC file \'{}\' is locked. Its dependencies are ' \ - 'not going to be checked out.' - self.logger.warn(msg.format(stage.relpath)) + logger.warning( + "DVC file '{path}' is locked. Its dependencies are" + " not going to be checked out." + .format(path=stage.relpath) + ) stage.checkout(force=force) @@ -612,16 +617,18 @@ class Project(object): show_checksums=False) except DvcException as exc: msg = "Failed to pull cache for '{}': {}" - self.logger.debug(msg.format(out, exc)) + logger.debug(msg.format(out, exc)) if self.cache.local.changed_cache_file(md5): msg = "Missing cache for directory '{}'. " \ "Cache for files inside will be lost. " \ "Would you like to continue? Use '-f' to force. " if not force and not prompt.confirm(msg): - raise DvcException("Unable to fully collect " - "used cache without cache " - "for directory '{}'".format(out)) + raise DvcException( + "unable to fully collect used cache" + " without cache for directory '{}'" + .format(out) + ) else: return ret @@ -689,9 +696,11 @@ class Project(object): for stage in stages: if active and not target and stage.locked: - msg = 'DVC file \'{}\' is locked. Its dependencies are ' \ - 'not going to be pushed/pulled/fetched.' - self.logger.warn(msg.format(stage.relpath)) + logger.warning( + "DVC file '{path}' is locked. Its dependencies are" + " not going to be pushed/pulled/fetched." + .format(path=stage.relpath) + ) for out in stage.outs: scheme = out.path_info['scheme'] @@ -745,7 +754,7 @@ class Project(object): def _do_gc(self, typ, func, clist): removed = func(clist) if not removed: - self.logger.info("No unused {} cache to remove.".format(typ)) + logger.info("No unused {} cache to remove.".format(typ)) def gc(self, all_branches=False, @@ -870,9 +879,11 @@ class Project(object): for stage in stages: if stage.locked: - msg = 'DVC file \'{}\' is locked. Its dependencies are not ' \ - 'going to be shown in the status output.' - self.logger.warn(msg.format(stage.relpath)) + logger.warning( + "DVC file '{path}' is locked. Its dependencies are" + " not going to be shown in the status output." + .format(path=stage.relpath) + ) status.update(stage.status()) @@ -990,9 +1001,8 @@ class Project(object): ret = self._read_metric_hxsv(fd, xpath, '\t') else: ret = fd.read() - except Exception as exc: - self.logger.error('Unable to read metric in \'{}\''.format(path), - exc) + except Exception: + logger.error("unable to read metric in '{}'".format(path)) return ret @@ -1065,32 +1075,35 @@ class Project(object): for branch, val in res.items(): if all_branches or all_tags: - self.logger.info('{}:'.format(branch)) + logger.info('{}:'.format(branch)) for fname, metric in val.items(): - self.logger.info('\t{}: {}'.format(fname, metric)) + logger.info('\t{}: {}'.format(fname, metric)) if res: return res if path: - msg = 'File \'{}\' does not exist'.format(path) + msg = "file '{}' does not exist".format(path) else: - msg = 'No metric files in this repository. ' \ - 'Use \'dvc metrics add\' to add a metric file to track.' + msg = ( + "no metric files in this repository." + " use 'dvc metrics add' to add a metric file to track." + ) + raise DvcException(msg) def _metrics_modify(self, path, typ=None, xpath=None, delete=False): out = self._find_output_by_path(path) if not out: - msg = 'Unable to find file \'{}\' in the pipeline'.format(path) + msg = "unable to find file '{}' in the pipeline".format(path) raise DvcException(msg) if out.path_info['scheme'] != 'local': - msg = 'Output \'{}\' scheme \'{}\' is not supported for metrics' + msg = "output '{}' scheme '{}' is not supported for metrics" raise DvcException(msg.format(out.path, out.path_info['scheme'])) if out.use_cache: - msg = 'Cached output \'{}\' is not supported for metrics' + msg = "cached output '{}' is not supported for metrics" raise DvcException(msg.format(out.rel_path)) if typ: @@ -1169,7 +1182,10 @@ class Project(object): G, G_active = self.graph() - return list(nx.weakly_connected_component_subgraphs(G_active)) + return [ + G.subgraph(c).copy() + for c in nx.weakly_connected_components(G) + ] def stages(self): """ @@ -1218,7 +1234,7 @@ class Project(object): def _welcome_message(self): import colorama - self.logger.info( + logger.info( "\n" "DVC has enabled anonymous aggregate usage analytics.\n" "Read the analytics documentation (and how to opt-out) here:\n" diff --git a/dvc/remote/azure.py b/dvc/remote/azure.py index 482c254dd..38ee994bf 100644 --- a/dvc/remote/azure.py +++ b/dvc/remote/azure.py @@ -12,7 +12,7 @@ try: except ImportError: from urllib.parse import urlparse -from dvc.logger import logger +import dvc.logger as logger from dvc.progress import progress from dvc.config import Config from dvc.remote.base import RemoteBase @@ -56,10 +56,10 @@ class RemoteAzure(RemoteBase): or os.getenv('AZURE_STORAGE_CONNECTION_STRING')) if not self.bucket: - raise ValueError('Azure Storage container name missing') + raise ValueError('azure storage container name missing') if not self.connection_string: - raise ValueError('Azure Storage connection string missing') + raise ValueError('azure storage connection string missing') self.__blob_service = None @@ -125,9 +125,9 @@ class RemoteAzure(RemoteBase): try: self.blob_service.create_blob_from_path( bucket, path, from_info['path'], progress_callback=cb) - except Exception as ex: - msg = "Failed to upload '{}'".format(from_info['path']) - logger.warn(msg, ex) + except Exception: + msg = "failed to upload '{}'".format(from_info['path']) + logger.warning(msg) else: progress.finish_target(name) @@ -162,9 +162,9 @@ class RemoteAzure(RemoteBase): try: self.blob_service.get_blob_to_path( bucket, path, tmp_file, progress_callback=cb) - except Exception as exc: - msg = "Failed to download '{}/{}'".format(bucket, path) - logger.warn(msg, exc) + except Exception: + msg = "failed to download '{}/{}'".format(bucket, path) + logger.warning(msg) else: os.rename(tmp_file, to_info['path']) diff --git a/dvc/remote/base.py b/dvc/remote/base.py index 53eb27beb..4b76b4e08 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -5,8 +5,8 @@ import posixpath from multiprocessing import cpu_count import dvc.prompt as prompt +import dvc.logger as logger from dvc.config import Config -from dvc.logger import logger from dvc.exceptions import DvcException @@ -60,24 +60,25 @@ class RemoteBase(object): deps_ok = all(cls.REQUIRES.values()) if url_ok and not deps_ok: missing = [k for k, v in cls.REQUIRES.items() if v is None] - msg = "URL \'{}\' is supported but requires these missing " \ - "dependencies: {}. If you have installed dvc using pip, " \ - "choose one of these options to proceed: \n" \ - "\n" \ - " 1) Install specific missing dependencies:\n" \ - " pip install {}\n" \ - " 2) Install dvc package that includes those missing " \ - "dependencies: \n" \ - " pip install dvc[{}]\n" \ - " 3) Install dvc package with all possible " \ - "dependencies included: \n" \ - " pip install dvc[all]\n" \ - "\n" \ - "If you have installed dvc from a binary package and you " \ - "are still seeing this message, please report it to us " \ - "using https://github.com/iterative/dvc/issues. Thank you!" - msg = msg.format(url, missing, " ".join(missing), cls.scheme) - logger.warn(msg) + logger.warning( + "URL '{}' is supported but requires these missing " + "dependencies: {}. If you have installed dvc using pip, " + "choose one of these options to proceed: \n" + "\n" + " 1) Install specific missing dependencies:\n" + " pip install {}\n" + " 2) Install dvc package that includes those missing " + "dependencies: \n" + " pip install dvc[{}]\n" + " 3) Install dvc package with all possible " + "dependencies included: \n" + " pip install dvc[all]\n" + "\n" + "If you have installed dvc from a binary package and you " + "are still seeing this message, please report it to us " + "using https://github.com/iterative/dvc/issues. Thank you!" + .format(url, missing, " ".join(missing), cls.scheme) + ) return url_ok and deps_ok @@ -209,8 +210,8 @@ class RemoteBase(object): if expected != actual: if self.exists(cache): - msg = 'Corrupted cache file {}' - logger.warn(msg.format(str(cache))) + msg = 'corrupted cache file {}' + logger.warning(msg.format(str(cache))) self.remove(cache) return True @@ -241,22 +242,24 @@ class RemoteBase(object): if not force and not self.already_cached(path_info): msg = ( - 'File "{}" is going to be removed. ' - 'Are you sure you want to proceed?' + "file '{}' is going to be removed." + ' Are you sure you want to proceed?' .format(str(path_info)) ) if not prompt.confirm(msg): - raise DvcException("Unable to remove {} without a confirmation" - " from the user. Use '-f' to force." - .format(str(path_info))) + raise DvcException( + "unable to remove '{}' without a confirmation" + " from the user. Use '-f' to force." + .format(str(path_info)) + ) self.remove(path_info) def do_checkout(self, path_info, checksum, force=False): if self.exists(path_info): - msg = "Data '{}' exists. Removing before checkout." - logger.warn(msg.format(str(path_info))) + msg = "data '{}' exists. Removing before checkout." + logger.warning(msg.format(str(path_info))) self.safe_remove(path_info, force=force) from_info = self.checksum_to_path_info(checksum) @@ -280,7 +283,7 @@ class RemoteBase(object): if self.changed_cache(checksum): msg = "Cache '{}' not found. File '{}' won't be created." - logger.warn(msg.format(checksum, str(path_info))) + logger.warning(msg.format(checksum, str(path_info))) self.safe_remove(path_info, force=force) return diff --git a/dvc/remote/gs.py b/dvc/remote/gs.py index 531e4e7e8..7fc8ee923 100644 --- a/dvc/remote/gs.py +++ b/dvc/remote/gs.py @@ -10,7 +10,7 @@ try: except ImportError: from urllib.parse import urlparse -from dvc.logger import logger +import dvc.logger as logger from dvc.remote.base import RemoteBase from dvc.config import Config from dvc.progress import progress @@ -85,8 +85,8 @@ class RemoteGS(RemoteBase): blob = gs.bucket(from_info['bucket']).get_blob(from_info['path']) if not blob: - msg = '{} doesn\'t exist in the cloud' - raise DvcException(msg.format(from_info['path'])) + msg = "'{}' doesn't exist in the cloud".format(from_info['path']) + raise DvcException(msg) bucket = self.gs.bucket(to_info['bucket']) bucket.copy_blob(blob, @@ -157,11 +157,11 @@ class RemoteGS(RemoteBase): bucket = gs.bucket(to_info['bucket']) blob = bucket.blob(to_info['path']) blob.upload_from_filename(from_info['path']) - except Exception as exc: - msg = "Failed to upload '{}' to '{}/{}'" - logger.warn(msg.format(from_info['path'], - to_info['bucket'], - to_info['path']), exc) + except Exception: + msg = "failed to upload '{}' to '{}/{}'" + logger.error(msg.format(from_info['path'], + to_info['bucket'], + to_info['path'])) continue progress.finish_target(name) @@ -206,11 +206,11 @@ class RemoteGS(RemoteBase): bucket = gs.bucket(from_info['bucket']) blob = bucket.get_blob(from_info['path']) blob.download_to_filename(tmp_file) - except Exception as exc: - msg = "Failed to download '{}/{}' to '{}'" - logger.warn(msg.format(from_info['bucket'], - from_info['path'], - to_info['path']), exc) + except Exception: + msg = "failed to download '{}/{}' to '{}'" + logger.error(msg.format(from_info['bucket'], + from_info['path'], + to_info['path'])) continue os.rename(tmp_file, to_info['path']) diff --git a/dvc/remote/hdfs.py b/dvc/remote/hdfs.py index dc114f894..07aae9664 100644 --- a/dvc/remote/hdfs.py +++ b/dvc/remote/hdfs.py @@ -4,9 +4,9 @@ import getpass import posixpath from subprocess import Popen, PIPE +import dvc.logger as logger from dvc.config import Config from dvc.remote.base import RemoteBase, RemoteBaseCmdError -from dvc.logger import logger from dvc.utils import fix_env diff --git a/dvc/remote/http.py b/dvc/remote/http.py index d2d80a0cf..d939121b9 100644 --- a/dvc/remote/http.py +++ b/dvc/remote/http.py @@ -2,7 +2,7 @@ import os import threading import requests -from dvc.logger import logger +import dvc.logger as logger from dvc.progress import progress from dvc.exceptions import DvcException from dvc.config import Config @@ -72,9 +72,9 @@ class RemoteHTTP(RemoteBase): try: self._download_to(from_info['path'], tmp_file, callback=cb) - except Exception as exc: - msg = "Failed to download '{}'".format(from_info['path']) - logger.warn(msg, exc) + except Exception: + msg = "failed to download '{}'".format(from_info['path']) + logger.error(msg) continue os.rename(tmp_file, to_info['path']) @@ -108,7 +108,7 @@ class RemoteHTTP(RemoteBase): etag = self._request('HEAD', url).headers.get('ETag') if not etag: - raise DvcException("Could not find an ETag for '{}'".format(url)) + raise DvcException("could not find an ETag for '{}'".format(url)) if etag.startswith('W/'): raise DvcException( @@ -138,7 +138,7 @@ class RemoteHTTP(RemoteBase): try: return requests.request(method, url, **kwargs) except requests.exceptions.RequestException: - raise DvcException('Could not perform a {} request'.format(method)) + raise DvcException('could not perform a {} request'.format(method)) def gc(self): raise NotImplementedError diff --git a/dvc/remote/local.py b/dvc/remote/local.py index 18c53f34e..fedb8b863 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -7,9 +7,9 @@ import shutil import posixpath from operator import itemgetter +import dvc.logger as logger from dvc.system import System from dvc.remote.base import RemoteBase, STATUS_MAP, STATUS_NEW, STATUS_DELETED -from dvc.logger import logger from dvc.utils import remove, move, copyfile, dict_md5, to_chunks from dvc.utils import LARGE_DIR_SIZE from dvc.config import Config @@ -92,7 +92,7 @@ class RemoteLOCAL(RemoteBase): if self.state.changed(cache, md5=md5): if os.path.exists(cache): msg = 'Corrupted cache file {}.' - logger.warn(msg.format(os.path.relpath(cache))) + logger.warning(msg.format(os.path.relpath(cache))) remove(cache) return True return False @@ -153,13 +153,14 @@ class RemoteLOCAL(RemoteBase): logger.debug(msg) return + except DvcException as exc: - msg = 'Cache type \'{}\' is not supported: {}' + msg = "Cache type '{}' is not supported: {}" logger.debug(msg.format(self.cache_types[0], str(exc))) del self.cache_types[0] i -= 1 - raise DvcException('No possible cache types left to try out.') + raise DvcException('no possible cache types left to try out.') @property def ospath(self): @@ -233,13 +234,13 @@ class RemoteLOCAL(RemoteBase): try: with open(path, 'r') as fd: d = json.load(fd) - except Exception as exc: - msg = u'Failed to load dir cache \'{}\'' - logger.error(msg.format(os.path.relpath(path)), exc) + except Exception: + msg = u"Failed to load dir cache '{}'" + logger.error(msg.format(os.path.relpath(path))) return [] if not isinstance(d, list): - msg = u'Dir cache file format error \'{}\': skipping the file' + msg = u"dir cache file format error '{}' [skipping the file]" logger.error(msg.format(os.path.relpath(path))) return [] @@ -480,10 +481,11 @@ class RemoteLOCAL(RemoteBase): try: copyfile(from_info['path'], to_info['path'], name=name) - except Exception as exc: - msg = "Failed to upload '{}' tp '{}'" - logger.warn(msg.format(from_info['path'], - to_info['path']), exc) + except Exception: + logger.error( + "failed to upload '{}' to '{}'" + .format(from_info['path'], to_info['path']) + ) def download(self, from_infos, @@ -512,10 +514,12 @@ class RemoteLOCAL(RemoteBase): tmp_file, no_progress_bar=no_progress_bar, name=name) - except Exception as exc: - msg = "Failed to download '{}' to '{}'" - logger.warn(msg.format(from_info['path'], - to_info['path']), exc) + except Exception: + logger.error( + "failed to download '{}' to '{}'" + .format(from_info['path'], to_info['path']) + ) + continue os.rename(tmp_file, to_info['path']) diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index 3ca2ac5e9..578d50f31 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -11,7 +11,7 @@ try: except ImportError: from urllib.parse import urlparse -from dvc.logger import logger +import dvc.logger as logger from dvc.progress import progress from dvc.config import Config from dvc.remote.base import RemoteBase @@ -205,9 +205,9 @@ class RemoteS3(RemoteBase): to_info['bucket'], to_info['path'], Callback=cb) - except Exception as exc: - msg = "Failed to upload '{}'".format(from_info['path']) - logger.warn(msg, exc) + except Exception: + msg = "failed to upload '{}'".format(from_info['path']) + logger.error(msg) continue progress.finish_target(name) @@ -256,10 +256,10 @@ class RemoteS3(RemoteBase): from_info['path'], tmp_file, Callback=cb) - except Exception as exc: - msg = "Failed to download '{}/{}'".format(from_info['bucket'], + except Exception: + msg = "failed to download '{}/{}'".format(from_info['bucket'], from_info['path']) - logger.warn(msg, exc) + logger.error(msg) continue os.rename(tmp_file, to_info['path']) diff --git a/dvc/remote/ssh.py b/dvc/remote/ssh.py index 8d576cad1..7c98243e9 100644 --- a/dvc/remote/ssh.py +++ b/dvc/remote/ssh.py @@ -8,7 +8,7 @@ except ImportError: paramiko = None import dvc.prompt as prompt -from dvc.logger import logger +import dvc.logger as logger from dvc.progress import progress from dvc.remote.base import RemoteBase, RemoteBaseCmdError from dvc.config import Config @@ -177,7 +177,7 @@ class RemoteSSH(RemoteBase): md5cmd = 'md5sum' index = 0 else: - msg = '\'{}\' is not supported as a remote'.format(stdout) + msg = "'{}' is not supported as a remote".format(stdout) raise DvcException(msg) stdout = self._exec(ssh, '{} {}'.format(md5cmd, path_info['path'])) @@ -290,11 +290,11 @@ class RemoteSSH(RemoteBase): ssh.open_sftp().get(from_info['path'], tmp_file, callback=create_cb(name)) - except Exception as exc: - msg = "Failed to download '{}/{}' to '{}'" - logger.warn(msg.format(from_info['host'], - from_info['path'], - to_info['path']), exc) + except Exception: + msg = "failed to download '{}/{}' to '{}'" + logger.error(msg.format(from_info['host'], + from_info['path'], + to_info['path'])) continue os.rename(tmp_file, to_info['path']) @@ -331,11 +331,11 @@ class RemoteSSH(RemoteBase): sftp.put(from_info['path'], to_info['path'], callback=create_cb(name)) - except Exception as exc: - msg = "Failed to upload '{}' to '{}/{}'" - logger.warn(msg.format(from_info['path'], - to_info['host'], - to_info['path'], exc)) + except Exception: + msg = "failed to upload '{}' to '{}/{}'" + logger.error(msg.format(from_info['path'], + to_info['host'], + to_info['path'])) continue progress.finish_target(name) diff --git a/dvc/scm.py b/dvc/scm.py index b293688aa..b903d3905 100644 --- a/dvc/scm.py +++ b/dvc/scm.py @@ -1,7 +1,7 @@ import os +import dvc.logger as logger from dvc.exceptions import DvcException -from dvc.logger import logger from dvc.utils import fix_env @@ -213,12 +213,16 @@ class Git(Base): # See https://github.com/iterative/dvc/issues/610 for more details. try: self.repo.index.add(paths) - except AssertionError as exc: - msg = 'Failed to add \'{}\' to git. You can add those files ' - msg += 'manually using \'git add\'. ' - msg += 'See \'https://github.com/iterative/dvc/issues/610\' ' - msg += 'for more details.' - logger.error(msg.format(str(paths)), exc) + except AssertionError: + msg = ( + "failed to add '{}' to git. You can add those files" + " manually using 'git add'." + " See 'https://github.com/iterative/dvc/issues/610'" + " for more details." + .format(str(paths)) + ) + + logger.error(msg) def commit(self, msg): self.repo.index.commit(msg) @@ -257,7 +261,7 @@ class Git(Base): 'hooks', name) if os.path.isfile(hook): - msg = 'Git hook \'{}\' already exists.' + msg = "git hook '{}' already exists." raise SCMError(msg.format(os.path.relpath(hook))) with open(hook, 'w+') as fd: fd.write('#!/bin/sh\nexec dvc {}\n'.format(cmd)) diff --git a/dvc/stage.py b/dvc/stage.py index 761378600..59da076c1 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -5,16 +5,16 @@ import subprocess from schema import Schema, SchemaError, Optional, Or, And import dvc.prompt as prompt +import dvc.logger as logger import dvc.dependency as dependency import dvc.output as output from dvc.exceptions import DvcException -from dvc.logger import logger from dvc.utils import dict_md5, fix_env class StageCmdFailedError(DvcException): def __init__(self, stage): - msg = u'Stage \'{}\' cmd {} failed'.format(stage.relpath, stage.cmd) + msg = u"Stage '{}' cmd {} failed".format(stage.relpath, stage.cmd) super(StageCmdFailedError, self).__init__(msg) @@ -125,9 +125,7 @@ class Stage(object): @property def is_data_source(self): - """ - Whether the stage file was created with `dvc add`. - """ + """Whether the stage file was created with `dvc add`.""" return self.cmd is None @staticmethod @@ -149,10 +147,7 @@ class Stage(object): md5 = self._get_md5() assert md5 is not None - if self.md5 == md5: - return False - - return True + return self.md5 != md5 @property def is_callback(self): @@ -164,55 +159,50 @@ class Stage(object): @property def is_import(self): - """ - Whether the stage file was created with `dvc import`. - """ + """Whether the stage file was created with `dvc import`.""" return (not self.cmd and len(self.deps) == 1 and len(self.outs) == 1) - def _changed_deps(self, log): + def _changed_deps(self): if self.locked: return False if self.is_callback: msg = "Dvc file '{}' is a 'callback' stage (has a command and " \ "no dependencies) and thus always considered as changed." - self.project.logger.warn(msg.format(self.relpath)) + logger.warning(msg.format(self.relpath)) return True for dep in self.deps: if not dep.changed(): continue - log("Dependency '{}' of '{}' changed.".format(dep, self.relpath)) + msg = "Dependency '{}' of '{}' changed.".format(dep, self.relpath) + logger.warning(msg) return True return False - def _changed_outs(self, log): + def _changed_outs(self): for out in self.outs: if not out.changed(): continue - log("Output '{}' of '{}' changed.".format(out, self.relpath)) + msg = "Output '{}' of '{}' changed.".format(out, self.relpath) + logger.warning(msg) return True return False - def _changed_md5(self, log): + def _changed_md5(self): if self.changed_md5(): - log("Dvc file '{}' changed.".format(self.relpath)) + logger.warning("Dvc file '{}' changed.".format(self.relpath)) return True return False - def changed(self, print_info=False): - if print_info: - log = self.project.logger.info - else: - log = self.project.logger.debug - - ret = any([self._changed_deps(log), - self._changed_outs(log), - self._changed_md5(log)]) + def changed(self): + ret = any([self._changed_deps(), + self._changed_outs(), + self._changed_md5()]) if ret: msg = "Stage '{}' changed.".format(self.relpath) @@ -221,7 +211,7 @@ class Stage(object): msg = "Stage '{}' didn't change.".format(self.relpath) color = 'green' - log(logger.colorize(msg, color)) + logger.info(logger.colorize(msg, color)) return ret @@ -243,7 +233,7 @@ class Stage(object): os.unlink(self.path) def reproduce(self, force=False, dry=False, interactive=False): - if not self.changed(print_info=True) and not force: + if not self.changed() and not force: return None if (self.cmd or self.is_import) and not self.locked and not dry: @@ -254,14 +244,14 @@ class Stage(object): msg = msg.format(self.relpath) if interactive and not prompt.confirm(msg): - raise DvcException('Reproduction aborted by the user') + raise DvcException('reproduction aborted by the user') - self.project.logger.info(u'Reproducing \'{}\''.format(self.relpath)) + logger.info(u"Reproducing '{}'".format(self.relpath)) self.run(dry=dry) - msg = u'\'{}\' was reproduced'.format(self.relpath) - self.project.logger.debug(msg) + msg = u"'{}' was reproduced".format(self.relpath) + logger.debug(msg) return self @@ -331,7 +321,7 @@ class Stage(object): from dvc.remote.s3 import RemoteS3 old = Stage.load(self.project, self.path) - if old._changed_outs(log=self.project.logger.debug): + if old._changed_outs(): return False # NOTE: need to save checksums for deps in order to compare them @@ -383,7 +373,7 @@ class Stage(object): stage._check_duplicated_arguments() if fname is not None and os.path.basename(fname) != fname: - msg = "Stage file name '{}' should not contain subdirectories. " \ + msg = "stage file name '{}' should not contain subdirectories. " \ "Use '-c|--cwd' to change location of the stage file." raise StageFileBadNameError(msg.format(fname)) @@ -400,15 +390,14 @@ class Stage(object): # NOTE: remove outs before we check build cache if remove_outs: stage.remove_outs(ignore_remove=False) - project.logger.warn("Build cache is ignored when using " - "--remove-outs.") + logger.warning("Build cache is ignored when using --remove-outs.") ignore_build_cache = True else: stage.unprotect_outs() if os.path.exists(path): if not ignore_build_cache and stage.is_cached: - project.logger.info('Stage is cached, skipping.') + logger.info('Stage is cached, skipping.') return None msg = "'{}' already exists. Do you wish to run the command and " \ @@ -422,7 +411,7 @@ class Stage(object): @staticmethod def _check_dvc_filename(fname): if not Stage.is_stage_filename(fname): - msg = "Bad stage filename '{}'. Stage files should be named " \ + msg = "bad stage filename '{}'. Stage files should be named " \ "'Dvcfile' or have a '.dvc' suffix(e.g. '{}.dvc')." raise StageFileBadNameError(msg.format(os.path.relpath(fname), os.path.basename(fname))) @@ -521,12 +510,13 @@ class Stage(object): or os.path.basename(os.path.realpath(executable)) != 'fish': return - msg = "DVC detected that you are using fish as your default " \ - "shell. Be aware that it might cause problems by overwriting " \ - "your current environment variables with values defined " \ - "in '.fishrc', which might affect your command. See " \ - "https://github.com/iterative/dvc/issues/1307. " - self.project.logger.warn(msg) + logger.warning( + "DVC detected that you are using fish as your default " + "shell. Be aware that it might cause problems by overwriting " + "your current environment variables with values defined " + "in '.fishrc', which might affect your command. See " + "https://github.com/iterative/dvc/issues/1307. " + ) def _check_circular_dependency(self): from dvc.exceptions import CircularDependencyError @@ -566,25 +556,24 @@ class Stage(object): def run(self, dry=False): if self.locked: - msg = u'Verifying outputs in locked stage \'{}\'' - self.project.logger.info(msg.format(self.relpath)) + msg = u"Verifying outputs in locked stage '{}'" + logger.info(msg.format(self.relpath)) if not dry: self.check_missing_outputs() elif self.is_import: - msg = u'Importing \'{}\' -> \'{}\'' - self.project.logger.info(msg.format(self.deps[0].path, - self.outs[0].path)) + msg = u"Importing '{}' -> '{}'" + logger.info(msg.format(self.deps[0].path, self.outs[0].path)) if not dry: self.deps[0].download(self.outs[0].path_info) elif self.is_data_source: - msg = u'Verifying data sources in \'{}\''.format(self.relpath) - self.project.logger.info(msg) + msg = u"Verifying data sources in '{}'".format(self.relpath) + logger.info(msg) if not dry: self.check_missing_outputs() else: msg = u'Running command:\n\t{}'.format(self.cmd) - self.project.logger.info(msg) + logger.info(msg) if not dry: self._run() diff --git a/dvc/state.py b/dvc/state.py index 3a0ab984a..b2ca2a3cd 100644 --- a/dvc/state.py +++ b/dvc/state.py @@ -3,11 +3,11 @@ import time import sqlite3 import nanotime +import dvc.logger as logger from dvc.config import Config from dvc.system import System from dvc.utils import file_md5, remove from dvc.exceptions import DvcException -from dvc.logger import logger class StateDuplicateError(DvcException): @@ -134,17 +134,15 @@ class State(object): version = ret[0][0] if version > self.VERSION: - msg = "You are using an old version '{}' of dvc that is " \ + msg = "you are using an old version '{}' of dvc that is " \ "using state file version '{}' which is not " \ "compatible with the state file version '{}' that " \ "is used in this projet. Please upgrade right now!" - raise DvcException(msg.format(VERSION, - self.VERSION, - version)) + raise DvcException(msg.format(VERSION, self.VERSION, version)) elif version < self.VERSION: msg = "State file version '{}' is too old. " \ "Reformatting to the current version '{}'." - self.project.logger.warn(msg.format(version, self.VERSION)) + logger.warning(msg.format(version, self.VERSION)) cmd = "DROP TABLE IF EXISTS {};" self._execute(cmd.format(self.STATE_TABLE)) self._execute(cmd.format(self.STATE_INFO_TABLE)) @@ -212,8 +210,8 @@ class State(object): count = self._from_sqlite(ret[0][0]) + self.inserts if count > self.row_limit: - msg = "Cleaning up state. This might take a while." - self.project.logger.warn(msg) + msg = "cleaning up state, this might take a while." + logger.warning(msg) delete = (count - self.row_limit) delete += int(self.row_limit * (self.row_cleanup_quota/100.)) @@ -372,7 +370,7 @@ class State(object): mtime, _ = self.mtime_and_size(path) if i == inode and m == mtime: - logger.debug('Removing \'{}\' as unused link.'.format(path)) + logger.debug("Removing '{}' as unused link.".format(path)) remove(path) unused.append(p) diff --git a/dvc/system.py b/dvc/system.py index 767b12607..9c584aa37 100644 --- a/dvc/system.py +++ b/dvc/system.py @@ -38,7 +38,7 @@ class System(object): os.symlink(source, link_name) return except Exception as exc: - msg = "Failed to symlink '{}' -> '{}': {}" + msg = "failed to symlink '{}' -> '{}': {}" raise DvcException(msg.format(source, link_name, str(exc))) flags = 0 @@ -119,7 +119,7 @@ class System(object): ret = -1 if ret != 0: - raise DvcException('Reflink is not supported') + raise DvcException('reflink is not supported') @staticmethod def getdirinfo(path): diff --git a/dvc/updater.py b/dvc/updater.py index c2abb0de5..5523067d9 100644 --- a/dvc/updater.py +++ b/dvc/updater.py @@ -4,9 +4,9 @@ import time import requests import colorama +import dvc.logger as logger from dvc import VERSION_BASE from dvc.lock import Lock, LockError -from dvc.logger import logger from dvc.utils import is_binary @@ -99,6 +99,9 @@ class Updater(object): # pragma: no cover return l_patch > c_patch def _notify(self): + if not sys.stdout.isatty(): + return + message = ( 'Update available {red}{current}{reset} -> {green}{latest}{reset}' + '\n' @@ -111,8 +114,7 @@ class Updater(object): # pragma: no cover current=self.current, latest=self.latest) - if sys.stdout.isatty(): - logger.box(message, border_color='yellow') + logger.box(message, border_color='yellow') def _get_update_instructions(self): instructions = { @@ -186,6 +188,6 @@ class Updater(object): # pragma: no cover system = platform.system() func = m.get(system) if func is None: - raise DvcException("Not supported system '{}'".format(system)) + raise DvcException("not supported system '{}'".format(system)) return func() diff --git a/dvc/utils.py b/dvc/utils.py index ec07698de..120fff3ef 100644 --- a/dvc/utils.py +++ b/dvc/utils.py @@ -9,9 +9,9 @@ import json import shutil import hashlib +import dvc.logger as logger from dvc.progress import progress from dvc.istextfile import istextfile -from dvc.logger import logger LOCAL_CHUNK_SIZE = 1024*1024 @@ -143,7 +143,7 @@ def remove(path): if not os.path.exists(path): return - logger.debug(u'Removing \'{}\''.format(os.path.relpath(path))) + logger.debug(u"Removing '{}'".format(os.path.relpath(path))) def _chmod(func, p, excinfo): perm = os.stat(p).st_mode diff --git a/tests/basic_env.py b/tests/basic_env.py index cc9b3d513..4df285da8 100644 --- a/tests/basic_env.py +++ b/tests/basic_env.py @@ -4,6 +4,7 @@ import tempfile from git import Repo from unittest import TestCase +import dvc.logger as logger from dvc.project import Project @@ -84,4 +85,4 @@ class TestDvc(TestGit): def setUp(self): super(TestDvc, self).setUp() self.dvc = Project.init(self._root_dir) - self.dvc.logger.be_verbose() + logger.be_verbose() diff --git a/tests/test_add.py b/tests/test_add.py index 2c898a626..444264ce7 100644 --- a/tests/test_add.py +++ b/tests/test_add.py @@ -77,7 +77,7 @@ class TestAddDirectoryWithForwardSlash(TestDvc): self.assertEqual(len(stages), 1) stage = stages[0] self.assertTrue(stage is not None) - self.assertEquals(os.path.abspath('directory.dvc'), stage.path) + self.assertEqual(os.path.abspath('directory.dvc'), stage.path) class TestAddTrackedFile(TestDvc): diff --git a/tests/test_cache.py b/tests/test_cache.py index 17fb37569..a7a33a520 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -23,13 +23,13 @@ class TestCache(TestDvc): def test_all(self): md5_list = list(Cache(self.dvc).local.all()) - self.assertEquals(len(md5_list), 2) + self.assertEqual(len(md5_list), 2) self.assertIn(self.cache1_md5, md5_list) self.assertIn(self.cache2_md5, md5_list) def test_get(self): cache = Cache(self.dvc).local.get(self.cache1_md5) - self.assertEquals(cache, self.cache1) + self.assertEqual(cache, self.cache1) class TestCacheLoadBadDirCache(TestDvc): @@ -63,7 +63,7 @@ class TestExternalCacheDir(TestDvc): self.assertEqual(ret, 0) self.assertFalse(os.path.exists('.dvc/cache')) - self.assertNotEquals(len(os.listdir(cache_dir)), 0) + self.assertNotEqual(len(os.listdir(cache_dir)), 0) class TestSharedCacheDir(TestDir): diff --git a/tests/test_logger.py b/tests/test_logger.py index bfa988b11..bd619489b 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -1,4 +1,5 @@ from mock import patch +from unittest import TestCase try: from StringIO import StringIO @@ -6,160 +7,121 @@ except ImportError: from io import StringIO import logging - +import dvc.logger as logger from dvc.command.base import CmdBase -from dvc.logger import Logger, logger - -from tests.basic_env import TestDvc -class StringContaining(str): - def __eq__(self, other): - return self in other +class TestLogger(TestCase): + handlers = logger.logger.handlers + color_patch = patch.object(logger, 'colorize', new=lambda x, color='': x) + def setUp(self): + logger.logger.handlers[0].stream = StringIO() + logger.logger.handlers[1].stream = StringIO() + logger._set_default_level() + self.color_patch.start() -class TestLogger(TestDvc): - def test_config(self): - logger.set_level() - logger.set_level('debug') - self.assertEqual(logger.logger.getEffectiveLevel(), logging.DEBUG) - - logger.set_level('error') - self.assertEqual(logger.logger.getEffectiveLevel(), logging.ERROR) - - def test_set_level(self): - logger.set_level() - logger.set_level('debug') - self.assertEqual(logger.logger.getEffectiveLevel(), logging.DEBUG) - - def test_be_quiet(self): - logger.set_level() - logger.be_quiet() - self.assertEqual(logger.logger.getEffectiveLevel(), logging.CRITICAL) - - def test_be_verbose(self): - logger.set_level() + def tearDown(self): + logger.logger.handlers = self.handlers logger.be_verbose() - self.assertEqual(logger.logger.getEffectiveLevel(), logging.DEBUG) + self.color_patch.stop() - def test_colorize(self): - for name, color in logger.COLOR_MAP.items(): - msg = logger.colorize(name, name) - # This is not a tty, so it should not colorize anything - self.assertEqual(msg, name) + @property + def stdout(self): + return logger.logger.handlers[0].stream.getvalue() - def test_handlers(self): - Logger() - Logger() - self.assertEqual(len(logger.logger.handlers), 2) + @property + def stderr(self): + return logger.logger.handlers[1].stream.getvalue() - @patch('sys.stderr', new_callable=StringIO) - @patch('sys.stdout', new_callable=StringIO) - def test_stderr(self, mock_stdout, mock_stderr): - logger = Logger(force=True) - error_message = 'error msg' - logger.error(error_message) - self.assertEqual('', mock_stdout.getvalue()) - self.assertEqual('Error: {}\n\nHaving any troubles? ' - 'Hit us up at dvc.org/support, we ' - 'are always happy to help!\n'.format(error_message), - mock_stderr.getvalue()) + def test_info(self): + logger.info('message') - @patch('sys.stderr', new_callable=StringIO) - @patch('sys.stdout', new_callable=StringIO) - def test_stdout(self, mock_stdout, mock_stderr): - logger = Logger(force=True) - non_error_message = 'non-error message' - logger.info(non_error_message) - self.assertEqual('', mock_stderr.getvalue()) - self.assertEqual('{}\n'.format(non_error_message), - mock_stdout.getvalue()) + self.assertEqual(self.stdout, 'message\n') + def test_debug(self): + with logger.verbose(): + logger.debug('message') -class TestLoggerException(TestDvc): - def setUp(self): - super(TestLoggerException, self).setUp() + self.assertEqual(self.stdout, 'Debug: message\n') - @patch('sys.stderr', new_callable=StringIO) - @patch('sys.stdout', new_callable=StringIO) - def test_empty_err_msg(self, mock_stdout, mock_stderr): - from dvc.exceptions import DvcException + def test_warning(self): + logger.warning('message') - logger = Logger(force=True) + self.assertEqual(self.stdout, 'Warning: message\n') - exc_msg = 'msg' - logger.error("", exc=DvcException(exc_msg)) + def test_error(self): + logger.error('message') - self.assertEqual('', mock_stdout.getvalue()) - self.assertEqual('Error: {}\n\nHaving any troubles? ' - 'Hit us up at dvc.org/support, we ' - 'are always happy to help!\n'.format(exc_msg), - mock_stderr.getvalue()) + output = ( + 'Error: message\n' + '\n' + 'Having any troubles? Hit us up at https://dvc.org/support,' + ' we are always happy to help!\n' + ) - @patch('sys.stderr', new_callable=StringIO) - @patch('sys.stdout', new_callable=StringIO) - def test_empty_exc_msg(self, mock_stdout, mock_stderr): - from dvc.exceptions import DvcException + self.assertEqual(self.stderr, output) - logger = Logger(force=True) + def test_error_with_exception(self): + try: + raise Exception('exception description') + except Exception: + logger.error('') - err_msg = 'msg' - logger.error(err_msg, exc=DvcException("")) + output = ( + 'Error: exception description\n' + '\n' + 'Having any troubles? Hit us up at https://dvc.org/support,' + ' we are always happy to help!\n' + ) - self.assertEqual('', mock_stdout.getvalue()) - self.assertEqual('Error: {}\n\nHaving any troubles? ' - 'Hit us up at dvc.org/support, we ' - 'are always happy to help!\n'.format(err_msg), - mock_stderr.getvalue()) + self.assertEqual(self.stderr, output) - @patch('sys.stderr', new_callable=StringIO) - @patch('sys.stdout', new_callable=StringIO) - def test_msg_and_exc(self, mock_stdout, mock_stderr): - from dvc.exceptions import DvcException + def test_error_with_exception_and_message(self): + try: + raise Exception('exception description') + except Exception: + logger.error('message') - logger = Logger(force=True) + output = ( + 'Error: message - exception description\n' + '\n' + 'Having any troubles? Hit us up at https://dvc.org/support,' + ' we are always happy to help!\n' + ) - err_msg = 'error' - exc_msg = 'exception' - logger.error(err_msg, exc=DvcException(exc_msg)) + self.assertEqual(self.stderr, output) - self.assertEqual('', mock_stdout.getvalue()) - self.assertEqual('Error: {}: {}\n\nHaving any troubles? ' - 'Hit us up at dvc.org/support, we are always ' - 'happy to help!\n'.format(err_msg, exc_msg), - mock_stderr.getvalue()) - - -class TestLoggerQuiet(TestDvc): - def setUp(self): - super(TestLoggerQuiet, self).setUp() - - class A(object): - quiet = True - verbose = False - - CmdBase._set_loglevel(A()) - - @patch('sys.stderr', new_callable=StringIO) - @patch('sys.stdout', new_callable=StringIO) - def test_progress(self, mock_stdout, mock_stderr): - from dvc.progress import progress - progress.update_target('target', 50, 100) - self.assertEqual('', mock_stdout.getvalue()) - self.assertEqual('', mock_stderr.getvalue()) - - @patch('sys.stderr', new_callable=StringIO) - @patch('sys.stdout', new_callable=StringIO) - def test_box(self, mock_stdout, mock_stderr): + def test_box(self): logger.box('message') - self.assertEqual('', mock_stdout.getvalue()) - self.assertEqual('', mock_stderr.getvalue()) + output = ( + '+-----------------+\n' + '| |\n' + '| message |\n' + '| |\n' + '+-----------------+\n' + '\n' + ) -class TestLoggerCLI(TestDvc): - def test(self): - from dvc.logger import logger + self.assertEqual(self.stdout, output) + def test_level(self): + self.assertEqual(logger.level(), logging.INFO) + + def test_is_verbose(self): + self.assertFalse(logger.is_verbose()) + + with logger.verbose(): + self.assertTrue(logger.is_verbose()) + + def test_is_quiet(self): + self.assertFalse(logger.is_quiet()) + + with logger.quiet(): + self.assertFalse(logger.is_verbose()) + + def test_cli(self): class A(object): quiet = True verbose = False diff --git a/tests/test_repro.py b/tests/test_repro.py index 460863ae0..6d91e060d 100644 --- a/tests/test_repro.py +++ b/tests/test_repro.py @@ -642,7 +642,6 @@ class TestReproExternalBase(TestDvc): self.assertEqual(ret, 0) self.dvc = Project('.') - self.dvc.logger.be_verbose() foo_key = remote_key + self.sep + self.FOO bar_key = remote_key + self.sep + self.BAR