Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

logger [rfc]: Use a module instead of a class

Browse Source 
- Remove deprecation warnings
- Format error output and remove duplicated info

Close #1086
This commit is contained in:
Mr. Outis
2018-12-23 21:06:39 -06:00
parent 33257a3efd
commit 4c420d5f4d
58 changed files with 739 additions and 765 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/analytics.py
+1 -1
View File
@@ -2,8 +2,8 @@ import os
import json
import errno
import dvc.logger as logger
from dvc import VERSION
from dvc.logger import logger
class Analytics(object):
dvc/cli.py
+10 -12
View File
@@ -4,6 +4,7 @@ import os
import sys
import argparse
import dvc.logger as logger
from dvc.command.init import CmdInit
from dvc.command.destroy import CmdDestroy
from dvc.command.remove import CmdRemove
@@ -28,7 +29,6 @@ from dvc.command.lock import CmdLock, CmdUnlock
from dvc.command.pipeline import CmdPipelineShow, CmdPipelineList
from dvc.command.daemon import CmdDaemonUpdater, CmdDaemonAnalytics
from dvc.exceptions import DvcParserError
from dvc.logger import logger
from dvc import VERSION
@@ -44,7 +44,7 @@ def _fix_subparsers(subparsers):
class DvcParser(argparse.ArgumentParser):
def error(self, message):
sys.stderr.write('{}: {}\n'.format(logger.error_prefix(), message))
logger.error(message)
self.print_help()
raise DvcParserError()
@@ -86,7 +86,7 @@ def parse_args(argv=None):
'--version',
action=VersionAction,
nargs=0,
help='Show program\'s version.')
help="Show program's version.")
# Sub commands
subparsers = parser.add_subparsers(
@@ -462,7 +462,7 @@ def parse_args(argv=None):
status_parser.set_defaults(func=CmdDataStatus)
# Repro
REPRO_HELP = 'Reproduce DVC file. Default file name - \'Dvcfile\'.'
REPRO_HELP = "Reproduce DVC file. Default file name - 'Dvcfile'."
repro_parser = subparsers.add_parser(
'repro',
parents=[parent_parser],
@@ -529,7 +529,7 @@ def parse_args(argv=None):
action='store_true',
default=False,
help='Reproduce all descendants of a changed stage'
'even if their direct dependencies didn\'t '
"even if their direct dependencies didn't "
'change.')
repro_parser.set_defaults(func=CmdRepro)
@@ -897,18 +897,16 @@ def parse_args(argv=None):
help='JSON path.')
metrics_show_group.add_argument(
'--tsv-path',
help='TSV path \'row,column\'(e.g. \'1,2\').')
help="TSV path 'row,column' (e.g. '1,2').")
metrics_show_group.add_argument(
'--htsv-path',
help='Headed TSV path \'row,column\''
'(e.g. \'Name,3\').')
help="Headed TSV path 'row,column (e.g. 'Name,3').")
metrics_show_group.add_argument(
'--csv-path',
help='CSV path \'row,column\'(e.g. \'1,2\').')
help="CSV path 'row,column' (e.g. '1,2').")
metrics_show_group.add_argument(
'--hcsv-path',
help='Headed CSV path \'row,column\''
'(e.g. \'Name,3\').')
help="Headed CSV path 'row,column' (e.g. 'Name,3').")
metrics_show_parser.add_argument(
'-a',
'--all-branches',
@@ -982,7 +980,7 @@ def parse_args(argv=None):
install_parser.set_defaults(func=CmdInstall)
# Root
ROOT_HELP = 'Relative path to project\'s directory.'
ROOT_HELP = "Relative path to project's directory."
root_parser = subparsers.add_parser(
'root',
parents=[parent_parser],
dvc/command/add.py
+3 -3
View File
@@ -1,6 +1,6 @@
import dvc.logger as logger
from dvc.exceptions import DvcException
from dvc.command.base import CmdBase
from dvc.logger import logger
class CmdAdd(CmdBase):
@@ -8,7 +8,7 @@ class CmdAdd(CmdBase):
for target in self.args.targets:
try:
self.project.add(target, recursive=self.args.recursive)
except DvcException as ex:
logger.error('Failed to add \'{}\''.format(target), ex)
except DvcException:
logger.error('failed to add file')
return 1
return 0
dvc/command/base.py
+3 -3
View File
@@ -1,4 +1,4 @@
from dvc.logger import logger
import dvc.logger as logger
class CmdBase(object):
@@ -24,8 +24,8 @@ class CmdBase(object):
try:
with self.project.lock:
return self.run()
except LockError as ex:
logger.error('Failed to lock before running a command', ex)
except LockError:
logger.error('failed to lock before running a command')
return 1
# Abstract methods that have to be implemented by any inheritance class
dvc/command/config.py
+8 -10
View File
@@ -1,7 +1,7 @@
import os
import dvc.logger as logger
from dvc.command.base import CmdBase
from dvc.logger import logger
from dvc.config import Config
from dvc.exceptions import DvcException
@@ -43,16 +43,16 @@ class CmdConfig(CmdBase):
try:
self.config.unset(configobj, section, opt)
self.config.save(configobj)
except DvcException as exc:
logger.error("Failed to unset '{}'".format(self.args.name), exc)
except DvcException:
logger.error("failed to unset '{}'".format(self.args.name))
return 1
return 0
def _show(self, section, opt):
try:
self.config.show(self.configobj, section, opt)
except DvcException as exc:
logger.error("Failed to show '{}'".format(self.args.name), exc)
except DvcException:
logger.error("failed to show '{}'".format(self.args.name))
return 1
return 0
@@ -60,11 +60,9 @@ class CmdConfig(CmdBase):
try:
self.config.set(self.configobj, section, opt, value)
self.config.save(self.configobj)
except DvcException as exc:
logger.error("Failed to set '{}.{}' to '{}'".format(section,
opt,
value),
exc)
except DvcException:
logger.error("failed to set '{}.{}' to '{}'"
.format(section, opt, value))
return 1
return 0
dvc/command/data_sync.py
+7 -8
View File
@@ -1,4 +1,5 @@
from dvc.command.base import CmdBase
import dvc.logger as logger
class CmdDataBase(CmdBase):
@@ -27,9 +28,8 @@ class CmdDataPull(CmdDataBase):
all_tags=self.args.all_tags,
with_deps=self.args.with_deps,
force=self.args.force)
except Exception as exc:
self.project.logger.error('Failed to pull data from the cloud',
exc)
except Exception:
logger.error('failed to pull data from the cloud')
return 1
return 0
@@ -44,8 +44,8 @@ class CmdDataPush(CmdDataBase):
all_branches=self.args.all_branches,
all_tags=self.args.all_tags,
with_deps=self.args.with_deps)
except Exception as exc:
self.project.logger.error('Failed to push data to the cloud', exc)
except Exception:
logger.error('failed to push data to the cloud')
return 1
return 0
@@ -60,8 +60,7 @@ class CmdDataFetch(CmdDataBase):
all_branches=self.args.all_branches,
all_tags=self.args.all_tags,
with_deps=self.args.with_deps)
except Exception as exc:
self.project.logger.error('Failed to fetch data from the cloud',
exc)
except Exception:
logger.error('failed to fetch data from the cloud')
return 1
return 0
dvc/command/destroy.py
+14 -9
View File
@@ -1,4 +1,5 @@
import dvc.prompt as prompt
import dvc.logger as logger
from dvc.command.base import CmdBase
from dvc.exceptions import DvcException
@@ -6,17 +7,21 @@ from dvc.exceptions import DvcException
class CmdDestroy(CmdBase):
def run_cmd(self):
try:
msg = u'This will destroy all information about your pipelines, ' \
u'all data files, as well as cache in .dvc/cache.\n' \
u'Are you sure you want to continue?'
statement = (
'This will destroy all information about your pipelines,'
' all data files, as well as cache in .dvc/cache.'
'\n'
'Are you sure you want to continue?'
)
if not self.args.force and not prompt.confirm(msg):
msg = u'Cannot destroy without a confirmation from the ' \
u'user. Use \'-f\' to force.'
raise DvcException(msg)
if not self.args.force and not prompt.confirm(statement):
raise DvcException(
"cannot destroy without a confirmation from the user."
" Use '-f' to force."
)
self.project.destroy()
except Exception as exc:
self.project.logger.error('Failed to destroy DVC', exc)
except Exception:
logger.error('failed to destroy DVC')
return 1
return 0
dvc/command/gc.py
+3 -2
View File
@@ -1,12 +1,13 @@
import os
import dvc.prompt as prompt
import dvc.logger as logger
from dvc.command.base import CmdBase
class CmdGC(CmdBase):
def run(self):
msg = 'This will remove all cache except the cache that is used in '
msg = 'this will remove all cache except the cache that is used in '
if not self.args.all_branches and not self.args.all_tags:
msg += 'the current git branch'
elif self.args.all_branches and not self.args.all_tags:
@@ -24,7 +25,7 @@ class CmdGC(CmdBase):
else:
msg += ' of the current project.'
self.project.logger.warn(msg)
logger.warning(msg)
msg = 'Are you sure you want to proceed?'
if not self.args.force and not prompt.confirm(msg):
dvc/command/imp.py
+3 -3
View File
@@ -5,9 +5,9 @@ try:
except ImportError:
from urllib.parse import urlparse
import dvc.logger as logger
from dvc.exceptions import DvcException
from dvc.command.base import CmdBase
from dvc.logger import logger
class CmdImport(CmdBase):
@@ -20,7 +20,7 @@ class CmdImport(CmdBase):
out = self.args.out or default_out
self.project.imp(self.args.url, out)
except DvcException as ex:
logger.error('Failed to import {}'.format(self.args.url), ex)
except DvcException:
logger.error('failed to import {}'.format(self.args.url))
return 1
return 0
dvc/command/init.py
+3 -3
View File
@@ -1,4 +1,4 @@
from dvc.logger import logger
import dvc.logger as logger
class CmdInit(object):
@@ -13,7 +13,7 @@ class CmdInit(object):
no_scm=self.args.no_scm,
force=self.args.force)
self.config = self.project.config
except InitError as e:
logger.error('Failed to initiate dvc', e)
except InitError:
logger.error('failed to initiate dvc')
return 1
return 0
dvc/command/install.py
+3 -3
View File
@@ -1,4 +1,4 @@
from dvc.logger import logger
import dvc.logger as logger
from dvc.command.base import CmdBase
@@ -6,7 +6,7 @@ class CmdInstall(CmdBase):
def run_cmd(self):
try:
self.project.install()
except Exception as e:
logger.error('Failed to install dvc hooks', e)
except Exception:
logger.error('failed to install dvc hooks')
return 1
return 0
dvc/command/lock.py
+7 -4
View File
@@ -1,3 +1,4 @@
import dvc.logger as logger
from dvc.exceptions import DvcException
from dvc.command.base import CmdBase
@@ -7,10 +8,12 @@ class CmdLockBase(CmdBase):
for target in self.args.targets:
try:
self.project.lock_stage(target, unlock=unlock)
except DvcException as ex:
msg = 'Failed to {}lock \'{}\''
msg = msg.format('un' if unlock else '', target)
self.project.logger.error(msg, ex)
except DvcException:
logger.error(
"failed to {}lock '{}'"
.format('un' if unlock else '', target)
)
return 1
return 0
dvc/command/metrics.py
+11 -10
View File
@@ -1,3 +1,4 @@
import dvc.logger as logger
from dvc.command.base import CmdBase
from dvc.exceptions import DvcException
@@ -30,8 +31,8 @@ class CmdMetricsShow(CmdBase):
xpath=xpath,
all_branches=self.args.all_branches,
all_tags=self.args.all_tags)
except DvcException as exc:
self.project.logger.error('Failed to show metrics', exc)
except DvcException:
logger.error('failed to show metrics')
return 1
return 0
@@ -43,8 +44,8 @@ class CmdMetricsModify(CmdBase):
self.project.metrics_modify(self.args.path,
typ=self.args.type,
xpath=self.args.xpath)
except DvcException as exc:
self.project.logger.error('Failed to modify metrics', exc)
except DvcException:
logger.error('failed to modify metrics')
return 1
return 0
@@ -56,9 +57,9 @@ class CmdMetricsAdd(CmdBase):
self.project.metrics_add(self.args.path,
self.args.type,
self.args.xpath)
except DvcException as exc:
msg = 'Failed to add metric file \'{}\''.format(self.args.path)
self.project.logger.error(msg, exc)
except DvcException:
msg = "failed to add metric file '{}'".format(self.args.path)
logger.error(msg)
return 1
return 0
@@ -68,9 +69,9 @@ class CmdMetricsRemove(CmdBase):
def run(self):
try:
self.project.metrics_remove(self.args.path)
except DvcException as exc:
msg = 'Failed to remove metric file \'{}\''.format(self.args.path)
self.project.logger.error(msg, exc)
except DvcException:
msg = "failed to remove metric file '{}'".format(self.args.path)
logger.error(msg)
return 1
return 0
dvc/command/move.py
+5 -4
View File
@@ -1,3 +1,4 @@
import dvc.logger as logger
from dvc.exceptions import DvcException
from dvc.command.base import CmdBase
@@ -6,9 +7,9 @@ class CmdMove(CmdBase):
def run(self):
try:
self.project.move(self.args.src, self.args.dst)
except DvcException as ex:
msg = 'Failed to move \'{}\' -> \'{}\''.format(self.args.src,
self.args.dst)
self.project.logger.error(msg, ex)
except DvcException:
msg = "failed to move '{}' -> '{}'".format(self.args.src,
self.args.dst)
logger.error(msg)
return 1
return 0
dvc/command/pipeline.py
+10 -9
View File
@@ -1,5 +1,6 @@
import os
import dvc.logger as logger
from dvc.exceptions import DvcException
from dvc.command.base import CmdBase
@@ -16,12 +17,12 @@ class CmdPipelineShow(CmdBase):
for n in networkx.dfs_postorder_nodes(G, node):
if commands:
self.project.logger.info(stages[n].cmd)
logger.info(stages[n].cmd)
elif outs:
for out in stages[n].outs:
self.project.logger.info(str(out))
logger.info(str(out))
else:
self.project.logger.info(n)
logger.info(n)
def __build_graph(self, target, commands, outs):
import networkx
@@ -106,9 +107,9 @@ class CmdPipelineShow(CmdBase):
self._show(target,
self.args.commands,
self.args.outs)
except DvcException as ex:
msg = 'Failed to show pipeline for \'{}\''.format(target)
self.project.logger.error(msg, ex)
except DvcException:
msg = "failed to show pipeline for '{}'".format(target)
logger.error(msg)
return 1
return 0
@@ -121,9 +122,9 @@ class CmdPipelineList(CmdBase):
for p in pipelines:
stages = networkx.get_node_attributes(p, 'stage')
for stage in stages:
self.project.logger.info(stage)
logger.info(stage)
if len(stages) != 0:
self.project.logger.info("="*80)
self.project.logger.info("{} pipeline(s) total".format(len(pipelines)))
logger.info('=' * 80)
logger.info('{} pipeline(s) total'.format(len(pipelines)))
return 0
dvc/command/remote.py
+2 -2
View File
@@ -1,8 +1,8 @@
import re
import dvc.logger as logger
from dvc.config import Config
from dvc.command.config import CmdConfig
from dvc.logger import logger
class CmdRemoteAdd(CmdConfig):
@@ -13,7 +13,7 @@ class CmdRemoteAdd(CmdConfig):
return ret
if self.args.default:
msg = 'Setting \'{}\' as a default remote.'.format(self.args.name)
msg = "Setting '{}' as a default remote.".format(self.args.name)
logger.info(msg)
ret = self._set(Config.SECTION_CORE,
Config.SECTION_CORE_REMOTE,
dvc/command/remove.py
+3 -3
View File
@@ -1,7 +1,7 @@
import dvc.prompt as prompt
import dvc.logger as logger
from dvc.exceptions import DvcException
from dvc.command.base import CmdBase
from dvc.logger import logger
class CmdRemove(CmdBase):
@@ -30,7 +30,7 @@ class CmdRemove(CmdBase):
try:
outs_only = self._is_outs_only(target)
self.project.remove(target, outs_only=outs_only)
except DvcException as ex:
logger.error('Failed to remove {}'.format(target), ex)
except DvcException:
logger.error('failed to remove {}'.format(target))
return 1
return 0
dvc/command/repro.py
+4 -4
View File
@@ -1,5 +1,6 @@
import os
import dvc.logger as logger
from dvc.command.base import CmdBase
from dvc.command.status import CmdDataStatus
from dvc.exceptions import DvcException
@@ -30,13 +31,12 @@ class CmdRepro(CmdBase):
ignore_build_cache=self.args.ignore_build_cache)
if len(stages) == 0:
self.project.logger.info(CmdDataStatus.UP_TO_DATE_MSG)
logger.info(CmdDataStatus.UP_TO_DATE_MSG)
if self.args.metrics:
self.project.metrics_show()
except DvcException as ex:
msg = 'Failed to reproduce \'{}\''.format(target)
self.project.logger.error(msg, ex)
except DvcException:
logger.error()
ret = 1
break
dvc/command/root.py
+2 -1
View File
@@ -1,5 +1,6 @@
import os
import dvc.logger as logger
from dvc.command.base import CmdBase
@@ -8,5 +9,5 @@ class CmdRoot(CmdBase):
return self.run()
def run(self):
self.project.logger.info(os.path.relpath(self.project.root_dir))
logger.info(os.path.relpath(self.project.root_dir))
return 0
dvc/command/run.py
+3 -2
View File
@@ -1,3 +1,4 @@
import dvc.logger as logger
from dvc.command.base import CmdBase
from dvc.exceptions import DvcException
@@ -33,8 +34,8 @@ class CmdRun(CmdBase):
overwrite=overwrite,
ignore_build_cache=self.args.ignore_build_cache,
remove_outs=self.args.remove_outs)
except DvcException as ex:
self.project.logger.error('Failed to run command', ex)
except DvcException:
logger.error('failed to run command')
return 1
return 0
dvc/command/status.py
+6 -5
View File
@@ -1,3 +1,4 @@
import dvc.logger as logger
from dvc.command.data_sync import CmdDataBase
@@ -16,11 +17,11 @@ class CmdDataStatus(CmdDataBase):
for key, value in status.items():
if isinstance(value, dict):
self.project.logger.info('{}{}'.format(ind, key))
logger.info('{}{}'.format(ind, key))
self._show(value, indent+1)
else:
msg = '{}{}{}'.format(ind, self._normalize(value), key)
self.project.logger.info(msg)
logger.info(msg)
def do_run(self, target=None):
indent = 1 if self.args.cloud else 0
@@ -36,9 +37,9 @@ class CmdDataStatus(CmdDataBase):
if st:
self._show(st, indent)
else:
self.project.logger.info(self.UP_TO_DATE_MSG)
logger.info(self.UP_TO_DATE_MSG)
except Exception as exc:
self.project.logger.error('Failed to obtain data status', exc)
except Exception:
logger.error('failed to obtain data status')
return 1
return 0
dvc/command/unprotect.py
+4 -3
View File
@@ -1,3 +1,4 @@
import dvc.logger as logger
from dvc.exceptions import DvcException
from dvc.command.base import CmdBase
@@ -7,8 +8,8 @@ class CmdUnprotect(CmdBase):
for target in self.args.targets:
try:
self.project.unprotect(target)
except DvcException as ex:
msg = "Failed to unprotect '{}'".format(target)
self.project.logger.error(msg, ex)
except DvcException:
msg = "failed to unprotect '{}'".format(target)
logger.error(msg)
return 1
return 0
dvc/config.py
+7 -7
View File
@@ -6,14 +6,14 @@ import errno
import configobj
from schema import Schema, Optional, And, Use, Regex
from dvc.logger import logger
import dvc.logger as logger
from dvc.exceptions import DvcException
class ConfigError(DvcException):
""" DVC config exception """
def __init__(self, msg, ex=None):
super(ConfigError, self).__init__('Config file error: {}'.format(msg),
super(ConfigError, self).__init__('config file error: {}'.format(msg),
ex)
@@ -312,19 +312,19 @@ class Config(object):
raise
conf.write()
except Exception as exc:
msg = "Failed to write config '{}'".format(conf.filename)
msg = "failed to write config '{}'".format(conf.filename)
raise ConfigError(msg, exc)
def unset(self, config, section, opt=None):
if section not in config.keys():
raise ConfigError("Section '{}' doesn't exist".format(section))
raise ConfigError("section '{}' doesn't exist".format(section))
if opt is None:
del config[section]
return
if opt not in config[section].keys():
raise ConfigError("Option '{}.{}' doesn't exist".format(section,
raise ConfigError("option '{}.{}' doesn't exist".format(section,
opt))
del config[section][opt]
@@ -339,10 +339,10 @@ class Config(object):
def show(self, config, section, opt):
if section not in config.keys():
raise ConfigError("Section '{}' doesn't exist".format(section))
raise ConfigError("section '{}' doesn't exist".format(section))
if opt not in config[section].keys():
raise ConfigError("Option '{}.{}' doesn't exist".format(section,
raise ConfigError("option '{}.{}' doesn't exist".format(section,
opt))
logger.info(config[section][opt])
dvc/daemon.py
+5 -5
View File
@@ -2,7 +2,7 @@ import os
import sys
from subprocess import Popen
from dvc.logger import logger
import dvc.logger as logger
from dvc.utils import is_binary, fix_env
@@ -34,8 +34,8 @@ class Daemon(object): # pragma: no cover
pid = os.fork()
if pid > 0:
return
except OSError as exc:
logger.error("Failed at first fork", exc)
except OSError:
logger.error("failed at first fork")
os._exit(1)
os.setsid()
@@ -45,8 +45,8 @@ class Daemon(object): # pragma: no cover
pid = os.fork()
if pid > 0:
os._exit(0)
except OSError as exc:
logger.error("Failed at second fork", exc)
except OSError:
logger.error("failed at second fork")
os._exit(1)
sys.stdin.close()
dvc/data_cloud.py
+21 -26
View File
@@ -1,4 +1,4 @@
from dvc.logger import logger
import dvc.logger as logger
from dvc.config import Config, ConfigError
from dvc.remote.s3 import RemoteS3
@@ -35,8 +35,8 @@ class DataCloud(object):
if self._core.get(Config.SECTION_CORE_CLOUD, None):
# backward compatibility
msg = 'Using obsoleted config format. Consider updating.'
logger.warn(msg)
msg = 'using obsoleted config format. Consider updating.'
logger.warning(msg)
return self._init_compat()
return None
@@ -52,12 +52,12 @@ class DataCloud(object):
section = Config.SECTION_REMOTE_FMT.format(remote)
cloud_config = self._config.get(section, None)
if not cloud_config:
msg = "Can't find remote section '{}' in config"
msg = "can't find remote section '{}' in config"
raise ConfigError(msg.format(section))
cloud_type = self.supported(cloud_config)
if not cloud_type:
raise ConfigError("Unsupported cloud '{}'".format(cloud_config))
raise ConfigError("unsupported cloud '{}'".format(cloud_config))
return self._init_cloud(cloud_config, cloud_type)
@@ -69,25 +69,25 @@ class DataCloud(object):
cloud_type = self.CLOUD_MAP.get(name, None)
if not cloud_type:
msg = "Wrong cloud type {} specified"
raise ConfigError(msg.format(name))
msg = "wrong cloud type '{}' specified".format(name)
raise ConfigError(msg)
cloud_config = self._config.get(name, None)
if not cloud_config:
msg = "Can't find cloud section '{}' in config"
raise ConfigError(msg.format(name))
msg = "can't find cloud section '{}' in config".format(name)
raise ConfigError(msg)
# NOTE: check if the class itself has everything needed for operation.
# E.g. all the imported packages.
if not cloud_type.supported(cloud_type.compat_config(cloud_config)):
raise ConfigError("Unsupported cloud '{}'".format(name))
raise ConfigError("unsupported cloud '{}'".format(name))
return self._init_cloud(cloud_config, cloud_type)
def _init_cloud(self, cloud_config, cloud_type):
global_storage_path = self._core.get(Config.SECTION_CORE_STORAGEPATH)
if global_storage_path:
logger.warn('Using obsoleted config format. Consider updating.')
logger.warning('using obsoleted config format. Consider updating.')
cloud = cloud_type(self.project, cloud_config)
return cloud
@@ -99,17 +99,16 @@ class DataCloud(object):
if self._cloud:
return self._cloud
msg = "No remote repository specified. Setup default " \
"repository with\n"\
" dvc config core.remote <name>\n" \
"or use:\n" \
" dvc {} -r <name>".format(cmd)
raise ConfigError(msg)
raise ConfigError(
'No remote repository specified. Setup default repository with\n'
' dvc config core.remote <name>\n'
'or use:\n'
' dvc {} -r <name>\n'
.format(cmd)
)
def push(self, targets, jobs=1, remote=None, show_checksums=False):
"""
Push data items in a cloud-agnostic way.
"""
"""Push data items in a cloud-agnostic way."""
return self.project.cache.local.push(targets,
jobs=jobs,
remote=self._get_cloud(remote,
@@ -117,9 +116,7 @@ class DataCloud(object):
show_checksums=show_checksums)
def pull(self, targets, jobs=1, remote=None, show_checksums=False):
"""
Pull data items in a cloud-agnostic way.
"""
"""Pull data items in a cloud-agnostic way."""
return self.project.cache.local.pull(targets,
jobs=jobs,
remote=self._get_cloud(remote,
@@ -127,9 +124,7 @@ class DataCloud(object):
show_checksums=show_checksums)
def status(self, targets, jobs=1, remote=None, show_checksums=False):
"""
Check status of data items in a cloud-agnostic way.
"""
"""Check status of data items in a cloud-agnostic way."""
cloud = self._get_cloud(remote, 'status')
return self.project.cache.local.status(targets,
jobs=jobs,
dvc/dependency/base.py
+6 -12
View File
@@ -3,22 +3,16 @@ import re
from dvc.exceptions import DvcException
class DependencyError(DvcException):
def __init__(self, path, msg):
msg = 'Dependency \'{}\' error: {}'.format(path, msg)
super(DependencyError, self).__init__(msg)
class DependencyDoesNotExistError(DependencyError):
class DependencyDoesNotExistError(DvcException):
def __init__(self, path):
msg = 'does not exist'
super(DependencyDoesNotExistError, self).__init__(path, msg)
msg = "dependency '{}' does not exist".format(path)
super(DependencyDoesNotExistError, self).__init__(msg)
class DependencyIsNotFileOrDirError(DependencyError):
class DependencyIsNotFileOrDirError(DvcException):
def __init__(self, path):
msg = 'not a file or directory'
super(DependencyIsNotFileOrDirError, self).__init__(path, msg)
msg = "dependency '{}' is not a file or directory".format(path)
super(DependencyIsNotFileOrDirError, self).__init__(msg)
class DependencyBase(object):
dvc/dependency/local.py
+3 -2
View File
@@ -5,6 +5,7 @@ try:
except ImportError:
from urllib.parse import urlparse
import dvc.logger as logger
from dvc.dependency.base import DependencyBase
from dvc.dependency.base import DependencyDoesNotExistError
from dvc.dependency.base import DependencyIsNotFileOrDirError
@@ -72,8 +73,8 @@ class DependencyLOCAL(DependencyBase):
if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
(os.path.isdir(self.path) and len(os.listdir(self.path)) == 0):
msg = "File/directory '{}' is empty.".format(self.rel_path)
self.project.logger.warn(msg)
msg = "file/directory '{}' is empty.".format(self.rel_path)
logger.warning(msg)
self.info = self.remote.save_info(self.path_info)
dvc/exceptions.py
+8 -8
View File
@@ -15,7 +15,7 @@ class DvcException(Exception):
class UnsupportedRemoteError(DvcException):
def __init__(self, config):
msg = "Remote '{}' is not supported.".format(config)
msg = "remote '{}' is not supported.".format(config)
super(UnsupportedRemoteError, self).__init__(msg)
@@ -24,7 +24,7 @@ class OutputDuplicationError(DvcException):
assert isinstance(output, str)
assert isinstance(stages, list)
assert all(isinstance(stage, str) for stage in stages)
msg = "File '{}' is specified as an output in more than one stage:{}"
msg = "file '{}' is specified as an output in more than one stage:{}"
s = ""
for stage in stages:
s += "\n " + stage
@@ -36,7 +36,7 @@ class WorkingDirectoryAsOutputError(DvcException):
assert isinstance(cwd, str)
assert isinstance(fname, str)
msg = (
"Current working directory '{cwd}' is specified as an output in"
"current working directory '{cwd}' is specified as an output in"
" '{fname}'. Use another CWD to prevent any data removal."
.format(cwd=cwd, fname=fname)
)
@@ -46,20 +46,20 @@ class WorkingDirectoryAsOutputError(DvcException):
class CircularDependencyError(DvcException):
def __init__(self, dependency):
assert isinstance(dependency, str)
msg = "File '{}' is specified as an output and as a dependency."
msg = "file '{}' is specified as an output and as a dependency."
super(CircularDependencyError, self).__init__(msg.format(dependency))
class ArgumentDuplicationError(DvcException):
def __init__(self, path):
assert isinstance(path, str)
msg = "File '{}' is specified more than once."
msg = "file '{}' is specified more than once."
super(ArgumentDuplicationError, self).__init__(msg.format(path))
class MoveNotDataSourceError(DvcException):
def __init__(self, path):
msg = "Move is not permitted for stages that are not data sources. " \
msg = "move is not permitted for stages that are not data sources. " \
"You need to either move '{path}' to a new location and edit " \
"it by hand, or remove '{path}' and create a new one at the " \
"desired location."
@@ -68,10 +68,10 @@ class MoveNotDataSourceError(DvcException):
class NotDvcProjectError(DvcException):
def __init__(self, root):
msg = "Not a dvc repository (checked up to mount point {})"
msg = "not a dvc repository (checked up to mount point '{}')"
super(NotDvcProjectError, self).__init__(msg.format(root))
class DvcParserError(DvcException):
def __init__(self):
super(DvcException, self).__init__("Parser error")
super(DvcException, self).__init__("parser error")
dvc/lock.py
+1 -1
View File
@@ -21,7 +21,7 @@ class Lock(object):
try:
self._lock = zc.lockfile.LockFile(self.lock_file)
except zc.lockfile.LockError:
raise LockError('Cannot perform the cmd since DVC is busy and '
raise LockError('cannot perform the cmd since DVC is busy and '
'locked. Please retry the cmd later.')
def lock(self):
dvc/logger.py
+218 -217
View File
@@ -4,239 +4,240 @@ import colorama
import traceback
import re
colorama.init()
from contextlib import contextmanager
def visual_width(line):
""" Get the the number of columns required to display a string """
return len(re.sub(colorama.ansitowin32.AnsiToWin32.ANSI_CSI_RE, '', line))
def info(message):
logger.info(message)
def visual_center(line, width):
""" Center align string according to it's visual width """
def debug(message):
prefix = colorize('Debug', color='blue')
spaces = max(width - visual_width(line), 0)
left_padding = int(spaces / 2)
right_padding = (spaces - left_padding)
out = '{prefix}: {message}'.format(prefix=prefix, message=message)
return (left_padding * ' ') + line + (right_padding * ' ')
logger.debug(out)
class Logger(object):
FMT = '%(message)s'
DEFAULT_LEVEL = logging.INFO
DEFAULT_LEVEL_NAME = 'info'
def warning(message):
prefix = colorize('Warning', color='yellow')
LEVEL_MAP = {
'debug': logging.DEBUG,
'info': logging.INFO,
'warn': logging.WARNING,
'error': logging.ERROR
out = '{prefix}: {message}'.format(prefix=prefix, message=message)
logger.warning(out)
def error(message=None):
prefix = colorize('Error', color='red')
out = (
'{prefix}: {description}'
'\n'
'{stack_trace}'
'\n'
'{footer}'.format(
prefix=prefix,
description=_description(message),
stack_trace=_stack_trace(),
footer=_footer(),
)
)
logger.error(out)
def box(message, border_color=None):
lines = message.split('\n')
max_width = max(_visual_width(line) for line in lines)
padding_horizontal = 5
padding_vertical = 1
box_size_horizontal = (max_width + (padding_horizontal * 2))
chars = {
'corner': '+',
'horizontal': '-',
'vertical': '|',
'empty': ' ',
}
COLOR_MAP = {
margin = "{corner}{line}{corner}\n".format(
corner=chars['corner'],
line=chars['horizontal'] * box_size_horizontal,
)
padding_lines = [
"{border}{space}{border}\n".format(
border=colorize(chars['vertical'], color=border_color),
space=chars['empty'] * box_size_horizontal,
) * padding_vertical
]
content_lines = [
"{border}{space}{content}{space}{border}\n".format(
border=colorize(chars['vertical'], color=border_color),
space=chars['empty'] * padding_horizontal,
content=_visual_center(line, max_width),
) for line in lines
]
box = "{margin}{padding}{content}{padding}{margin}".format(
margin=colorize(margin, color=border_color),
padding=''.join(padding_lines),
content=''.join(content_lines),
)
logger.info(box)
def level():
return logger.getEffectiveLevel()
def set_level(level_name):
if not level_name:
return
levels = {
'info': logging.INFO,
'debug': logging.DEBUG,
'warning': logging.WARNING,
'error': logging.ERROR,
'critical': logging.CRITICAL,
}
logger.setLevel(levels.get(level_name))
def be_quiet():
logger.setLevel(logging.CRITICAL)
def be_verbose():
logger.setLevel(logging.DEBUG)
@contextmanager
def verbose():
previous_level = level()
be_verbose()
yield
logger.setLevel(previous_level)
@contextmanager
def quiet():
previous_level = level()
be_quiet()
yield
logger.setLevel(previous_level)
def is_quiet():
return level() == logging.CRITICAL
def is_verbose():
return level() == logging.DEBUG
def colorize(message, color=None):
if not color:
return message
colors = {
'green': colorama.Fore.GREEN,
'yellow': colorama.Fore.YELLOW,
'blue': colorama.Fore.BLUE,
'red': colorama.Fore.RED,
}
LEVEL_COLOR_MAP = {
'debug': 'blue',
'warn': 'yellow',
'error': 'red',
}
def __init__(self, loglevel=None, force=False):
self.logger = logging.getLogger('dvc')
if force or not self._already_initialized():
class LogLevelFilter(logging.Filter):
def filter(self, record):
return record.levelno <= logging.WARNING
sh_out = logging.StreamHandler(sys.stdout)
sh_out.setFormatter(logging.Formatter(self.FMT))
sh_out.setLevel(logging.DEBUG)
sh_out.addFilter(LogLevelFilter())
sh_err = logging.StreamHandler(sys.stderr)
sh_err.setFormatter(logging.Formatter(self.FMT))
sh_err.setLevel(logging.ERROR)
self.logger.addHandler(sh_out)
self.logger.addHandler(sh_err)
self.set_level(loglevel)
def set_level(self, level=None):
if not level:
lvl = self.DEFAULT_LEVEL
else:
level = level.lower()
lvl = self.LEVEL_MAP.get(level, self.DEFAULT_LEVEL)
self.logger.setLevel(lvl)
self.lvl = lvl
def be_quiet(self):
self.lvl = logging.CRITICAL
self.logger.setLevel(logging.CRITICAL)
def be_verbose(self):
self.lvl = logging.DEBUG
self.logger.setLevel(logging.DEBUG)
def colorize(self, msg, color):
header = ''
footer = ''
if sys.stdout.isatty(): # pragma: no cover
header = self.COLOR_MAP.get(color.lower(), '')
footer = colorama.Style.RESET_ALL
return u'{}{}{}'.format(header, msg, footer)
def parse_exc(self, exc, tb=None):
str_tb = tb if tb else None
str_exc = str(exc) if exc else ""
l_str_exc = []
if len(str_exc) != 0:
l_str_exc.append(str_exc)
if exc and hasattr(exc, 'cause') and exc.cause:
cause_tb = exc.cause_tb if hasattr(exc, 'cause_tb') else None
l_cause_str_exc, cause_str_tb = self.parse_exc(exc.cause, cause_tb)
str_tb = cause_str_tb
l_str_exc += l_cause_str_exc
return (l_str_exc, str_tb)
def _prefix(self, msg, typ):
color = self.LEVEL_COLOR_MAP.get(typ.lower(), '')
return self.colorize('{}'.format(msg), color)
def error_prefix(self):
return self._prefix('Error', 'error')
def warning_prefix(self):
return self._prefix('Warning', 'warn')
def debug_prefix(self):
return self._prefix('Debug', 'debug')
def _with_progress(self, func, msg):
from dvc.progress import progress
with progress:
func(msg)
def _with_exc(self, func, prefix, msg, suffix="", exc=None):
l_str_exc, str_tb = self.parse_exc(exc)
if exc is not None and self.is_verbose():
str_tb = str_tb if str_tb else traceback.format_exc()
self._with_progress(self.logger.error, str_tb)
l_msg = [prefix]
if msg is not None and len(msg) != 0:
l_msg.append(msg)
l_msg += l_str_exc
self._with_progress(func, ': '.join(l_msg) + suffix)
def error(self, msg, exc=None):
if self.is_quiet():
return
chat = "\n\nHaving any troubles? Hit us up at dvc.org/support, " \
"we are always happy to help!"
self._with_exc(self.logger.error,
self.error_prefix(),
msg,
suffix=chat,
exc=exc)
def warn(self, msg, exc=None):
if self.is_quiet():
return
self._with_exc(self.logger.warning,
self.warning_prefix(),
msg,
exc=exc)
def debug(self, msg, exc=None):
if not self.is_verbose():
return
self._with_exc(self.logger.debug,
self.debug_prefix(),
msg,
exc=exc)
def info(self, msg):
if self.is_quiet():
return
self._with_progress(self.logger.info, msg)
def is_quiet(self):
return self.lvl == logging.CRITICAL
def is_verbose(self):
return self.lvl == logging.DEBUG
def box(self, msg, border_color=''):
if self.is_quiet():
return
lines = msg.split('\n')
max_width = max(visual_width(line) for line in lines)
padding_horizontal = 5
padding_vertical = 1
box_size_horizontal = (max_width + (padding_horizontal * 2))
chars = {
'corner': '+',
'horizontal': '-',
'vertical': '|',
'empty': ' ',
}
margin = "{corner}{line}{corner}\n".format(
corner=chars['corner'],
line=chars['horizontal'] * box_size_horizontal,
)
padding_lines = [
"{border}{space}{border}\n".format(
border=self.colorize(chars['vertical'], border_color),
space=chars['empty'] * box_size_horizontal,
) * padding_vertical
]
content_lines = [
"{border}{space}{content}{space}{border}\n".format(
border=self.colorize(chars['vertical'], border_color),
space=chars['empty'] * padding_horizontal,
content=visual_center(line, max_width),
) for line in lines
]
box = "{margin}{padding}{content}{padding}{margin}".format(
margin=self.colorize(margin, border_color),
padding=''.join(padding_lines),
content=''.join(content_lines),
)
print(box)
def _already_initialized(self):
return bool(self.logger.handlers)
return u'{color}{message}{nc}'.format(
color=colors.get(color, ''),
message=message,
nc=colorama.Fore.RESET,
)
logger = Logger()
def _init_colorama():
colorama.init()
def _set_default_level():
logger.setLevel(logging.INFO)
def _add_handlers():
formatter = '%(message)s'
class LogLevelFilter(logging.Filter):
def filter(self, record):
return record.levelno <= logging.WARNING
sh_out = logging.StreamHandler(sys.stdout)
sh_out.setFormatter(logging.Formatter(formatter))
sh_out.setLevel(logging.DEBUG)
sh_out.addFilter(LogLevelFilter())
sh_err = logging.StreamHandler(sys.stderr)
sh_err.setFormatter(logging.Formatter(formatter))
sh_err.setLevel(logging.ERROR)
logger.addHandler(sh_out)
logger.addHandler(sh_err)
def _stack_trace():
trace = sys.exc_info()[2]
if not is_verbose() or not trace:
return ''
return '{line}\n{stack_trace}{line}\n'.format(
line=colorize('-' * 60, color='red'),
stack_trace=traceback.format_exc(),
)
def _description(message):
exception = sys.exc_info()[1]
if exception and message:
description = '{message} - {exception}'
elif exception:
description = '{exception}'
elif message:
description = '{message}'
return description.format(message=message, exception=str(exception))
def _footer():
return '{phrase} Hit us up at {url}, we are always happy to help!'.format(
phrase=colorize('Having any troubles?', 'yellow'),
url=colorize('https://dvc.org/support', 'blue'),
)
def _visual_width(line):
"""Get the the number of columns required to display a string"""
return len(re.sub(colorama.ansitowin32.AnsiToWin32.ANSI_CSI_RE, '', line))
def _visual_center(line, width):
"""Center align string according to it's visual width"""
spaces = max(width - _visual_width(line), 0)
left_padding = int(spaces / 2)
right_padding = (spaces - left_padding)
return (left_padding * ' ') + line + (right_padding * ' ')
logger = logging.getLogger('dvc')
_set_default_level()
_add_handlers()
_init_colorama()
dvc/main.py
+7 -7
View File
@@ -1,4 +1,4 @@
from dvc.logger import logger
import dvc.logger as logger
from dvc.cli import parse_args
from dvc.command.base import CmdBase
from dvc.analytics import Analytics
@@ -18,16 +18,16 @@ def main(argv=None):
cmd = args.func(args)
ret = cmd.run_cmd()
except KeyboardInterrupt as ex:
logger.error("Interrupted by the user", ex)
except KeyboardInterrupt:
logger.error("interrupted by the user")
ret = 252
except NotDvcProjectError as ex:
logger.error("", ex)
except NotDvcProjectError:
logger.error()
ret = 253
except DvcParserError:
ret = 254
except Exception as ex:
logger.error('Unexpected error', ex)
except Exception:
logger.error('unexpected error')
ret = 255
Analytics().send_cmd(cmd, args, ret)
dvc/output/base.py
+9 -14
View File
@@ -1,24 +1,19 @@
from dvc.exceptions import DvcException
class OutputError(DvcException):
def __init__(self, path, msg):
msg = 'Output \'{}\' error: {}'.format(path, msg)
super(OutputError, self).__init__(msg)
class OutputDoesNotExistError(OutputError):
class OutputDoesNotExistError(DvcException):
def __init__(self, path):
super(OutputDoesNotExistError, self).__init__(path, 'does not exist')
msg = "output '{}' does not exists".format(path)
super(OutputDoesNotExistError, self).__init__(msg)
class OutputIsNotFileOrDirError(OutputError):
class OutputIsNotFileOrDirError(DvcException):
def __init__(self, path):
msg = 'not a file or directory'
super(OutputIsNotFileOrDirError, self).__init__(path, msg)
msg = "output '{}' is not a file or directory".format(path)
super(OutputIsNotFileOrDirError, self).__init__(msg)
class OutputAlreadyTrackedError(OutputError):
class OutputAlreadyTrackedError(DvcException):
def __init__(self, path):
msg = 'already tracked by scm(e.g. git)'
super(OutputAlreadyTrackedError, self).__init__(path, msg)
msg = "output '{}' is already tracked by scm (e.g. git)".format(path)
super(OutputAlreadyTrackedError, self).__init__(msg)
dvc/output/gs.py
+1 -1
View File
@@ -17,7 +17,7 @@ class OutputGS(DependencyGS):
self.use_cache = cache
self.metric = metric
if cache and self.project.cache.gs is None:
raise DvcException("No cache location setup for \'gs\' outputs.")
raise DvcException("no cache location setup for 'gs' outputs.")
def dumpd(self):
ret = super(OutputGS, self).dumpd()
dvc/output/hdfs.py
+1 -1
View File
@@ -17,7 +17,7 @@ class OutputHDFS(DependencyHDFS):
self.use_cache = cache
self.metric = metric
if cache and self.project.cache.hdfs is None:
raise DvcException("No cache location setup for \'hdfs\' outputs.")
raise DvcException("no cache location setup for 'hdfs' outputs.")
def dumpd(self):
ret = super(OutputHDFS, self).dumpd()
dvc/output/local.py
+10 -10
View File
@@ -1,6 +1,7 @@
import os
from schema import Optional, Or
import dvc.logger as logger
from dvc.output.base import OutputDoesNotExistError, OutputIsNotFileOrDirError
from dvc.output.base import OutputAlreadyTrackedError
from dvc.dependency.local import DependencyLOCAL
@@ -76,36 +77,35 @@ class OutputLOCAL(DependencyLOCAL):
return
if os.path.isdir(self.path):
msg = 'Directory \'{}\' cannot be used as metrics.'
msg = "directory '{}' cannot be used as metrics."
raise DvcException(msg.format(self.rel_path))
if not istextfile(self.path):
msg = 'Binary file \'{}\' cannot be used as metrics.'
msg = "binary file '{}' cannot be used as metrics."
raise DvcException(msg.format(self.rel_path))
def save(self):
if not self.use_cache:
super(OutputLOCAL, self).save()
self._verify_metric()
msg = 'Output \'{}\' doesn\'t use cache. Skipping saving.'
self.project.logger.info(msg.format(self.rel_path))
msg = "Output '{}' doesn't use cache. Skipping saving."
logger.info(msg.format(self.rel_path))
return
if not os.path.exists(self.path):
raise self.DoesNotExistError(self.rel_path)
if not os.path.isfile(self.path) \
and not os.path.isdir(self.path): # pragma: no cover
if (not os.path.isfile(self.path) and not os.path.isdir(self.path)):
raise self.IsNotFileOrDirError(self.rel_path)
if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
(os.path.isdir(self.path) and len(os.listdir(self.path)) == 0):
msg = "File/directory '{}' is empty.".format(self.rel_path)
self.project.logger.warn(msg)
msg = "file/directory '{}' is empty.".format(self.rel_path)
logger.warning(msg)
if not self.changed():
msg = 'Output \'{}\' didn\'t change. Skipping saving.'
self.project.logger.info(msg.format(self.rel_path))
msg = "Output '{}' didn't change. Skipping saving."
logger.info(msg.format(self.rel_path))
return
if self.is_local:
dvc/output/s3.py
+1 -1
View File
@@ -17,7 +17,7 @@ class OutputS3(DependencyS3):
self.use_cache = cache
self.metric = metric
if cache and self.project.cache.s3 is None:
raise DvcException("No cache location setup for \'s3\' outputs.")
raise DvcException("no cache location setup for 's3' outputs.")
def dumpd(self):
ret = super(OutputS3, self).dumpd()
dvc/output/ssh.py
+1 -1
View File
@@ -17,7 +17,7 @@ class OutputSSH(DependencySSH):
self.use_cache = cache
self.metric = metric
if cache and self.project.cache.ssh is None:
raise DvcException("No cache location setup for \'ssh\' outputs.")
raise DvcException("no cache location setup for 'ssh' outputs.")
def dumpd(self):
ret = super(OutputSSH, self).dumpd()
dvc/progress.py
+1 -2
View File
@@ -1,6 +1,7 @@
from __future__ import print_function
import sys
import threading
import dvc.logger as logger
class Progress(object):
@@ -86,8 +87,6 @@ class Progress(object):
return num + bar + percent + target_name
def _print(self, *args, **kwargs):
from dvc.logger import logger
if logger.is_quiet():
return
dvc/project.py
+58 -42
View File
@@ -1,6 +1,7 @@
import collections
import os
import dvc.prompt as prompt
import dvc.logger as logger
from dvc.exceptions import DvcException, MoveNotDataSourceError
from dvc.exceptions import NotDvcProjectError
@@ -14,7 +15,7 @@ class InitError(DvcException):
class ReproductionError(DvcException):
def __init__(self, dvc_file_name, ex):
self.path = dvc_file_name
msg = 'Failed to reproduce \'{}\''.format(dvc_file_name)
msg = "failed to reproduce '{}'".format(dvc_file_name)
super(ReproductionError, self).__init__(msg, cause=ex)
@@ -22,7 +23,6 @@ class Project(object):
DVC_DIR = '.dvc'
def __init__(self, root_dir=None):
from dvc.logger import logger
from dvc.config import Config
from dvc.state import State
from dvc.lock import Lock
@@ -45,8 +45,8 @@ class Project(object):
self.state = State(self, self.config._config)
core = self.config._config[Config.SECTION_CORE]
self.logger = logger
self.logger.set_level(core.get(Config.SECTION_CORE_LOGLEVEL, None))
logger.set_level(core.get(Config.SECTION_CORE_LOGLEVEL))
self.cache = Cache(self)
self.cloud = DataCloud(self, config=self.config._config)
@@ -89,7 +89,7 @@ class Project(object):
msg = '\nTo track the changes with git run:\n\n'
msg += '\tgit add ' + " ".join(self._files_to_git_add)
self.logger.info(msg)
logger.info(msg)
@staticmethod
def init(root_dir=os.curdir, no_scm=False, force=False):
@@ -108,7 +108,6 @@ class Project(object):
import shutil
from dvc.scm import SCM, Base
from dvc.config import Config
from dvc.logger import logger
root_dir = os.path.abspath(root_dir)
dvc_dir = os.path.join(root_dir, Project.DVC_DIR)
@@ -302,7 +301,7 @@ class Project(object):
self._remind_to_git_add()
if not found:
msg = 'Unable to find dvcfile with output \'{}\''
msg = "unable to find dvcfile with output '{}'"
raise DvcException(msg.format(from_path))
def _unprotect_file(self, path):
@@ -312,7 +311,7 @@ class Project(object):
from dvc.utils import copyfile, move, remove
if System.is_symlink(path) or System.is_hardlink(path):
self.logger.debug("Unprotecting '{}'".format(path))
logger.debug("Unprotecting '{}'".format(path))
tmp = os.path.join(os.path.dirname(path), '.' + str(uuid.uuid4()))
move(path, tmp)
@@ -321,8 +320,8 @@ class Project(object):
remove(tmp)
else:
self.logger.debug("Skipping copying for '{}', since it is not "
"a symlink or a hardlink.".format(path))
logger.debug("Skipping copying for '{}', since it is not "
"a symlink or a hardlink.".format(path))
os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
@@ -334,8 +333,10 @@ class Project(object):
def unprotect(self, path):
if not os.path.exists(path):
raise DvcException("Can't unprotect non-existing "
"data '{}'".format(path))
raise DvcException(
"can't unprotect non-existing data '{}'"
.format(path)
)
if os.path.isdir(path):
self._unprotect_dir(path)
@@ -415,9 +416,11 @@ class Project(object):
stage = stages[node]
if stage.locked:
msg = 'DVC file \'{}\' is locked. Its dependencies are not ' \
'going to be reproduced.'
self.logger.warn(msg.format(stage.relpath))
logger.warning(
"DVC file '{path}' is locked. Its dependencies are"
" not going to be reproduced."
.format(path=stage.relpath)
)
stage = stage.reproduce(force=force, dry=dry, interactive=interactive)
if not stage:
@@ -563,9 +566,11 @@ class Project(object):
for stage in stages:
if stage.locked:
msg = 'DVC file \'{}\' is locked. Its dependencies are ' \
'not going to be checked out.'
self.logger.warn(msg.format(stage.relpath))
logger.warning(
"DVC file '{path}' is locked. Its dependencies are"
" not going to be checked out."
.format(path=stage.relpath)
)
stage.checkout(force=force)
@@ -612,16 +617,18 @@ class Project(object):
show_checksums=False)
except DvcException as exc:
msg = "Failed to pull cache for '{}': {}"
self.logger.debug(msg.format(out, exc))
logger.debug(msg.format(out, exc))
if self.cache.local.changed_cache_file(md5):
msg = "Missing cache for directory '{}'. " \
"Cache for files inside will be lost. " \
"Would you like to continue? Use '-f' to force. "
if not force and not prompt.confirm(msg):
raise DvcException("Unable to fully collect "
"used cache without cache "
"for directory '{}'".format(out))
raise DvcException(
"unable to fully collect used cache"
" without cache for directory '{}'"
.format(out)
)
else:
return ret
@@ -689,9 +696,11 @@ class Project(object):
for stage in stages:
if active and not target and stage.locked:
msg = 'DVC file \'{}\' is locked. Its dependencies are ' \
'not going to be pushed/pulled/fetched.'
self.logger.warn(msg.format(stage.relpath))
logger.warning(
"DVC file '{path}' is locked. Its dependencies are"
" not going to be pushed/pulled/fetched."
.format(path=stage.relpath)
)
for out in stage.outs:
scheme = out.path_info['scheme']
@@ -745,7 +754,7 @@ class Project(object):
def _do_gc(self, typ, func, clist):
removed = func(clist)
if not removed:
self.logger.info("No unused {} cache to remove.".format(typ))
logger.info("No unused {} cache to remove.".format(typ))
def gc(self,
all_branches=False,
@@ -870,9 +879,11 @@ class Project(object):
for stage in stages:
if stage.locked:
msg = 'DVC file \'{}\' is locked. Its dependencies are not ' \
'going to be shown in the status output.'
self.logger.warn(msg.format(stage.relpath))
logger.warning(
"DVC file '{path}' is locked. Its dependencies are"
" not going to be shown in the status output."
.format(path=stage.relpath)
)
status.update(stage.status())
@@ -990,9 +1001,8 @@ class Project(object):
ret = self._read_metric_hxsv(fd, xpath, '\t')
else:
ret = fd.read()
except Exception as exc:
self.logger.error('Unable to read metric in \'{}\''.format(path),
exc)
except Exception:
logger.error("unable to read metric in '{}'".format(path))
return ret
@@ -1065,32 +1075,35 @@ class Project(object):
for branch, val in res.items():
if all_branches or all_tags:
self.logger.info('{}:'.format(branch))
logger.info('{}:'.format(branch))
for fname, metric in val.items():
self.logger.info('\t{}: {}'.format(fname, metric))
logger.info('\t{}: {}'.format(fname, metric))
if res:
return res
if path:
msg = 'File \'{}\' does not exist'.format(path)
msg = "file '{}' does not exist".format(path)
else:
msg = 'No metric files in this repository. ' \
'Use \'dvc metrics add\' to add a metric file to track.'
msg = (
"no metric files in this repository."
" use 'dvc metrics add' to add a metric file to track."
)
raise DvcException(msg)
def _metrics_modify(self, path, typ=None, xpath=None, delete=False):
out = self._find_output_by_path(path)
if not out:
msg = 'Unable to find file \'{}\' in the pipeline'.format(path)
msg = "unable to find file '{}' in the pipeline".format(path)
raise DvcException(msg)
if out.path_info['scheme'] != 'local':
msg = 'Output \'{}\' scheme \'{}\' is not supported for metrics'
msg = "output '{}' scheme '{}' is not supported for metrics"
raise DvcException(msg.format(out.path, out.path_info['scheme']))
if out.use_cache:
msg = 'Cached output \'{}\' is not supported for metrics'
msg = "cached output '{}' is not supported for metrics"
raise DvcException(msg.format(out.rel_path))
if typ:
@@ -1169,7 +1182,10 @@ class Project(object):
G, G_active = self.graph()
return list(nx.weakly_connected_component_subgraphs(G_active))
return [
G.subgraph(c).copy()
for c in nx.weakly_connected_components(G)
]
def stages(self):
"""
@@ -1218,7 +1234,7 @@ class Project(object):
def _welcome_message(self):
import colorama
self.logger.info(
logger.info(
"\n"
"DVC has enabled anonymous aggregate usage analytics.\n"
"Read the analytics documentation (and how to opt-out) here:\n"
dvc/remote/azure.py
+9 -9
View File
@@ -12,7 +12,7 @@ try:
except ImportError:
from urllib.parse import urlparse
from dvc.logger import logger
import dvc.logger as logger
from dvc.progress import progress
from dvc.config import Config
from dvc.remote.base import RemoteBase
@@ -56,10 +56,10 @@ class RemoteAzure(RemoteBase):
or os.getenv('AZURE_STORAGE_CONNECTION_STRING'))
if not self.bucket:
raise ValueError('Azure Storage container name missing')
raise ValueError('azure storage container name missing')
if not self.connection_string:
raise ValueError('Azure Storage connection string missing')
raise ValueError('azure storage connection string missing')
self.__blob_service = None
@@ -125,9 +125,9 @@ class RemoteAzure(RemoteBase):
try:
self.blob_service.create_blob_from_path(
bucket, path, from_info['path'], progress_callback=cb)
except Exception as ex:
msg = "Failed to upload '{}'".format(from_info['path'])
logger.warn(msg, ex)
except Exception:
msg = "failed to upload '{}'".format(from_info['path'])
logger.warning(msg)
else:
progress.finish_target(name)
@@ -162,9 +162,9 @@ class RemoteAzure(RemoteBase):
try:
self.blob_service.get_blob_to_path(
bucket, path, tmp_file, progress_callback=cb)
except Exception as exc:
msg = "Failed to download '{}/{}'".format(bucket, path)
logger.warn(msg, exc)
except Exception:
msg = "failed to download '{}/{}'".format(bucket, path)
logger.warning(msg)
else:
os.rename(tmp_file, to_info['path'])
dvc/remote/base.py
+32 -29
View File
@@ -5,8 +5,8 @@ import posixpath
from multiprocessing import cpu_count
import dvc.prompt as prompt
import dvc.logger as logger
from dvc.config import Config
from dvc.logger import logger
from dvc.exceptions import DvcException
@@ -60,24 +60,25 @@ class RemoteBase(object):
deps_ok = all(cls.REQUIRES.values())
if url_ok and not deps_ok:
missing = [k for k, v in cls.REQUIRES.items() if v is None]
msg = "URL \'{}\' is supported but requires these missing " \
"dependencies: {}. If you have installed dvc using pip, " \
"choose one of these options to proceed: \n" \
"\n" \
" 1) Install specific missing dependencies:\n" \
" pip install {}\n" \
" 2) Install dvc package that includes those missing " \
"dependencies: \n" \
" pip install dvc[{}]\n" \
" 3) Install dvc package with all possible " \
"dependencies included: \n" \
" pip install dvc[all]\n" \
"\n" \
"If you have installed dvc from a binary package and you " \
"are still seeing this message, please report it to us " \
"using https://github.com/iterative/dvc/issues. Thank you!"
msg = msg.format(url, missing, " ".join(missing), cls.scheme)
logger.warn(msg)
logger.warning(
"URL '{}' is supported but requires these missing "
"dependencies: {}. If you have installed dvc using pip, "
"choose one of these options to proceed: \n"
"\n"
" 1) Install specific missing dependencies:\n"
" pip install {}\n"
" 2) Install dvc package that includes those missing "
"dependencies: \n"
" pip install dvc[{}]\n"
" 3) Install dvc package with all possible "
"dependencies included: \n"
" pip install dvc[all]\n"
"\n"
"If you have installed dvc from a binary package and you "
"are still seeing this message, please report it to us "
"using https://github.com/iterative/dvc/issues. Thank you!"
.format(url, missing, " ".join(missing), cls.scheme)
)
return url_ok and deps_ok
@@ -209,8 +210,8 @@ class RemoteBase(object):
if expected != actual:
if self.exists(cache):
msg = 'Corrupted cache file {}'
logger.warn(msg.format(str(cache)))
msg = 'corrupted cache file {}'
logger.warning(msg.format(str(cache)))
self.remove(cache)
return True
@@ -241,22 +242,24 @@ class RemoteBase(object):
if not force and not self.already_cached(path_info):
msg = (
'File "{}" is going to be removed. '
'Are you sure you want to proceed?'
"file '{}' is going to be removed."
' Are you sure you want to proceed?'
.format(str(path_info))
)
if not prompt.confirm(msg):
raise DvcException("Unable to remove {} without a confirmation"
" from the user. Use '-f' to force."
.format(str(path_info)))
raise DvcException(
"unable to remove '{}' without a confirmation"
" from the user. Use '-f' to force."
.format(str(path_info))
)
self.remove(path_info)
def do_checkout(self, path_info, checksum, force=False):
if self.exists(path_info):
msg = "Data '{}' exists. Removing before checkout."
logger.warn(msg.format(str(path_info)))
msg = "data '{}' exists. Removing before checkout."
logger.warning(msg.format(str(path_info)))
self.safe_remove(path_info, force=force)
from_info = self.checksum_to_path_info(checksum)
@@ -280,7 +283,7 @@ class RemoteBase(object):
if self.changed_cache(checksum):
msg = "Cache '{}' not found. File '{}' won't be created."
logger.warn(msg.format(checksum, str(path_info)))
logger.warning(msg.format(checksum, str(path_info)))
self.safe_remove(path_info, force=force)
return
dvc/remote/gs.py
+13 -13
View File
@@ -10,7 +10,7 @@ try:
except ImportError:
from urllib.parse import urlparse
from dvc.logger import logger
import dvc.logger as logger
from dvc.remote.base import RemoteBase
from dvc.config import Config
from dvc.progress import progress
@@ -85,8 +85,8 @@ class RemoteGS(RemoteBase):
blob = gs.bucket(from_info['bucket']).get_blob(from_info['path'])
if not blob:
msg = '{} doesn\'t exist in the cloud'
raise DvcException(msg.format(from_info['path']))
msg = "'{}' doesn't exist in the cloud".format(from_info['path'])
raise DvcException(msg)
bucket = self.gs.bucket(to_info['bucket'])
bucket.copy_blob(blob,
@@ -157,11 +157,11 @@ class RemoteGS(RemoteBase):
bucket = gs.bucket(to_info['bucket'])
blob = bucket.blob(to_info['path'])
blob.upload_from_filename(from_info['path'])
except Exception as exc:
msg = "Failed to upload '{}' to '{}/{}'"
logger.warn(msg.format(from_info['path'],
to_info['bucket'],
to_info['path']), exc)
except Exception:
msg = "failed to upload '{}' to '{}/{}'"
logger.error(msg.format(from_info['path'],
to_info['bucket'],
to_info['path']))
continue
progress.finish_target(name)
@@ -206,11 +206,11 @@ class RemoteGS(RemoteBase):
bucket = gs.bucket(from_info['bucket'])
blob = bucket.get_blob(from_info['path'])
blob.download_to_filename(tmp_file)
except Exception as exc:
msg = "Failed to download '{}/{}' to '{}'"
logger.warn(msg.format(from_info['bucket'],
from_info['path'],
to_info['path']), exc)
except Exception:
msg = "failed to download '{}/{}' to '{}'"
logger.error(msg.format(from_info['bucket'],
from_info['path'],
to_info['path']))
continue
os.rename(tmp_file, to_info['path'])
dvc/remote/hdfs.py
+1 -1
View File
@@ -4,9 +4,9 @@ import getpass
import posixpath
from subprocess import Popen, PIPE
import dvc.logger as logger
from dvc.config import Config
from dvc.remote.base import RemoteBase, RemoteBaseCmdError
from dvc.logger import logger
from dvc.utils import fix_env
dvc/remote/http.py
+6 -6
View File
@@ -2,7 +2,7 @@ import os
import threading
import requests
from dvc.logger import logger
import dvc.logger as logger
from dvc.progress import progress
from dvc.exceptions import DvcException
from dvc.config import Config
@@ -72,9 +72,9 @@ class RemoteHTTP(RemoteBase):
try:
self._download_to(from_info['path'], tmp_file, callback=cb)
except Exception as exc:
msg = "Failed to download '{}'".format(from_info['path'])
logger.warn(msg, exc)
except Exception:
msg = "failed to download '{}'".format(from_info['path'])
logger.error(msg)
continue
os.rename(tmp_file, to_info['path'])
@@ -108,7 +108,7 @@ class RemoteHTTP(RemoteBase):
etag = self._request('HEAD', url).headers.get('ETag')
if not etag:
raise DvcException("Could not find an ETag for '{}'".format(url))
raise DvcException("could not find an ETag for '{}'".format(url))
if etag.startswith('W/'):
raise DvcException(
@@ -138,7 +138,7 @@ class RemoteHTTP(RemoteBase):
try:
return requests.request(method, url, **kwargs)
except requests.exceptions.RequestException:
raise DvcException('Could not perform a {} request'.format(method))
raise DvcException('could not perform a {} request'.format(method))
def gc(self):
raise NotImplementedError
dvc/remote/local.py
+20 -16
View File
@@ -7,9 +7,9 @@ import shutil
import posixpath
from operator import itemgetter
import dvc.logger as logger
from dvc.system import System
from dvc.remote.base import RemoteBase, STATUS_MAP, STATUS_NEW, STATUS_DELETED
from dvc.logger import logger
from dvc.utils import remove, move, copyfile, dict_md5, to_chunks
from dvc.utils import LARGE_DIR_SIZE
from dvc.config import Config
@@ -92,7 +92,7 @@ class RemoteLOCAL(RemoteBase):
if self.state.changed(cache, md5=md5):
if os.path.exists(cache):
msg = 'Corrupted cache file {}.'
logger.warn(msg.format(os.path.relpath(cache)))
logger.warning(msg.format(os.path.relpath(cache)))
remove(cache)
return True
return False
@@ -153,13 +153,14 @@ class RemoteLOCAL(RemoteBase):
logger.debug(msg)
return
except DvcException as exc:
msg = 'Cache type \'{}\' is not supported: {}'
msg = "Cache type '{}' is not supported: {}"
logger.debug(msg.format(self.cache_types[0], str(exc)))
del self.cache_types[0]
i -= 1
raise DvcException('No possible cache types left to try out.')
raise DvcException('no possible cache types left to try out.')
@property
def ospath(self):
@@ -233,13 +234,13 @@ class RemoteLOCAL(RemoteBase):
try:
with open(path, 'r') as fd:
d = json.load(fd)
except Exception as exc:
msg = u'Failed to load dir cache \'{}\''
logger.error(msg.format(os.path.relpath(path)), exc)
except Exception:
msg = u"Failed to load dir cache '{}'"
logger.error(msg.format(os.path.relpath(path)))
return []
if not isinstance(d, list):
msg = u'Dir cache file format error \'{}\': skipping the file'
msg = u"dir cache file format error '{}' [skipping the file]"
logger.error(msg.format(os.path.relpath(path)))
return []
@@ -480,10 +481,11 @@ class RemoteLOCAL(RemoteBase):
try:
copyfile(from_info['path'], to_info['path'], name=name)
except Exception as exc:
msg = "Failed to upload '{}' tp '{}'"
logger.warn(msg.format(from_info['path'],
to_info['path']), exc)
except Exception:
logger.error(
"failed to upload '{}' to '{}'"
.format(from_info['path'], to_info['path'])
)
def download(self,
from_infos,
@@ -512,10 +514,12 @@ class RemoteLOCAL(RemoteBase):
tmp_file,
no_progress_bar=no_progress_bar,
name=name)
except Exception as exc:
msg = "Failed to download '{}' to '{}'"
logger.warn(msg.format(from_info['path'],
to_info['path']), exc)
except Exception:
logger.error(
"failed to download '{}' to '{}'"
.format(from_info['path'], to_info['path'])
)
continue
os.rename(tmp_file, to_info['path'])
dvc/remote/s3.py
+7 -7
View File
@@ -11,7 +11,7 @@ try:
except ImportError:
from urllib.parse import urlparse
from dvc.logger import logger
import dvc.logger as logger
from dvc.progress import progress
from dvc.config import Config
from dvc.remote.base import RemoteBase
@@ -205,9 +205,9 @@ class RemoteS3(RemoteBase):
to_info['bucket'],
to_info['path'],
Callback=cb)
except Exception as exc:
msg = "Failed to upload '{}'".format(from_info['path'])
logger.warn(msg, exc)
except Exception:
msg = "failed to upload '{}'".format(from_info['path'])
logger.error(msg)
continue
progress.finish_target(name)
@@ -256,10 +256,10 @@ class RemoteS3(RemoteBase):
from_info['path'],
tmp_file,
Callback=cb)
except Exception as exc:
msg = "Failed to download '{}/{}'".format(from_info['bucket'],
except Exception:
msg = "failed to download '{}/{}'".format(from_info['bucket'],
from_info['path'])
logger.warn(msg, exc)
logger.error(msg)
continue
os.rename(tmp_file, to_info['path'])
dvc/remote/ssh.py
+12 -12
View File
@@ -8,7 +8,7 @@ except ImportError:
paramiko = None
import dvc.prompt as prompt
from dvc.logger import logger
import dvc.logger as logger
from dvc.progress import progress
from dvc.remote.base import RemoteBase, RemoteBaseCmdError
from dvc.config import Config
@@ -177,7 +177,7 @@ class RemoteSSH(RemoteBase):
md5cmd = 'md5sum'
index = 0
else:
msg = '\'{}\' is not supported as a remote'.format(stdout)
msg = "'{}' is not supported as a remote".format(stdout)
raise DvcException(msg)
stdout = self._exec(ssh, '{} {}'.format(md5cmd, path_info['path']))
@@ -290,11 +290,11 @@ class RemoteSSH(RemoteBase):
ssh.open_sftp().get(from_info['path'],
tmp_file,
callback=create_cb(name))
except Exception as exc:
msg = "Failed to download '{}/{}' to '{}'"
logger.warn(msg.format(from_info['host'],
from_info['path'],
to_info['path']), exc)
except Exception:
msg = "failed to download '{}/{}' to '{}'"
logger.error(msg.format(from_info['host'],
from_info['path'],
to_info['path']))
continue
os.rename(tmp_file, to_info['path'])
@@ -331,11 +331,11 @@ class RemoteSSH(RemoteBase):
sftp.put(from_info['path'],
to_info['path'],
callback=create_cb(name))
except Exception as exc:
msg = "Failed to upload '{}' to '{}/{}'"
logger.warn(msg.format(from_info['path'],
to_info['host'],
to_info['path'], exc))
except Exception:
msg = "failed to upload '{}' to '{}/{}'"
logger.error(msg.format(from_info['path'],
to_info['host'],
to_info['path']))
continue
progress.finish_target(name)
dvc/scm.py
+12 -8
View File
@@ -1,7 +1,7 @@
import os
import dvc.logger as logger
from dvc.exceptions import DvcException
from dvc.logger import logger
from dvc.utils import fix_env
@@ -213,12 +213,16 @@ class Git(Base):
# See https://github.com/iterative/dvc/issues/610 for more details.
try:
self.repo.index.add(paths)
except AssertionError as exc:
msg = 'Failed to add \'{}\' to git. You can add those files '
msg += 'manually using \'git add\'. '
msg += 'See \'https://github.com/iterative/dvc/issues/610\' '
msg += 'for more details.'
logger.error(msg.format(str(paths)), exc)
except AssertionError:
msg = (
"failed to add '{}' to git. You can add those files"
" manually using 'git add'."
" See 'https://github.com/iterative/dvc/issues/610'"
" for more details."
.format(str(paths))
)
logger.error(msg)
def commit(self, msg):
self.repo.index.commit(msg)
@@ -257,7 +261,7 @@ class Git(Base):
'hooks',
name)
if os.path.isfile(hook):
msg = 'Git hook \'{}\' already exists.'
msg = "git hook '{}' already exists."
raise SCMError(msg.format(os.path.relpath(hook)))
with open(hook, 'w+') as fd:
fd.write('#!/bin/sh\nexec dvc {}\n'.format(cmd))
dvc/stage.py
+43 -54
View File
@@ -5,16 +5,16 @@ import subprocess
from schema import Schema, SchemaError, Optional, Or, And
import dvc.prompt as prompt
import dvc.logger as logger
import dvc.dependency as dependency
import dvc.output as output
from dvc.exceptions import DvcException
from dvc.logger import logger
from dvc.utils import dict_md5, fix_env
class StageCmdFailedError(DvcException):
def __init__(self, stage):
msg = u'Stage \'{}\' cmd {} failed'.format(stage.relpath, stage.cmd)
msg = u"Stage '{}' cmd {} failed".format(stage.relpath, stage.cmd)
super(StageCmdFailedError, self).__init__(msg)
@@ -125,9 +125,7 @@ class Stage(object):
@property
def is_data_source(self):
"""
Whether the stage file was created with `dvc add`.
"""
"""Whether the stage file was created with `dvc add`."""
return self.cmd is None
@staticmethod
@@ -149,10 +147,7 @@ class Stage(object):
md5 = self._get_md5()
assert md5 is not None
if self.md5 == md5:
return False
return True
return self.md5 != md5
@property
def is_callback(self):
@@ -164,55 +159,50 @@ class Stage(object):
@property
def is_import(self):
"""
Whether the stage file was created with `dvc import`.
"""
"""Whether the stage file was created with `dvc import`."""
return (not self.cmd
and len(self.deps) == 1
and len(self.outs) == 1)
def _changed_deps(self, log):
def _changed_deps(self):
if self.locked:
return False
if self.is_callback:
msg = "Dvc file '{}' is a 'callback' stage (has a command and " \
"no dependencies) and thus always considered as changed."
self.project.logger.warn(msg.format(self.relpath))
logger.warning(msg.format(self.relpath))
return True
for dep in self.deps:
if not dep.changed():
continue
log("Dependency '{}' of '{}' changed.".format(dep, self.relpath))
msg = "Dependency '{}' of '{}' changed.".format(dep, self.relpath)
logger.warning(msg)
return True
return False
def _changed_outs(self, log):
def _changed_outs(self):
for out in self.outs:
if not out.changed():
continue
log("Output '{}' of '{}' changed.".format(out, self.relpath))
msg = "Output '{}' of '{}' changed.".format(out, self.relpath)
logger.warning(msg)
return True
return False
def _changed_md5(self, log):
def _changed_md5(self):
if self.changed_md5():
log("Dvc file '{}' changed.".format(self.relpath))
logger.warning("Dvc file '{}' changed.".format(self.relpath))
return True
return False
def changed(self, print_info=False):
if print_info:
log = self.project.logger.info
else:
log = self.project.logger.debug
ret = any([self._changed_deps(log),
self._changed_outs(log),
self._changed_md5(log)])
def changed(self):
ret = any([self._changed_deps(),
self._changed_outs(),
self._changed_md5()])
if ret:
msg = "Stage '{}' changed.".format(self.relpath)
@@ -221,7 +211,7 @@ class Stage(object):
msg = "Stage '{}' didn't change.".format(self.relpath)
color = 'green'
log(logger.colorize(msg, color))
logger.info(logger.colorize(msg, color))
return ret
@@ -243,7 +233,7 @@ class Stage(object):
os.unlink(self.path)
def reproduce(self, force=False, dry=False, interactive=False):
if not self.changed(print_info=True) and not force:
if not self.changed() and not force:
return None
if (self.cmd or self.is_import) and not self.locked and not dry:
@@ -254,14 +244,14 @@ class Stage(object):
msg = msg.format(self.relpath)
if interactive and not prompt.confirm(msg):
raise DvcException('Reproduction aborted by the user')
raise DvcException('reproduction aborted by the user')
self.project.logger.info(u'Reproducing \'{}\''.format(self.relpath))
logger.info(u"Reproducing '{}'".format(self.relpath))
self.run(dry=dry)
msg = u'\'{}\' was reproduced'.format(self.relpath)
self.project.logger.debug(msg)
msg = u"'{}' was reproduced".format(self.relpath)
logger.debug(msg)
return self
@@ -331,7 +321,7 @@ class Stage(object):
from dvc.remote.s3 import RemoteS3
old = Stage.load(self.project, self.path)
if old._changed_outs(log=self.project.logger.debug):
if old._changed_outs():
return False
# NOTE: need to save checksums for deps in order to compare them
@@ -383,7 +373,7 @@ class Stage(object):
stage._check_duplicated_arguments()
if fname is not None and os.path.basename(fname) != fname:
msg = "Stage file name '{}' should not contain subdirectories. " \
msg = "stage file name '{}' should not contain subdirectories. " \
"Use '-c|--cwd' to change location of the stage file."
raise StageFileBadNameError(msg.format(fname))
@@ -400,15 +390,14 @@ class Stage(object):
# NOTE: remove outs before we check build cache
if remove_outs:
stage.remove_outs(ignore_remove=False)
project.logger.warn("Build cache is ignored when using "
"--remove-outs.")
logger.warning("Build cache is ignored when using --remove-outs.")
ignore_build_cache = True
else:
stage.unprotect_outs()
if os.path.exists(path):
if not ignore_build_cache and stage.is_cached:
project.logger.info('Stage is cached, skipping.')
logger.info('Stage is cached, skipping.')
return None
msg = "'{}' already exists. Do you wish to run the command and " \
@@ -422,7 +411,7 @@ class Stage(object):
@staticmethod
def _check_dvc_filename(fname):
if not Stage.is_stage_filename(fname):
msg = "Bad stage filename '{}'. Stage files should be named " \
msg = "bad stage filename '{}'. Stage files should be named " \
"'Dvcfile' or have a '.dvc' suffix(e.g. '{}.dvc')."
raise StageFileBadNameError(msg.format(os.path.relpath(fname),
os.path.basename(fname)))
@@ -521,12 +510,13 @@ class Stage(object):
or os.path.basename(os.path.realpath(executable)) != 'fish':
return
msg = "DVC detected that you are using fish as your default " \
"shell. Be aware that it might cause problems by overwriting " \
"your current environment variables with values defined " \
"in '.fishrc', which might affect your command. See " \
"https://github.com/iterative/dvc/issues/1307. "
self.project.logger.warn(msg)
logger.warning(
"DVC detected that you are using fish as your default "
"shell. Be aware that it might cause problems by overwriting "
"your current environment variables with values defined "
"in '.fishrc', which might affect your command. See "
"https://github.com/iterative/dvc/issues/1307. "
)
def _check_circular_dependency(self):
from dvc.exceptions import CircularDependencyError
@@ -566,25 +556,24 @@ class Stage(object):
def run(self, dry=False):
if self.locked:
msg = u'Verifying outputs in locked stage \'{}\''
self.project.logger.info(msg.format(self.relpath))
msg = u"Verifying outputs in locked stage '{}'"
logger.info(msg.format(self.relpath))
if not dry:
self.check_missing_outputs()
elif self.is_import:
msg = u'Importing \'{}\' -> \'{}\''
self.project.logger.info(msg.format(self.deps[0].path,
self.outs[0].path))
msg = u"Importing '{}' -> '{}'"
logger.info(msg.format(self.deps[0].path, self.outs[0].path))
if not dry:
self.deps[0].download(self.outs[0].path_info)
elif self.is_data_source:
msg = u'Verifying data sources in \'{}\''.format(self.relpath)
self.project.logger.info(msg)
msg = u"Verifying data sources in '{}'".format(self.relpath)
logger.info(msg)
if not dry:
self.check_missing_outputs()
else:
msg = u'Running command:\n\t{}'.format(self.cmd)
self.project.logger.info(msg)
logger.info(msg)
if not dry:
self._run()
dvc/state.py
+7 -9
View File
@@ -3,11 +3,11 @@ import time
import sqlite3
import nanotime
import dvc.logger as logger
from dvc.config import Config
from dvc.system import System
from dvc.utils import file_md5, remove
from dvc.exceptions import DvcException
from dvc.logger import logger
class StateDuplicateError(DvcException):
@@ -134,17 +134,15 @@ class State(object):
version = ret[0][0]
if version > self.VERSION:
msg = "You are using an old version '{}' of dvc that is " \
msg = "you are using an old version '{}' of dvc that is " \
"using state file version '{}' which is not " \
"compatible with the state file version '{}' that " \
"is used in this projet. Please upgrade right now!"
raise DvcException(msg.format(VERSION,
self.VERSION,
version))
raise DvcException(msg.format(VERSION, self.VERSION, version))
elif version < self.VERSION:
msg = "State file version '{}' is too old. " \
"Reformatting to the current version '{}'."
self.project.logger.warn(msg.format(version, self.VERSION))
logger.warning(msg.format(version, self.VERSION))
cmd = "DROP TABLE IF EXISTS {};"
self._execute(cmd.format(self.STATE_TABLE))
self._execute(cmd.format(self.STATE_INFO_TABLE))
@@ -212,8 +210,8 @@ class State(object):
count = self._from_sqlite(ret[0][0]) + self.inserts
if count > self.row_limit:
msg = "Cleaning up state. This might take a while."
self.project.logger.warn(msg)
msg = "cleaning up state, this might take a while."
logger.warning(msg)
delete = (count - self.row_limit)
delete += int(self.row_limit * (self.row_cleanup_quota/100.))
@@ -372,7 +370,7 @@ class State(object):
mtime, _ = self.mtime_and_size(path)
if i == inode and m == mtime:
logger.debug('Removing \'{}\' as unused link.'.format(path))
logger.debug("Removing '{}' as unused link.".format(path))
remove(path)
unused.append(p)
dvc/system.py
+2 -2
View File
@@ -38,7 +38,7 @@ class System(object):
os.symlink(source, link_name)
return
except Exception as exc:
msg = "Failed to symlink '{}' -> '{}': {}"
msg = "failed to symlink '{}' -> '{}': {}"
raise DvcException(msg.format(source, link_name, str(exc)))
flags = 0
@@ -119,7 +119,7 @@ class System(object):
ret = -1
if ret != 0:
raise DvcException('Reflink is not supported')
raise DvcException('reflink is not supported')
@staticmethod
def getdirinfo(path):
dvc/updater.py
+6 -4
View File
@@ -4,9 +4,9 @@ import time
import requests
import colorama
import dvc.logger as logger
from dvc import VERSION_BASE
from dvc.lock import Lock, LockError
from dvc.logger import logger
from dvc.utils import is_binary
@@ -99,6 +99,9 @@ class Updater(object): # pragma: no cover
return l_patch > c_patch
def _notify(self):
if not sys.stdout.isatty():
return
message = (
'Update available {red}{current}{reset} -> {green}{latest}{reset}'
+ '\n'
@@ -111,8 +114,7 @@ class Updater(object): # pragma: no cover
current=self.current,
latest=self.latest)
if sys.stdout.isatty():
logger.box(message, border_color='yellow')
logger.box(message, border_color='yellow')
def _get_update_instructions(self):
instructions = {
@@ -186,6 +188,6 @@ class Updater(object): # pragma: no cover
system = platform.system()
func = m.get(system)
if func is None:
raise DvcException("Not supported system '{}'".format(system))
raise DvcException("not supported system '{}'".format(system))
return func()
dvc/utils.py
+2 -2
View File
@@ -9,9 +9,9 @@ import json
import shutil
import hashlib
import dvc.logger as logger
from dvc.progress import progress
from dvc.istextfile import istextfile
from dvc.logger import logger
LOCAL_CHUNK_SIZE = 1024*1024
@@ -143,7 +143,7 @@ def remove(path):
if not os.path.exists(path):
return
logger.debug(u'Removing \'{}\''.format(os.path.relpath(path)))
logger.debug(u"Removing '{}'".format(os.path.relpath(path)))
def _chmod(func, p, excinfo):
perm = os.stat(p).st_mode
tests/basic_env.py
+2 -1
View File
@@ -4,6 +4,7 @@ import tempfile
from git import Repo
from unittest import TestCase
import dvc.logger as logger
from dvc.project import Project
@@ -84,4 +85,4 @@ class TestDvc(TestGit):
def setUp(self):
super(TestDvc, self).setUp()
self.dvc = Project.init(self._root_dir)
self.dvc.logger.be_verbose()
logger.be_verbose()
tests/test_add.py
+1 -1
View File
@@ -77,7 +77,7 @@ class TestAddDirectoryWithForwardSlash(TestDvc):
self.assertEqual(len(stages), 1)
stage = stages[0]
self.assertTrue(stage is not None)
self.assertEquals(os.path.abspath('directory.dvc'), stage.path)
self.assertEqual(os.path.abspath('directory.dvc'), stage.path)
class TestAddTrackedFile(TestDvc):
tests/test_cache.py
+3 -3
View File
@@ -23,13 +23,13 @@ class TestCache(TestDvc):
def test_all(self):
md5_list = list(Cache(self.dvc).local.all())
self.assertEquals(len(md5_list), 2)
self.assertEqual(len(md5_list), 2)
self.assertIn(self.cache1_md5, md5_list)
self.assertIn(self.cache2_md5, md5_list)
def test_get(self):
cache = Cache(self.dvc).local.get(self.cache1_md5)
self.assertEquals(cache, self.cache1)
self.assertEqual(cache, self.cache1)
class TestCacheLoadBadDirCache(TestDvc):
@@ -63,7 +63,7 @@ class TestExternalCacheDir(TestDvc):
self.assertEqual(ret, 0)
self.assertFalse(os.path.exists('.dvc/cache'))
self.assertNotEquals(len(os.listdir(cache_dir)), 0)
self.assertNotEqual(len(os.listdir(cache_dir)), 0)
class TestSharedCacheDir(TestDir):
tests/test_logger.py
+88 -126
View File
@@ -1,4 +1,5 @@
from mock import patch
from unittest import TestCase
try:
from StringIO import StringIO
@@ -6,160 +7,121 @@ except ImportError:
from io import StringIO
import logging
import dvc.logger as logger
from dvc.command.base import CmdBase
from dvc.logger import Logger, logger
from tests.basic_env import TestDvc
class StringContaining(str):
def __eq__(self, other):
return self in other
class TestLogger(TestCase):
handlers = logger.logger.handlers
color_patch = patch.object(logger, 'colorize', new=lambda x, color='': x)
def setUp(self):
logger.logger.handlers[0].stream = StringIO()
logger.logger.handlers[1].stream = StringIO()
logger._set_default_level()
self.color_patch.start()
class TestLogger(TestDvc):
def test_config(self):
logger.set_level()
logger.set_level('debug')
self.assertEqual(logger.logger.getEffectiveLevel(), logging.DEBUG)
logger.set_level('error')
self.assertEqual(logger.logger.getEffectiveLevel(), logging.ERROR)
def test_set_level(self):
logger.set_level()
logger.set_level('debug')
self.assertEqual(logger.logger.getEffectiveLevel(), logging.DEBUG)
def test_be_quiet(self):
logger.set_level()
logger.be_quiet()
self.assertEqual(logger.logger.getEffectiveLevel(), logging.CRITICAL)
def test_be_verbose(self):
logger.set_level()
def tearDown(self):
logger.logger.handlers = self.handlers
logger.be_verbose()
self.assertEqual(logger.logger.getEffectiveLevel(), logging.DEBUG)
self.color_patch.stop()
def test_colorize(self):
for name, color in logger.COLOR_MAP.items():
msg = logger.colorize(name, name)
# This is not a tty, so it should not colorize anything
self.assertEqual(msg, name)
@property
def stdout(self):
return logger.logger.handlers[0].stream.getvalue()
def test_handlers(self):
Logger()
Logger()
self.assertEqual(len(logger.logger.handlers), 2)
@property
def stderr(self):
return logger.logger.handlers[1].stream.getvalue()
@patch('sys.stderr', new_callable=StringIO)
@patch('sys.stdout', new_callable=StringIO)
def test_stderr(self, mock_stdout, mock_stderr):
logger = Logger(force=True)
error_message = 'error msg'
logger.error(error_message)
self.assertEqual('', mock_stdout.getvalue())
self.assertEqual('Error: {}\n\nHaving any troubles? '
'Hit us up at dvc.org/support, we '
'are always happy to help!\n'.format(error_message),
mock_stderr.getvalue())
def test_info(self):
logger.info('message')
@patch('sys.stderr', new_callable=StringIO)
@patch('sys.stdout', new_callable=StringIO)
def test_stdout(self, mock_stdout, mock_stderr):
logger = Logger(force=True)
non_error_message = 'non-error message'
logger.info(non_error_message)
self.assertEqual('', mock_stderr.getvalue())
self.assertEqual('{}\n'.format(non_error_message),
mock_stdout.getvalue())
self.assertEqual(self.stdout, 'message\n')
def test_debug(self):
with logger.verbose():
logger.debug('message')
class TestLoggerException(TestDvc):
def setUp(self):
super(TestLoggerException, self).setUp()
self.assertEqual(self.stdout, 'Debug: message\n')
@patch('sys.stderr', new_callable=StringIO)
@patch('sys.stdout', new_callable=StringIO)
def test_empty_err_msg(self, mock_stdout, mock_stderr):
from dvc.exceptions import DvcException
def test_warning(self):
logger.warning('message')
logger = Logger(force=True)
self.assertEqual(self.stdout, 'Warning: message\n')
exc_msg = 'msg'
logger.error("", exc=DvcException(exc_msg))
def test_error(self):
logger.error('message')
self.assertEqual('', mock_stdout.getvalue())
self.assertEqual('Error: {}\n\nHaving any troubles? '
'Hit us up at dvc.org/support, we '
'are always happy to help!\n'.format(exc_msg),
mock_stderr.getvalue())
output = (
'Error: message\n'
'\n'
'Having any troubles? Hit us up at https://dvc.org/support,'
' we are always happy to help!\n'
)
@patch('sys.stderr', new_callable=StringIO)
@patch('sys.stdout', new_callable=StringIO)
def test_empty_exc_msg(self, mock_stdout, mock_stderr):
from dvc.exceptions import DvcException
self.assertEqual(self.stderr, output)
logger = Logger(force=True)
def test_error_with_exception(self):
try:
raise Exception('exception description')
except Exception:
logger.error('')
err_msg = 'msg'
logger.error(err_msg, exc=DvcException(""))
output = (
'Error: exception description\n'
'\n'
'Having any troubles? Hit us up at https://dvc.org/support,'
' we are always happy to help!\n'
)
self.assertEqual('', mock_stdout.getvalue())
self.assertEqual('Error: {}\n\nHaving any troubles? '
'Hit us up at dvc.org/support, we '
'are always happy to help!\n'.format(err_msg),
mock_stderr.getvalue())
self.assertEqual(self.stderr, output)
@patch('sys.stderr', new_callable=StringIO)
@patch('sys.stdout', new_callable=StringIO)
def test_msg_and_exc(self, mock_stdout, mock_stderr):
from dvc.exceptions import DvcException
def test_error_with_exception_and_message(self):
try:
raise Exception('exception description')
except Exception:
logger.error('message')
logger = Logger(force=True)
output = (
'Error: message - exception description\n'
'\n'
'Having any troubles? Hit us up at https://dvc.org/support,'
' we are always happy to help!\n'
)
err_msg = 'error'
exc_msg = 'exception'
logger.error(err_msg, exc=DvcException(exc_msg))
self.assertEqual(self.stderr, output)
self.assertEqual('', mock_stdout.getvalue())
self.assertEqual('Error: {}: {}\n\nHaving any troubles? '
'Hit us up at dvc.org/support, we are always '
'happy to help!\n'.format(err_msg, exc_msg),
mock_stderr.getvalue())
class TestLoggerQuiet(TestDvc):
def setUp(self):
super(TestLoggerQuiet, self).setUp()
class A(object):
quiet = True
verbose = False
CmdBase._set_loglevel(A())
@patch('sys.stderr', new_callable=StringIO)
@patch('sys.stdout', new_callable=StringIO)
def test_progress(self, mock_stdout, mock_stderr):
from dvc.progress import progress
progress.update_target('target', 50, 100)
self.assertEqual('', mock_stdout.getvalue())
self.assertEqual('', mock_stderr.getvalue())
@patch('sys.stderr', new_callable=StringIO)
@patch('sys.stdout', new_callable=StringIO)
def test_box(self, mock_stdout, mock_stderr):
def test_box(self):
logger.box('message')
self.assertEqual('', mock_stdout.getvalue())
self.assertEqual('', mock_stderr.getvalue())
output = (
'+-----------------+\n'
'| |\n'
'| message |\n'
'| |\n'
'+-----------------+\n'
'\n'
)
class TestLoggerCLI(TestDvc):
def test(self):
from dvc.logger import logger
self.assertEqual(self.stdout, output)
def test_level(self):
self.assertEqual(logger.level(), logging.INFO)
def test_is_verbose(self):
self.assertFalse(logger.is_verbose())
with logger.verbose():
self.assertTrue(logger.is_verbose())
def test_is_quiet(self):
self.assertFalse(logger.is_quiet())
with logger.quiet():
self.assertFalse(logger.is_verbose())
def test_cli(self):
class A(object):
quiet = True
verbose = False
tests/test_repro.py
-1
View File
@@ -642,7 +642,6 @@ class TestReproExternalBase(TestDvc):
self.assertEqual(ret, 0)
self.dvc = Project('.')
self.dvc.logger.be_verbose()
foo_key = remote_key + self.sep + self.FOO
bar_key = remote_key + self.sep + self.BAR