dvc: comply with both flake8 and black

flake8 provides some additional features like unused vars check, etc, so
it makes total sense to use it as well.

Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
Ruslan Kuprieiev
2019-02-04 00:10:36 +02:00
parent 1cc2e0edcb
commit 4896f08ddf
69 changed files with 940 additions and 268 deletions
+4
View File
@@ -0,0 +1,4 @@
[flake8]
ignore = E203, E266, E501, W503, F403, F401
max-line-length = 79
select = B,C,E,F,W,T4,B9
+5
View File
@@ -4,3 +4,8 @@ repos:
hooks:
- id: black
language_version: python3.6
- repo: https://gitlab.com/pycqa/flake8
rev: master
hooks:
- id: flake8
language_version: python3.6
+3 -1
View File
@@ -228,7 +228,9 @@ class Analytics(object):
core = config.config.get(Config.SECTION_CORE, {})
enabled = core.get(Config.SECTION_CORE_ANALYTICS, True)
logger.debug("Analytics is {}abled.".format("en" if enabled else "dis"))
logger.debug(
"Analytics is {}abled.".format("en" if enabled else "dis")
)
return enabled
@staticmethod
+3 -1
View File
@@ -35,7 +35,9 @@ class Cache(object):
protected = config.get(Config.SECTION_CACHE_PROTECTED)
sect = {
Config.PRIVATE_CWD: config.get(Config.PRIVATE_CWD, project.dvc_dir),
Config.PRIVATE_CWD: config.get(
Config.PRIVATE_CWD, project.dvc_dir
),
Config.SECTION_REMOTE_URL: cache_dir,
Config.SECTION_CACHE_TYPE: cache_type,
Config.SECTION_CACHE_PROTECTED: protected,
+10 -2
View File
@@ -101,7 +101,11 @@ def parse_args(argv=None):
"-q", "--quiet", action="store_true", default=False, help="Be quiet."
)
log_level_group.add_argument(
"-v", "--verbose", action="store_true", default=False, help="Be verbose."
"-v",
"--verbose",
action="store_true",
default=False,
help="Be verbose.",
)
# Main parser
@@ -116,7 +120,11 @@ def parse_args(argv=None):
# NOTE: On some python versions action='version' prints to stderr
# instead of stdout https://bugs.python.org/issue18920
parser.add_argument(
"-V", "--version", action=VersionAction, nargs=0, help="Show program's version."
"-V",
"--version",
action=VersionAction,
nargs=0,
help="Show program's version.",
)
# Sub commands
+3 -1
View File
@@ -28,5 +28,7 @@ def add_parser(subparsers, parent_parser):
default=False,
help="Recursively add each file under the directory.",
)
add_parser.add_argument("targets", nargs="+", help="Input files/directories.")
add_parser.add_argument(
"targets", nargs="+", help="Input files/directories."
)
add_parser.set_defaults(func=CmdAdd)
+9 -2
View File
@@ -22,7 +22,10 @@ def add_parser(subparsers, parent_parser):
CACHE_HELP = "Manage cache settings."
cache_parser = subparsers.add_parser(
"cache", parents=[parent_parser], description=CACHE_HELP, help=CACHE_HELP
"cache",
parents=[parent_parser],
description=CACHE_HELP,
help=CACHE_HELP,
)
cache_subparsers = cache_parser.add_subparsers(
@@ -42,7 +45,11 @@ def add_parser(subparsers, parent_parser):
help=CACHE_DIR_HELP,
)
cache_dir_parser.add_argument(
"-u", "--unset", default=False, action="store_true", help="Unset option."
"-u",
"--unset",
default=False,
action="store_true",
help="Unset option.",
)
cache_dir_parser.add_argument(
"value",
+3 -1
View File
@@ -10,7 +10,9 @@ class CmdCheckout(CmdBase):
else:
for target in self.args.targets:
self.project.checkout(
target=target, with_deps=self.args.with_deps, force=self.args.force
target=target,
with_deps=self.args.with_deps,
force=self.args.force,
)
return 0
+11 -3
View File
@@ -64,7 +64,9 @@ class CmdConfig(CmdBase):
self.config.set(self.configobj, section, opt, value)
self.config.save(self.configobj)
except DvcException:
logger.error("failed to set '{}.{}' to '{}'".format(section, opt, value))
logger.error(
"failed to set '{}.{}' to '{}'".format(section, opt, value)
)
return 1
return 0
@@ -104,8 +106,14 @@ def add_parser(subparsers, parent_parser):
help=CONFIG_HELP,
)
config_parser.add_argument(
"-u", "--unset", default=False, action="store_true", help="Unset option."
"-u",
"--unset",
default=False,
action="store_true",
help="Unset option.",
)
config_parser.add_argument("name", help="Option name.")
config_parser.add_argument("value", nargs="?", default=None, help="Option value.")
config_parser.add_argument(
"value", nargs="?", default=None, help="Option value."
)
config_parser.set_defaults(func=CmdConfig)
+4 -1
View File
@@ -40,7 +40,10 @@ class CmdDaemonAnalytics(CmdDaemonBase):
def add_parser(subparsers, parent_parser):
DAEMON_HELP = "Service daemon."
daemon_parser = subparsers.add_parser(
"daemon", parents=[parent_parser], description=DAEMON_HELP, help=DAEMON_HELP
"daemon",
parents=[parent_parser],
description=DAEMON_HELP,
help=DAEMON_HELP,
)
daemon_subparsers = daemon_parser.add_subparsers(
+21 -6
View File
@@ -106,9 +106,14 @@ def add_parser(subparsers, parent_parser):
# Pull
PULL_HELP = "Pull data files from the cloud."
pull_parser = subparsers.add_parser(
"pull", parents=[parent_cache_parser], description=PULL_HELP, help=PULL_HELP
"pull",
parents=[parent_cache_parser],
description=PULL_HELP,
help=PULL_HELP,
)
pull_parser.add_argument(
"-r", "--remote", help="Remote repository to pull from."
)
pull_parser.add_argument("-r", "--remote", help="Remote repository to pull from.")
pull_parser.add_argument(
"-a",
"--all-branches",
@@ -149,9 +154,14 @@ def add_parser(subparsers, parent_parser):
# Push
PUSH_HELP = "Push data files to the cloud."
push_parser = subparsers.add_parser(
"push", parents=[parent_cache_parser], description=PUSH_HELP, help=PUSH_HELP
"push",
parents=[parent_cache_parser],
description=PUSH_HELP,
help=PUSH_HELP,
)
push_parser.add_argument(
"-r", "--remote", help="Remote repository to push to."
)
push_parser.add_argument("-r", "--remote", help="Remote repository to push to.")
push_parser.add_argument(
"-a",
"--all-branches",
@@ -185,9 +195,14 @@ def add_parser(subparsers, parent_parser):
# Fetch
FETCH_HELP = "Fetch data files from the cloud."
fetch_parser = subparsers.add_parser(
"fetch", parents=[parent_cache_parser], description=FETCH_HELP, help=FETCH_HELP
"fetch",
parents=[parent_cache_parser],
description=FETCH_HELP,
help=FETCH_HELP,
)
fetch_parser.add_argument(
"-r", "--remote", help="Remote repository to fetch from."
)
fetch_parser.add_argument("-r", "--remote", help="Remote repository to fetch from.")
fetch_parser.add_argument(
"-a",
"--all-branches",
+11 -3
View File
@@ -31,12 +31,20 @@ class CmdDestroy(CmdBase):
def add_parser(subparsers, parent_parser):
DESTROY_HELP = (
"Destroy dvc. Will remove all project's information, " "data files and cache."
"Destroy dvc. Will remove all project's information, "
"data files and cache."
)
destroy_parser = subparsers.add_parser(
"destroy", parents=[parent_parser], description=DESTROY_HELP, help=DESTROY_HELP
"destroy",
parents=[parent_parser],
description=DESTROY_HELP,
help=DESTROY_HELP,
)
destroy_parser.add_argument(
"-f", "--force", action="store_true", default=False, help="Force destruction."
"-f",
"--force",
action="store_true",
default=False,
help="Force destruction.",
)
destroy_parser.set_defaults(func=CmdDestroy)
+4 -1
View File
@@ -30,7 +30,10 @@ class CmdImport(CmdBase):
def add_parser(subparsers, parent_parser):
IMPORT_HELP = "Import files from URL."
import_parser = subparsers.add_parser(
"import", parents=[parent_parser], description=IMPORT_HELP, help=IMPORT_HELP
"import",
parents=[parent_parser],
description=IMPORT_HELP,
help=IMPORT_HELP,
)
import_parser.add_argument(
"url",
+3 -1
View File
@@ -23,7 +23,9 @@ class CmdInit(object):
def add_parser(subparsers, parent_parser):
"""Setup parser for `dvc init`."""
INIT_HELP = "Initialize dvc over a directory " "(should already be a git dir)."
INIT_HELP = (
"Initialize dvc over a directory " "(should already be a git dir)."
)
init_parser = subparsers.add_parser(
"init", parents=[parent_parser], description=INIT_HELP, help=INIT_HELP
)
+4 -1
View File
@@ -17,6 +17,9 @@ class CmdInstall(CmdBase):
def add_parser(subparsers, parent_parser):
INSTALL_HELP = "Install dvc hooks into the repository."
install_parser = subparsers.add_parser(
"install", parents=[parent_parser], description=INSTALL_HELP, help=INSTALL_HELP
"install",
parents=[parent_parser],
description=INSTALL_HELP,
help=INSTALL_HELP,
)
install_parser.set_defaults(func=CmdInstall)
+7 -2
View File
@@ -12,7 +12,9 @@ class CmdLockBase(CmdBase):
self.project.lock_stage(target, unlock=unlock)
except DvcException:
logger.error(
"failed to {}lock '{}'".format("un" if unlock else "", target)
"failed to {}lock '{}'".format(
"un" if unlock else "", target
)
)
return 1
@@ -39,7 +41,10 @@ def add_parser(subparsers, parent_parser):
UNLOCK_HELP = "Unlock DVC file."
unlock_parser = subparsers.add_parser(
"unlock", parents=[parent_parser], description=UNLOCK_HELP, help=UNLOCK_HELP
"unlock",
parents=[parent_parser],
description=UNLOCK_HELP,
help=UNLOCK_HELP,
)
unlock_parser.add_argument("targets", nargs="+", help="DVC files.")
unlock_parser.set_defaults(func=CmdUnlock)
+9 -3
View File
@@ -59,7 +59,9 @@ class CmdMetricsModify(CmdBase):
class CmdMetricsAdd(CmdBase):
def run(self):
try:
self.project.metrics_add(self.args.path, self.args.type, self.args.xpath)
self.project.metrics_add(
self.args.path, self.args.type, self.args.xpath
)
except DvcException:
msg = "failed to add metric file '{}'".format(self.args.path)
logger.error(msg)
@@ -83,11 +85,15 @@ class CmdMetricsRemove(CmdBase):
def add_parser(subparsers, parent_parser):
METRICS_HELP = "Get metrics from all branches."
metrics_parser = subparsers.add_parser(
"metrics", parents=[parent_parser], description=METRICS_HELP, help=METRICS_HELP
"metrics",
parents=[parent_parser],
description=METRICS_HELP,
help=METRICS_HELP,
)
metrics_subparsers = metrics_parser.add_subparsers(
dest="cmd", help="Use dvc metrics CMD --help for command-specific help."
dest="cmd",
help="Use dvc metrics CMD --help for command-specific help.",
)
fix_subparsers(metrics_subparsers)
+6 -2
View File
@@ -10,7 +10,9 @@ class CmdMove(CmdBase):
try:
self.project.move(self.args.src, self.args.dst)
except DvcException:
msg = "failed to move '{}' -> '{}'".format(self.args.src, self.args.dst)
msg = "failed to move '{}' -> '{}'".format(
self.args.src, self.args.dst
)
logger.error(msg)
return 1
return 0
@@ -26,6 +28,8 @@ def add_parser(subparsers, parent_parser):
move_parser = subparsers.add_parser(
"move", parents=[parent_parser], description=description, help=help
)
move_parser.add_argument("src", help="Source path to a data file or directory.")
move_parser.add_argument(
"src", help="Source path to a data file or directory."
)
move_parser.add_argument("dst", help="Destination path.")
move_parser.set_defaults(func=CmdMove)
+19 -6
View File
@@ -35,7 +35,9 @@ class CmdPipelineShow(CmdBase):
stage = Stage.load(self.project, target)
node = os.path.relpath(stage.path, self.project.root_dir)
pipelines = list(filter(lambda g: node in g.nodes(), self.project.pipelines()))
pipelines = list(
filter(lambda g: node in g.nodes(), self.project.pipelines())
)
assert len(pipelines) == 1
G = pipelines[0]
@@ -99,10 +101,15 @@ class CmdPipelineShow(CmdBase):
for target in self.args.targets:
try:
if self.args.ascii:
self._show_ascii(target, self.args.commands, self.args.outs)
self._show_ascii(
target, self.args.commands, self.args.outs
)
elif self.args.dot:
self.__write_dot(
target, self.args.commands, self.args.outs, self.args.dot
target,
self.args.commands,
self.args.outs,
self.args.dot,
)
else:
self._show(target, self.args.commands, self.args.outs)
@@ -139,7 +146,8 @@ def add_parser(subparsers, parent_parser):
)
pipeline_subparsers = pipeline_parser.add_subparsers(
dest="cmd", help="Use dvc pipeline CMD --help for command-specific help."
dest="cmd",
help="Use dvc pipeline CMD --help for command-specific help.",
)
fix_subparsers(pipeline_subparsers)
@@ -167,9 +175,14 @@ def add_parser(subparsers, parent_parser):
help="Print output files instead of paths to DVC files.",
)
pipeline_show_parser.add_argument(
"--ascii", action="store_true", default=False, help="Output DAG as ASCII."
"--ascii",
action="store_true",
default=False,
help="Output DAG as ASCII.",
)
pipeline_show_parser.add_argument(
"--dot", help="Write DAG in .dot format."
)
pipeline_show_parser.add_argument("--dot", help="Write DAG in .dot format.")
pipeline_show_parser.add_argument(
"targets", nargs="*", help="DVC files. 'Dvcfile' by default."
)
+23 -7
View File
@@ -32,7 +32,9 @@ class CmdRemoteAdd(CmdConfig):
def run(self):
remote = _get({Config.SECTION_REMOTE_URL: self.args.url})
if remote == RemoteLOCAL:
self.args.url = self.resolve_path(self.args.url, self.configobj.filename)
self.args.url = self.resolve_path(
self.args.url, self.configobj.filename
)
section = Config.SECTION_REMOTE_FMT.format(self.args.name)
ret = self._set(section, Config.SECTION_REMOTE_URL, self.args.url)
@@ -61,7 +63,9 @@ class CmdRemoteRemove(CmdConfig):
if default == self.args.name:
return self._unset(
Config.SECTION_CORE, opt=Config.SECTION_CORE_REMOTE, configobj=config
Config.SECTION_CORE,
opt=Config.SECTION_CORE_REMOTE,
configobj=config,
)
def run(self):
@@ -104,7 +108,9 @@ class CmdRemoteList(CmdConfig):
r = re.match(Config.SECTION_REMOTE_REGEX, section)
if r:
name = r.group("name")
url = self.configobj[section].get(Config.SECTION_REMOTE_URL, "")
url = self.configobj[section].get(
Config.SECTION_REMOTE_URL, ""
)
logger.info("{}\t{}".format(name, url))
return 0
@@ -114,11 +120,15 @@ def add_parser(subparsers, parent_parser):
REMOTE_HELP = "Manage set of tracked repositories."
remote_parser = subparsers.add_parser(
"remote", parents=[parent_parser], description=REMOTE_HELP, help=REMOTE_HELP
"remote",
parents=[parent_parser],
description=REMOTE_HELP,
help=REMOTE_HELP,
)
remote_subparsers = remote_parser.add_subparsers(
dest="cmd", help="Use dvc remote CMD --help for " "command-specific help."
dest="cmd",
help="Use dvc remote CMD --help for " "command-specific help.",
)
fix_subparsers(remote_subparsers)
@@ -152,7 +162,9 @@ def add_parser(subparsers, parent_parser):
description=REMOTE_DEFAULT_HELP,
help=REMOTE_DEFAULT_HELP,
)
remote_default_parser.add_argument("name", nargs="?", help="Name of the remote.")
remote_default_parser.add_argument(
"name", nargs="?", help="Name of the remote."
)
remote_default_parser.add_argument(
"-u",
"--unset",
@@ -183,7 +195,11 @@ def add_parser(subparsers, parent_parser):
remote_modify_parser.add_argument("option", help="Option.")
remote_modify_parser.add_argument("value", nargs="?", help="Value.")
remote_modify_parser.add_argument(
"-u", "--unset", default=False, action="store_true", help="Unset option."
"-u",
"--unset",
default=False,
action="store_true",
help="Unset option.",
)
remote_modify_parser.set_defaults(func=CmdRemoteModify)
+14 -4
View File
@@ -14,13 +14,16 @@ class CmdRemove(CmdBase):
if self.args.force:
return False
msg = "Are you sure you want to remove {} with its outputs?".format(target)
msg = "Are you sure you want to remove {} with its outputs?".format(
target
)
if prompt.confirm(msg):
return False
raise DvcException(
"Cannot purge without a confirmation from the user." " Use '-f' to force."
"Cannot purge without a confirmation from the user."
" Use '-f' to force."
)
def run(self):
@@ -37,7 +40,10 @@ class CmdRemove(CmdBase):
def add_parser(subparsers, parent_parser):
REMOVE_HELP = "Remove outputs of DVC file."
remove_parser = subparsers.add_parser(
"remove", parents=[parent_parser], description=REMOVE_HELP, help=REMOVE_HELP
"remove",
parents=[parent_parser],
description=REMOVE_HELP,
help=REMOVE_HELP,
)
remove_parser_group = remove_parser.add_mutually_exclusive_group()
remove_parser_group.add_argument(
@@ -55,7 +61,11 @@ def add_parser(subparsers, parent_parser):
help="Remove DVC file and all its outputs.",
)
remove_parser.add_argument(
"-f", "--force", action="store_true", default=False, help="Force purge."
"-f",
"--force",
action="store_true",
default=False,
help="Force purge.",
)
remove_parser.add_argument("targets", nargs="+", help="DVC files.")
remove_parser.set_defaults(func=CmdRemove)
+9 -3
View File
@@ -52,9 +52,14 @@ class CmdRepro(CmdBase):
def add_parser(subparsers, parent_parser):
REPRO_HELP = "Reproduce DVC file. Default file name - 'Dvcfile'."
repro_parser = subparsers.add_parser(
"repro", parents=[parent_parser], description=REPRO_HELP, help=REPRO_HELP
"repro",
parents=[parent_parser],
description=REPRO_HELP,
help=REPRO_HELP,
)
repro_parser.add_argument(
"targets", nargs="*", help="DVC file to reproduce."
)
repro_parser.add_argument("targets", nargs="*", help="DVC file to reproduce.")
repro_parser.add_argument(
"-f",
"--force",
@@ -67,7 +72,8 @@ def add_parser(subparsers, parent_parser):
"--single-item",
action="store_true",
default=False,
help="Reproduce only single data item without recursive dependencies " "check.",
help="Reproduce only single data item without recursive dependencies "
"check.",
)
repro_parser.add_argument(
"-c",
+10 -2
View File
@@ -13,7 +13,12 @@ class CmdRun(CmdBase):
overwrite = self.args.yes or self.args.overwrite_dvcfile
if not any(
[self.args.deps, self.args.outs, self.args.outs_no_cache, self.args.command]
[
self.args.deps,
self.args.outs,
self.args.outs_no_cache,
self.args.command,
]
): # pragma: no cover
logger.error(
"too few arguments. Specify at least one: '-d', "
@@ -64,7 +69,10 @@ class CmdRun(CmdBase):
def add_parser(subparsers, parent_parser):
RUN_HELP = "Generate a stage file from a given " "command and execute the command."
RUN_HELP = (
"Generate a stage file from a given "
"command and execute the command."
)
run_parser = subparsers.add_parser(
"run", parents=[parent_parser], description=RUN_HELP, help=RUN_HELP
)
+3 -1
View File
@@ -25,5 +25,7 @@ def add_parser(subparsers, parent_parser):
description=UNPROTECT_HELP,
help=UNPROTECT_HELP,
)
unprotect_parser.add_argument("targets", nargs="+", help="Data files/directory.")
unprotect_parser.add_argument(
"targets", nargs="+", help="Data files/directory."
)
unprotect_parser.set_defaults(func=CmdUnprotect)
+27 -9
View File
@@ -22,7 +22,9 @@ class ConfigError(DvcException):
"""
def __init__(self, msg, ex=None):
super(ConfigError, self).__init__("config file error: {}".format(msg), ex)
super(ConfigError, self).__init__(
"config file error: {}".format(msg), ex
)
def supported_cache_type(types):
@@ -178,7 +180,9 @@ class Config(object): # pylint: disable=too-many-instance-attributes
Optional(
SECTION_CORE_INTERACTIVE, default=False
): SECTION_CORE_INTERACTIVE_SCHEMA,
Optional(SECTION_CORE_ANALYTICS, default=True): SECTION_CORE_ANALYTICS_SCHEMA,
Optional(
SECTION_CORE_ANALYTICS, default=True
): SECTION_CORE_ANALYTICS_SCHEMA,
# backward compatibility
Optional(SECTION_CORE_CLOUD, default=""): SECTION_CORE_CLOUD_SCHEMA,
Optional(SECTION_CORE_STORAGEPATH, default=""): str,
@@ -198,7 +202,9 @@ class Config(object): # pylint: disable=too-many-instance-attributes
Optional(SECTION_AWS_PROFILE): str,
Optional(SECTION_AWS_CREDENTIALPATH): str,
Optional(SECTION_AWS_ENDPOINT_URL): str,
Optional(SECTION_AWS_USE_SSL, default=True): And(str, is_bool, Use(to_bool)),
Optional(SECTION_AWS_USE_SSL, default=True): And(
str, is_bool, Use(to_bool)
),
}
# backward compatibility
@@ -232,7 +238,9 @@ class Config(object): # pylint: disable=too-many-instance-attributes
Optional(SECTION_AWS_PROFILE): str,
Optional(SECTION_AWS_CREDENTIALPATH): str,
Optional(SECTION_AWS_ENDPOINT_URL): str,
Optional(SECTION_AWS_USE_SSL, default=True): And(str, is_bool, Use(to_bool)),
Optional(SECTION_AWS_USE_SSL, default=True): And(
str, is_bool, Use(to_bool)
),
Optional(SECTION_GCP_PROJECTNAME): str,
Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA,
Optional(SECTION_CACHE_PROTECTED, default=False): And(
@@ -302,7 +310,9 @@ class Config(object): # pylint: disable=too-many-instance-attributes
"""
from appdirs import user_config_dir
return user_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR)
return user_config_dir(
appname=Config.APPNAME, appauthor=Config.APPAUTHOR
)
@staticmethod
def get_system_config_dir():
@@ -313,7 +323,9 @@ class Config(object): # pylint: disable=too-many-instance-attributes
"""
from appdirs import site_config_dir
return site_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR)
return site_config_dir(
appname=Config.APPNAME, appauthor=Config.APPAUTHOR
)
@staticmethod
def init(dvc_dir):
@@ -403,7 +415,9 @@ class Config(object): # pylint: disable=too-many-instance-attributes
self.config = Schema(self.SCHEMA).validate(self.config)
# NOTE: now converting back to ConfigObj
self.config = configobj.ConfigObj(self.config, write_empty_values=True)
self.config = configobj.ConfigObj(
self.config, write_empty_values=True
)
self.config.filename = self.config_file
self._resolve_paths(self.config, self.config_file)
except Exception as ex:
@@ -474,7 +488,9 @@ class Config(object): # pylint: disable=too-many-instance-attributes
return
if opt not in config[section].keys():
raise ConfigError("option '{}.{}' doesn't exist".format(section, opt))
raise ConfigError(
"option '{}.{}' doesn't exist".format(section, opt)
)
del config[section][opt]
if not config[section]:
@@ -508,7 +524,9 @@ class Config(object): # pylint: disable=too-many-instance-attributes
raise ConfigError("section '{}' doesn't exist".format(section))
if opt not in config[section].keys():
raise ConfigError("option '{}.{}' doesn't exist".format(section, opt))
raise ConfigError(
"option '{}.{}' doesn't exist".format(section, opt)
)
logger.info(config[section][opt])
+7 -2
View File
@@ -357,9 +357,14 @@ def draw(vertexes, edges):
y = vertex.view.xy[1]
canvas.box(
int(round(x - minx)), int(round(y - miny)), vertex.view.w, vertex.view.h
int(round(x - minx)),
int(round(y - miny)),
vertex.view.w,
vertex.view.h,
)
canvas.text(int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data)
canvas.text(
int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data
)
canvas.draw()
+3 -1
View File
@@ -10,7 +10,9 @@ class DependencyHTTP(DependencyBase, OutputBase):
REMOTE = RemoteHTTP
def __init__(self, stage, path, info=None, remote=None):
super(DependencyHTTP, self).__init__(stage, path, info=info, remote=remote)
super(DependencyHTTP, self).__init__(
stage, path, info=info, remote=remote
)
if path.startswith("remote"):
path = urljoin(self.remote.cache_dir, urlparse(path).path)
+11 -4
View File
@@ -41,10 +41,12 @@ class OutputDuplicationError(DvcException):
assert isinstance(output, str) or isinstance(output, builtin_str)
assert isinstance(stages, list)
assert all(
isinstance(stage, str) or isinstance(stage, builtin_str) for stage in stages
isinstance(stage, str) or isinstance(stage, builtin_str)
for stage in stages
)
msg = (
"file/directory '{}' is specified as an output in more than one" "stage: {}"
"file/directory '{}' is specified as an output in more than one"
"stage: {}"
).format(output, "\n ".join(stages))
super(OutputDuplicationError, self).__init__(msg)
@@ -80,9 +82,14 @@ class CircularDependencyError(DvcException):
"""
def __init__(self, dependency):
assert isinstance(dependency, str) or isinstance(dependency, builtin_str)
assert isinstance(dependency, str) or isinstance(
dependency, builtin_str
)
msg = "file/directory '{}' is specified as an output and as a " "dependency."
msg = (
"file/directory '{}' is specified as an output and as a "
"dependency."
)
super(CircularDependencyError, self).__init__(msg.format(dependency))
+2 -1
View File
@@ -244,7 +244,8 @@ def _parse_exc():
if is_verbose():
stack_trace = "{line}\n{stack_trace}{line}\n".format(
line=colorize("-" * 60, color="red"), stack_trace="\n".join(tb_list)
line=colorize("-" * 60, color="red"),
stack_trace="\n".join(tb_list),
)
else:
stack_trace = ""
+13 -4
View File
@@ -48,7 +48,9 @@ class OutputBase(object):
DoesNotExistError = OutputDoesNotExistError
IsNotFileOrDirError = OutputIsNotFileOrDirError
def __init__(self, stage, path, info=None, remote=None, cache=True, metric=False):
def __init__(
self, stage, path, info=None, remote=None, cache=True, metric=False
):
self.stage = stage
self.project = stage.project
self.url = path
@@ -57,9 +59,14 @@ class OutputBase(object):
self.use_cache = False if self.IS_DEPENDENCY else cache
self.metric = False if self.IS_DEPENDENCY else metric
if self.use_cache and getattr(self.project.cache, self.REMOTE.scheme) is None:
if (
self.use_cache
and getattr(self.project.cache, self.REMOTE.scheme) is None
):
raise DvcException(
"no cache location setup for '{}' outputs.".format(self.REMOTE.scheme)
"no cache location setup for '{}' outputs.".format(
self.REMOTE.scheme
)
)
def __repr__(self):
@@ -133,7 +140,9 @@ class OutputBase(object):
if not self.use_cache:
self.info = self.remote.save_info(self.path_info)
else:
self.info = getattr(self.project.cache, self.scheme).save(self.path_info)
self.info = getattr(self.project.cache, self.scheme).save(
self.path_info
)
def dumpd(self):
ret = self.info.copy()
+3 -1
View File
@@ -10,7 +10,9 @@ from dvc.remote.hdfs import RemoteHDFS
class OutputHDFS(OutputBase):
REMOTE = RemoteHDFS
def __init__(self, stage, path, info=None, remote=None, cache=True, metric=False):
def __init__(
self, stage, path, info=None, remote=None, cache=True, metric=False
):
super(OutputHDFS, self).__init__(
stage, path, info=info, remote=remote, cache=cache, metric=metric
)
+12 -4
View File
@@ -13,12 +13,16 @@ from dvc.output.base import OutputBase, OutputAlreadyTrackedError
class OutputLOCAL(OutputBase):
REMOTE = RemoteLOCAL
def __init__(self, stage, path, info=None, remote=None, cache=True, metric=False):
def __init__(
self, stage, path, info=None, remote=None, cache=True, metric=False
):
super(OutputLOCAL, self).__init__(
stage, path, info, remote=remote, cache=cache, metric=metric
)
if remote:
p = os.path.join(remote.prefix, urlparse(self.url).path.lstrip("/"))
p = os.path.join(
remote.prefix, urlparse(self.url).path.lstrip("/")
)
else:
p = path
@@ -34,7 +38,9 @@ class OutputLOCAL(OutputBase):
@property
def is_local(self):
return urlparse(self.url).scheme != "remote" and not os.path.isabs(self.url)
return urlparse(self.url).scheme != "remote" and not os.path.isabs(
self.url
)
@property
def sep(self):
@@ -51,7 +57,9 @@ class OutputLOCAL(OutputBase):
def dumpd(self):
ret = super(OutputLOCAL, self).dumpd()
if self.is_local:
path = self.remote.unixpath(os.path.relpath(self.path, self.stage.cwd))
path = self.remote.unixpath(
os.path.relpath(self.path, self.stage.cwd)
)
else:
path = self.url
+8 -2
View File
@@ -10,7 +10,9 @@ from dvc.output.base import OutputBase
class OutputS3(OutputBase):
REMOTE = RemoteS3
def __init__(self, stage, path, info=None, remote=None, cache=True, metric=False):
def __init__(
self, stage, path, info=None, remote=None, cache=True, metric=False
):
super(OutputS3, self).__init__(
stage, path, info=info, remote=remote, cache=cache, metric=metric
)
@@ -18,4 +20,8 @@ class OutputS3(OutputBase):
path = urlparse(path).path.lstrip("/")
if remote:
path = posixpath.join(remote.prefix, path)
self.path_info = {"scheme": self.scheme, "bucket": bucket, "path": path}
self.path_info = {
"scheme": self.scheme,
"bucket": bucket,
"path": path,
}
+12 -4
View File
@@ -11,17 +11,25 @@ from dvc.remote.ssh import RemoteSSH
class OutputSSH(OutputBase):
REMOTE = RemoteSSH
def __init__(self, stage, path, info=None, remote=None, cache=True, metric=False):
def __init__(
self, stage, path, info=None, remote=None, cache=True, metric=False
):
super(OutputSSH, self).__init__(
stage, path, info=info, remote=remote, cache=cache, metric=metric
)
parsed = urlparse(path)
host = remote.host if remote else parsed.hostname
port = remote.port if remote else (parsed.port or RemoteSSH.DEFAULT_PORT)
user = remote.user if remote else (parsed.username or getpass.getuser())
port = (
remote.port if remote else (parsed.port or RemoteSSH.DEFAULT_PORT)
)
user = (
remote.user if remote else (parsed.username or getpass.getuser())
)
if remote:
path = posixpath.join(remote.prefix, urlparse(path).path.lstrip("/"))
path = posixpath.join(
remote.prefix, urlparse(path).path.lstrip("/")
)
else:
path = parsed.path
+76 -23
View File
@@ -7,7 +7,11 @@ import os
import dvc.prompt as prompt
import dvc.logger as logger
from dvc.exceptions import DvcException, MoveNotDataSourceError, NotDvcProjectError
from dvc.exceptions import (
DvcException,
MoveNotDataSourceError,
NotDvcProjectError,
)
class InitError(DvcException):
@@ -276,7 +280,9 @@ class Project(object):
import dvc.output as Output
from dvc.stage import Stage
from_out = Output.loads_from(Stage(self, cwd=os.curdir), [from_path])[0]
from_out = Output.loads_from(Stage(self, cwd=os.curdir), [from_path])[
0
]
to_path = self._expand_target_path(from_path, to_path)
@@ -289,7 +295,9 @@ class Project(object):
)
except StopIteration:
raise DvcException(
"unable to find stage file with output '{path}'".format(path=from_path)
"unable to find stage file with output '{path}'".format(
path=from_path
)
)
if not stage.is_data_source:
@@ -349,7 +357,9 @@ class Project(object):
def unprotect(self, path):
if not os.path.exists(path):
raise DvcException("can't unprotect non-existing data '{}'".format(path))
raise DvcException(
"can't unprotect non-existing data '{}'".format(path)
)
if os.path.isdir(path):
self._unprotect_dir(path)
@@ -553,7 +563,9 @@ class Project(object):
used.append(out.path)
self.state.remove_unused_links(used)
def checkout(self, target=None, with_deps=False, force=False, recursive=False):
def checkout(
self, target=None, with_deps=False, force=False, recursive=False
):
if target and not recursive:
all_stages = self.active_stages()
@@ -569,7 +581,9 @@ class Project(object):
if stage.locked:
logger.warning(
"DVC file '{path}' is locked. Its dependencies are"
" not going to be checked out.".format(path=stage.relpath)
" not going to be checked out.".format(
path=stage.relpath
)
)
stage.checkout(force=force)
@@ -597,7 +611,9 @@ class Project(object):
return ret
def _collect_dir_cache(self, out, branch=None, remote=None, force=False, jobs=None):
def _collect_dir_cache(
self, out, branch=None, remote=None, force=False, jobs=None
):
info = out.dumpd()
ret = [info]
r = out.remote
@@ -605,7 +621,9 @@ class Project(object):
if self.cache.local.changed_cache_file(md5):
try:
self.cloud.pull(ret, jobs=jobs, remote=remote, show_checksums=False)
self.cloud.pull(
ret, jobs=jobs, remote=remote, show_checksums=False
)
except DvcException as exc:
msg = "Failed to pull cache for '{}': {}"
logger.debug(msg.format(out, exc))
@@ -626,7 +644,9 @@ class Project(object):
for i in self.cache.local.load_dir_cache(md5):
i["branch"] = branch
i[r.PARAM_PATH] = os.path.join(info[r.PARAM_PATH], i[r.PARAM_RELPATH])
i[r.PARAM_PATH] = os.path.join(
info[r.PARAM_PATH], i[r.PARAM_RELPATH]
)
ret.append(i)
return ret
@@ -680,7 +700,9 @@ class Project(object):
cache["ssh"] = []
cache["azure"] = []
for branch in self.scm.brancher(all_branches=all_branches, all_tags=all_tags):
for branch in self.scm.brancher(
all_branches=all_branches, all_tags=all_tags
):
if target:
if recursive:
stages = self.stages(target)
@@ -703,7 +725,11 @@ class Project(object):
for out in stage.outs:
scheme = out.path_info["scheme"]
cache[scheme] += self._collect_used_cache(
out, branch=branch, remote=remote, force=force, jobs=jobs
out,
branch=branch,
remote=remote,
force=force,
jobs=jobs,
)
return cache
@@ -809,7 +835,9 @@ class Project(object):
self._do_gc("azure", self.cache.azure.gc, clist)
if cloud:
self._do_gc("remote", self.cloud._get_cloud(remote, "gc -c").gc, clist)
self._do_gc(
"remote", self.cloud._get_cloud(remote, "gc -c").gc, clist
)
def push(
self,
@@ -833,7 +861,9 @@ class Project(object):
jobs=jobs,
recursive=recursive,
)["local"]
self.cloud.push(used, jobs, remote=remote, show_checksums=show_checksums)
self.cloud.push(
used, jobs, remote=remote, show_checksums=show_checksums
)
def fetch(
self,
@@ -857,7 +887,9 @@ class Project(object):
jobs=jobs,
recursive=recursive,
)["local"]
self.cloud.pull(used, jobs, remote=remote, show_checksums=show_checksums)
self.cloud.pull(
used, jobs, remote=remote, show_checksums=show_checksums
)
def pull(
self,
@@ -882,7 +914,10 @@ class Project(object):
recursive=recursive,
)
self.checkout(
target=target, with_deps=with_deps, force=force, recursive=recursive
target=target,
with_deps=with_deps,
force=force,
recursive=recursive,
)
def _local_status(self, target=None, with_deps=False):
@@ -938,7 +973,10 @@ class Project(object):
if status == cloud.STATUS_OK:
continue
prefix_map = {cloud.STATUS_DELETED: "deleted", cloud.STATUS_NEW: "new"}
prefix_map = {
cloud.STATUS_DELETED: "deleted",
cloud.STATUS_NEW: "new",
}
ret[name] = prefix_map[status]
@@ -1036,7 +1074,9 @@ class Project(object):
abs_path = os.path.abspath(path)
if os.path.isdir(abs_path) and recursive:
matched = [
out for out in outs if os.path.abspath(out.path).startswith(abs_path)
out
for out in outs
if os.path.abspath(out.path).startswith(abs_path)
]
else:
matched = [out for out in outs if out.path == abs_path]
@@ -1056,17 +1096,23 @@ class Project(object):
recursive=False,
):
res = {}
for branch in self.scm.brancher(all_branches=all_branches, all_tags=all_tags):
for branch in self.scm.brancher(
all_branches=all_branches, all_tags=all_tags
):
astages = self.active_stages()
outs = [out for stage in astages for out in stage.outs]
if path:
outs = self._find_output_by_path(path, outs=outs, recursive=recursive)
outs = self._find_output_by_path(
path, outs=outs, recursive=recursive
)
stages = [out.stage.path for out in outs]
entries = []
for out in outs:
if all([out.metric, not typ, isinstance(out.metric, dict)]):
if all(
[out.metric, not typ, isinstance(out.metric, dict)]
):
entries += [
(
out.path,
@@ -1192,7 +1238,10 @@ class Project(object):
def graph(self, stages=None, from_directory=None):
import networkx as nx
from dvc.exceptions import OutputDuplicationError, WorkingDirectoryAsOutputError
from dvc.exceptions import (
OutputDuplicationError,
WorkingDirectoryAsOutputError,
)
G = nx.DiGraph()
G_active = nx.DiGraph()
@@ -1217,7 +1266,9 @@ class Project(object):
)
if overlaps:
raise WorkingDirectoryAsOutputError(stage.cwd, stage.relpath)
raise WorkingDirectoryAsOutputError(
stage.cwd, stage.relpath
)
# collect the whole DAG
for stage in stages:
@@ -1252,7 +1303,9 @@ class Project(object):
G, G_active = self.graph(from_directory=from_directory)
return [G.subgraph(c).copy() for c in nx.weakly_connected_components(G)]
return [
G.subgraph(c).copy() for c in nx.weakly_connected_components(G)
]
def stages(self, from_directory=None):
"""
+15 -5
View File
@@ -84,7 +84,9 @@ class RemoteAzure(RemoteBase):
raise NotImplementedError
logger.debug(
"Removing azure://{}/{}".format(path_info["bucket"], path_info["path"])
"Removing azure://{}/{}".format(
path_info["bucket"], path_info["path"]
)
)
self.blob_service.delete_blob(path_info["bucket"], path_info["path"])
@@ -93,7 +95,9 @@ class RemoteAzure(RemoteBase):
blob_service = self.blob_service
next_marker = None
while True:
blobs = blob_service.list_blobs(bucket, prefix=prefix, marker=next_marker)
blobs = blob_service.list_blobs(
bucket, prefix=prefix, marker=next_marker
)
for blob in blobs:
yield blob.name
@@ -120,7 +124,9 @@ class RemoteAzure(RemoteBase):
path = to_info["path"]
logger.debug(
"Uploading '{}' to '{}/{}'".format(from_info["path"], bucket, path)
"Uploading '{}' to '{}/{}'".format(
from_info["path"], bucket, path
)
)
if not name:
@@ -138,7 +144,9 @@ class RemoteAzure(RemoteBase):
else:
progress.finish_target(name)
def download(self, from_infos, to_infos, no_progress_bar=False, names=None):
def download(
self, from_infos, to_infos, no_progress_bar=False, names=None
):
names = self._verify_path_args(from_infos, to_infos, names)
for to_info, from_info, name in zip(to_infos, from_infos, names):
@@ -152,7 +160,9 @@ class RemoteAzure(RemoteBase):
path = from_info["path"]
logger.debug(
"Downloading '{}/{}' to '{}'".format(bucket, path, to_info["path"])
"Downloading '{}/{}' to '{}'".format(
bucket, path, to_info["path"]
)
)
tmp_file = self.tmp_file(to_info["path"])
+12 -4
View File
@@ -56,7 +56,9 @@ class RemoteBase(object):
deps_ok = all(self.REQUIRES.values())
if not deps_ok:
missing = [k for k, v in self.REQUIRES.items() if v is None]
url = config.get(Config.SECTION_REMOTE_URL, "{}://".format(self.scheme))
url = config.get(
Config.SECTION_REMOTE_URL, "{}://".format(self.scheme)
)
msg = (
"URL '{}' is supported but requires these missing "
"dependencies: {}. If you have installed dvc using pip, "
@@ -130,7 +132,9 @@ class RemoteBase(object):
"""
logger.debug(
"checking if '{}'('{}') has changed.".format(path_info, checksum_info)
"checking if '{}'('{}') has changed.".format(
path_info, checksum_info
)
)
if not self.exists(path_info):
@@ -231,7 +235,9 @@ class RemoteBase(object):
# is 32 bytes, so ~3200Mb list) and we don't really need all of it at
# the same time, so it makes sense to use a generator to gradually
# iterate over it, without keeping all of it in memory.
return (self.path_to_checksum(path) for path in self.list_cache_paths())
return (
self.path_to_checksum(path) for path in self.list_cache_paths()
)
def gc(self, cinfos):
from dvc.remote.local import RemoteLOCAL
@@ -255,7 +261,9 @@ class RemoteBase(object):
expected = {self.PARAM_CHECKSUM: checksum}
actual = self.save_info(cache)
logger.debug("Cache '{}' actual '{}'.".format(str(expected), str(actual)))
logger.debug(
"Cache '{}' actual '{}'.".format(str(expected), str(actual))
)
if expected != actual:
if self.exists(cache):
+15 -5
View File
@@ -63,7 +63,9 @@ class RemoteGS(RemoteBase):
raise NotImplementedError
return {
self.PARAM_CHECKSUM: self.get_md5(path_info["bucket"], path_info["path"])
self.PARAM_CHECKSUM: self.get_md5(
path_info["bucket"], path_info["path"]
)
}
def copy(self, from_info, to_info, gs=None):
@@ -96,7 +98,9 @@ class RemoteGS(RemoteBase):
raise NotImplementedError
logger.debug(
"Removing gs://{}/{}".format(path_info["bucket"], path_info["path"])
"Removing gs://{}/{}".format(
path_info["bucket"], path_info["path"]
)
)
blob = self.gs.bucket(path_info["bucket"]).get_blob(path_info["path"])
@@ -149,13 +153,17 @@ class RemoteGS(RemoteBase):
except Exception:
msg = "failed to upload '{}' to '{}/{}'"
logger.error(
msg.format(from_info["path"], to_info["bucket"], to_info["path"])
msg.format(
from_info["path"], to_info["bucket"], to_info["path"]
)
)
continue
progress.finish_target(name)
def download(self, from_infos, to_infos, no_progress_bar=False, names=None):
def download(
self, from_infos, to_infos, no_progress_bar=False, names=None
):
names = self._verify_path_args(from_infos, to_infos, names)
gs = self.gs
@@ -194,7 +202,9 @@ class RemoteGS(RemoteBase):
except Exception:
msg = "failed to download '{}/{}' to '{}'"
logger.error(
msg.format(from_info["bucket"], from_info["path"], to_info["path"])
msg.format(
from_info["bucket"], from_info["path"], to_info["path"]
)
)
continue
+15 -5
View File
@@ -27,7 +27,9 @@ class RemoteHDFS(RemoteBase):
self.prefix = self.url
self.user = self.group("user")
if not self.user:
self.user = config.get(Config.SECTION_REMOTE_USER, getpass.getuser())
self.user = config.get(
Config.SECTION_REMOTE_USER, getpass.getuser()
)
self.path_info = {"scheme": "hdfs", "user": self.user}
@@ -78,7 +80,9 @@ class RemoteHDFS(RemoteBase):
)
def rm(self, path_info):
self.hadoop_fs("rm {}".format(path_info["path"]), user=path_info["user"])
self.hadoop_fs(
"rm {}".format(path_info["path"]), user=path_info["user"]
)
def save_info(self, path_info):
if path_info["scheme"] != "hdfs":
@@ -135,10 +139,14 @@ class RemoteHDFS(RemoteBase):
cmd = "mkdir -p {}".format(posixpath.dirname(to_info["path"]))
self.hadoop_fs(cmd, user=to_info["user"])
cmd = "copyFromLocal {} {}".format(from_info["path"], to_info["path"])
cmd = "copyFromLocal {} {}".format(
from_info["path"], to_info["path"]
)
self.hadoop_fs(cmd, user=to_info["user"])
def download(self, from_infos, to_infos, no_progress_bar=False, names=None):
def download(
self, from_infos, to_infos, no_progress_bar=False, names=None
):
names = self._verify_path_args(from_infos, to_infos, names)
for to_info, from_info, name in zip(to_infos, from_infos, names):
@@ -156,7 +164,9 @@ class RemoteHDFS(RemoteBase):
if not os.path.exists(dname):
os.makedirs(dname)
cmd = "copyToLocal {} {}".format(from_info["path"], to_info["path"])
cmd = "copyToLocal {} {}".format(
from_info["path"], to_info["path"]
)
self.hadoop_fs(cmd, user=from_info["user"])
def list_cache_paths(self):
+6 -2
View File
@@ -41,7 +41,9 @@ class RemoteHTTP(RemoteBase):
def prefix(self):
return self.cache_dir
def download(self, from_infos, to_infos, no_progress_bar=False, names=None):
def download(
self, from_infos, to_infos, no_progress_bar=False, names=None
):
names = self._verify_path_args(to_infos, from_infos, names)
for to_info, from_info, name in zip(to_infos, from_infos, names):
@@ -51,7 +53,9 @@ class RemoteHTTP(RemoteBase):
if to_info["scheme"] != "local":
raise NotImplementedError
msg = "Downloading '{}' to '{}'".format(from_info["path"], to_info["path"])
msg = "Downloading '{}' to '{}'".format(
from_info["path"], to_info["path"]
)
logger.debug(msg)
tmp_file = self.tmp_file(to_info["path"])
+29 -9
View File
@@ -205,7 +205,8 @@ class RemoteLOCAL(RemoteBase):
if len(files) > LARGE_DIR_SIZE:
msg = (
"Computing md5 for a large directory {}. " "This is only done once."
"Computing md5 for a large directory {}. "
"This is only done once."
)
logger.info(msg.format(os.path.relpath(root)))
bar = True
@@ -223,7 +224,9 @@ class RemoteLOCAL(RemoteBase):
processed += 1
md5 = self.state.update(path)
dir_info.append({self.PARAM_RELPATH: relpath, self.PARAM_CHECKSUM: md5})
dir_info.append(
{self.PARAM_RELPATH: relpath, self.PARAM_CHECKSUM: md5}
)
if bar:
progress.finish_target(title)
@@ -349,7 +352,9 @@ class RemoteLOCAL(RemoteBase):
for file in files
)
cached_files = set(os.path.join(path, file["relpath"]) for file in dir_info)
cached_files = set(
os.path.join(path, file["relpath"]) for file in dir_info
)
delta = working_dir_files - cached_files
@@ -420,7 +425,9 @@ class RemoteLOCAL(RemoteBase):
path = path_info["path"]
msg = "Saving '{}' to cache '{}'."
logger.info(msg.format(os.path.relpath(path), os.path.relpath(self.cache_dir)))
logger.info(
msg.format(os.path.relpath(path), os.path.relpath(self.cache_dir))
)
if os.path.isdir(path):
return self._save_dir(path_info)
@@ -460,7 +467,9 @@ class RemoteLOCAL(RemoteBase):
raise NotImplementedError
logger.debug(
"Uploading '{}' to '{}'".format(from_info["path"], to_info["path"])
"Uploading '{}' to '{}'".format(
from_info["path"], to_info["path"]
)
)
if not name:
@@ -477,7 +486,9 @@ class RemoteLOCAL(RemoteBase):
)
)
def download(self, from_infos, to_infos, no_progress_bar=False, names=None):
def download(
self, from_infos, to_infos, no_progress_bar=False, names=None
):
names = self._verify_path_args(from_infos, to_infos, names)
for to_info, from_info, name in zip(to_infos, from_infos, names):
@@ -488,7 +499,9 @@ class RemoteLOCAL(RemoteBase):
raise NotImplementedError
logger.debug(
"Downloading '{}' to '{}'".format(from_info["path"], to_info["path"])
"Downloading '{}' to '{}'".format(
from_info["path"], to_info["path"]
)
)
if not name:
@@ -561,7 +574,9 @@ class RemoteLOCAL(RemoteBase):
progress.finish_target(title)
for md5, info in ret.items():
info["status"] = STATUS_MAP[(md5 in local_exists, md5 in remote_exists)]
info["status"] = STATUS_MAP[
(md5 in local_exists, md5 in remote_exists)
]
return ret
@@ -603,7 +618,12 @@ class RemoteLOCAL(RemoteBase):
)
def _process(
self, checksum_infos, remote, jobs=None, show_checksums=False, download=False
self,
checksum_infos,
remote,
jobs=None,
show_checksums=False,
download=False,
):
msg = "Preparing to {} data {} '{}'"
logger.info(
+23 -7
View File
@@ -87,7 +87,9 @@ class RemoteS3(RemoteBase):
try:
obj = self.s3.head_object(Bucket=bucket, Key=path)
except Exception:
raise DvcException("s3://{}/{} does not exist".format(bucket, path))
raise DvcException(
"s3://{}/{} does not exist".format(bucket, path)
)
return obj["ETag"].strip('"')
@@ -96,7 +98,9 @@ class RemoteS3(RemoteBase):
raise NotImplementedError
return {
self.PARAM_CHECKSUM: self.get_etag(path_info["bucket"], path_info["path"])
self.PARAM_CHECKSUM: self.get_etag(
path_info["bucket"], path_info["path"]
)
}
def copy(self, from_info, to_info, s3=None):
@@ -122,10 +126,14 @@ class RemoteS3(RemoteBase):
raise NotImplementedError
logger.debug(
"Removing s3://{}/{}".format(path_info["bucket"], path_info["path"])
"Removing s3://{}/{}".format(
path_info["bucket"], path_info["path"]
)
)
self.s3.delete_object(Bucket=path_info["bucket"], Key=path_info["path"])
self.s3.delete_object(
Bucket=path_info["bucket"], Key=path_info["path"]
)
def _list_paths(self, bucket, prefix):
s3 = self.s3
@@ -186,7 +194,10 @@ class RemoteS3(RemoteBase):
try:
s3.upload_file(
from_info["path"], to_info["bucket"], to_info["path"], Callback=cb
from_info["path"],
to_info["bucket"],
to_info["path"],
Callback=cb,
)
except Exception:
msg = "failed to upload '{}'".format(from_info["path"])
@@ -195,7 +206,9 @@ class RemoteS3(RemoteBase):
progress.finish_target(name)
def download(self, from_infos, to_infos, no_progress_bar=False, names=None):
def download(
self, from_infos, to_infos, no_progress_bar=False, names=None
):
names = self._verify_path_args(from_infos, to_infos, names)
s3 = self.s3
@@ -232,7 +245,10 @@ class RemoteS3(RemoteBase):
cb = Callback(name, total)
s3.download_file(
from_info["bucket"], from_info["path"], tmp_file, Callback=cb
from_info["bucket"],
from_info["path"],
tmp_file,
Callback=cb,
)
except Exception:
msg = "failed to download '{}/{}'".format(
+44 -13
View File
@@ -76,12 +76,16 @@ class RemoteSSH(RemoteBase):
)
self.prefix = parsed.path
self.port = (
config.get(Config.SECTION_REMOTE_PORT) or parsed.port or self.DEFAULT_PORT
config.get(Config.SECTION_REMOTE_PORT)
or parsed.port
or self.DEFAULT_PORT
)
self.keyfile = config.get(Config.SECTION_REMOTE_KEY_FILE, None)
self.timeout = config.get(Config.SECTION_REMOTE_TIMEOUT, self.TIMEOUT)
self.password = config.get(Config.SECTION_REMOTE_PASSWORD, None)
self.ask_password = config.get(Config.SECTION_REMOTE_ASK_PASSWORD, False)
self.ask_password = config.get(
Config.SECTION_REMOTE_ASK_PASSWORD, False
)
self.path_info = {
"scheme": "ssh",
@@ -91,7 +95,10 @@ class RemoteSSH(RemoteBase):
}
def ssh(self, host=None, user=None, port=None):
msg = "Establishing ssh connection with '{}' " "through port '{}' as user '{}'"
msg = (
"Establishing ssh connection with '{}' "
"through port '{}' as user '{}'"
)
logger.debug(msg.format(host, port, user))
ssh = paramiko.SSHClient()
@@ -121,7 +128,9 @@ class RemoteSSH(RemoteBase):
assert not isinstance(path_info, list)
assert path_info["scheme"] == "ssh"
with self.ssh(path_info["host"], path_info["user"], path_info["port"]) as ssh:
with self.ssh(
path_info["host"], path_info["user"], path_info["port"]
) as ssh:
try:
self._exec(ssh, "test -e {}".format(path_info["path"]))
exists = True
@@ -139,7 +148,11 @@ class RemoteSSH(RemoteBase):
stdout_chunks = []
stderr_chunks = []
while not channel.closed or channel.recv_ready() or channel.recv_stderr_ready():
while (
not channel.closed
or channel.recv_ready()
or channel.recv_stderr_ready()
):
import select
got_chunk = False
@@ -180,7 +193,9 @@ class RemoteSSH(RemoteBase):
raise NotImplementedError
ssh = self.ssh(
host=path_info["host"], user=path_info["user"], port=path_info["port"]
host=path_info["host"],
user=path_info["user"],
port=path_info["port"],
)
# Use different md5 commands depending on os
@@ -214,7 +229,9 @@ class RemoteSSH(RemoteBase):
ssh
if ssh
else self.ssh(
host=from_info["host"], user=from_info["user"], port=from_info["port"]
host=from_info["host"],
user=from_info["user"],
port=from_info["port"],
)
)
@@ -254,12 +271,16 @@ class RemoteSSH(RemoteBase):
)
ssh = self.ssh(
host=path_info["host"], user=path_info["user"], port=path_info["port"]
host=path_info["host"],
user=path_info["user"],
port=path_info["port"],
)
ssh.open_sftp().remove(path_info["path"])
ssh.close()
def download(self, from_infos, to_infos, no_progress_bar=False, names=None):
def download(
self, from_infos, to_infos, no_progress_bar=False, names=None
):
names = self._verify_path_args(from_infos, to_infos, names)
ssh = self.ssh(
@@ -299,7 +320,9 @@ class RemoteSSH(RemoteBase):
except Exception:
msg = "failed to download '{}/{}' to '{}'"
logger.error(
msg.format(from_info["host"], from_info["path"], to_info["path"])
msg.format(
from_info["host"], from_info["path"], to_info["path"]
)
)
continue
@@ -312,7 +335,9 @@ class RemoteSSH(RemoteBase):
names = self._verify_path_args(to_infos, from_infos, names)
ssh = self.ssh(
host=to_infos[0]["host"], user=to_infos[0]["user"], port=to_infos[0]["port"]
host=to_infos[0]["host"],
user=to_infos[0]["user"],
port=to_infos[0]["port"],
)
sftp = ssh.open_sftp()
@@ -336,11 +361,17 @@ class RemoteSSH(RemoteBase):
self._exec(ssh, "mkdir -p {}".format(dname))
try:
sftp.put(from_info["path"], to_info["path"], callback=create_cb(name))
sftp.put(
from_info["path"],
to_info["path"],
callback=create_cb(name),
)
except Exception:
msg = "failed to upload '{}' to '{}/{}'"
logger.error(
msg.format(from_info["path"], to_info["host"], to_info["path"])
msg.format(
from_info["path"], to_info["host"], to_info["path"]
)
)
continue
+3 -1
View File
@@ -76,7 +76,9 @@ class Base(object):
def tag(self, tag):
"""Makes SCM create a tag with a specified name."""
def brancher(self, branches=None, all_branches=False, tags=None, all_tags=False):
def brancher(
self, branches=None, all_branches=False, tags=None, all_tags=False
):
"""Generator that iterates over specified revisions.
Args:
+3 -1
View File
@@ -76,7 +76,9 @@ class Git(Base):
ignore_list = []
if os.path.exists(gitignore):
ignore_list = open(gitignore, "r").readlines()
filtered = list(filter(lambda x: x.strip() == entry.strip(), ignore_list))
filtered = list(
filter(lambda x: x.strip() == entry.strip(), ignore_list)
)
if filtered:
return
+15 -5
View File
@@ -131,7 +131,9 @@ class Stage(object):
self.locked = locked
def __repr__(self):
return "Stage: '{path}'".format(path=self.relpath if self.path else "No path")
return "Stage: '{path}'".format(
path=self.relpath if self.path else "No path"
)
@property
def relpath(self):
@@ -212,7 +214,9 @@ class Stage(object):
return False
def changed(self):
ret = any([self._changed_deps(), self._changed_outs(), self._changed_md5()])
ret = any(
[self._changed_deps(), self._changed_outs(), self._changed_md5()]
)
if ret:
msg = "Stage '{}' changed.".format(self.relpath)
@@ -467,7 +471,9 @@ class Stage(object):
self._check_dvc_filename(fname)
logger.info(
"Saving information to '{file}'.".format(file=os.path.relpath(fname))
"Saving information to '{file}'.".format(
file=os.path.relpath(fname)
)
)
with open(fname, "w") as fd:
@@ -487,7 +493,9 @@ class Stage(object):
# NOTE: excluding parameters that don't affect the state of the
# pipeline. Not excluding `OutputLOCAL.PARAM_CACHE`, because if
# it has changed, we might not have that output in our cache.
return dict_md5(d, exclude=[self.PARAM_LOCKED, OutputLOCAL.PARAM_METRIC])
return dict_md5(
d, exclude=[self.PARAM_LOCKED, OutputLOCAL.PARAM_METRIC]
)
def save(self):
for dep in self.deps:
@@ -559,7 +567,9 @@ class Stage(object):
def run(self, dry=False):
if self.locked:
logger.info(
"Verifying outputs in locked stage '{stage}'".format(stage=self.relpath)
"Verifying outputs in locked stage '{stage}'".format(
stage=self.relpath
)
)
if not dry:
self.check_missing_outputs()
+26 -9
View File
@@ -60,7 +60,9 @@ class State(object): # pylint: disable=too-many-instance-attributes
LINK_STATE_TABLE = "link_state"
LINK_STATE_TABLE_LAYOUT = (
"path TEXT PRIMARY KEY, " "inode INTEGER NOT NULL, " "mtime TEXT NOT NULL"
"path TEXT PRIMARY KEY, "
"inode INTEGER NOT NULL, "
"mtime TEXT NOT NULL"
)
STATE_ROW_LIMIT = 100000000
@@ -82,7 +84,8 @@ class State(object): # pylint: disable=too-many-instance-attributes
Config.SECTION_STATE_ROW_LIMIT, self.STATE_ROW_LIMIT
)
self.row_cleanup_quota = state_config.get(
Config.SECTION_STATE_ROW_CLEANUP_QUOTA, self.STATE_ROW_CLEANUP_QUOTA
Config.SECTION_STATE_ROW_CLEANUP_QUOTA,
self.STATE_ROW_CLEANUP_QUOTA,
)
if not self.dvc_dir:
@@ -92,7 +95,10 @@ class State(object): # pylint: disable=too-many-instance-attributes
self.state_file = os.path.join(self.dvc_dir, self.STATE_FILE)
# https://www.sqlite.org/tempfiles.html
self.temp_files = [self.state_file + "-journal", self.state_file + "-wal"]
self.temp_files = [
self.state_file + "-journal",
self.state_file + "-wal",
]
self.database = None
self.cursor = None
@@ -186,8 +192,12 @@ class State(object): # pylint: disable=too-many-instance-attributes
# Check that the state file is indeed a database
cmd = "CREATE TABLE IF NOT EXISTS {} ({})"
self._execute(cmd.format(self.STATE_TABLE, self.STATE_TABLE_LAYOUT))
self._execute(cmd.format(self.STATE_INFO_TABLE, self.STATE_INFO_TABLE_LAYOUT))
self._execute(cmd.format(self.LINK_STATE_TABLE, self.LINK_STATE_TABLE_LAYOUT))
self._execute(
cmd.format(self.STATE_INFO_TABLE, self.STATE_INFO_TABLE_LAYOUT)
)
self._execute(
cmd.format(self.LINK_STATE_TABLE, self.LINK_STATE_TABLE_LAYOUT)
)
cmd = (
"INSERT OR IGNORE INTO {} (count) SELECT 0 "
@@ -253,7 +263,9 @@ class State(object): # pylint: disable=too-many-instance-attributes
"DELETE FROM {} WHERE timestamp IN ("
"SELECT timestamp FROM {} ORDER BY timestamp ASC LIMIT {});"
)
self._execute(cmd.format(self.STATE_TABLE, self.STATE_TABLE, delete))
self._execute(
cmd.format(self.STATE_TABLE, self.STATE_TABLE, delete)
)
self._vacuum()
@@ -268,7 +280,9 @@ class State(object): # pylint: disable=too-many-instance-attributes
cmd = "UPDATE {} SET count = {} WHERE rowid = {}"
self._execute(
cmd.format(
self.STATE_INFO_TABLE, self._to_sqlite(count), self.STATE_INFO_ROW
self.STATE_INFO_TABLE,
self._to_sqlite(count),
self.STATE_INFO_ROW,
)
)
@@ -423,8 +437,11 @@ class State(object): # pylint: disable=too-many-instance-attributes
inode = self._inode(path)
relpath = os.path.relpath(path, self.root_dir)
cmd = "REPLACE INTO {}(path, inode, mtime) " 'VALUES ("{}", {}, "{}")'.format(
self.LINK_STATE_TABLE, relpath, self._to_sqlite(inode), mtime
cmd = (
"REPLACE INTO {}(path, inode, mtime) "
'VALUES ("{}", {}, "{}")'.format(
self.LINK_STATE_TABLE, relpath, self._to_sqlite(inode), mtime
)
)
self._execute(cmd)
+22 -4
View File
@@ -23,7 +23,11 @@ class System(object):
raise DvcException("link", cause=exc)
CreateHardLink = ctypes.windll.kernel32.CreateHardLinkW
CreateHardLink.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_void_p]
CreateHardLink.argtypes = [
ctypes.c_wchar_p,
ctypes.c_wchar_p,
ctypes.c_void_p,
]
CreateHardLink.restype = ctypes.wintypes.BOOL
res = CreateHardLink(link_name, source, None)
@@ -158,10 +162,20 @@ class System(object):
flags = FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT
func = ctypes.windll.kernel32.CreateFileW
func.argtypes = [c_wchar_p, DWORD, DWORD, c_void_p, DWORD, DWORD, HANDLE]
func.argtypes = [
c_wchar_p,
DWORD,
DWORD,
c_void_p,
DWORD,
DWORD,
HANDLE,
]
func.restype = HANDLE
hfile = func(path, 0, FILE_SHARE_READ, None, OPEN_EXISTING, flags, None)
hfile = func(
path, 0, FILE_SHARE_READ, None, OPEN_EXISTING, flags, None
)
if hfile is None:
raise WinError()
@@ -198,7 +212,11 @@ class System(object):
info = System.getdirinfo(path)
inode = abs(
hash(
(info.dwVolumeSerialNumber, info.nFileIndexHigh, info.nFileIndexLow)
(
info.dwVolumeSerialNumber,
info.nFileIndexHigh,
info.nFileIndexLow,
)
)
)
assert inode >= 0
+2 -1
View File
@@ -128,7 +128,8 @@ class Updater(object): # pragma: no cover
"formula": "Run {yellow}brew{reset} upgrade dvc",
"cask": "Run {yellow}brew cask{reset} upgrade dvc",
"apt": (
"Run {yellow}apt-get{reset} install" " {blue}--only-upgrade{reset} dvc"
"Run {yellow}apt-get{reset} install"
" {blue}--only-upgrade{reset} dvc"
),
"binary": (
"To upgrade follow this steps:\n"
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.black]
line-length = 88
line-length = 79
include = '\.pyi?$'
exclude = '''
/(
+3 -3
View File
@@ -20,9 +20,9 @@ pip install -r scripts/ci/requirements.txt
# stop the build if there are any readme formatting errors
python setup.py checkdocs
# stop the build if there are any black errors
black dvc --check
black tests --check
# stop the build if there are any black or flake8 errors
black ./ --check
flake8 ./
#NOTE: ssh keys for ssh test to be able to ssh to the localhost
ls -la ~/.ssh/
+1
View File
@@ -1,3 +1,4 @@
Pygments # required by collective.checkdocs
collective.checkdocs
black
flake8
+3 -1
View File
@@ -4,7 +4,9 @@ from subprocess import check_call
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
os.chdir(REPO_ROOT)
os.putenv("PATH", "{}:{}".format(os.path.join(REPO_ROOT, "bin"), os.getenv("PATH")))
os.putenv(
"PATH", "{}:{}".format(os.path.join(REPO_ROOT, "bin"), os.getenv("PATH"))
)
os.putenv("DVC_HOME", REPO_ROOT)
os.putenv("DVC_TEST", "true")
+6 -3
View File
@@ -10,7 +10,9 @@ from dvc.project import Project
class TestDir(TestCase):
GCP_CREDS_FILE = os.path.abspath(os.path.join("scripts", "ci", "gcp-creds.json"))
GCP_CREDS_FILE = os.path.abspath(
os.path.join("scripts", "ci", "gcp-creds.json")
)
DATA_DIR = "data_dir"
DATA_SUB_DIR = os.path.join(DATA_DIR, "data_sub_dir")
DATA = os.path.join(DATA_DIR, "data")
@@ -32,7 +34,8 @@ class TestDir(TestCase):
BAR_CONTENTS = BAR + "r"
CODE = "code.py"
CODE_CONTENTS = (
"import sys\nimport shutil\n" "shutil.copyfile(sys.argv[1], sys.argv[2])"
"import sys\nimport shutil\n"
"shutil.copyfile(sys.argv[1], sys.argv[2])"
)
def _pushd(self, d):
@@ -87,7 +90,7 @@ class TestGit(TestDir):
while retries:
try:
self.git = Repo.init()
except GitCommandNotFound as exc:
except GitCommandNotFound:
retries -= 1
continue
break
+15 -5
View File
@@ -14,10 +14,14 @@ class TestCache(TestDvc):
self.cache1_md5 = "123"
self.cache2_md5 = "234"
self.cache1 = os.path.join(
self.dvc.cache.local.cache_dir, self.cache1_md5[0:2], self.cache1_md5[2:]
self.dvc.cache.local.cache_dir,
self.cache1_md5[0:2],
self.cache1_md5[2:],
)
self.cache2 = os.path.join(
self.dvc.cache.local.cache_dir, self.cache2_md5[0:2], self.cache2_md5[2:]
self.dvc.cache.local.cache_dir,
self.cache2_md5[0:2],
self.cache2_md5[2:],
)
self.create(self.cache1, "1")
self.create(self.cache2, "2")
@@ -103,9 +107,15 @@ class TestSharedCacheDir(TestDir):
)
)
self.assertEqual(len(subdirs), 3)
self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[0]))), 1)
self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[1]))), 1)
self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[2]))), 1)
self.assertEqual(
len(os.listdir(os.path.join(cache_dir, subdirs[0]))), 1
)
self.assertEqual(
len(os.listdir(os.path.join(cache_dir, subdirs[1]))), 1
)
self.assertEqual(
len(os.listdir(os.path.join(cache_dir, subdirs[2]))), 1
)
class TestCacheLinkType(TestDvc):
+9 -3
View File
@@ -113,7 +113,9 @@ class CheckoutBase(TestDvc):
self.dvc.scm.commit("adding " + fname)
def read_ignored(self):
return list(map(lambda s: s.strip("\n"), open(self.GIT_IGNORE).readlines()))
return list(
map(lambda s: s.strip("\n"), open(self.GIT_IGNORE).readlines())
)
def outs_info(self, stage):
FileInfo = collections.namedtuple("FileInfo", "path inode")
@@ -125,7 +127,9 @@ class CheckoutBase(TestDvc):
for file in files
]
return [FileInfo(path=path, inode=System.inode(path)) for path in paths]
return [
FileInfo(path=path, inode=System.inode(path)) for path in paths
]
class TestRemoveFilesWhenCheckout(CheckoutBase):
@@ -314,7 +318,9 @@ class TestCheckoutNotCachedFile(TestDvc):
cmd = "python {} {} {}".format(self.CODE, self.FOO, "out")
self.dvc.add(self.FOO)
stage = self.dvc.run(cmd=cmd, deps=[self.FOO, self.CODE], outs_no_cache=["out"])
stage = self.dvc.run(
cmd=cmd, deps=[self.FOO, self.CODE], outs_no_cache=["out"]
)
self.assertTrue(stage is not None)
self.dvc.checkout(force=True)
+30 -9
View File
@@ -70,7 +70,13 @@ def _should_test_gcp():
shutil.copyfile(TestDvc.GCP_CREDS_FILE, creds)
try:
check_output(
["gcloud", "auth", "activate-service-account", "--key-file", creds]
[
"gcloud",
"auth",
"activate-service-account",
"--key-file",
creds,
]
)
except (CalledProcessError, OSError):
return False
@@ -111,12 +117,16 @@ def _should_test_hdfs():
return False
try:
check_output(["hadoop", "version"], shell=True, executable=os.getenv("SHELL"))
check_output(
["hadoop", "version"], shell=True, executable=os.getenv("SHELL")
)
except (CalledProcessError, IOError):
return False
p = Popen(
"hadoop fs -ls hdfs://127.0.0.1/", shell=True, executable=os.getenv("SHELL")
"hadoop fs -ls hdfs://127.0.0.1/",
shell=True,
executable=os.getenv("SHELL"),
)
p.communicate()
if p.returncode != 0:
@@ -134,11 +144,15 @@ def get_local_url():
def get_ssh_url():
return "ssh://{}@127.0.0.1:{}".format(getpass.getuser(), get_local_storagepath())
return "ssh://{}@127.0.0.1:{}".format(
getpass.getuser(), get_local_storagepath()
)
def get_hdfs_url():
return "hdfs://{}@127.0.0.1{}".format(getpass.getuser(), get_local_storagepath())
return "hdfs://{}@127.0.0.1{}".format(
getpass.getuser(), get_local_storagepath()
)
def get_aws_storagepath():
@@ -161,7 +175,9 @@ def get_azure_url_compat():
container_name = os.getenv("AZURE_STORAGE_CONTAINER_NAME")
connection_string = os.getenv("AZURE_STORAGE_CONNECTION_STRING")
assert container_name is not None
return "azure://ContainerName={};{}".format(container_name, connection_string)
return "azure://ContainerName={};{}".format(
container_name, connection_string
)
def get_azure_url():
@@ -205,7 +221,9 @@ class TestDataCloudBase(TestDvc):
def _ensure_should_run(self):
if not self._should_test():
raise SkipTest("Test {} is disabled".format(self.__class__.__name__))
raise SkipTest(
"Test {} is disabled".format(self.__class__.__name__)
)
def _setup_cloud(self):
self._ensure_should_run()
@@ -433,7 +451,8 @@ class TestDataCloudCLIBase(TestDvc):
# NOTE: check if remote gc works correctly on directories
self.main(["gc", "-c", "-f"] + args)
shutil.move(
self.dvc.cache.local.cache_dir, self.dvc.cache.local.cache_dir + ".back"
self.dvc.cache.local.cache_dir,
self.dvc.cache.local.cache_dir + ".back",
)
self.main(["fetch"] + args)
@@ -625,7 +644,9 @@ class TestRecursiveSyncOperations(TestDataCloudBase):
return RemoteLOCAL
def _prepare_repo(self):
self.main(["remote", "add", "-d", TEST_REMOTE, self.cloud._cloud.cache_dir])
self.main(
["remote", "add", "-d", TEST_REMOTE, self.cloud._cloud.cache_dir]
)
self.create(self.DATA, self.DATA_CONTENTS)
self.create(self.DATA_SUB, self.DATA_SUB_CONTENTS)
+3 -1
View File
@@ -118,7 +118,9 @@ class TestGCMultipleProjects(TestDvc):
self.additional_dvc = Project.init(self.additional_path)
cache_path = os.path.join(self._root_dir, ".dvc", "cache")
config_path = os.path.join(self.additional_path, ".dvc", "config.local")
config_path = os.path.join(
self.additional_path, ".dvc", "config.local"
)
cfg = configobj.ConfigObj()
cfg.filename = config_path
cfg["cache"] = {"dir": cache_path}
+63 -12
View File
@@ -142,13 +142,19 @@ class TestMetricsRecursive(TestDvc):
def test(self):
ret = self.dvc.metrics_show("nested", all_branches=True, recursive=False)
ret = self.dvc.metrics_show(
"nested", all_branches=True, recursive=False
)
self.assertEqual(len(ret), 0)
ret = self.dvc.metrics_show("nested", all_branches=True, recursive=True)
ret = self.dvc.metrics_show(
"nested", all_branches=True, recursive=True
)
self.assertEqual(len(ret), 1)
self.assertEqual(
ret["nested"][os.path.join("nested", "subnested", "metric_subnested")],
ret["nested"][
os.path.join("nested", "subnested", "metric_subnested")
],
"subnested",
)
self.assertEqual(
@@ -214,7 +220,14 @@ class TestMetricsCLICompat(TestMetrics):
self.assertEqual(ret, 0)
ret = main(
["metrics", "show", "--all-branches", "metric_tsv", "--tsv-path", "0,0"]
[
"metrics",
"show",
"--all-branches",
"metric_tsv",
"--tsv-path",
"0,0",
]
)
self.assertEqual(ret, 0)
@@ -231,7 +244,14 @@ class TestMetricsCLICompat(TestMetrics):
self.assertEqual(ret, 0)
ret = main(
["metrics", "show", "--all-branches", "metric_csv", "--csv-path", "0,0"]
[
"metrics",
"show",
"--all-branches",
"metric_csv",
"--csv-path",
"0,0",
]
)
self.assertEqual(ret, 0)
@@ -255,23 +275,54 @@ class TestMetricsCLI(TestMetrics):
self.assertEqual(ret, 0)
ret = main(
["metrics", "show", "-a", "metric_json", "-t", "json", "-x", "branch"]
[
"metrics",
"show",
"-a",
"metric_json",
"-t",
"json",
"-x",
"branch",
]
)
self.assertEqual(ret, 0)
ret = main(["metrics", "show", "-a", "metric_tsv", "-t", "tsv", "-x", "0,0"])
self.assertEqual(ret, 0)
ret = main(
["metrics", "show", "-a", "metric_htsv", "-t", "htsv", "-x", "branch,0"]
["metrics", "show", "-a", "metric_tsv", "-t", "tsv", "-x", "0,0"]
)
self.assertEqual(ret, 0)
ret = main(["metrics", "show", "-a", "metric_csv", "-t", "csv", "-x", "0,0"])
ret = main(
[
"metrics",
"show",
"-a",
"metric_htsv",
"-t",
"htsv",
"-x",
"branch,0",
]
)
self.assertEqual(ret, 0)
ret = main(
["metrics", "show", "-a", "metric_hcsv", "-t", "hcsv", "-x", "branch,0"]
["metrics", "show", "-a", "metric_csv", "-t", "csv", "-x", "0,0"]
)
self.assertEqual(ret, 0)
ret = main(
[
"metrics",
"show",
"-a",
"metric_hcsv",
"-t",
"hcsv",
"-x",
"branch,0",
]
)
self.assertEqual(ret, 0)
+6 -2
View File
@@ -132,7 +132,9 @@ class TestMoveDirectoryShouldNotOverwriteExisting(TestDvc):
self.assertEqual(ret, 0)
self.assertFalse(os.path.exists(self.DATA_DIR))
self.assertFalse(os.path.exists(self.DATA_DIR + Stage.STAGE_FILE_SUFFIX))
self.assertFalse(
os.path.exists(self.DATA_DIR + Stage.STAGE_FILE_SUFFIX)
)
self.assertTrue(os.path.exists(new_dir_name))
self.assertTrue(os.path.isfile(new_dir_name + Stage.STAGE_FILE_SUFFIX))
@@ -163,4 +165,6 @@ class TestMoveFileBetweenDirectories(TestDvc):
self.assertTrue(os.path.exists(new_data_stage_file))
new_stage_file = load_stage_file(new_data_stage_file)
self.assertEqual(os.path.basename(self.DATA), new_stage_file["outs"][0]["path"])
self.assertEqual(
os.path.basename(self.DATA), new_stage_file["outs"][0]["path"]
)
+55 -10
View File
@@ -44,13 +44,22 @@ class TestPipelineShowSingle(TestDvc):
def test_dot_commands(self):
ret = main(
["pipeline", "show", "--dot", self.dotFile, self.stage, "--commands"]
[
"pipeline",
"show",
"--dot",
self.dotFile,
self.stage,
"--commands",
]
)
self.assertEqual(ret, 0)
self.assertTrue(os.path.isfile(self.dotFile))
def test_dot_outs(self):
ret = main(["pipeline", "show", "--dot", self.dotFile, self.stage, "--outs"])
ret = main(
["pipeline", "show", "--dot", self.dotFile, self.stage, "--outs"]
)
self.assertEqual(ret, 0)
self.assertTrue(os.path.isfile(self.dotFile))
@@ -85,12 +94,16 @@ class TestPipelineShow(TestRepro):
self.assertEqual(ret, 0)
def test_dot(self):
ret = main(["pipeline", "show", "--dot", self.dotFile, self.file1_stage])
ret = main(
["pipeline", "show", "--dot", self.dotFile, self.file1_stage]
)
self.assertEqual(ret, 0)
self.assertTrue(os.path.isfile(self.dotFile))
def test_ascii_commands(self):
ret = main(["pipeline", "show", "--ascii", self.file1_stage, "--commands"])
ret = main(
["pipeline", "show", "--ascii", self.file1_stage, "--commands"]
)
self.assertEqual(ret, 0)
def test_ascii_outs(self):
@@ -99,14 +112,28 @@ class TestPipelineShow(TestRepro):
def test_dot_commands(self):
ret = main(
["pipeline", "show", "--dot", self.dotFile, self.file1_stage, "--commands"]
[
"pipeline",
"show",
"--dot",
self.dotFile,
self.file1_stage,
"--commands",
]
)
self.assertEqual(ret, 0)
self.assertTrue(os.path.isfile(self.dotFile))
def test_dot_outs(self):
ret = main(
["pipeline", "show", "--dot", self.dotFile, self.file1_stage, "--outs"]
[
"pipeline",
"show",
"--dot",
self.dotFile,
self.file1_stage,
"--outs",
]
)
self.assertEqual(ret, 0)
self.assertTrue(os.path.isfile(self.dotFile))
@@ -142,12 +169,16 @@ class TestPipelineShowDeep(TestReproChangedDeepData):
self.assertEqual(ret, 0)
def test_dot(self):
ret = main(["pipeline", "show", "--dot", self.dotFile, self.file1_stage])
ret = main(
["pipeline", "show", "--dot", self.dotFile, self.file1_stage]
)
self.assertEqual(ret, 0)
self.assertTrue(os.path.isfile(self.dotFile))
def test_ascii_commands(self):
ret = main(["pipeline", "show", "--ascii", self.file1_stage, "--commands"])
ret = main(
["pipeline", "show", "--ascii", self.file1_stage, "--commands"]
)
self.assertEqual(ret, 0)
def test_ascii_outs(self):
@@ -156,14 +187,28 @@ class TestPipelineShowDeep(TestReproChangedDeepData):
def test_dot_commands(self):
ret = main(
["pipeline", "show", "--dot", self.dotFile, self.file1_stage, "--commands"]
[
"pipeline",
"show",
"--dot",
self.dotFile,
self.file1_stage,
"--commands",
]
)
self.assertEqual(ret, 0)
self.assertTrue(os.path.isfile(self.dotFile))
def test_dot_outs(self):
ret = main(
["pipeline", "show", "--dot", self.dotFile, self.file1_stage, "--outs"]
[
"pipeline",
"show",
"--dot",
self.dotFile,
self.file1_stage,
"--outs",
]
)
self.assertEqual(ret, 0)
self.assertTrue(os.path.isfile(self.dotFile))
+14 -5
View File
@@ -23,7 +23,9 @@ class TestRemote(TestDvc):
self.assertEqual(main(["remote", "list"]), 0)
self.assertEqual(main(["remote", "remove", remotes[0]]), 0)
self.assertEqual(main(["remote", "modify", remotes[0], "option", "value"]), 0)
self.assertEqual(
main(["remote", "modify", remotes[0], "option", "value"]), 0
)
self.assertEqual(main(["remote", "list"]), 0)
@@ -61,10 +63,14 @@ class TestRemoteRemoveDefault(TestDvc):
self.assertEqual(
main(["remote", "add", "--default", remote, "s3://bucket/name"]), 0
)
self.assertEqual(main(["remote", "modify", remote, "profile", "default"]), 0)
self.assertEqual(
main(["remote", "modify", remote, "profile", "default"]), 0
)
self.assertEqual(main(["config", "--local", "core.remote", remote]), 0)
config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, Config.CONFIG))
config = configobj.ConfigObj(
os.path.join(self.dvc.dvc_dir, Config.CONFIG)
)
local_config = configobj.ConfigObj(
os.path.join(self.dvc.dvc_dir, Config.CONFIG_LOCAL)
)
@@ -72,11 +78,14 @@ class TestRemoteRemoveDefault(TestDvc):
config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE], remote
)
self.assertEqual(
local_config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE], remote
local_config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE],
remote,
)
self.assertEqual(main(["remote", "remove", remote]), 0)
config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, Config.CONFIG))
config = configobj.ConfigObj(
os.path.join(self.dvc.dvc_dir, Config.CONFIG)
)
local_config = configobj.ConfigObj(
os.path.join(self.dvc.dvc_dir, Config.CONFIG_LOCAL)
)
+44 -14
View File
@@ -58,9 +58,13 @@ class TestReproFail(TestRepro):
class TestReproCyclicGraph(TestDvc):
def test(self):
self.dvc.run(deps=[self.FOO], outs=["bar.txt"], cmd="echo bar > bar.txt")
self.dvc.run(
deps=[self.FOO], outs=["bar.txt"], cmd="echo bar > bar.txt"
)
self.dvc.run(deps=["bar.txt"], outs=["baz.txt"], cmd="echo baz > baz.txt")
self.dvc.run(
deps=["bar.txt"], outs=["baz.txt"], cmd="echo baz > baz.txt"
)
with open("cycle.dvc", "w") as fd:
stage_dump = {
@@ -257,7 +261,9 @@ class TestReproNoDeps(TestRepro):
)
with open(code_file, "w+") as fd:
fd.write(code)
self.dvc.run(fname=stage_file, outs=[out], cmd="python {}".format(code_file))
self.dvc.run(
fname=stage_file, outs=[out], cmd="python {}".format(code_file)
)
stages = self.dvc.reproduce(stage_file)
self.assertEqual(len(stages), 1)
@@ -281,7 +287,9 @@ class TestReproChangedCode(TestRepro):
def swap_code(self):
os.unlink(self.CODE)
new_contents = self.CODE_CONTENTS
new_contents += "\nshutil.copyfile('{}', " "sys.argv[2])\n".format(self.BAR)
new_contents += "\nshutil.copyfile('{}', " "sys.argv[2])\n".format(
self.BAR
)
self.create(self.CODE, new_contents)
@@ -422,7 +430,9 @@ class TestReproIgnoreBuildCache(TestDvc):
class TestReproPipeline(TestReproChangedDeepData):
def test(self):
stages = self.dvc.reproduce(self.file1_stage, force=True, pipeline=True)
stages = self.dvc.reproduce(
self.file1_stage, force=True, pipeline=True
)
self.assertEqual(len(stages), 3)
def test_cli(self):
@@ -793,7 +803,9 @@ class TestReproExternalBase(TestDvc):
def corrupted_cache(self):
os.unlink("bar.dvc")
stage = self.dvc.run(deps=[self.FOO], outs=[self.BAR], cmd="echo bar > bar")
stage = self.dvc.run(
deps=[self.FOO], outs=[self.BAR], cmd="echo bar > bar"
)
with open(self.BAR, "w") as fd:
fd.write("corrupting the cache")
@@ -817,7 +829,11 @@ class TestReproExternalBase(TestDvc):
return
cache = (
self.scheme + self.scheme_sep + self.bucket + self.sep + str(uuid.uuid4())
self.scheme
+ self.scheme_sep
+ self.bucket
+ self.sep
+ str(uuid.uuid4())
)
ret = main(["config", "cache." + self.cache_scheme, "myrepo"])
@@ -827,7 +843,9 @@ class TestReproExternalBase(TestDvc):
remote_name = "myremote"
remote_key = str(uuid.uuid4())
remote = self.scheme + self.scheme_sep + self.bucket + self.sep + remote_key
remote = (
self.scheme + self.scheme_sep + self.bucket + self.sep + remote_key
)
ret = main(["remote", "add", remote_name, remote])
self.assertEqual(ret, 0)
@@ -837,8 +855,12 @@ class TestReproExternalBase(TestDvc):
foo_key = remote_key + self.sep + self.FOO
bar_key = remote_key + self.sep + self.BAR
foo_path = self.scheme + self.scheme_sep + self.bucket + self.sep + foo_key
bar_path = self.scheme + self.scheme_sep + self.bucket + self.sep + bar_key
foo_path = (
self.scheme + self.scheme_sep + self.bucket + self.sep + foo_key
)
bar_path = (
self.scheme + self.scheme_sep + self.bucket + self.sep + bar_key
)
# Using both plain and remote notation
out_foo_path = "remote://" + remote_name + "/" + self.FOO
@@ -853,11 +875,15 @@ class TestReproExternalBase(TestDvc):
self.assertEqual(self.dvc.status(import_stage.path), {})
self.check_already_cached(import_stage)
import_remote_stage = self.dvc.imp(out_foo_path, out_foo_path + "_imported")
import_remote_stage = self.dvc.imp(
out_foo_path, out_foo_path + "_imported"
)
self.assertEqual(self.dvc.status(import_remote_stage.path), {})
cmd_stage = self.dvc.run(
outs=[out_bar_path], deps=[out_foo_path], cmd=self.cmd(foo_path, bar_path)
outs=[out_bar_path],
deps=[out_foo_path],
cmd=self.cmd(foo_path, bar_path),
)
self.assertEqual(self.dvc.status(cmd_stage.path), {})
@@ -1161,7 +1187,9 @@ class TestReproExternalHTTP(TestReproExternalBase):
fd.write(cmd)
run_stage = self.dvc.run(
deps=[run_dependency], outs=[run_output], cmd="python create-output.py"
deps=[run_dependency],
outs=[run_output],
cmd="python create-output.py",
)
self.assertTrue(run_stage is not None)
@@ -1188,7 +1216,9 @@ class TestReproShell(TestDvc):
fname = "shell.txt"
stage = fname + ".dvc"
self.dvc.run(fname=stage, outs=[fname], cmd="echo $SHELL > {}".format(fname))
self.dvc.run(
fname=stage, outs=[fname], cmd="echo $SHELL > {}".format(fname)
)
with open(fname, "r") as fd:
self.assertEqual(os.getenv("SHELL"), fd.read().strip())
+68 -13
View File
@@ -63,7 +63,12 @@ class TestRun(TestDvc):
class TestRunEmpty(TestDvc):
def test(self):
self.dvc.run(
cmd="", deps=[], outs=[], outs_no_cache=[], fname="empty.dvc", cwd=os.curdir
cmd="",
deps=[],
outs=[],
outs_no_cache=[],
fname="empty.dvc",
cwd=os.curdir,
)
@@ -84,7 +89,12 @@ class TestRunBadStageFilename(TestDvc):
def test(self):
with self.assertRaises(StageFileBadNameError):
self.dvc.run(
cmd="", deps=[], outs=[], outs_no_cache=[], fname="empty", cwd=os.curdir
cmd="",
deps=[],
outs=[],
outs_no_cache=[],
fname="empty",
cwd=os.curdir,
)
with self.assertRaises(StageFileBadNameError):
@@ -101,7 +111,8 @@ class TestRunBadStageFilename(TestDvc):
class TestRunNoExec(TestDvc):
def test(self):
self.dvc.run(
cmd="python {} {} {}".format(self.CODE, self.FOO, "out"), no_exec=True
cmd="python {} {} {}".format(self.CODE, self.FOO, "out"),
no_exec=True,
)
self.assertFalse(os.path.exists("out"))
@@ -128,16 +139,25 @@ class TestRunCircularDependency(TestDvc):
def test_non_normalized_paths(self):
with self.assertRaises(CircularDependencyError):
self.dvc.run(
cmd="", deps=["./foo"], outs=["foo"], fname="circular-dependency.dvc"
cmd="",
deps=["./foo"],
outs=["foo"],
fname="circular-dependency.dvc",
)
def test_graph(self):
self.dvc.run(deps=[self.FOO], outs=["bar.txt"], cmd="echo bar > bar.txt")
self.dvc.run(
deps=[self.FOO], outs=["bar.txt"], cmd="echo bar > bar.txt"
)
self.dvc.run(deps=["bar.txt"], outs=["baz.txt"], cmd="echo baz > baz.txt")
self.dvc.run(
deps=["bar.txt"], outs=["baz.txt"], cmd="echo baz > baz.txt"
)
with self.assertRaises(CyclicGraphError):
self.dvc.run(deps=["baz.txt"], outs=[self.FOO], cmd="echo baz > foo")
self.dvc.run(
deps=["baz.txt"], outs=[self.FOO], cmd="echo baz > foo"
)
class TestRunDuplicatedArguments(TestDvc):
@@ -162,7 +182,10 @@ class TestRunDuplicatedArguments(TestDvc):
def test_non_normalized_paths(self):
with self.assertRaises(ArgumentDuplicationError):
self.dvc.run(
cmd="", deps=[], outs=["foo", "./foo"], fname="circular-dependency.dvc"
cmd="",
deps=[],
outs=["foo", "./foo"],
fname="circular-dependency.dvc",
)
@@ -172,7 +195,10 @@ class TestRunWorkingDirectoryAsOutput(TestDvc):
with self.assertRaises(WorkingDirectoryAsOutputError):
self.dvc.run(
cmd="", cwd=self.DATA_DIR, outs=[self.FOO], fname="inside-cwd.dvc"
cmd="",
cwd=self.DATA_DIR,
outs=[self.FOO],
fname="inside-cwd.dvc",
)
@@ -271,7 +297,16 @@ class TestRunUnprotectOutsCopy(TestDvc):
self.assertEqual(ret, 0)
ret = main(
["run", "-d", self.CODE, "-o", self.FOO, "python", self.CODE, self.FOO]
[
"run",
"-d",
self.CODE,
"-o",
self.FOO,
"python",
self.CODE,
self.FOO,
]
)
self.assertEqual(ret, 0)
self.assertFalse(os.access(self.FOO, os.W_OK))
@@ -313,7 +348,16 @@ class TestRunUnprotectOutsSymlink(TestDvc):
self.assertEqual(ret, 0)
ret = main(
["run", "-d", self.CODE, "-o", self.FOO, "python", self.CODE, self.FOO]
[
"run",
"-d",
self.CODE,
"-o",
self.FOO,
"python",
self.CODE,
self.FOO,
]
)
self.assertEqual(ret, 0)
self.assertFalse(os.access(self.FOO, os.W_OK))
@@ -357,7 +401,16 @@ class TestRunUnprotectOutsHardlink(TestDvc):
self.assertEqual(ret, 0)
ret = main(
["run", "-d", self.CODE, "-o", self.FOO, "python", self.CODE, self.FOO]
[
"run",
"-d",
self.CODE,
"-o",
self.FOO,
"python",
self.CODE,
self.FOO,
]
)
self.assertEqual(ret, 0)
self.assertFalse(os.access(self.FOO, os.W_OK))
@@ -468,7 +521,9 @@ class TestCmdRunOverwrite(TestDvc):
time.sleep(1)
ret = main(["run", "--overwrite-dvcfile", "-f", "out.dvc", "-d", self.BAR])
ret = main(
["run", "--overwrite-dvcfile", "-f", "out.dvc", "-d", self.BAR]
)
self.assertEqual(ret, 0)
# NOTE: check that dvcfile was overwritten
+3 -1
View File
@@ -45,7 +45,9 @@ class TestIgnore(TestGit):
def _count_gitignore(self):
with open(Git.GITIGNORE, "r") as fd:
lines = fd.readlines()
return len(list(filter(lambda x: x.strip() == "/" + self.FOO, lines)))
return len(
list(filter(lambda x: x.strip() == "/" + self.FOO, lines))
)
def test(self):
git = Git(self._root_dir)
+6 -2
View File
@@ -15,10 +15,14 @@ class TestUnprotect(TestDvc):
self.assertEqual(ret, 0)
self.assertNotEqual(
os.stat(self.FOO).st_mode & (stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH), 0
os.stat(self.FOO).st_mode
& (stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH),
0,
)
self.assertEqual(
os.stat(self.FOO).st_mode & (stat.S_IWRITE | stat.S_IWGRP | stat.S_IWOTH), 0
os.stat(self.FOO).st_mode
& (stat.S_IWRITE | stat.S_IWGRP | stat.S_IWOTH),
0,
)
ret = main(["unprotect", self.FOO])
+3 -1
View File
@@ -21,7 +21,9 @@ class TestUtils(TestDvc):
self.assertTrue(filecmp.cmp(src, dest, shallow=False))
utils.copyfile(src, dest_dir)
self.assertTrue(filecmp.cmp(src, "{}/{}".format(dest_dir, src), shallow=False))
self.assertTrue(
filecmp.cmp(src, "{}/{}".format(dest_dir, src), shallow=False)
)
shutil.rmtree(dest_dir)
os.remove(src)
+2 -1
View File
@@ -14,7 +14,8 @@ class TestRemoteAzure(TestCase):
def test_init_compat(self):
url = "azure://ContainerName={container_name};{connection_string}".format(
container_name=self.container_name, connection_string=self.connection_string
container_name=self.container_name,
connection_string=self.connection_string,
)
config = {"url": url}
remote = RemoteAzure(None, config)
+3 -1
View File
@@ -12,4 +12,6 @@ class TestStageChecksum(TestCase):
d = {"md5": "123456", "cmd": "mycmd", "outs": outs, "deps": deps}
with mock.patch.object(stage, "dumpd", return_value=d):
self.assertEqual(stage._compute_md5(), "e9521a22111493406ea64a88cda63e0b")
self.assertEqual(
stage._compute_md5(), "e9521a22111493406ea64a88cda63e0b"
)