mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 07:18:27 +01:00
dvc: comply with both flake8 and black
flake8 provides some additional features like unused vars check, etc, so it makes total sense to use it as well. Signed-off-by: Ruslan Kuprieiev <[email protected]>
This commit is contained in:
@@ -0,0 +1,4 @@
|
||||
[flake8]
|
||||
ignore = E203, E266, E501, W503, F403, F401
|
||||
max-line-length = 79
|
||||
select = B,C,E,F,W,T4,B9
|
||||
@@ -4,3 +4,8 @@ repos:
|
||||
hooks:
|
||||
- id: black
|
||||
language_version: python3.6
|
||||
- repo: https://gitlab.com/pycqa/flake8
|
||||
rev: master
|
||||
hooks:
|
||||
- id: flake8
|
||||
language_version: python3.6
|
||||
|
||||
+3
-1
@@ -228,7 +228,9 @@ class Analytics(object):
|
||||
|
||||
core = config.config.get(Config.SECTION_CORE, {})
|
||||
enabled = core.get(Config.SECTION_CORE_ANALYTICS, True)
|
||||
logger.debug("Analytics is {}abled.".format("en" if enabled else "dis"))
|
||||
logger.debug(
|
||||
"Analytics is {}abled.".format("en" if enabled else "dis")
|
||||
)
|
||||
return enabled
|
||||
|
||||
@staticmethod
|
||||
|
||||
+3
-1
@@ -35,7 +35,9 @@ class Cache(object):
|
||||
protected = config.get(Config.SECTION_CACHE_PROTECTED)
|
||||
|
||||
sect = {
|
||||
Config.PRIVATE_CWD: config.get(Config.PRIVATE_CWD, project.dvc_dir),
|
||||
Config.PRIVATE_CWD: config.get(
|
||||
Config.PRIVATE_CWD, project.dvc_dir
|
||||
),
|
||||
Config.SECTION_REMOTE_URL: cache_dir,
|
||||
Config.SECTION_CACHE_TYPE: cache_type,
|
||||
Config.SECTION_CACHE_PROTECTED: protected,
|
||||
|
||||
+10
-2
@@ -101,7 +101,11 @@ def parse_args(argv=None):
|
||||
"-q", "--quiet", action="store_true", default=False, help="Be quiet."
|
||||
)
|
||||
log_level_group.add_argument(
|
||||
"-v", "--verbose", action="store_true", default=False, help="Be verbose."
|
||||
"-v",
|
||||
"--verbose",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Be verbose.",
|
||||
)
|
||||
|
||||
# Main parser
|
||||
@@ -116,7 +120,11 @@ def parse_args(argv=None):
|
||||
# NOTE: On some python versions action='version' prints to stderr
|
||||
# instead of stdout https://bugs.python.org/issue18920
|
||||
parser.add_argument(
|
||||
"-V", "--version", action=VersionAction, nargs=0, help="Show program's version."
|
||||
"-V",
|
||||
"--version",
|
||||
action=VersionAction,
|
||||
nargs=0,
|
||||
help="Show program's version.",
|
||||
)
|
||||
|
||||
# Sub commands
|
||||
|
||||
+3
-1
@@ -28,5 +28,7 @@ def add_parser(subparsers, parent_parser):
|
||||
default=False,
|
||||
help="Recursively add each file under the directory.",
|
||||
)
|
||||
add_parser.add_argument("targets", nargs="+", help="Input files/directories.")
|
||||
add_parser.add_argument(
|
||||
"targets", nargs="+", help="Input files/directories."
|
||||
)
|
||||
add_parser.set_defaults(func=CmdAdd)
|
||||
|
||||
@@ -22,7 +22,10 @@ def add_parser(subparsers, parent_parser):
|
||||
|
||||
CACHE_HELP = "Manage cache settings."
|
||||
cache_parser = subparsers.add_parser(
|
||||
"cache", parents=[parent_parser], description=CACHE_HELP, help=CACHE_HELP
|
||||
"cache",
|
||||
parents=[parent_parser],
|
||||
description=CACHE_HELP,
|
||||
help=CACHE_HELP,
|
||||
)
|
||||
|
||||
cache_subparsers = cache_parser.add_subparsers(
|
||||
@@ -42,7 +45,11 @@ def add_parser(subparsers, parent_parser):
|
||||
help=CACHE_DIR_HELP,
|
||||
)
|
||||
cache_dir_parser.add_argument(
|
||||
"-u", "--unset", default=False, action="store_true", help="Unset option."
|
||||
"-u",
|
||||
"--unset",
|
||||
default=False,
|
||||
action="store_true",
|
||||
help="Unset option.",
|
||||
)
|
||||
cache_dir_parser.add_argument(
|
||||
"value",
|
||||
|
||||
@@ -10,7 +10,9 @@ class CmdCheckout(CmdBase):
|
||||
else:
|
||||
for target in self.args.targets:
|
||||
self.project.checkout(
|
||||
target=target, with_deps=self.args.with_deps, force=self.args.force
|
||||
target=target,
|
||||
with_deps=self.args.with_deps,
|
||||
force=self.args.force,
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
+11
-3
@@ -64,7 +64,9 @@ class CmdConfig(CmdBase):
|
||||
self.config.set(self.configobj, section, opt, value)
|
||||
self.config.save(self.configobj)
|
||||
except DvcException:
|
||||
logger.error("failed to set '{}.{}' to '{}'".format(section, opt, value))
|
||||
logger.error(
|
||||
"failed to set '{}.{}' to '{}'".format(section, opt, value)
|
||||
)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
@@ -104,8 +106,14 @@ def add_parser(subparsers, parent_parser):
|
||||
help=CONFIG_HELP,
|
||||
)
|
||||
config_parser.add_argument(
|
||||
"-u", "--unset", default=False, action="store_true", help="Unset option."
|
||||
"-u",
|
||||
"--unset",
|
||||
default=False,
|
||||
action="store_true",
|
||||
help="Unset option.",
|
||||
)
|
||||
config_parser.add_argument("name", help="Option name.")
|
||||
config_parser.add_argument("value", nargs="?", default=None, help="Option value.")
|
||||
config_parser.add_argument(
|
||||
"value", nargs="?", default=None, help="Option value."
|
||||
)
|
||||
config_parser.set_defaults(func=CmdConfig)
|
||||
|
||||
@@ -40,7 +40,10 @@ class CmdDaemonAnalytics(CmdDaemonBase):
|
||||
def add_parser(subparsers, parent_parser):
|
||||
DAEMON_HELP = "Service daemon."
|
||||
daemon_parser = subparsers.add_parser(
|
||||
"daemon", parents=[parent_parser], description=DAEMON_HELP, help=DAEMON_HELP
|
||||
"daemon",
|
||||
parents=[parent_parser],
|
||||
description=DAEMON_HELP,
|
||||
help=DAEMON_HELP,
|
||||
)
|
||||
|
||||
daemon_subparsers = daemon_parser.add_subparsers(
|
||||
|
||||
@@ -106,9 +106,14 @@ def add_parser(subparsers, parent_parser):
|
||||
# Pull
|
||||
PULL_HELP = "Pull data files from the cloud."
|
||||
pull_parser = subparsers.add_parser(
|
||||
"pull", parents=[parent_cache_parser], description=PULL_HELP, help=PULL_HELP
|
||||
"pull",
|
||||
parents=[parent_cache_parser],
|
||||
description=PULL_HELP,
|
||||
help=PULL_HELP,
|
||||
)
|
||||
pull_parser.add_argument(
|
||||
"-r", "--remote", help="Remote repository to pull from."
|
||||
)
|
||||
pull_parser.add_argument("-r", "--remote", help="Remote repository to pull from.")
|
||||
pull_parser.add_argument(
|
||||
"-a",
|
||||
"--all-branches",
|
||||
@@ -149,9 +154,14 @@ def add_parser(subparsers, parent_parser):
|
||||
# Push
|
||||
PUSH_HELP = "Push data files to the cloud."
|
||||
push_parser = subparsers.add_parser(
|
||||
"push", parents=[parent_cache_parser], description=PUSH_HELP, help=PUSH_HELP
|
||||
"push",
|
||||
parents=[parent_cache_parser],
|
||||
description=PUSH_HELP,
|
||||
help=PUSH_HELP,
|
||||
)
|
||||
push_parser.add_argument(
|
||||
"-r", "--remote", help="Remote repository to push to."
|
||||
)
|
||||
push_parser.add_argument("-r", "--remote", help="Remote repository to push to.")
|
||||
push_parser.add_argument(
|
||||
"-a",
|
||||
"--all-branches",
|
||||
@@ -185,9 +195,14 @@ def add_parser(subparsers, parent_parser):
|
||||
# Fetch
|
||||
FETCH_HELP = "Fetch data files from the cloud."
|
||||
fetch_parser = subparsers.add_parser(
|
||||
"fetch", parents=[parent_cache_parser], description=FETCH_HELP, help=FETCH_HELP
|
||||
"fetch",
|
||||
parents=[parent_cache_parser],
|
||||
description=FETCH_HELP,
|
||||
help=FETCH_HELP,
|
||||
)
|
||||
fetch_parser.add_argument(
|
||||
"-r", "--remote", help="Remote repository to fetch from."
|
||||
)
|
||||
fetch_parser.add_argument("-r", "--remote", help="Remote repository to fetch from.")
|
||||
fetch_parser.add_argument(
|
||||
"-a",
|
||||
"--all-branches",
|
||||
|
||||
+11
-3
@@ -31,12 +31,20 @@ class CmdDestroy(CmdBase):
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
DESTROY_HELP = (
|
||||
"Destroy dvc. Will remove all project's information, " "data files and cache."
|
||||
"Destroy dvc. Will remove all project's information, "
|
||||
"data files and cache."
|
||||
)
|
||||
destroy_parser = subparsers.add_parser(
|
||||
"destroy", parents=[parent_parser], description=DESTROY_HELP, help=DESTROY_HELP
|
||||
"destroy",
|
||||
parents=[parent_parser],
|
||||
description=DESTROY_HELP,
|
||||
help=DESTROY_HELP,
|
||||
)
|
||||
destroy_parser.add_argument(
|
||||
"-f", "--force", action="store_true", default=False, help="Force destruction."
|
||||
"-f",
|
||||
"--force",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Force destruction.",
|
||||
)
|
||||
destroy_parser.set_defaults(func=CmdDestroy)
|
||||
|
||||
+4
-1
@@ -30,7 +30,10 @@ class CmdImport(CmdBase):
|
||||
def add_parser(subparsers, parent_parser):
|
||||
IMPORT_HELP = "Import files from URL."
|
||||
import_parser = subparsers.add_parser(
|
||||
"import", parents=[parent_parser], description=IMPORT_HELP, help=IMPORT_HELP
|
||||
"import",
|
||||
parents=[parent_parser],
|
||||
description=IMPORT_HELP,
|
||||
help=IMPORT_HELP,
|
||||
)
|
||||
import_parser.add_argument(
|
||||
"url",
|
||||
|
||||
+3
-1
@@ -23,7 +23,9 @@ class CmdInit(object):
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
"""Setup parser for `dvc init`."""
|
||||
INIT_HELP = "Initialize dvc over a directory " "(should already be a git dir)."
|
||||
INIT_HELP = (
|
||||
"Initialize dvc over a directory " "(should already be a git dir)."
|
||||
)
|
||||
init_parser = subparsers.add_parser(
|
||||
"init", parents=[parent_parser], description=INIT_HELP, help=INIT_HELP
|
||||
)
|
||||
|
||||
@@ -17,6 +17,9 @@ class CmdInstall(CmdBase):
|
||||
def add_parser(subparsers, parent_parser):
|
||||
INSTALL_HELP = "Install dvc hooks into the repository."
|
||||
install_parser = subparsers.add_parser(
|
||||
"install", parents=[parent_parser], description=INSTALL_HELP, help=INSTALL_HELP
|
||||
"install",
|
||||
parents=[parent_parser],
|
||||
description=INSTALL_HELP,
|
||||
help=INSTALL_HELP,
|
||||
)
|
||||
install_parser.set_defaults(func=CmdInstall)
|
||||
|
||||
+7
-2
@@ -12,7 +12,9 @@ class CmdLockBase(CmdBase):
|
||||
self.project.lock_stage(target, unlock=unlock)
|
||||
except DvcException:
|
||||
logger.error(
|
||||
"failed to {}lock '{}'".format("un" if unlock else "", target)
|
||||
"failed to {}lock '{}'".format(
|
||||
"un" if unlock else "", target
|
||||
)
|
||||
)
|
||||
|
||||
return 1
|
||||
@@ -39,7 +41,10 @@ def add_parser(subparsers, parent_parser):
|
||||
|
||||
UNLOCK_HELP = "Unlock DVC file."
|
||||
unlock_parser = subparsers.add_parser(
|
||||
"unlock", parents=[parent_parser], description=UNLOCK_HELP, help=UNLOCK_HELP
|
||||
"unlock",
|
||||
parents=[parent_parser],
|
||||
description=UNLOCK_HELP,
|
||||
help=UNLOCK_HELP,
|
||||
)
|
||||
unlock_parser.add_argument("targets", nargs="+", help="DVC files.")
|
||||
unlock_parser.set_defaults(func=CmdUnlock)
|
||||
|
||||
@@ -59,7 +59,9 @@ class CmdMetricsModify(CmdBase):
|
||||
class CmdMetricsAdd(CmdBase):
|
||||
def run(self):
|
||||
try:
|
||||
self.project.metrics_add(self.args.path, self.args.type, self.args.xpath)
|
||||
self.project.metrics_add(
|
||||
self.args.path, self.args.type, self.args.xpath
|
||||
)
|
||||
except DvcException:
|
||||
msg = "failed to add metric file '{}'".format(self.args.path)
|
||||
logger.error(msg)
|
||||
@@ -83,11 +85,15 @@ class CmdMetricsRemove(CmdBase):
|
||||
def add_parser(subparsers, parent_parser):
|
||||
METRICS_HELP = "Get metrics from all branches."
|
||||
metrics_parser = subparsers.add_parser(
|
||||
"metrics", parents=[parent_parser], description=METRICS_HELP, help=METRICS_HELP
|
||||
"metrics",
|
||||
parents=[parent_parser],
|
||||
description=METRICS_HELP,
|
||||
help=METRICS_HELP,
|
||||
)
|
||||
|
||||
metrics_subparsers = metrics_parser.add_subparsers(
|
||||
dest="cmd", help="Use dvc metrics CMD --help for command-specific help."
|
||||
dest="cmd",
|
||||
help="Use dvc metrics CMD --help for command-specific help.",
|
||||
)
|
||||
|
||||
fix_subparsers(metrics_subparsers)
|
||||
|
||||
+6
-2
@@ -10,7 +10,9 @@ class CmdMove(CmdBase):
|
||||
try:
|
||||
self.project.move(self.args.src, self.args.dst)
|
||||
except DvcException:
|
||||
msg = "failed to move '{}' -> '{}'".format(self.args.src, self.args.dst)
|
||||
msg = "failed to move '{}' -> '{}'".format(
|
||||
self.args.src, self.args.dst
|
||||
)
|
||||
logger.error(msg)
|
||||
return 1
|
||||
return 0
|
||||
@@ -26,6 +28,8 @@ def add_parser(subparsers, parent_parser):
|
||||
move_parser = subparsers.add_parser(
|
||||
"move", parents=[parent_parser], description=description, help=help
|
||||
)
|
||||
move_parser.add_argument("src", help="Source path to a data file or directory.")
|
||||
move_parser.add_argument(
|
||||
"src", help="Source path to a data file or directory."
|
||||
)
|
||||
move_parser.add_argument("dst", help="Destination path.")
|
||||
move_parser.set_defaults(func=CmdMove)
|
||||
|
||||
+19
-6
@@ -35,7 +35,9 @@ class CmdPipelineShow(CmdBase):
|
||||
stage = Stage.load(self.project, target)
|
||||
node = os.path.relpath(stage.path, self.project.root_dir)
|
||||
|
||||
pipelines = list(filter(lambda g: node in g.nodes(), self.project.pipelines()))
|
||||
pipelines = list(
|
||||
filter(lambda g: node in g.nodes(), self.project.pipelines())
|
||||
)
|
||||
|
||||
assert len(pipelines) == 1
|
||||
G = pipelines[0]
|
||||
@@ -99,10 +101,15 @@ class CmdPipelineShow(CmdBase):
|
||||
for target in self.args.targets:
|
||||
try:
|
||||
if self.args.ascii:
|
||||
self._show_ascii(target, self.args.commands, self.args.outs)
|
||||
self._show_ascii(
|
||||
target, self.args.commands, self.args.outs
|
||||
)
|
||||
elif self.args.dot:
|
||||
self.__write_dot(
|
||||
target, self.args.commands, self.args.outs, self.args.dot
|
||||
target,
|
||||
self.args.commands,
|
||||
self.args.outs,
|
||||
self.args.dot,
|
||||
)
|
||||
else:
|
||||
self._show(target, self.args.commands, self.args.outs)
|
||||
@@ -139,7 +146,8 @@ def add_parser(subparsers, parent_parser):
|
||||
)
|
||||
|
||||
pipeline_subparsers = pipeline_parser.add_subparsers(
|
||||
dest="cmd", help="Use dvc pipeline CMD --help for command-specific help."
|
||||
dest="cmd",
|
||||
help="Use dvc pipeline CMD --help for command-specific help.",
|
||||
)
|
||||
|
||||
fix_subparsers(pipeline_subparsers)
|
||||
@@ -167,9 +175,14 @@ def add_parser(subparsers, parent_parser):
|
||||
help="Print output files instead of paths to DVC files.",
|
||||
)
|
||||
pipeline_show_parser.add_argument(
|
||||
"--ascii", action="store_true", default=False, help="Output DAG as ASCII."
|
||||
"--ascii",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Output DAG as ASCII.",
|
||||
)
|
||||
pipeline_show_parser.add_argument(
|
||||
"--dot", help="Write DAG in .dot format."
|
||||
)
|
||||
pipeline_show_parser.add_argument("--dot", help="Write DAG in .dot format.")
|
||||
pipeline_show_parser.add_argument(
|
||||
"targets", nargs="*", help="DVC files. 'Dvcfile' by default."
|
||||
)
|
||||
|
||||
+23
-7
@@ -32,7 +32,9 @@ class CmdRemoteAdd(CmdConfig):
|
||||
def run(self):
|
||||
remote = _get({Config.SECTION_REMOTE_URL: self.args.url})
|
||||
if remote == RemoteLOCAL:
|
||||
self.args.url = self.resolve_path(self.args.url, self.configobj.filename)
|
||||
self.args.url = self.resolve_path(
|
||||
self.args.url, self.configobj.filename
|
||||
)
|
||||
|
||||
section = Config.SECTION_REMOTE_FMT.format(self.args.name)
|
||||
ret = self._set(section, Config.SECTION_REMOTE_URL, self.args.url)
|
||||
@@ -61,7 +63,9 @@ class CmdRemoteRemove(CmdConfig):
|
||||
|
||||
if default == self.args.name:
|
||||
return self._unset(
|
||||
Config.SECTION_CORE, opt=Config.SECTION_CORE_REMOTE, configobj=config
|
||||
Config.SECTION_CORE,
|
||||
opt=Config.SECTION_CORE_REMOTE,
|
||||
configobj=config,
|
||||
)
|
||||
|
||||
def run(self):
|
||||
@@ -104,7 +108,9 @@ class CmdRemoteList(CmdConfig):
|
||||
r = re.match(Config.SECTION_REMOTE_REGEX, section)
|
||||
if r:
|
||||
name = r.group("name")
|
||||
url = self.configobj[section].get(Config.SECTION_REMOTE_URL, "")
|
||||
url = self.configobj[section].get(
|
||||
Config.SECTION_REMOTE_URL, ""
|
||||
)
|
||||
logger.info("{}\t{}".format(name, url))
|
||||
return 0
|
||||
|
||||
@@ -114,11 +120,15 @@ def add_parser(subparsers, parent_parser):
|
||||
|
||||
REMOTE_HELP = "Manage set of tracked repositories."
|
||||
remote_parser = subparsers.add_parser(
|
||||
"remote", parents=[parent_parser], description=REMOTE_HELP, help=REMOTE_HELP
|
||||
"remote",
|
||||
parents=[parent_parser],
|
||||
description=REMOTE_HELP,
|
||||
help=REMOTE_HELP,
|
||||
)
|
||||
|
||||
remote_subparsers = remote_parser.add_subparsers(
|
||||
dest="cmd", help="Use dvc remote CMD --help for " "command-specific help."
|
||||
dest="cmd",
|
||||
help="Use dvc remote CMD --help for " "command-specific help.",
|
||||
)
|
||||
|
||||
fix_subparsers(remote_subparsers)
|
||||
@@ -152,7 +162,9 @@ def add_parser(subparsers, parent_parser):
|
||||
description=REMOTE_DEFAULT_HELP,
|
||||
help=REMOTE_DEFAULT_HELP,
|
||||
)
|
||||
remote_default_parser.add_argument("name", nargs="?", help="Name of the remote.")
|
||||
remote_default_parser.add_argument(
|
||||
"name", nargs="?", help="Name of the remote."
|
||||
)
|
||||
remote_default_parser.add_argument(
|
||||
"-u",
|
||||
"--unset",
|
||||
@@ -183,7 +195,11 @@ def add_parser(subparsers, parent_parser):
|
||||
remote_modify_parser.add_argument("option", help="Option.")
|
||||
remote_modify_parser.add_argument("value", nargs="?", help="Value.")
|
||||
remote_modify_parser.add_argument(
|
||||
"-u", "--unset", default=False, action="store_true", help="Unset option."
|
||||
"-u",
|
||||
"--unset",
|
||||
default=False,
|
||||
action="store_true",
|
||||
help="Unset option.",
|
||||
)
|
||||
remote_modify_parser.set_defaults(func=CmdRemoteModify)
|
||||
|
||||
|
||||
+14
-4
@@ -14,13 +14,16 @@ class CmdRemove(CmdBase):
|
||||
if self.args.force:
|
||||
return False
|
||||
|
||||
msg = "Are you sure you want to remove {} with its outputs?".format(target)
|
||||
msg = "Are you sure you want to remove {} with its outputs?".format(
|
||||
target
|
||||
)
|
||||
|
||||
if prompt.confirm(msg):
|
||||
return False
|
||||
|
||||
raise DvcException(
|
||||
"Cannot purge without a confirmation from the user." " Use '-f' to force."
|
||||
"Cannot purge without a confirmation from the user."
|
||||
" Use '-f' to force."
|
||||
)
|
||||
|
||||
def run(self):
|
||||
@@ -37,7 +40,10 @@ class CmdRemove(CmdBase):
|
||||
def add_parser(subparsers, parent_parser):
|
||||
REMOVE_HELP = "Remove outputs of DVC file."
|
||||
remove_parser = subparsers.add_parser(
|
||||
"remove", parents=[parent_parser], description=REMOVE_HELP, help=REMOVE_HELP
|
||||
"remove",
|
||||
parents=[parent_parser],
|
||||
description=REMOVE_HELP,
|
||||
help=REMOVE_HELP,
|
||||
)
|
||||
remove_parser_group = remove_parser.add_mutually_exclusive_group()
|
||||
remove_parser_group.add_argument(
|
||||
@@ -55,7 +61,11 @@ def add_parser(subparsers, parent_parser):
|
||||
help="Remove DVC file and all its outputs.",
|
||||
)
|
||||
remove_parser.add_argument(
|
||||
"-f", "--force", action="store_true", default=False, help="Force purge."
|
||||
"-f",
|
||||
"--force",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Force purge.",
|
||||
)
|
||||
remove_parser.add_argument("targets", nargs="+", help="DVC files.")
|
||||
remove_parser.set_defaults(func=CmdRemove)
|
||||
|
||||
@@ -52,9 +52,14 @@ class CmdRepro(CmdBase):
|
||||
def add_parser(subparsers, parent_parser):
|
||||
REPRO_HELP = "Reproduce DVC file. Default file name - 'Dvcfile'."
|
||||
repro_parser = subparsers.add_parser(
|
||||
"repro", parents=[parent_parser], description=REPRO_HELP, help=REPRO_HELP
|
||||
"repro",
|
||||
parents=[parent_parser],
|
||||
description=REPRO_HELP,
|
||||
help=REPRO_HELP,
|
||||
)
|
||||
repro_parser.add_argument(
|
||||
"targets", nargs="*", help="DVC file to reproduce."
|
||||
)
|
||||
repro_parser.add_argument("targets", nargs="*", help="DVC file to reproduce.")
|
||||
repro_parser.add_argument(
|
||||
"-f",
|
||||
"--force",
|
||||
@@ -67,7 +72,8 @@ def add_parser(subparsers, parent_parser):
|
||||
"--single-item",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Reproduce only single data item without recursive dependencies " "check.",
|
||||
help="Reproduce only single data item without recursive dependencies "
|
||||
"check.",
|
||||
)
|
||||
repro_parser.add_argument(
|
||||
"-c",
|
||||
|
||||
+10
-2
@@ -13,7 +13,12 @@ class CmdRun(CmdBase):
|
||||
overwrite = self.args.yes or self.args.overwrite_dvcfile
|
||||
|
||||
if not any(
|
||||
[self.args.deps, self.args.outs, self.args.outs_no_cache, self.args.command]
|
||||
[
|
||||
self.args.deps,
|
||||
self.args.outs,
|
||||
self.args.outs_no_cache,
|
||||
self.args.command,
|
||||
]
|
||||
): # pragma: no cover
|
||||
logger.error(
|
||||
"too few arguments. Specify at least one: '-d', "
|
||||
@@ -64,7 +69,10 @@ class CmdRun(CmdBase):
|
||||
|
||||
|
||||
def add_parser(subparsers, parent_parser):
|
||||
RUN_HELP = "Generate a stage file from a given " "command and execute the command."
|
||||
RUN_HELP = (
|
||||
"Generate a stage file from a given "
|
||||
"command and execute the command."
|
||||
)
|
||||
run_parser = subparsers.add_parser(
|
||||
"run", parents=[parent_parser], description=RUN_HELP, help=RUN_HELP
|
||||
)
|
||||
|
||||
@@ -25,5 +25,7 @@ def add_parser(subparsers, parent_parser):
|
||||
description=UNPROTECT_HELP,
|
||||
help=UNPROTECT_HELP,
|
||||
)
|
||||
unprotect_parser.add_argument("targets", nargs="+", help="Data files/directory.")
|
||||
unprotect_parser.add_argument(
|
||||
"targets", nargs="+", help="Data files/directory."
|
||||
)
|
||||
unprotect_parser.set_defaults(func=CmdUnprotect)
|
||||
|
||||
+27
-9
@@ -22,7 +22,9 @@ class ConfigError(DvcException):
|
||||
"""
|
||||
|
||||
def __init__(self, msg, ex=None):
|
||||
super(ConfigError, self).__init__("config file error: {}".format(msg), ex)
|
||||
super(ConfigError, self).__init__(
|
||||
"config file error: {}".format(msg), ex
|
||||
)
|
||||
|
||||
|
||||
def supported_cache_type(types):
|
||||
@@ -178,7 +180,9 @@ class Config(object): # pylint: disable=too-many-instance-attributes
|
||||
Optional(
|
||||
SECTION_CORE_INTERACTIVE, default=False
|
||||
): SECTION_CORE_INTERACTIVE_SCHEMA,
|
||||
Optional(SECTION_CORE_ANALYTICS, default=True): SECTION_CORE_ANALYTICS_SCHEMA,
|
||||
Optional(
|
||||
SECTION_CORE_ANALYTICS, default=True
|
||||
): SECTION_CORE_ANALYTICS_SCHEMA,
|
||||
# backward compatibility
|
||||
Optional(SECTION_CORE_CLOUD, default=""): SECTION_CORE_CLOUD_SCHEMA,
|
||||
Optional(SECTION_CORE_STORAGEPATH, default=""): str,
|
||||
@@ -198,7 +202,9 @@ class Config(object): # pylint: disable=too-many-instance-attributes
|
||||
Optional(SECTION_AWS_PROFILE): str,
|
||||
Optional(SECTION_AWS_CREDENTIALPATH): str,
|
||||
Optional(SECTION_AWS_ENDPOINT_URL): str,
|
||||
Optional(SECTION_AWS_USE_SSL, default=True): And(str, is_bool, Use(to_bool)),
|
||||
Optional(SECTION_AWS_USE_SSL, default=True): And(
|
||||
str, is_bool, Use(to_bool)
|
||||
),
|
||||
}
|
||||
|
||||
# backward compatibility
|
||||
@@ -232,7 +238,9 @@ class Config(object): # pylint: disable=too-many-instance-attributes
|
||||
Optional(SECTION_AWS_PROFILE): str,
|
||||
Optional(SECTION_AWS_CREDENTIALPATH): str,
|
||||
Optional(SECTION_AWS_ENDPOINT_URL): str,
|
||||
Optional(SECTION_AWS_USE_SSL, default=True): And(str, is_bool, Use(to_bool)),
|
||||
Optional(SECTION_AWS_USE_SSL, default=True): And(
|
||||
str, is_bool, Use(to_bool)
|
||||
),
|
||||
Optional(SECTION_GCP_PROJECTNAME): str,
|
||||
Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA,
|
||||
Optional(SECTION_CACHE_PROTECTED, default=False): And(
|
||||
@@ -302,7 +310,9 @@ class Config(object): # pylint: disable=too-many-instance-attributes
|
||||
"""
|
||||
from appdirs import user_config_dir
|
||||
|
||||
return user_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR)
|
||||
return user_config_dir(
|
||||
appname=Config.APPNAME, appauthor=Config.APPAUTHOR
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_system_config_dir():
|
||||
@@ -313,7 +323,9 @@ class Config(object): # pylint: disable=too-many-instance-attributes
|
||||
"""
|
||||
from appdirs import site_config_dir
|
||||
|
||||
return site_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR)
|
||||
return site_config_dir(
|
||||
appname=Config.APPNAME, appauthor=Config.APPAUTHOR
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def init(dvc_dir):
|
||||
@@ -403,7 +415,9 @@ class Config(object): # pylint: disable=too-many-instance-attributes
|
||||
self.config = Schema(self.SCHEMA).validate(self.config)
|
||||
|
||||
# NOTE: now converting back to ConfigObj
|
||||
self.config = configobj.ConfigObj(self.config, write_empty_values=True)
|
||||
self.config = configobj.ConfigObj(
|
||||
self.config, write_empty_values=True
|
||||
)
|
||||
self.config.filename = self.config_file
|
||||
self._resolve_paths(self.config, self.config_file)
|
||||
except Exception as ex:
|
||||
@@ -474,7 +488,9 @@ class Config(object): # pylint: disable=too-many-instance-attributes
|
||||
return
|
||||
|
||||
if opt not in config[section].keys():
|
||||
raise ConfigError("option '{}.{}' doesn't exist".format(section, opt))
|
||||
raise ConfigError(
|
||||
"option '{}.{}' doesn't exist".format(section, opt)
|
||||
)
|
||||
del config[section][opt]
|
||||
|
||||
if not config[section]:
|
||||
@@ -508,7 +524,9 @@ class Config(object): # pylint: disable=too-many-instance-attributes
|
||||
raise ConfigError("section '{}' doesn't exist".format(section))
|
||||
|
||||
if opt not in config[section].keys():
|
||||
raise ConfigError("option '{}.{}' doesn't exist".format(section, opt))
|
||||
raise ConfigError(
|
||||
"option '{}.{}' doesn't exist".format(section, opt)
|
||||
)
|
||||
|
||||
logger.info(config[section][opt])
|
||||
|
||||
|
||||
+7
-2
@@ -357,9 +357,14 @@ def draw(vertexes, edges):
|
||||
y = vertex.view.xy[1]
|
||||
|
||||
canvas.box(
|
||||
int(round(x - minx)), int(round(y - miny)), vertex.view.w, vertex.view.h
|
||||
int(round(x - minx)),
|
||||
int(round(y - miny)),
|
||||
vertex.view.w,
|
||||
vertex.view.h,
|
||||
)
|
||||
|
||||
canvas.text(int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data)
|
||||
canvas.text(
|
||||
int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data
|
||||
)
|
||||
|
||||
canvas.draw()
|
||||
|
||||
@@ -10,7 +10,9 @@ class DependencyHTTP(DependencyBase, OutputBase):
|
||||
REMOTE = RemoteHTTP
|
||||
|
||||
def __init__(self, stage, path, info=None, remote=None):
|
||||
super(DependencyHTTP, self).__init__(stage, path, info=info, remote=remote)
|
||||
super(DependencyHTTP, self).__init__(
|
||||
stage, path, info=info, remote=remote
|
||||
)
|
||||
if path.startswith("remote"):
|
||||
path = urljoin(self.remote.cache_dir, urlparse(path).path)
|
||||
|
||||
|
||||
+11
-4
@@ -41,10 +41,12 @@ class OutputDuplicationError(DvcException):
|
||||
assert isinstance(output, str) or isinstance(output, builtin_str)
|
||||
assert isinstance(stages, list)
|
||||
assert all(
|
||||
isinstance(stage, str) or isinstance(stage, builtin_str) for stage in stages
|
||||
isinstance(stage, str) or isinstance(stage, builtin_str)
|
||||
for stage in stages
|
||||
)
|
||||
msg = (
|
||||
"file/directory '{}' is specified as an output in more than one" "stage: {}"
|
||||
"file/directory '{}' is specified as an output in more than one"
|
||||
"stage: {}"
|
||||
).format(output, "\n ".join(stages))
|
||||
super(OutputDuplicationError, self).__init__(msg)
|
||||
|
||||
@@ -80,9 +82,14 @@ class CircularDependencyError(DvcException):
|
||||
"""
|
||||
|
||||
def __init__(self, dependency):
|
||||
assert isinstance(dependency, str) or isinstance(dependency, builtin_str)
|
||||
assert isinstance(dependency, str) or isinstance(
|
||||
dependency, builtin_str
|
||||
)
|
||||
|
||||
msg = "file/directory '{}' is specified as an output and as a " "dependency."
|
||||
msg = (
|
||||
"file/directory '{}' is specified as an output and as a "
|
||||
"dependency."
|
||||
)
|
||||
super(CircularDependencyError, self).__init__(msg.format(dependency))
|
||||
|
||||
|
||||
|
||||
+2
-1
@@ -244,7 +244,8 @@ def _parse_exc():
|
||||
|
||||
if is_verbose():
|
||||
stack_trace = "{line}\n{stack_trace}{line}\n".format(
|
||||
line=colorize("-" * 60, color="red"), stack_trace="\n".join(tb_list)
|
||||
line=colorize("-" * 60, color="red"),
|
||||
stack_trace="\n".join(tb_list),
|
||||
)
|
||||
else:
|
||||
stack_trace = ""
|
||||
|
||||
+13
-4
@@ -48,7 +48,9 @@ class OutputBase(object):
|
||||
DoesNotExistError = OutputDoesNotExistError
|
||||
IsNotFileOrDirError = OutputIsNotFileOrDirError
|
||||
|
||||
def __init__(self, stage, path, info=None, remote=None, cache=True, metric=False):
|
||||
def __init__(
|
||||
self, stage, path, info=None, remote=None, cache=True, metric=False
|
||||
):
|
||||
self.stage = stage
|
||||
self.project = stage.project
|
||||
self.url = path
|
||||
@@ -57,9 +59,14 @@ class OutputBase(object):
|
||||
self.use_cache = False if self.IS_DEPENDENCY else cache
|
||||
self.metric = False if self.IS_DEPENDENCY else metric
|
||||
|
||||
if self.use_cache and getattr(self.project.cache, self.REMOTE.scheme) is None:
|
||||
if (
|
||||
self.use_cache
|
||||
and getattr(self.project.cache, self.REMOTE.scheme) is None
|
||||
):
|
||||
raise DvcException(
|
||||
"no cache location setup for '{}' outputs.".format(self.REMOTE.scheme)
|
||||
"no cache location setup for '{}' outputs.".format(
|
||||
self.REMOTE.scheme
|
||||
)
|
||||
)
|
||||
|
||||
def __repr__(self):
|
||||
@@ -133,7 +140,9 @@ class OutputBase(object):
|
||||
if not self.use_cache:
|
||||
self.info = self.remote.save_info(self.path_info)
|
||||
else:
|
||||
self.info = getattr(self.project.cache, self.scheme).save(self.path_info)
|
||||
self.info = getattr(self.project.cache, self.scheme).save(
|
||||
self.path_info
|
||||
)
|
||||
|
||||
def dumpd(self):
|
||||
ret = self.info.copy()
|
||||
|
||||
+3
-1
@@ -10,7 +10,9 @@ from dvc.remote.hdfs import RemoteHDFS
|
||||
class OutputHDFS(OutputBase):
|
||||
REMOTE = RemoteHDFS
|
||||
|
||||
def __init__(self, stage, path, info=None, remote=None, cache=True, metric=False):
|
||||
def __init__(
|
||||
self, stage, path, info=None, remote=None, cache=True, metric=False
|
||||
):
|
||||
super(OutputHDFS, self).__init__(
|
||||
stage, path, info=info, remote=remote, cache=cache, metric=metric
|
||||
)
|
||||
|
||||
+12
-4
@@ -13,12 +13,16 @@ from dvc.output.base import OutputBase, OutputAlreadyTrackedError
|
||||
class OutputLOCAL(OutputBase):
|
||||
REMOTE = RemoteLOCAL
|
||||
|
||||
def __init__(self, stage, path, info=None, remote=None, cache=True, metric=False):
|
||||
def __init__(
|
||||
self, stage, path, info=None, remote=None, cache=True, metric=False
|
||||
):
|
||||
super(OutputLOCAL, self).__init__(
|
||||
stage, path, info, remote=remote, cache=cache, metric=metric
|
||||
)
|
||||
if remote:
|
||||
p = os.path.join(remote.prefix, urlparse(self.url).path.lstrip("/"))
|
||||
p = os.path.join(
|
||||
remote.prefix, urlparse(self.url).path.lstrip("/")
|
||||
)
|
||||
else:
|
||||
p = path
|
||||
|
||||
@@ -34,7 +38,9 @@ class OutputLOCAL(OutputBase):
|
||||
|
||||
@property
|
||||
def is_local(self):
|
||||
return urlparse(self.url).scheme != "remote" and not os.path.isabs(self.url)
|
||||
return urlparse(self.url).scheme != "remote" and not os.path.isabs(
|
||||
self.url
|
||||
)
|
||||
|
||||
@property
|
||||
def sep(self):
|
||||
@@ -51,7 +57,9 @@ class OutputLOCAL(OutputBase):
|
||||
def dumpd(self):
|
||||
ret = super(OutputLOCAL, self).dumpd()
|
||||
if self.is_local:
|
||||
path = self.remote.unixpath(os.path.relpath(self.path, self.stage.cwd))
|
||||
path = self.remote.unixpath(
|
||||
os.path.relpath(self.path, self.stage.cwd)
|
||||
)
|
||||
else:
|
||||
path = self.url
|
||||
|
||||
|
||||
+8
-2
@@ -10,7 +10,9 @@ from dvc.output.base import OutputBase
|
||||
class OutputS3(OutputBase):
|
||||
REMOTE = RemoteS3
|
||||
|
||||
def __init__(self, stage, path, info=None, remote=None, cache=True, metric=False):
|
||||
def __init__(
|
||||
self, stage, path, info=None, remote=None, cache=True, metric=False
|
||||
):
|
||||
super(OutputS3, self).__init__(
|
||||
stage, path, info=info, remote=remote, cache=cache, metric=metric
|
||||
)
|
||||
@@ -18,4 +20,8 @@ class OutputS3(OutputBase):
|
||||
path = urlparse(path).path.lstrip("/")
|
||||
if remote:
|
||||
path = posixpath.join(remote.prefix, path)
|
||||
self.path_info = {"scheme": self.scheme, "bucket": bucket, "path": path}
|
||||
self.path_info = {
|
||||
"scheme": self.scheme,
|
||||
"bucket": bucket,
|
||||
"path": path,
|
||||
}
|
||||
|
||||
+12
-4
@@ -11,17 +11,25 @@ from dvc.remote.ssh import RemoteSSH
|
||||
class OutputSSH(OutputBase):
|
||||
REMOTE = RemoteSSH
|
||||
|
||||
def __init__(self, stage, path, info=None, remote=None, cache=True, metric=False):
|
||||
def __init__(
|
||||
self, stage, path, info=None, remote=None, cache=True, metric=False
|
||||
):
|
||||
super(OutputSSH, self).__init__(
|
||||
stage, path, info=info, remote=remote, cache=cache, metric=metric
|
||||
)
|
||||
parsed = urlparse(path)
|
||||
host = remote.host if remote else parsed.hostname
|
||||
port = remote.port if remote else (parsed.port or RemoteSSH.DEFAULT_PORT)
|
||||
user = remote.user if remote else (parsed.username or getpass.getuser())
|
||||
port = (
|
||||
remote.port if remote else (parsed.port or RemoteSSH.DEFAULT_PORT)
|
||||
)
|
||||
user = (
|
||||
remote.user if remote else (parsed.username or getpass.getuser())
|
||||
)
|
||||
|
||||
if remote:
|
||||
path = posixpath.join(remote.prefix, urlparse(path).path.lstrip("/"))
|
||||
path = posixpath.join(
|
||||
remote.prefix, urlparse(path).path.lstrip("/")
|
||||
)
|
||||
else:
|
||||
path = parsed.path
|
||||
|
||||
|
||||
+76
-23
@@ -7,7 +7,11 @@ import os
|
||||
import dvc.prompt as prompt
|
||||
import dvc.logger as logger
|
||||
|
||||
from dvc.exceptions import DvcException, MoveNotDataSourceError, NotDvcProjectError
|
||||
from dvc.exceptions import (
|
||||
DvcException,
|
||||
MoveNotDataSourceError,
|
||||
NotDvcProjectError,
|
||||
)
|
||||
|
||||
|
||||
class InitError(DvcException):
|
||||
@@ -276,7 +280,9 @@ class Project(object):
|
||||
import dvc.output as Output
|
||||
from dvc.stage import Stage
|
||||
|
||||
from_out = Output.loads_from(Stage(self, cwd=os.curdir), [from_path])[0]
|
||||
from_out = Output.loads_from(Stage(self, cwd=os.curdir), [from_path])[
|
||||
0
|
||||
]
|
||||
|
||||
to_path = self._expand_target_path(from_path, to_path)
|
||||
|
||||
@@ -289,7 +295,9 @@ class Project(object):
|
||||
)
|
||||
except StopIteration:
|
||||
raise DvcException(
|
||||
"unable to find stage file with output '{path}'".format(path=from_path)
|
||||
"unable to find stage file with output '{path}'".format(
|
||||
path=from_path
|
||||
)
|
||||
)
|
||||
|
||||
if not stage.is_data_source:
|
||||
@@ -349,7 +357,9 @@ class Project(object):
|
||||
|
||||
def unprotect(self, path):
|
||||
if not os.path.exists(path):
|
||||
raise DvcException("can't unprotect non-existing data '{}'".format(path))
|
||||
raise DvcException(
|
||||
"can't unprotect non-existing data '{}'".format(path)
|
||||
)
|
||||
|
||||
if os.path.isdir(path):
|
||||
self._unprotect_dir(path)
|
||||
@@ -553,7 +563,9 @@ class Project(object):
|
||||
used.append(out.path)
|
||||
self.state.remove_unused_links(used)
|
||||
|
||||
def checkout(self, target=None, with_deps=False, force=False, recursive=False):
|
||||
def checkout(
|
||||
self, target=None, with_deps=False, force=False, recursive=False
|
||||
):
|
||||
|
||||
if target and not recursive:
|
||||
all_stages = self.active_stages()
|
||||
@@ -569,7 +581,9 @@ class Project(object):
|
||||
if stage.locked:
|
||||
logger.warning(
|
||||
"DVC file '{path}' is locked. Its dependencies are"
|
||||
" not going to be checked out.".format(path=stage.relpath)
|
||||
" not going to be checked out.".format(
|
||||
path=stage.relpath
|
||||
)
|
||||
)
|
||||
|
||||
stage.checkout(force=force)
|
||||
@@ -597,7 +611,9 @@ class Project(object):
|
||||
|
||||
return ret
|
||||
|
||||
def _collect_dir_cache(self, out, branch=None, remote=None, force=False, jobs=None):
|
||||
def _collect_dir_cache(
|
||||
self, out, branch=None, remote=None, force=False, jobs=None
|
||||
):
|
||||
info = out.dumpd()
|
||||
ret = [info]
|
||||
r = out.remote
|
||||
@@ -605,7 +621,9 @@ class Project(object):
|
||||
|
||||
if self.cache.local.changed_cache_file(md5):
|
||||
try:
|
||||
self.cloud.pull(ret, jobs=jobs, remote=remote, show_checksums=False)
|
||||
self.cloud.pull(
|
||||
ret, jobs=jobs, remote=remote, show_checksums=False
|
||||
)
|
||||
except DvcException as exc:
|
||||
msg = "Failed to pull cache for '{}': {}"
|
||||
logger.debug(msg.format(out, exc))
|
||||
@@ -626,7 +644,9 @@ class Project(object):
|
||||
|
||||
for i in self.cache.local.load_dir_cache(md5):
|
||||
i["branch"] = branch
|
||||
i[r.PARAM_PATH] = os.path.join(info[r.PARAM_PATH], i[r.PARAM_RELPATH])
|
||||
i[r.PARAM_PATH] = os.path.join(
|
||||
info[r.PARAM_PATH], i[r.PARAM_RELPATH]
|
||||
)
|
||||
ret.append(i)
|
||||
|
||||
return ret
|
||||
@@ -680,7 +700,9 @@ class Project(object):
|
||||
cache["ssh"] = []
|
||||
cache["azure"] = []
|
||||
|
||||
for branch in self.scm.brancher(all_branches=all_branches, all_tags=all_tags):
|
||||
for branch in self.scm.brancher(
|
||||
all_branches=all_branches, all_tags=all_tags
|
||||
):
|
||||
if target:
|
||||
if recursive:
|
||||
stages = self.stages(target)
|
||||
@@ -703,7 +725,11 @@ class Project(object):
|
||||
for out in stage.outs:
|
||||
scheme = out.path_info["scheme"]
|
||||
cache[scheme] += self._collect_used_cache(
|
||||
out, branch=branch, remote=remote, force=force, jobs=jobs
|
||||
out,
|
||||
branch=branch,
|
||||
remote=remote,
|
||||
force=force,
|
||||
jobs=jobs,
|
||||
)
|
||||
|
||||
return cache
|
||||
@@ -809,7 +835,9 @@ class Project(object):
|
||||
self._do_gc("azure", self.cache.azure.gc, clist)
|
||||
|
||||
if cloud:
|
||||
self._do_gc("remote", self.cloud._get_cloud(remote, "gc -c").gc, clist)
|
||||
self._do_gc(
|
||||
"remote", self.cloud._get_cloud(remote, "gc -c").gc, clist
|
||||
)
|
||||
|
||||
def push(
|
||||
self,
|
||||
@@ -833,7 +861,9 @@ class Project(object):
|
||||
jobs=jobs,
|
||||
recursive=recursive,
|
||||
)["local"]
|
||||
self.cloud.push(used, jobs, remote=remote, show_checksums=show_checksums)
|
||||
self.cloud.push(
|
||||
used, jobs, remote=remote, show_checksums=show_checksums
|
||||
)
|
||||
|
||||
def fetch(
|
||||
self,
|
||||
@@ -857,7 +887,9 @@ class Project(object):
|
||||
jobs=jobs,
|
||||
recursive=recursive,
|
||||
)["local"]
|
||||
self.cloud.pull(used, jobs, remote=remote, show_checksums=show_checksums)
|
||||
self.cloud.pull(
|
||||
used, jobs, remote=remote, show_checksums=show_checksums
|
||||
)
|
||||
|
||||
def pull(
|
||||
self,
|
||||
@@ -882,7 +914,10 @@ class Project(object):
|
||||
recursive=recursive,
|
||||
)
|
||||
self.checkout(
|
||||
target=target, with_deps=with_deps, force=force, recursive=recursive
|
||||
target=target,
|
||||
with_deps=with_deps,
|
||||
force=force,
|
||||
recursive=recursive,
|
||||
)
|
||||
|
||||
def _local_status(self, target=None, with_deps=False):
|
||||
@@ -938,7 +973,10 @@ class Project(object):
|
||||
if status == cloud.STATUS_OK:
|
||||
continue
|
||||
|
||||
prefix_map = {cloud.STATUS_DELETED: "deleted", cloud.STATUS_NEW: "new"}
|
||||
prefix_map = {
|
||||
cloud.STATUS_DELETED: "deleted",
|
||||
cloud.STATUS_NEW: "new",
|
||||
}
|
||||
|
||||
ret[name] = prefix_map[status]
|
||||
|
||||
@@ -1036,7 +1074,9 @@ class Project(object):
|
||||
abs_path = os.path.abspath(path)
|
||||
if os.path.isdir(abs_path) and recursive:
|
||||
matched = [
|
||||
out for out in outs if os.path.abspath(out.path).startswith(abs_path)
|
||||
out
|
||||
for out in outs
|
||||
if os.path.abspath(out.path).startswith(abs_path)
|
||||
]
|
||||
else:
|
||||
matched = [out for out in outs if out.path == abs_path]
|
||||
@@ -1056,17 +1096,23 @@ class Project(object):
|
||||
recursive=False,
|
||||
):
|
||||
res = {}
|
||||
for branch in self.scm.brancher(all_branches=all_branches, all_tags=all_tags):
|
||||
for branch in self.scm.brancher(
|
||||
all_branches=all_branches, all_tags=all_tags
|
||||
):
|
||||
|
||||
astages = self.active_stages()
|
||||
outs = [out for stage in astages for out in stage.outs]
|
||||
|
||||
if path:
|
||||
outs = self._find_output_by_path(path, outs=outs, recursive=recursive)
|
||||
outs = self._find_output_by_path(
|
||||
path, outs=outs, recursive=recursive
|
||||
)
|
||||
stages = [out.stage.path for out in outs]
|
||||
entries = []
|
||||
for out in outs:
|
||||
if all([out.metric, not typ, isinstance(out.metric, dict)]):
|
||||
if all(
|
||||
[out.metric, not typ, isinstance(out.metric, dict)]
|
||||
):
|
||||
entries += [
|
||||
(
|
||||
out.path,
|
||||
@@ -1192,7 +1238,10 @@ class Project(object):
|
||||
|
||||
def graph(self, stages=None, from_directory=None):
|
||||
import networkx as nx
|
||||
from dvc.exceptions import OutputDuplicationError, WorkingDirectoryAsOutputError
|
||||
from dvc.exceptions import (
|
||||
OutputDuplicationError,
|
||||
WorkingDirectoryAsOutputError,
|
||||
)
|
||||
|
||||
G = nx.DiGraph()
|
||||
G_active = nx.DiGraph()
|
||||
@@ -1217,7 +1266,9 @@ class Project(object):
|
||||
)
|
||||
|
||||
if overlaps:
|
||||
raise WorkingDirectoryAsOutputError(stage.cwd, stage.relpath)
|
||||
raise WorkingDirectoryAsOutputError(
|
||||
stage.cwd, stage.relpath
|
||||
)
|
||||
|
||||
# collect the whole DAG
|
||||
for stage in stages:
|
||||
@@ -1252,7 +1303,9 @@ class Project(object):
|
||||
|
||||
G, G_active = self.graph(from_directory=from_directory)
|
||||
|
||||
return [G.subgraph(c).copy() for c in nx.weakly_connected_components(G)]
|
||||
return [
|
||||
G.subgraph(c).copy() for c in nx.weakly_connected_components(G)
|
||||
]
|
||||
|
||||
def stages(self, from_directory=None):
|
||||
"""
|
||||
|
||||
+15
-5
@@ -84,7 +84,9 @@ class RemoteAzure(RemoteBase):
|
||||
raise NotImplementedError
|
||||
|
||||
logger.debug(
|
||||
"Removing azure://{}/{}".format(path_info["bucket"], path_info["path"])
|
||||
"Removing azure://{}/{}".format(
|
||||
path_info["bucket"], path_info["path"]
|
||||
)
|
||||
)
|
||||
|
||||
self.blob_service.delete_blob(path_info["bucket"], path_info["path"])
|
||||
@@ -93,7 +95,9 @@ class RemoteAzure(RemoteBase):
|
||||
blob_service = self.blob_service
|
||||
next_marker = None
|
||||
while True:
|
||||
blobs = blob_service.list_blobs(bucket, prefix=prefix, marker=next_marker)
|
||||
blobs = blob_service.list_blobs(
|
||||
bucket, prefix=prefix, marker=next_marker
|
||||
)
|
||||
|
||||
for blob in blobs:
|
||||
yield blob.name
|
||||
@@ -120,7 +124,9 @@ class RemoteAzure(RemoteBase):
|
||||
path = to_info["path"]
|
||||
|
||||
logger.debug(
|
||||
"Uploading '{}' to '{}/{}'".format(from_info["path"], bucket, path)
|
||||
"Uploading '{}' to '{}/{}'".format(
|
||||
from_info["path"], bucket, path
|
||||
)
|
||||
)
|
||||
|
||||
if not name:
|
||||
@@ -138,7 +144,9 @@ class RemoteAzure(RemoteBase):
|
||||
else:
|
||||
progress.finish_target(name)
|
||||
|
||||
def download(self, from_infos, to_infos, no_progress_bar=False, names=None):
|
||||
def download(
|
||||
self, from_infos, to_infos, no_progress_bar=False, names=None
|
||||
):
|
||||
names = self._verify_path_args(from_infos, to_infos, names)
|
||||
|
||||
for to_info, from_info, name in zip(to_infos, from_infos, names):
|
||||
@@ -152,7 +160,9 @@ class RemoteAzure(RemoteBase):
|
||||
path = from_info["path"]
|
||||
|
||||
logger.debug(
|
||||
"Downloading '{}/{}' to '{}'".format(bucket, path, to_info["path"])
|
||||
"Downloading '{}/{}' to '{}'".format(
|
||||
bucket, path, to_info["path"]
|
||||
)
|
||||
)
|
||||
|
||||
tmp_file = self.tmp_file(to_info["path"])
|
||||
|
||||
+12
-4
@@ -56,7 +56,9 @@ class RemoteBase(object):
|
||||
deps_ok = all(self.REQUIRES.values())
|
||||
if not deps_ok:
|
||||
missing = [k for k, v in self.REQUIRES.items() if v is None]
|
||||
url = config.get(Config.SECTION_REMOTE_URL, "{}://".format(self.scheme))
|
||||
url = config.get(
|
||||
Config.SECTION_REMOTE_URL, "{}://".format(self.scheme)
|
||||
)
|
||||
msg = (
|
||||
"URL '{}' is supported but requires these missing "
|
||||
"dependencies: {}. If you have installed dvc using pip, "
|
||||
@@ -130,7 +132,9 @@ class RemoteBase(object):
|
||||
"""
|
||||
|
||||
logger.debug(
|
||||
"checking if '{}'('{}') has changed.".format(path_info, checksum_info)
|
||||
"checking if '{}'('{}') has changed.".format(
|
||||
path_info, checksum_info
|
||||
)
|
||||
)
|
||||
|
||||
if not self.exists(path_info):
|
||||
@@ -231,7 +235,9 @@ class RemoteBase(object):
|
||||
# is 32 bytes, so ~3200Mb list) and we don't really need all of it at
|
||||
# the same time, so it makes sense to use a generator to gradually
|
||||
# iterate over it, without keeping all of it in memory.
|
||||
return (self.path_to_checksum(path) for path in self.list_cache_paths())
|
||||
return (
|
||||
self.path_to_checksum(path) for path in self.list_cache_paths()
|
||||
)
|
||||
|
||||
def gc(self, cinfos):
|
||||
from dvc.remote.local import RemoteLOCAL
|
||||
@@ -255,7 +261,9 @@ class RemoteBase(object):
|
||||
expected = {self.PARAM_CHECKSUM: checksum}
|
||||
actual = self.save_info(cache)
|
||||
|
||||
logger.debug("Cache '{}' actual '{}'.".format(str(expected), str(actual)))
|
||||
logger.debug(
|
||||
"Cache '{}' actual '{}'.".format(str(expected), str(actual))
|
||||
)
|
||||
|
||||
if expected != actual:
|
||||
if self.exists(cache):
|
||||
|
||||
+15
-5
@@ -63,7 +63,9 @@ class RemoteGS(RemoteBase):
|
||||
raise NotImplementedError
|
||||
|
||||
return {
|
||||
self.PARAM_CHECKSUM: self.get_md5(path_info["bucket"], path_info["path"])
|
||||
self.PARAM_CHECKSUM: self.get_md5(
|
||||
path_info["bucket"], path_info["path"]
|
||||
)
|
||||
}
|
||||
|
||||
def copy(self, from_info, to_info, gs=None):
|
||||
@@ -96,7 +98,9 @@ class RemoteGS(RemoteBase):
|
||||
raise NotImplementedError
|
||||
|
||||
logger.debug(
|
||||
"Removing gs://{}/{}".format(path_info["bucket"], path_info["path"])
|
||||
"Removing gs://{}/{}".format(
|
||||
path_info["bucket"], path_info["path"]
|
||||
)
|
||||
)
|
||||
|
||||
blob = self.gs.bucket(path_info["bucket"]).get_blob(path_info["path"])
|
||||
@@ -149,13 +153,17 @@ class RemoteGS(RemoteBase):
|
||||
except Exception:
|
||||
msg = "failed to upload '{}' to '{}/{}'"
|
||||
logger.error(
|
||||
msg.format(from_info["path"], to_info["bucket"], to_info["path"])
|
||||
msg.format(
|
||||
from_info["path"], to_info["bucket"], to_info["path"]
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
progress.finish_target(name)
|
||||
|
||||
def download(self, from_infos, to_infos, no_progress_bar=False, names=None):
|
||||
def download(
|
||||
self, from_infos, to_infos, no_progress_bar=False, names=None
|
||||
):
|
||||
names = self._verify_path_args(from_infos, to_infos, names)
|
||||
|
||||
gs = self.gs
|
||||
@@ -194,7 +202,9 @@ class RemoteGS(RemoteBase):
|
||||
except Exception:
|
||||
msg = "failed to download '{}/{}' to '{}'"
|
||||
logger.error(
|
||||
msg.format(from_info["bucket"], from_info["path"], to_info["path"])
|
||||
msg.format(
|
||||
from_info["bucket"], from_info["path"], to_info["path"]
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
|
||||
+15
-5
@@ -27,7 +27,9 @@ class RemoteHDFS(RemoteBase):
|
||||
self.prefix = self.url
|
||||
self.user = self.group("user")
|
||||
if not self.user:
|
||||
self.user = config.get(Config.SECTION_REMOTE_USER, getpass.getuser())
|
||||
self.user = config.get(
|
||||
Config.SECTION_REMOTE_USER, getpass.getuser()
|
||||
)
|
||||
|
||||
self.path_info = {"scheme": "hdfs", "user": self.user}
|
||||
|
||||
@@ -78,7 +80,9 @@ class RemoteHDFS(RemoteBase):
|
||||
)
|
||||
|
||||
def rm(self, path_info):
|
||||
self.hadoop_fs("rm {}".format(path_info["path"]), user=path_info["user"])
|
||||
self.hadoop_fs(
|
||||
"rm {}".format(path_info["path"]), user=path_info["user"]
|
||||
)
|
||||
|
||||
def save_info(self, path_info):
|
||||
if path_info["scheme"] != "hdfs":
|
||||
@@ -135,10 +139,14 @@ class RemoteHDFS(RemoteBase):
|
||||
cmd = "mkdir -p {}".format(posixpath.dirname(to_info["path"]))
|
||||
self.hadoop_fs(cmd, user=to_info["user"])
|
||||
|
||||
cmd = "copyFromLocal {} {}".format(from_info["path"], to_info["path"])
|
||||
cmd = "copyFromLocal {} {}".format(
|
||||
from_info["path"], to_info["path"]
|
||||
)
|
||||
self.hadoop_fs(cmd, user=to_info["user"])
|
||||
|
||||
def download(self, from_infos, to_infos, no_progress_bar=False, names=None):
|
||||
def download(
|
||||
self, from_infos, to_infos, no_progress_bar=False, names=None
|
||||
):
|
||||
names = self._verify_path_args(from_infos, to_infos, names)
|
||||
|
||||
for to_info, from_info, name in zip(to_infos, from_infos, names):
|
||||
@@ -156,7 +164,9 @@ class RemoteHDFS(RemoteBase):
|
||||
if not os.path.exists(dname):
|
||||
os.makedirs(dname)
|
||||
|
||||
cmd = "copyToLocal {} {}".format(from_info["path"], to_info["path"])
|
||||
cmd = "copyToLocal {} {}".format(
|
||||
from_info["path"], to_info["path"]
|
||||
)
|
||||
self.hadoop_fs(cmd, user=from_info["user"])
|
||||
|
||||
def list_cache_paths(self):
|
||||
|
||||
+6
-2
@@ -41,7 +41,9 @@ class RemoteHTTP(RemoteBase):
|
||||
def prefix(self):
|
||||
return self.cache_dir
|
||||
|
||||
def download(self, from_infos, to_infos, no_progress_bar=False, names=None):
|
||||
def download(
|
||||
self, from_infos, to_infos, no_progress_bar=False, names=None
|
||||
):
|
||||
names = self._verify_path_args(to_infos, from_infos, names)
|
||||
|
||||
for to_info, from_info, name in zip(to_infos, from_infos, names):
|
||||
@@ -51,7 +53,9 @@ class RemoteHTTP(RemoteBase):
|
||||
if to_info["scheme"] != "local":
|
||||
raise NotImplementedError
|
||||
|
||||
msg = "Downloading '{}' to '{}'".format(from_info["path"], to_info["path"])
|
||||
msg = "Downloading '{}' to '{}'".format(
|
||||
from_info["path"], to_info["path"]
|
||||
)
|
||||
logger.debug(msg)
|
||||
|
||||
tmp_file = self.tmp_file(to_info["path"])
|
||||
|
||||
+29
-9
@@ -205,7 +205,8 @@ class RemoteLOCAL(RemoteBase):
|
||||
|
||||
if len(files) > LARGE_DIR_SIZE:
|
||||
msg = (
|
||||
"Computing md5 for a large directory {}. " "This is only done once."
|
||||
"Computing md5 for a large directory {}. "
|
||||
"This is only done once."
|
||||
)
|
||||
logger.info(msg.format(os.path.relpath(root)))
|
||||
bar = True
|
||||
@@ -223,7 +224,9 @@ class RemoteLOCAL(RemoteBase):
|
||||
processed += 1
|
||||
|
||||
md5 = self.state.update(path)
|
||||
dir_info.append({self.PARAM_RELPATH: relpath, self.PARAM_CHECKSUM: md5})
|
||||
dir_info.append(
|
||||
{self.PARAM_RELPATH: relpath, self.PARAM_CHECKSUM: md5}
|
||||
)
|
||||
|
||||
if bar:
|
||||
progress.finish_target(title)
|
||||
@@ -349,7 +352,9 @@ class RemoteLOCAL(RemoteBase):
|
||||
for file in files
|
||||
)
|
||||
|
||||
cached_files = set(os.path.join(path, file["relpath"]) for file in dir_info)
|
||||
cached_files = set(
|
||||
os.path.join(path, file["relpath"]) for file in dir_info
|
||||
)
|
||||
|
||||
delta = working_dir_files - cached_files
|
||||
|
||||
@@ -420,7 +425,9 @@ class RemoteLOCAL(RemoteBase):
|
||||
path = path_info["path"]
|
||||
|
||||
msg = "Saving '{}' to cache '{}'."
|
||||
logger.info(msg.format(os.path.relpath(path), os.path.relpath(self.cache_dir)))
|
||||
logger.info(
|
||||
msg.format(os.path.relpath(path), os.path.relpath(self.cache_dir))
|
||||
)
|
||||
|
||||
if os.path.isdir(path):
|
||||
return self._save_dir(path_info)
|
||||
@@ -460,7 +467,9 @@ class RemoteLOCAL(RemoteBase):
|
||||
raise NotImplementedError
|
||||
|
||||
logger.debug(
|
||||
"Uploading '{}' to '{}'".format(from_info["path"], to_info["path"])
|
||||
"Uploading '{}' to '{}'".format(
|
||||
from_info["path"], to_info["path"]
|
||||
)
|
||||
)
|
||||
|
||||
if not name:
|
||||
@@ -477,7 +486,9 @@ class RemoteLOCAL(RemoteBase):
|
||||
)
|
||||
)
|
||||
|
||||
def download(self, from_infos, to_infos, no_progress_bar=False, names=None):
|
||||
def download(
|
||||
self, from_infos, to_infos, no_progress_bar=False, names=None
|
||||
):
|
||||
names = self._verify_path_args(from_infos, to_infos, names)
|
||||
|
||||
for to_info, from_info, name in zip(to_infos, from_infos, names):
|
||||
@@ -488,7 +499,9 @@ class RemoteLOCAL(RemoteBase):
|
||||
raise NotImplementedError
|
||||
|
||||
logger.debug(
|
||||
"Downloading '{}' to '{}'".format(from_info["path"], to_info["path"])
|
||||
"Downloading '{}' to '{}'".format(
|
||||
from_info["path"], to_info["path"]
|
||||
)
|
||||
)
|
||||
|
||||
if not name:
|
||||
@@ -561,7 +574,9 @@ class RemoteLOCAL(RemoteBase):
|
||||
progress.finish_target(title)
|
||||
|
||||
for md5, info in ret.items():
|
||||
info["status"] = STATUS_MAP[(md5 in local_exists, md5 in remote_exists)]
|
||||
info["status"] = STATUS_MAP[
|
||||
(md5 in local_exists, md5 in remote_exists)
|
||||
]
|
||||
|
||||
return ret
|
||||
|
||||
@@ -603,7 +618,12 @@ class RemoteLOCAL(RemoteBase):
|
||||
)
|
||||
|
||||
def _process(
|
||||
self, checksum_infos, remote, jobs=None, show_checksums=False, download=False
|
||||
self,
|
||||
checksum_infos,
|
||||
remote,
|
||||
jobs=None,
|
||||
show_checksums=False,
|
||||
download=False,
|
||||
):
|
||||
msg = "Preparing to {} data {} '{}'"
|
||||
logger.info(
|
||||
|
||||
+23
-7
@@ -87,7 +87,9 @@ class RemoteS3(RemoteBase):
|
||||
try:
|
||||
obj = self.s3.head_object(Bucket=bucket, Key=path)
|
||||
except Exception:
|
||||
raise DvcException("s3://{}/{} does not exist".format(bucket, path))
|
||||
raise DvcException(
|
||||
"s3://{}/{} does not exist".format(bucket, path)
|
||||
)
|
||||
|
||||
return obj["ETag"].strip('"')
|
||||
|
||||
@@ -96,7 +98,9 @@ class RemoteS3(RemoteBase):
|
||||
raise NotImplementedError
|
||||
|
||||
return {
|
||||
self.PARAM_CHECKSUM: self.get_etag(path_info["bucket"], path_info["path"])
|
||||
self.PARAM_CHECKSUM: self.get_etag(
|
||||
path_info["bucket"], path_info["path"]
|
||||
)
|
||||
}
|
||||
|
||||
def copy(self, from_info, to_info, s3=None):
|
||||
@@ -122,10 +126,14 @@ class RemoteS3(RemoteBase):
|
||||
raise NotImplementedError
|
||||
|
||||
logger.debug(
|
||||
"Removing s3://{}/{}".format(path_info["bucket"], path_info["path"])
|
||||
"Removing s3://{}/{}".format(
|
||||
path_info["bucket"], path_info["path"]
|
||||
)
|
||||
)
|
||||
|
||||
self.s3.delete_object(Bucket=path_info["bucket"], Key=path_info["path"])
|
||||
self.s3.delete_object(
|
||||
Bucket=path_info["bucket"], Key=path_info["path"]
|
||||
)
|
||||
|
||||
def _list_paths(self, bucket, prefix):
|
||||
s3 = self.s3
|
||||
@@ -186,7 +194,10 @@ class RemoteS3(RemoteBase):
|
||||
|
||||
try:
|
||||
s3.upload_file(
|
||||
from_info["path"], to_info["bucket"], to_info["path"], Callback=cb
|
||||
from_info["path"],
|
||||
to_info["bucket"],
|
||||
to_info["path"],
|
||||
Callback=cb,
|
||||
)
|
||||
except Exception:
|
||||
msg = "failed to upload '{}'".format(from_info["path"])
|
||||
@@ -195,7 +206,9 @@ class RemoteS3(RemoteBase):
|
||||
|
||||
progress.finish_target(name)
|
||||
|
||||
def download(self, from_infos, to_infos, no_progress_bar=False, names=None):
|
||||
def download(
|
||||
self, from_infos, to_infos, no_progress_bar=False, names=None
|
||||
):
|
||||
names = self._verify_path_args(from_infos, to_infos, names)
|
||||
|
||||
s3 = self.s3
|
||||
@@ -232,7 +245,10 @@ class RemoteS3(RemoteBase):
|
||||
cb = Callback(name, total)
|
||||
|
||||
s3.download_file(
|
||||
from_info["bucket"], from_info["path"], tmp_file, Callback=cb
|
||||
from_info["bucket"],
|
||||
from_info["path"],
|
||||
tmp_file,
|
||||
Callback=cb,
|
||||
)
|
||||
except Exception:
|
||||
msg = "failed to download '{}/{}'".format(
|
||||
|
||||
+44
-13
@@ -76,12 +76,16 @@ class RemoteSSH(RemoteBase):
|
||||
)
|
||||
self.prefix = parsed.path
|
||||
self.port = (
|
||||
config.get(Config.SECTION_REMOTE_PORT) or parsed.port or self.DEFAULT_PORT
|
||||
config.get(Config.SECTION_REMOTE_PORT)
|
||||
or parsed.port
|
||||
or self.DEFAULT_PORT
|
||||
)
|
||||
self.keyfile = config.get(Config.SECTION_REMOTE_KEY_FILE, None)
|
||||
self.timeout = config.get(Config.SECTION_REMOTE_TIMEOUT, self.TIMEOUT)
|
||||
self.password = config.get(Config.SECTION_REMOTE_PASSWORD, None)
|
||||
self.ask_password = config.get(Config.SECTION_REMOTE_ASK_PASSWORD, False)
|
||||
self.ask_password = config.get(
|
||||
Config.SECTION_REMOTE_ASK_PASSWORD, False
|
||||
)
|
||||
|
||||
self.path_info = {
|
||||
"scheme": "ssh",
|
||||
@@ -91,7 +95,10 @@ class RemoteSSH(RemoteBase):
|
||||
}
|
||||
|
||||
def ssh(self, host=None, user=None, port=None):
|
||||
msg = "Establishing ssh connection with '{}' " "through port '{}' as user '{}'"
|
||||
msg = (
|
||||
"Establishing ssh connection with '{}' "
|
||||
"through port '{}' as user '{}'"
|
||||
)
|
||||
logger.debug(msg.format(host, port, user))
|
||||
|
||||
ssh = paramiko.SSHClient()
|
||||
@@ -121,7 +128,9 @@ class RemoteSSH(RemoteBase):
|
||||
assert not isinstance(path_info, list)
|
||||
assert path_info["scheme"] == "ssh"
|
||||
|
||||
with self.ssh(path_info["host"], path_info["user"], path_info["port"]) as ssh:
|
||||
with self.ssh(
|
||||
path_info["host"], path_info["user"], path_info["port"]
|
||||
) as ssh:
|
||||
try:
|
||||
self._exec(ssh, "test -e {}".format(path_info["path"]))
|
||||
exists = True
|
||||
@@ -139,7 +148,11 @@ class RemoteSSH(RemoteBase):
|
||||
|
||||
stdout_chunks = []
|
||||
stderr_chunks = []
|
||||
while not channel.closed or channel.recv_ready() or channel.recv_stderr_ready():
|
||||
while (
|
||||
not channel.closed
|
||||
or channel.recv_ready()
|
||||
or channel.recv_stderr_ready()
|
||||
):
|
||||
import select
|
||||
|
||||
got_chunk = False
|
||||
@@ -180,7 +193,9 @@ class RemoteSSH(RemoteBase):
|
||||
raise NotImplementedError
|
||||
|
||||
ssh = self.ssh(
|
||||
host=path_info["host"], user=path_info["user"], port=path_info["port"]
|
||||
host=path_info["host"],
|
||||
user=path_info["user"],
|
||||
port=path_info["port"],
|
||||
)
|
||||
|
||||
# Use different md5 commands depending on os
|
||||
@@ -214,7 +229,9 @@ class RemoteSSH(RemoteBase):
|
||||
ssh
|
||||
if ssh
|
||||
else self.ssh(
|
||||
host=from_info["host"], user=from_info["user"], port=from_info["port"]
|
||||
host=from_info["host"],
|
||||
user=from_info["user"],
|
||||
port=from_info["port"],
|
||||
)
|
||||
)
|
||||
|
||||
@@ -254,12 +271,16 @@ class RemoteSSH(RemoteBase):
|
||||
)
|
||||
|
||||
ssh = self.ssh(
|
||||
host=path_info["host"], user=path_info["user"], port=path_info["port"]
|
||||
host=path_info["host"],
|
||||
user=path_info["user"],
|
||||
port=path_info["port"],
|
||||
)
|
||||
ssh.open_sftp().remove(path_info["path"])
|
||||
ssh.close()
|
||||
|
||||
def download(self, from_infos, to_infos, no_progress_bar=False, names=None):
|
||||
def download(
|
||||
self, from_infos, to_infos, no_progress_bar=False, names=None
|
||||
):
|
||||
names = self._verify_path_args(from_infos, to_infos, names)
|
||||
|
||||
ssh = self.ssh(
|
||||
@@ -299,7 +320,9 @@ class RemoteSSH(RemoteBase):
|
||||
except Exception:
|
||||
msg = "failed to download '{}/{}' to '{}'"
|
||||
logger.error(
|
||||
msg.format(from_info["host"], from_info["path"], to_info["path"])
|
||||
msg.format(
|
||||
from_info["host"], from_info["path"], to_info["path"]
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
@@ -312,7 +335,9 @@ class RemoteSSH(RemoteBase):
|
||||
names = self._verify_path_args(to_infos, from_infos, names)
|
||||
|
||||
ssh = self.ssh(
|
||||
host=to_infos[0]["host"], user=to_infos[0]["user"], port=to_infos[0]["port"]
|
||||
host=to_infos[0]["host"],
|
||||
user=to_infos[0]["user"],
|
||||
port=to_infos[0]["port"],
|
||||
)
|
||||
sftp = ssh.open_sftp()
|
||||
|
||||
@@ -336,11 +361,17 @@ class RemoteSSH(RemoteBase):
|
||||
self._exec(ssh, "mkdir -p {}".format(dname))
|
||||
|
||||
try:
|
||||
sftp.put(from_info["path"], to_info["path"], callback=create_cb(name))
|
||||
sftp.put(
|
||||
from_info["path"],
|
||||
to_info["path"],
|
||||
callback=create_cb(name),
|
||||
)
|
||||
except Exception:
|
||||
msg = "failed to upload '{}' to '{}/{}'"
|
||||
logger.error(
|
||||
msg.format(from_info["path"], to_info["host"], to_info["path"])
|
||||
msg.format(
|
||||
from_info["path"], to_info["host"], to_info["path"]
|
||||
)
|
||||
)
|
||||
continue
|
||||
|
||||
|
||||
+3
-1
@@ -76,7 +76,9 @@ class Base(object):
|
||||
def tag(self, tag):
|
||||
"""Makes SCM create a tag with a specified name."""
|
||||
|
||||
def brancher(self, branches=None, all_branches=False, tags=None, all_tags=False):
|
||||
def brancher(
|
||||
self, branches=None, all_branches=False, tags=None, all_tags=False
|
||||
):
|
||||
"""Generator that iterates over specified revisions.
|
||||
|
||||
Args:
|
||||
|
||||
+3
-1
@@ -76,7 +76,9 @@ class Git(Base):
|
||||
ignore_list = []
|
||||
if os.path.exists(gitignore):
|
||||
ignore_list = open(gitignore, "r").readlines()
|
||||
filtered = list(filter(lambda x: x.strip() == entry.strip(), ignore_list))
|
||||
filtered = list(
|
||||
filter(lambda x: x.strip() == entry.strip(), ignore_list)
|
||||
)
|
||||
if filtered:
|
||||
return
|
||||
|
||||
|
||||
+15
-5
@@ -131,7 +131,9 @@ class Stage(object):
|
||||
self.locked = locked
|
||||
|
||||
def __repr__(self):
|
||||
return "Stage: '{path}'".format(path=self.relpath if self.path else "No path")
|
||||
return "Stage: '{path}'".format(
|
||||
path=self.relpath if self.path else "No path"
|
||||
)
|
||||
|
||||
@property
|
||||
def relpath(self):
|
||||
@@ -212,7 +214,9 @@ class Stage(object):
|
||||
return False
|
||||
|
||||
def changed(self):
|
||||
ret = any([self._changed_deps(), self._changed_outs(), self._changed_md5()])
|
||||
ret = any(
|
||||
[self._changed_deps(), self._changed_outs(), self._changed_md5()]
|
||||
)
|
||||
|
||||
if ret:
|
||||
msg = "Stage '{}' changed.".format(self.relpath)
|
||||
@@ -467,7 +471,9 @@ class Stage(object):
|
||||
self._check_dvc_filename(fname)
|
||||
|
||||
logger.info(
|
||||
"Saving information to '{file}'.".format(file=os.path.relpath(fname))
|
||||
"Saving information to '{file}'.".format(
|
||||
file=os.path.relpath(fname)
|
||||
)
|
||||
)
|
||||
|
||||
with open(fname, "w") as fd:
|
||||
@@ -487,7 +493,9 @@ class Stage(object):
|
||||
# NOTE: excluding parameters that don't affect the state of the
|
||||
# pipeline. Not excluding `OutputLOCAL.PARAM_CACHE`, because if
|
||||
# it has changed, we might not have that output in our cache.
|
||||
return dict_md5(d, exclude=[self.PARAM_LOCKED, OutputLOCAL.PARAM_METRIC])
|
||||
return dict_md5(
|
||||
d, exclude=[self.PARAM_LOCKED, OutputLOCAL.PARAM_METRIC]
|
||||
)
|
||||
|
||||
def save(self):
|
||||
for dep in self.deps:
|
||||
@@ -559,7 +567,9 @@ class Stage(object):
|
||||
def run(self, dry=False):
|
||||
if self.locked:
|
||||
logger.info(
|
||||
"Verifying outputs in locked stage '{stage}'".format(stage=self.relpath)
|
||||
"Verifying outputs in locked stage '{stage}'".format(
|
||||
stage=self.relpath
|
||||
)
|
||||
)
|
||||
if not dry:
|
||||
self.check_missing_outputs()
|
||||
|
||||
+26
-9
@@ -60,7 +60,9 @@ class State(object): # pylint: disable=too-many-instance-attributes
|
||||
|
||||
LINK_STATE_TABLE = "link_state"
|
||||
LINK_STATE_TABLE_LAYOUT = (
|
||||
"path TEXT PRIMARY KEY, " "inode INTEGER NOT NULL, " "mtime TEXT NOT NULL"
|
||||
"path TEXT PRIMARY KEY, "
|
||||
"inode INTEGER NOT NULL, "
|
||||
"mtime TEXT NOT NULL"
|
||||
)
|
||||
|
||||
STATE_ROW_LIMIT = 100000000
|
||||
@@ -82,7 +84,8 @@ class State(object): # pylint: disable=too-many-instance-attributes
|
||||
Config.SECTION_STATE_ROW_LIMIT, self.STATE_ROW_LIMIT
|
||||
)
|
||||
self.row_cleanup_quota = state_config.get(
|
||||
Config.SECTION_STATE_ROW_CLEANUP_QUOTA, self.STATE_ROW_CLEANUP_QUOTA
|
||||
Config.SECTION_STATE_ROW_CLEANUP_QUOTA,
|
||||
self.STATE_ROW_CLEANUP_QUOTA,
|
||||
)
|
||||
|
||||
if not self.dvc_dir:
|
||||
@@ -92,7 +95,10 @@ class State(object): # pylint: disable=too-many-instance-attributes
|
||||
self.state_file = os.path.join(self.dvc_dir, self.STATE_FILE)
|
||||
|
||||
# https://www.sqlite.org/tempfiles.html
|
||||
self.temp_files = [self.state_file + "-journal", self.state_file + "-wal"]
|
||||
self.temp_files = [
|
||||
self.state_file + "-journal",
|
||||
self.state_file + "-wal",
|
||||
]
|
||||
|
||||
self.database = None
|
||||
self.cursor = None
|
||||
@@ -186,8 +192,12 @@ class State(object): # pylint: disable=too-many-instance-attributes
|
||||
# Check that the state file is indeed a database
|
||||
cmd = "CREATE TABLE IF NOT EXISTS {} ({})"
|
||||
self._execute(cmd.format(self.STATE_TABLE, self.STATE_TABLE_LAYOUT))
|
||||
self._execute(cmd.format(self.STATE_INFO_TABLE, self.STATE_INFO_TABLE_LAYOUT))
|
||||
self._execute(cmd.format(self.LINK_STATE_TABLE, self.LINK_STATE_TABLE_LAYOUT))
|
||||
self._execute(
|
||||
cmd.format(self.STATE_INFO_TABLE, self.STATE_INFO_TABLE_LAYOUT)
|
||||
)
|
||||
self._execute(
|
||||
cmd.format(self.LINK_STATE_TABLE, self.LINK_STATE_TABLE_LAYOUT)
|
||||
)
|
||||
|
||||
cmd = (
|
||||
"INSERT OR IGNORE INTO {} (count) SELECT 0 "
|
||||
@@ -253,7 +263,9 @@ class State(object): # pylint: disable=too-many-instance-attributes
|
||||
"DELETE FROM {} WHERE timestamp IN ("
|
||||
"SELECT timestamp FROM {} ORDER BY timestamp ASC LIMIT {});"
|
||||
)
|
||||
self._execute(cmd.format(self.STATE_TABLE, self.STATE_TABLE, delete))
|
||||
self._execute(
|
||||
cmd.format(self.STATE_TABLE, self.STATE_TABLE, delete)
|
||||
)
|
||||
|
||||
self._vacuum()
|
||||
|
||||
@@ -268,7 +280,9 @@ class State(object): # pylint: disable=too-many-instance-attributes
|
||||
cmd = "UPDATE {} SET count = {} WHERE rowid = {}"
|
||||
self._execute(
|
||||
cmd.format(
|
||||
self.STATE_INFO_TABLE, self._to_sqlite(count), self.STATE_INFO_ROW
|
||||
self.STATE_INFO_TABLE,
|
||||
self._to_sqlite(count),
|
||||
self.STATE_INFO_ROW,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -423,8 +437,11 @@ class State(object): # pylint: disable=too-many-instance-attributes
|
||||
inode = self._inode(path)
|
||||
relpath = os.path.relpath(path, self.root_dir)
|
||||
|
||||
cmd = "REPLACE INTO {}(path, inode, mtime) " 'VALUES ("{}", {}, "{}")'.format(
|
||||
self.LINK_STATE_TABLE, relpath, self._to_sqlite(inode), mtime
|
||||
cmd = (
|
||||
"REPLACE INTO {}(path, inode, mtime) "
|
||||
'VALUES ("{}", {}, "{}")'.format(
|
||||
self.LINK_STATE_TABLE, relpath, self._to_sqlite(inode), mtime
|
||||
)
|
||||
)
|
||||
self._execute(cmd)
|
||||
|
||||
|
||||
+22
-4
@@ -23,7 +23,11 @@ class System(object):
|
||||
raise DvcException("link", cause=exc)
|
||||
|
||||
CreateHardLink = ctypes.windll.kernel32.CreateHardLinkW
|
||||
CreateHardLink.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_void_p]
|
||||
CreateHardLink.argtypes = [
|
||||
ctypes.c_wchar_p,
|
||||
ctypes.c_wchar_p,
|
||||
ctypes.c_void_p,
|
||||
]
|
||||
CreateHardLink.restype = ctypes.wintypes.BOOL
|
||||
|
||||
res = CreateHardLink(link_name, source, None)
|
||||
@@ -158,10 +162,20 @@ class System(object):
|
||||
flags = FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT
|
||||
|
||||
func = ctypes.windll.kernel32.CreateFileW
|
||||
func.argtypes = [c_wchar_p, DWORD, DWORD, c_void_p, DWORD, DWORD, HANDLE]
|
||||
func.argtypes = [
|
||||
c_wchar_p,
|
||||
DWORD,
|
||||
DWORD,
|
||||
c_void_p,
|
||||
DWORD,
|
||||
DWORD,
|
||||
HANDLE,
|
||||
]
|
||||
func.restype = HANDLE
|
||||
|
||||
hfile = func(path, 0, FILE_SHARE_READ, None, OPEN_EXISTING, flags, None)
|
||||
hfile = func(
|
||||
path, 0, FILE_SHARE_READ, None, OPEN_EXISTING, flags, None
|
||||
)
|
||||
if hfile is None:
|
||||
raise WinError()
|
||||
|
||||
@@ -198,7 +212,11 @@ class System(object):
|
||||
info = System.getdirinfo(path)
|
||||
inode = abs(
|
||||
hash(
|
||||
(info.dwVolumeSerialNumber, info.nFileIndexHigh, info.nFileIndexLow)
|
||||
(
|
||||
info.dwVolumeSerialNumber,
|
||||
info.nFileIndexHigh,
|
||||
info.nFileIndexLow,
|
||||
)
|
||||
)
|
||||
)
|
||||
assert inode >= 0
|
||||
|
||||
+2
-1
@@ -128,7 +128,8 @@ class Updater(object): # pragma: no cover
|
||||
"formula": "Run {yellow}brew{reset} upgrade dvc",
|
||||
"cask": "Run {yellow}brew cask{reset} upgrade dvc",
|
||||
"apt": (
|
||||
"Run {yellow}apt-get{reset} install" " {blue}--only-upgrade{reset} dvc"
|
||||
"Run {yellow}apt-get{reset} install"
|
||||
" {blue}--only-upgrade{reset} dvc"
|
||||
),
|
||||
"binary": (
|
||||
"To upgrade follow this steps:\n"
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
[tool.black]
|
||||
line-length = 88
|
||||
line-length = 79
|
||||
include = '\.pyi?$'
|
||||
exclude = '''
|
||||
/(
|
||||
|
||||
@@ -20,9 +20,9 @@ pip install -r scripts/ci/requirements.txt
|
||||
# stop the build if there are any readme formatting errors
|
||||
python setup.py checkdocs
|
||||
|
||||
# stop the build if there are any black errors
|
||||
black dvc --check
|
||||
black tests --check
|
||||
# stop the build if there are any black or flake8 errors
|
||||
black ./ --check
|
||||
flake8 ./
|
||||
|
||||
#NOTE: ssh keys for ssh test to be able to ssh to the localhost
|
||||
ls -la ~/.ssh/
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
Pygments # required by collective.checkdocs
|
||||
collective.checkdocs
|
||||
black
|
||||
flake8
|
||||
|
||||
+3
-1
@@ -4,7 +4,9 @@ from subprocess import check_call
|
||||
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
os.chdir(REPO_ROOT)
|
||||
|
||||
os.putenv("PATH", "{}:{}".format(os.path.join(REPO_ROOT, "bin"), os.getenv("PATH")))
|
||||
os.putenv(
|
||||
"PATH", "{}:{}".format(os.path.join(REPO_ROOT, "bin"), os.getenv("PATH"))
|
||||
)
|
||||
os.putenv("DVC_HOME", REPO_ROOT)
|
||||
os.putenv("DVC_TEST", "true")
|
||||
|
||||
|
||||
+6
-3
@@ -10,7 +10,9 @@ from dvc.project import Project
|
||||
|
||||
|
||||
class TestDir(TestCase):
|
||||
GCP_CREDS_FILE = os.path.abspath(os.path.join("scripts", "ci", "gcp-creds.json"))
|
||||
GCP_CREDS_FILE = os.path.abspath(
|
||||
os.path.join("scripts", "ci", "gcp-creds.json")
|
||||
)
|
||||
DATA_DIR = "data_dir"
|
||||
DATA_SUB_DIR = os.path.join(DATA_DIR, "data_sub_dir")
|
||||
DATA = os.path.join(DATA_DIR, "data")
|
||||
@@ -32,7 +34,8 @@ class TestDir(TestCase):
|
||||
BAR_CONTENTS = BAR + "r"
|
||||
CODE = "code.py"
|
||||
CODE_CONTENTS = (
|
||||
"import sys\nimport shutil\n" "shutil.copyfile(sys.argv[1], sys.argv[2])"
|
||||
"import sys\nimport shutil\n"
|
||||
"shutil.copyfile(sys.argv[1], sys.argv[2])"
|
||||
)
|
||||
|
||||
def _pushd(self, d):
|
||||
@@ -87,7 +90,7 @@ class TestGit(TestDir):
|
||||
while retries:
|
||||
try:
|
||||
self.git = Repo.init()
|
||||
except GitCommandNotFound as exc:
|
||||
except GitCommandNotFound:
|
||||
retries -= 1
|
||||
continue
|
||||
break
|
||||
|
||||
+15
-5
@@ -14,10 +14,14 @@ class TestCache(TestDvc):
|
||||
self.cache1_md5 = "123"
|
||||
self.cache2_md5 = "234"
|
||||
self.cache1 = os.path.join(
|
||||
self.dvc.cache.local.cache_dir, self.cache1_md5[0:2], self.cache1_md5[2:]
|
||||
self.dvc.cache.local.cache_dir,
|
||||
self.cache1_md5[0:2],
|
||||
self.cache1_md5[2:],
|
||||
)
|
||||
self.cache2 = os.path.join(
|
||||
self.dvc.cache.local.cache_dir, self.cache2_md5[0:2], self.cache2_md5[2:]
|
||||
self.dvc.cache.local.cache_dir,
|
||||
self.cache2_md5[0:2],
|
||||
self.cache2_md5[2:],
|
||||
)
|
||||
self.create(self.cache1, "1")
|
||||
self.create(self.cache2, "2")
|
||||
@@ -103,9 +107,15 @@ class TestSharedCacheDir(TestDir):
|
||||
)
|
||||
)
|
||||
self.assertEqual(len(subdirs), 3)
|
||||
self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[0]))), 1)
|
||||
self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[1]))), 1)
|
||||
self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[2]))), 1)
|
||||
self.assertEqual(
|
||||
len(os.listdir(os.path.join(cache_dir, subdirs[0]))), 1
|
||||
)
|
||||
self.assertEqual(
|
||||
len(os.listdir(os.path.join(cache_dir, subdirs[1]))), 1
|
||||
)
|
||||
self.assertEqual(
|
||||
len(os.listdir(os.path.join(cache_dir, subdirs[2]))), 1
|
||||
)
|
||||
|
||||
|
||||
class TestCacheLinkType(TestDvc):
|
||||
|
||||
@@ -113,7 +113,9 @@ class CheckoutBase(TestDvc):
|
||||
self.dvc.scm.commit("adding " + fname)
|
||||
|
||||
def read_ignored(self):
|
||||
return list(map(lambda s: s.strip("\n"), open(self.GIT_IGNORE).readlines()))
|
||||
return list(
|
||||
map(lambda s: s.strip("\n"), open(self.GIT_IGNORE).readlines())
|
||||
)
|
||||
|
||||
def outs_info(self, stage):
|
||||
FileInfo = collections.namedtuple("FileInfo", "path inode")
|
||||
@@ -125,7 +127,9 @@ class CheckoutBase(TestDvc):
|
||||
for file in files
|
||||
]
|
||||
|
||||
return [FileInfo(path=path, inode=System.inode(path)) for path in paths]
|
||||
return [
|
||||
FileInfo(path=path, inode=System.inode(path)) for path in paths
|
||||
]
|
||||
|
||||
|
||||
class TestRemoveFilesWhenCheckout(CheckoutBase):
|
||||
@@ -314,7 +318,9 @@ class TestCheckoutNotCachedFile(TestDvc):
|
||||
cmd = "python {} {} {}".format(self.CODE, self.FOO, "out")
|
||||
|
||||
self.dvc.add(self.FOO)
|
||||
stage = self.dvc.run(cmd=cmd, deps=[self.FOO, self.CODE], outs_no_cache=["out"])
|
||||
stage = self.dvc.run(
|
||||
cmd=cmd, deps=[self.FOO, self.CODE], outs_no_cache=["out"]
|
||||
)
|
||||
self.assertTrue(stage is not None)
|
||||
|
||||
self.dvc.checkout(force=True)
|
||||
|
||||
@@ -70,7 +70,13 @@ def _should_test_gcp():
|
||||
shutil.copyfile(TestDvc.GCP_CREDS_FILE, creds)
|
||||
try:
|
||||
check_output(
|
||||
["gcloud", "auth", "activate-service-account", "--key-file", creds]
|
||||
[
|
||||
"gcloud",
|
||||
"auth",
|
||||
"activate-service-account",
|
||||
"--key-file",
|
||||
creds,
|
||||
]
|
||||
)
|
||||
except (CalledProcessError, OSError):
|
||||
return False
|
||||
@@ -111,12 +117,16 @@ def _should_test_hdfs():
|
||||
return False
|
||||
|
||||
try:
|
||||
check_output(["hadoop", "version"], shell=True, executable=os.getenv("SHELL"))
|
||||
check_output(
|
||||
["hadoop", "version"], shell=True, executable=os.getenv("SHELL")
|
||||
)
|
||||
except (CalledProcessError, IOError):
|
||||
return False
|
||||
|
||||
p = Popen(
|
||||
"hadoop fs -ls hdfs://127.0.0.1/", shell=True, executable=os.getenv("SHELL")
|
||||
"hadoop fs -ls hdfs://127.0.0.1/",
|
||||
shell=True,
|
||||
executable=os.getenv("SHELL"),
|
||||
)
|
||||
p.communicate()
|
||||
if p.returncode != 0:
|
||||
@@ -134,11 +144,15 @@ def get_local_url():
|
||||
|
||||
|
||||
def get_ssh_url():
|
||||
return "ssh://{}@127.0.0.1:{}".format(getpass.getuser(), get_local_storagepath())
|
||||
return "ssh://{}@127.0.0.1:{}".format(
|
||||
getpass.getuser(), get_local_storagepath()
|
||||
)
|
||||
|
||||
|
||||
def get_hdfs_url():
|
||||
return "hdfs://{}@127.0.0.1{}".format(getpass.getuser(), get_local_storagepath())
|
||||
return "hdfs://{}@127.0.0.1{}".format(
|
||||
getpass.getuser(), get_local_storagepath()
|
||||
)
|
||||
|
||||
|
||||
def get_aws_storagepath():
|
||||
@@ -161,7 +175,9 @@ def get_azure_url_compat():
|
||||
container_name = os.getenv("AZURE_STORAGE_CONTAINER_NAME")
|
||||
connection_string = os.getenv("AZURE_STORAGE_CONNECTION_STRING")
|
||||
assert container_name is not None
|
||||
return "azure://ContainerName={};{}".format(container_name, connection_string)
|
||||
return "azure://ContainerName={};{}".format(
|
||||
container_name, connection_string
|
||||
)
|
||||
|
||||
|
||||
def get_azure_url():
|
||||
@@ -205,7 +221,9 @@ class TestDataCloudBase(TestDvc):
|
||||
|
||||
def _ensure_should_run(self):
|
||||
if not self._should_test():
|
||||
raise SkipTest("Test {} is disabled".format(self.__class__.__name__))
|
||||
raise SkipTest(
|
||||
"Test {} is disabled".format(self.__class__.__name__)
|
||||
)
|
||||
|
||||
def _setup_cloud(self):
|
||||
self._ensure_should_run()
|
||||
@@ -433,7 +451,8 @@ class TestDataCloudCLIBase(TestDvc):
|
||||
# NOTE: check if remote gc works correctly on directories
|
||||
self.main(["gc", "-c", "-f"] + args)
|
||||
shutil.move(
|
||||
self.dvc.cache.local.cache_dir, self.dvc.cache.local.cache_dir + ".back"
|
||||
self.dvc.cache.local.cache_dir,
|
||||
self.dvc.cache.local.cache_dir + ".back",
|
||||
)
|
||||
|
||||
self.main(["fetch"] + args)
|
||||
@@ -625,7 +644,9 @@ class TestRecursiveSyncOperations(TestDataCloudBase):
|
||||
return RemoteLOCAL
|
||||
|
||||
def _prepare_repo(self):
|
||||
self.main(["remote", "add", "-d", TEST_REMOTE, self.cloud._cloud.cache_dir])
|
||||
self.main(
|
||||
["remote", "add", "-d", TEST_REMOTE, self.cloud._cloud.cache_dir]
|
||||
)
|
||||
self.create(self.DATA, self.DATA_CONTENTS)
|
||||
self.create(self.DATA_SUB, self.DATA_SUB_CONTENTS)
|
||||
|
||||
|
||||
+3
-1
@@ -118,7 +118,9 @@ class TestGCMultipleProjects(TestDvc):
|
||||
self.additional_dvc = Project.init(self.additional_path)
|
||||
|
||||
cache_path = os.path.join(self._root_dir, ".dvc", "cache")
|
||||
config_path = os.path.join(self.additional_path, ".dvc", "config.local")
|
||||
config_path = os.path.join(
|
||||
self.additional_path, ".dvc", "config.local"
|
||||
)
|
||||
cfg = configobj.ConfigObj()
|
||||
cfg.filename = config_path
|
||||
cfg["cache"] = {"dir": cache_path}
|
||||
|
||||
+63
-12
@@ -142,13 +142,19 @@ class TestMetricsRecursive(TestDvc):
|
||||
|
||||
def test(self):
|
||||
|
||||
ret = self.dvc.metrics_show("nested", all_branches=True, recursive=False)
|
||||
ret = self.dvc.metrics_show(
|
||||
"nested", all_branches=True, recursive=False
|
||||
)
|
||||
self.assertEqual(len(ret), 0)
|
||||
|
||||
ret = self.dvc.metrics_show("nested", all_branches=True, recursive=True)
|
||||
ret = self.dvc.metrics_show(
|
||||
"nested", all_branches=True, recursive=True
|
||||
)
|
||||
self.assertEqual(len(ret), 1)
|
||||
self.assertEqual(
|
||||
ret["nested"][os.path.join("nested", "subnested", "metric_subnested")],
|
||||
ret["nested"][
|
||||
os.path.join("nested", "subnested", "metric_subnested")
|
||||
],
|
||||
"subnested",
|
||||
)
|
||||
self.assertEqual(
|
||||
@@ -214,7 +220,14 @@ class TestMetricsCLICompat(TestMetrics):
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
ret = main(
|
||||
["metrics", "show", "--all-branches", "metric_tsv", "--tsv-path", "0,0"]
|
||||
[
|
||||
"metrics",
|
||||
"show",
|
||||
"--all-branches",
|
||||
"metric_tsv",
|
||||
"--tsv-path",
|
||||
"0,0",
|
||||
]
|
||||
)
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
@@ -231,7 +244,14 @@ class TestMetricsCLICompat(TestMetrics):
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
ret = main(
|
||||
["metrics", "show", "--all-branches", "metric_csv", "--csv-path", "0,0"]
|
||||
[
|
||||
"metrics",
|
||||
"show",
|
||||
"--all-branches",
|
||||
"metric_csv",
|
||||
"--csv-path",
|
||||
"0,0",
|
||||
]
|
||||
)
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
@@ -255,23 +275,54 @@ class TestMetricsCLI(TestMetrics):
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
ret = main(
|
||||
["metrics", "show", "-a", "metric_json", "-t", "json", "-x", "branch"]
|
||||
[
|
||||
"metrics",
|
||||
"show",
|
||||
"-a",
|
||||
"metric_json",
|
||||
"-t",
|
||||
"json",
|
||||
"-x",
|
||||
"branch",
|
||||
]
|
||||
)
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
ret = main(["metrics", "show", "-a", "metric_tsv", "-t", "tsv", "-x", "0,0"])
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
ret = main(
|
||||
["metrics", "show", "-a", "metric_htsv", "-t", "htsv", "-x", "branch,0"]
|
||||
["metrics", "show", "-a", "metric_tsv", "-t", "tsv", "-x", "0,0"]
|
||||
)
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
ret = main(["metrics", "show", "-a", "metric_csv", "-t", "csv", "-x", "0,0"])
|
||||
ret = main(
|
||||
[
|
||||
"metrics",
|
||||
"show",
|
||||
"-a",
|
||||
"metric_htsv",
|
||||
"-t",
|
||||
"htsv",
|
||||
"-x",
|
||||
"branch,0",
|
||||
]
|
||||
)
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
ret = main(
|
||||
["metrics", "show", "-a", "metric_hcsv", "-t", "hcsv", "-x", "branch,0"]
|
||||
["metrics", "show", "-a", "metric_csv", "-t", "csv", "-x", "0,0"]
|
||||
)
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
ret = main(
|
||||
[
|
||||
"metrics",
|
||||
"show",
|
||||
"-a",
|
||||
"metric_hcsv",
|
||||
"-t",
|
||||
"hcsv",
|
||||
"-x",
|
||||
"branch,0",
|
||||
]
|
||||
)
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
|
||||
+6
-2
@@ -132,7 +132,9 @@ class TestMoveDirectoryShouldNotOverwriteExisting(TestDvc):
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
self.assertFalse(os.path.exists(self.DATA_DIR))
|
||||
self.assertFalse(os.path.exists(self.DATA_DIR + Stage.STAGE_FILE_SUFFIX))
|
||||
self.assertFalse(
|
||||
os.path.exists(self.DATA_DIR + Stage.STAGE_FILE_SUFFIX)
|
||||
)
|
||||
|
||||
self.assertTrue(os.path.exists(new_dir_name))
|
||||
self.assertTrue(os.path.isfile(new_dir_name + Stage.STAGE_FILE_SUFFIX))
|
||||
@@ -163,4 +165,6 @@ class TestMoveFileBetweenDirectories(TestDvc):
|
||||
self.assertTrue(os.path.exists(new_data_stage_file))
|
||||
|
||||
new_stage_file = load_stage_file(new_data_stage_file)
|
||||
self.assertEqual(os.path.basename(self.DATA), new_stage_file["outs"][0]["path"])
|
||||
self.assertEqual(
|
||||
os.path.basename(self.DATA), new_stage_file["outs"][0]["path"]
|
||||
)
|
||||
|
||||
+55
-10
@@ -44,13 +44,22 @@ class TestPipelineShowSingle(TestDvc):
|
||||
|
||||
def test_dot_commands(self):
|
||||
ret = main(
|
||||
["pipeline", "show", "--dot", self.dotFile, self.stage, "--commands"]
|
||||
[
|
||||
"pipeline",
|
||||
"show",
|
||||
"--dot",
|
||||
self.dotFile,
|
||||
self.stage,
|
||||
"--commands",
|
||||
]
|
||||
)
|
||||
self.assertEqual(ret, 0)
|
||||
self.assertTrue(os.path.isfile(self.dotFile))
|
||||
|
||||
def test_dot_outs(self):
|
||||
ret = main(["pipeline", "show", "--dot", self.dotFile, self.stage, "--outs"])
|
||||
ret = main(
|
||||
["pipeline", "show", "--dot", self.dotFile, self.stage, "--outs"]
|
||||
)
|
||||
self.assertEqual(ret, 0)
|
||||
self.assertTrue(os.path.isfile(self.dotFile))
|
||||
|
||||
@@ -85,12 +94,16 @@ class TestPipelineShow(TestRepro):
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def test_dot(self):
|
||||
ret = main(["pipeline", "show", "--dot", self.dotFile, self.file1_stage])
|
||||
ret = main(
|
||||
["pipeline", "show", "--dot", self.dotFile, self.file1_stage]
|
||||
)
|
||||
self.assertEqual(ret, 0)
|
||||
self.assertTrue(os.path.isfile(self.dotFile))
|
||||
|
||||
def test_ascii_commands(self):
|
||||
ret = main(["pipeline", "show", "--ascii", self.file1_stage, "--commands"])
|
||||
ret = main(
|
||||
["pipeline", "show", "--ascii", self.file1_stage, "--commands"]
|
||||
)
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def test_ascii_outs(self):
|
||||
@@ -99,14 +112,28 @@ class TestPipelineShow(TestRepro):
|
||||
|
||||
def test_dot_commands(self):
|
||||
ret = main(
|
||||
["pipeline", "show", "--dot", self.dotFile, self.file1_stage, "--commands"]
|
||||
[
|
||||
"pipeline",
|
||||
"show",
|
||||
"--dot",
|
||||
self.dotFile,
|
||||
self.file1_stage,
|
||||
"--commands",
|
||||
]
|
||||
)
|
||||
self.assertEqual(ret, 0)
|
||||
self.assertTrue(os.path.isfile(self.dotFile))
|
||||
|
||||
def test_dot_outs(self):
|
||||
ret = main(
|
||||
["pipeline", "show", "--dot", self.dotFile, self.file1_stage, "--outs"]
|
||||
[
|
||||
"pipeline",
|
||||
"show",
|
||||
"--dot",
|
||||
self.dotFile,
|
||||
self.file1_stage,
|
||||
"--outs",
|
||||
]
|
||||
)
|
||||
self.assertEqual(ret, 0)
|
||||
self.assertTrue(os.path.isfile(self.dotFile))
|
||||
@@ -142,12 +169,16 @@ class TestPipelineShowDeep(TestReproChangedDeepData):
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def test_dot(self):
|
||||
ret = main(["pipeline", "show", "--dot", self.dotFile, self.file1_stage])
|
||||
ret = main(
|
||||
["pipeline", "show", "--dot", self.dotFile, self.file1_stage]
|
||||
)
|
||||
self.assertEqual(ret, 0)
|
||||
self.assertTrue(os.path.isfile(self.dotFile))
|
||||
|
||||
def test_ascii_commands(self):
|
||||
ret = main(["pipeline", "show", "--ascii", self.file1_stage, "--commands"])
|
||||
ret = main(
|
||||
["pipeline", "show", "--ascii", self.file1_stage, "--commands"]
|
||||
)
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
def test_ascii_outs(self):
|
||||
@@ -156,14 +187,28 @@ class TestPipelineShowDeep(TestReproChangedDeepData):
|
||||
|
||||
def test_dot_commands(self):
|
||||
ret = main(
|
||||
["pipeline", "show", "--dot", self.dotFile, self.file1_stage, "--commands"]
|
||||
[
|
||||
"pipeline",
|
||||
"show",
|
||||
"--dot",
|
||||
self.dotFile,
|
||||
self.file1_stage,
|
||||
"--commands",
|
||||
]
|
||||
)
|
||||
self.assertEqual(ret, 0)
|
||||
self.assertTrue(os.path.isfile(self.dotFile))
|
||||
|
||||
def test_dot_outs(self):
|
||||
ret = main(
|
||||
["pipeline", "show", "--dot", self.dotFile, self.file1_stage, "--outs"]
|
||||
[
|
||||
"pipeline",
|
||||
"show",
|
||||
"--dot",
|
||||
self.dotFile,
|
||||
self.file1_stage,
|
||||
"--outs",
|
||||
]
|
||||
)
|
||||
self.assertEqual(ret, 0)
|
||||
self.assertTrue(os.path.isfile(self.dotFile))
|
||||
|
||||
+14
-5
@@ -23,7 +23,9 @@ class TestRemote(TestDvc):
|
||||
self.assertEqual(main(["remote", "list"]), 0)
|
||||
|
||||
self.assertEqual(main(["remote", "remove", remotes[0]]), 0)
|
||||
self.assertEqual(main(["remote", "modify", remotes[0], "option", "value"]), 0)
|
||||
self.assertEqual(
|
||||
main(["remote", "modify", remotes[0], "option", "value"]), 0
|
||||
)
|
||||
|
||||
self.assertEqual(main(["remote", "list"]), 0)
|
||||
|
||||
@@ -61,10 +63,14 @@ class TestRemoteRemoveDefault(TestDvc):
|
||||
self.assertEqual(
|
||||
main(["remote", "add", "--default", remote, "s3://bucket/name"]), 0
|
||||
)
|
||||
self.assertEqual(main(["remote", "modify", remote, "profile", "default"]), 0)
|
||||
self.assertEqual(
|
||||
main(["remote", "modify", remote, "profile", "default"]), 0
|
||||
)
|
||||
self.assertEqual(main(["config", "--local", "core.remote", remote]), 0)
|
||||
|
||||
config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, Config.CONFIG))
|
||||
config = configobj.ConfigObj(
|
||||
os.path.join(self.dvc.dvc_dir, Config.CONFIG)
|
||||
)
|
||||
local_config = configobj.ConfigObj(
|
||||
os.path.join(self.dvc.dvc_dir, Config.CONFIG_LOCAL)
|
||||
)
|
||||
@@ -72,11 +78,14 @@ class TestRemoteRemoveDefault(TestDvc):
|
||||
config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE], remote
|
||||
)
|
||||
self.assertEqual(
|
||||
local_config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE], remote
|
||||
local_config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE],
|
||||
remote,
|
||||
)
|
||||
|
||||
self.assertEqual(main(["remote", "remove", remote]), 0)
|
||||
config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, Config.CONFIG))
|
||||
config = configobj.ConfigObj(
|
||||
os.path.join(self.dvc.dvc_dir, Config.CONFIG)
|
||||
)
|
||||
local_config = configobj.ConfigObj(
|
||||
os.path.join(self.dvc.dvc_dir, Config.CONFIG_LOCAL)
|
||||
)
|
||||
|
||||
+44
-14
@@ -58,9 +58,13 @@ class TestReproFail(TestRepro):
|
||||
|
||||
class TestReproCyclicGraph(TestDvc):
|
||||
def test(self):
|
||||
self.dvc.run(deps=[self.FOO], outs=["bar.txt"], cmd="echo bar > bar.txt")
|
||||
self.dvc.run(
|
||||
deps=[self.FOO], outs=["bar.txt"], cmd="echo bar > bar.txt"
|
||||
)
|
||||
|
||||
self.dvc.run(deps=["bar.txt"], outs=["baz.txt"], cmd="echo baz > baz.txt")
|
||||
self.dvc.run(
|
||||
deps=["bar.txt"], outs=["baz.txt"], cmd="echo baz > baz.txt"
|
||||
)
|
||||
|
||||
with open("cycle.dvc", "w") as fd:
|
||||
stage_dump = {
|
||||
@@ -257,7 +261,9 @@ class TestReproNoDeps(TestRepro):
|
||||
)
|
||||
with open(code_file, "w+") as fd:
|
||||
fd.write(code)
|
||||
self.dvc.run(fname=stage_file, outs=[out], cmd="python {}".format(code_file))
|
||||
self.dvc.run(
|
||||
fname=stage_file, outs=[out], cmd="python {}".format(code_file)
|
||||
)
|
||||
|
||||
stages = self.dvc.reproduce(stage_file)
|
||||
self.assertEqual(len(stages), 1)
|
||||
@@ -281,7 +287,9 @@ class TestReproChangedCode(TestRepro):
|
||||
def swap_code(self):
|
||||
os.unlink(self.CODE)
|
||||
new_contents = self.CODE_CONTENTS
|
||||
new_contents += "\nshutil.copyfile('{}', " "sys.argv[2])\n".format(self.BAR)
|
||||
new_contents += "\nshutil.copyfile('{}', " "sys.argv[2])\n".format(
|
||||
self.BAR
|
||||
)
|
||||
self.create(self.CODE, new_contents)
|
||||
|
||||
|
||||
@@ -422,7 +430,9 @@ class TestReproIgnoreBuildCache(TestDvc):
|
||||
|
||||
class TestReproPipeline(TestReproChangedDeepData):
|
||||
def test(self):
|
||||
stages = self.dvc.reproduce(self.file1_stage, force=True, pipeline=True)
|
||||
stages = self.dvc.reproduce(
|
||||
self.file1_stage, force=True, pipeline=True
|
||||
)
|
||||
self.assertEqual(len(stages), 3)
|
||||
|
||||
def test_cli(self):
|
||||
@@ -793,7 +803,9 @@ class TestReproExternalBase(TestDvc):
|
||||
def corrupted_cache(self):
|
||||
os.unlink("bar.dvc")
|
||||
|
||||
stage = self.dvc.run(deps=[self.FOO], outs=[self.BAR], cmd="echo bar > bar")
|
||||
stage = self.dvc.run(
|
||||
deps=[self.FOO], outs=[self.BAR], cmd="echo bar > bar"
|
||||
)
|
||||
|
||||
with open(self.BAR, "w") as fd:
|
||||
fd.write("corrupting the cache")
|
||||
@@ -817,7 +829,11 @@ class TestReproExternalBase(TestDvc):
|
||||
return
|
||||
|
||||
cache = (
|
||||
self.scheme + self.scheme_sep + self.bucket + self.sep + str(uuid.uuid4())
|
||||
self.scheme
|
||||
+ self.scheme_sep
|
||||
+ self.bucket
|
||||
+ self.sep
|
||||
+ str(uuid.uuid4())
|
||||
)
|
||||
|
||||
ret = main(["config", "cache." + self.cache_scheme, "myrepo"])
|
||||
@@ -827,7 +843,9 @@ class TestReproExternalBase(TestDvc):
|
||||
|
||||
remote_name = "myremote"
|
||||
remote_key = str(uuid.uuid4())
|
||||
remote = self.scheme + self.scheme_sep + self.bucket + self.sep + remote_key
|
||||
remote = (
|
||||
self.scheme + self.scheme_sep + self.bucket + self.sep + remote_key
|
||||
)
|
||||
|
||||
ret = main(["remote", "add", remote_name, remote])
|
||||
self.assertEqual(ret, 0)
|
||||
@@ -837,8 +855,12 @@ class TestReproExternalBase(TestDvc):
|
||||
foo_key = remote_key + self.sep + self.FOO
|
||||
bar_key = remote_key + self.sep + self.BAR
|
||||
|
||||
foo_path = self.scheme + self.scheme_sep + self.bucket + self.sep + foo_key
|
||||
bar_path = self.scheme + self.scheme_sep + self.bucket + self.sep + bar_key
|
||||
foo_path = (
|
||||
self.scheme + self.scheme_sep + self.bucket + self.sep + foo_key
|
||||
)
|
||||
bar_path = (
|
||||
self.scheme + self.scheme_sep + self.bucket + self.sep + bar_key
|
||||
)
|
||||
|
||||
# Using both plain and remote notation
|
||||
out_foo_path = "remote://" + remote_name + "/" + self.FOO
|
||||
@@ -853,11 +875,15 @@ class TestReproExternalBase(TestDvc):
|
||||
self.assertEqual(self.dvc.status(import_stage.path), {})
|
||||
self.check_already_cached(import_stage)
|
||||
|
||||
import_remote_stage = self.dvc.imp(out_foo_path, out_foo_path + "_imported")
|
||||
import_remote_stage = self.dvc.imp(
|
||||
out_foo_path, out_foo_path + "_imported"
|
||||
)
|
||||
self.assertEqual(self.dvc.status(import_remote_stage.path), {})
|
||||
|
||||
cmd_stage = self.dvc.run(
|
||||
outs=[out_bar_path], deps=[out_foo_path], cmd=self.cmd(foo_path, bar_path)
|
||||
outs=[out_bar_path],
|
||||
deps=[out_foo_path],
|
||||
cmd=self.cmd(foo_path, bar_path),
|
||||
)
|
||||
|
||||
self.assertEqual(self.dvc.status(cmd_stage.path), {})
|
||||
@@ -1161,7 +1187,9 @@ class TestReproExternalHTTP(TestReproExternalBase):
|
||||
fd.write(cmd)
|
||||
|
||||
run_stage = self.dvc.run(
|
||||
deps=[run_dependency], outs=[run_output], cmd="python create-output.py"
|
||||
deps=[run_dependency],
|
||||
outs=[run_output],
|
||||
cmd="python create-output.py",
|
||||
)
|
||||
self.assertTrue(run_stage is not None)
|
||||
|
||||
@@ -1188,7 +1216,9 @@ class TestReproShell(TestDvc):
|
||||
fname = "shell.txt"
|
||||
stage = fname + ".dvc"
|
||||
|
||||
self.dvc.run(fname=stage, outs=[fname], cmd="echo $SHELL > {}".format(fname))
|
||||
self.dvc.run(
|
||||
fname=stage, outs=[fname], cmd="echo $SHELL > {}".format(fname)
|
||||
)
|
||||
|
||||
with open(fname, "r") as fd:
|
||||
self.assertEqual(os.getenv("SHELL"), fd.read().strip())
|
||||
|
||||
+68
-13
@@ -63,7 +63,12 @@ class TestRun(TestDvc):
|
||||
class TestRunEmpty(TestDvc):
|
||||
def test(self):
|
||||
self.dvc.run(
|
||||
cmd="", deps=[], outs=[], outs_no_cache=[], fname="empty.dvc", cwd=os.curdir
|
||||
cmd="",
|
||||
deps=[],
|
||||
outs=[],
|
||||
outs_no_cache=[],
|
||||
fname="empty.dvc",
|
||||
cwd=os.curdir,
|
||||
)
|
||||
|
||||
|
||||
@@ -84,7 +89,12 @@ class TestRunBadStageFilename(TestDvc):
|
||||
def test(self):
|
||||
with self.assertRaises(StageFileBadNameError):
|
||||
self.dvc.run(
|
||||
cmd="", deps=[], outs=[], outs_no_cache=[], fname="empty", cwd=os.curdir
|
||||
cmd="",
|
||||
deps=[],
|
||||
outs=[],
|
||||
outs_no_cache=[],
|
||||
fname="empty",
|
||||
cwd=os.curdir,
|
||||
)
|
||||
|
||||
with self.assertRaises(StageFileBadNameError):
|
||||
@@ -101,7 +111,8 @@ class TestRunBadStageFilename(TestDvc):
|
||||
class TestRunNoExec(TestDvc):
|
||||
def test(self):
|
||||
self.dvc.run(
|
||||
cmd="python {} {} {}".format(self.CODE, self.FOO, "out"), no_exec=True
|
||||
cmd="python {} {} {}".format(self.CODE, self.FOO, "out"),
|
||||
no_exec=True,
|
||||
)
|
||||
self.assertFalse(os.path.exists("out"))
|
||||
|
||||
@@ -128,16 +139,25 @@ class TestRunCircularDependency(TestDvc):
|
||||
def test_non_normalized_paths(self):
|
||||
with self.assertRaises(CircularDependencyError):
|
||||
self.dvc.run(
|
||||
cmd="", deps=["./foo"], outs=["foo"], fname="circular-dependency.dvc"
|
||||
cmd="",
|
||||
deps=["./foo"],
|
||||
outs=["foo"],
|
||||
fname="circular-dependency.dvc",
|
||||
)
|
||||
|
||||
def test_graph(self):
|
||||
self.dvc.run(deps=[self.FOO], outs=["bar.txt"], cmd="echo bar > bar.txt")
|
||||
self.dvc.run(
|
||||
deps=[self.FOO], outs=["bar.txt"], cmd="echo bar > bar.txt"
|
||||
)
|
||||
|
||||
self.dvc.run(deps=["bar.txt"], outs=["baz.txt"], cmd="echo baz > baz.txt")
|
||||
self.dvc.run(
|
||||
deps=["bar.txt"], outs=["baz.txt"], cmd="echo baz > baz.txt"
|
||||
)
|
||||
|
||||
with self.assertRaises(CyclicGraphError):
|
||||
self.dvc.run(deps=["baz.txt"], outs=[self.FOO], cmd="echo baz > foo")
|
||||
self.dvc.run(
|
||||
deps=["baz.txt"], outs=[self.FOO], cmd="echo baz > foo"
|
||||
)
|
||||
|
||||
|
||||
class TestRunDuplicatedArguments(TestDvc):
|
||||
@@ -162,7 +182,10 @@ class TestRunDuplicatedArguments(TestDvc):
|
||||
def test_non_normalized_paths(self):
|
||||
with self.assertRaises(ArgumentDuplicationError):
|
||||
self.dvc.run(
|
||||
cmd="", deps=[], outs=["foo", "./foo"], fname="circular-dependency.dvc"
|
||||
cmd="",
|
||||
deps=[],
|
||||
outs=["foo", "./foo"],
|
||||
fname="circular-dependency.dvc",
|
||||
)
|
||||
|
||||
|
||||
@@ -172,7 +195,10 @@ class TestRunWorkingDirectoryAsOutput(TestDvc):
|
||||
|
||||
with self.assertRaises(WorkingDirectoryAsOutputError):
|
||||
self.dvc.run(
|
||||
cmd="", cwd=self.DATA_DIR, outs=[self.FOO], fname="inside-cwd.dvc"
|
||||
cmd="",
|
||||
cwd=self.DATA_DIR,
|
||||
outs=[self.FOO],
|
||||
fname="inside-cwd.dvc",
|
||||
)
|
||||
|
||||
|
||||
@@ -271,7 +297,16 @@ class TestRunUnprotectOutsCopy(TestDvc):
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
ret = main(
|
||||
["run", "-d", self.CODE, "-o", self.FOO, "python", self.CODE, self.FOO]
|
||||
[
|
||||
"run",
|
||||
"-d",
|
||||
self.CODE,
|
||||
"-o",
|
||||
self.FOO,
|
||||
"python",
|
||||
self.CODE,
|
||||
self.FOO,
|
||||
]
|
||||
)
|
||||
self.assertEqual(ret, 0)
|
||||
self.assertFalse(os.access(self.FOO, os.W_OK))
|
||||
@@ -313,7 +348,16 @@ class TestRunUnprotectOutsSymlink(TestDvc):
|
||||
|
||||
self.assertEqual(ret, 0)
|
||||
ret = main(
|
||||
["run", "-d", self.CODE, "-o", self.FOO, "python", self.CODE, self.FOO]
|
||||
[
|
||||
"run",
|
||||
"-d",
|
||||
self.CODE,
|
||||
"-o",
|
||||
self.FOO,
|
||||
"python",
|
||||
self.CODE,
|
||||
self.FOO,
|
||||
]
|
||||
)
|
||||
self.assertEqual(ret, 0)
|
||||
self.assertFalse(os.access(self.FOO, os.W_OK))
|
||||
@@ -357,7 +401,16 @@ class TestRunUnprotectOutsHardlink(TestDvc):
|
||||
|
||||
self.assertEqual(ret, 0)
|
||||
ret = main(
|
||||
["run", "-d", self.CODE, "-o", self.FOO, "python", self.CODE, self.FOO]
|
||||
[
|
||||
"run",
|
||||
"-d",
|
||||
self.CODE,
|
||||
"-o",
|
||||
self.FOO,
|
||||
"python",
|
||||
self.CODE,
|
||||
self.FOO,
|
||||
]
|
||||
)
|
||||
self.assertEqual(ret, 0)
|
||||
self.assertFalse(os.access(self.FOO, os.W_OK))
|
||||
@@ -468,7 +521,9 @@ class TestCmdRunOverwrite(TestDvc):
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
ret = main(["run", "--overwrite-dvcfile", "-f", "out.dvc", "-d", self.BAR])
|
||||
ret = main(
|
||||
["run", "--overwrite-dvcfile", "-f", "out.dvc", "-d", self.BAR]
|
||||
)
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
# NOTE: check that dvcfile was overwritten
|
||||
|
||||
+3
-1
@@ -45,7 +45,9 @@ class TestIgnore(TestGit):
|
||||
def _count_gitignore(self):
|
||||
with open(Git.GITIGNORE, "r") as fd:
|
||||
lines = fd.readlines()
|
||||
return len(list(filter(lambda x: x.strip() == "/" + self.FOO, lines)))
|
||||
return len(
|
||||
list(filter(lambda x: x.strip() == "/" + self.FOO, lines))
|
||||
)
|
||||
|
||||
def test(self):
|
||||
git = Git(self._root_dir)
|
||||
|
||||
@@ -15,10 +15,14 @@ class TestUnprotect(TestDvc):
|
||||
self.assertEqual(ret, 0)
|
||||
|
||||
self.assertNotEqual(
|
||||
os.stat(self.FOO).st_mode & (stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH), 0
|
||||
os.stat(self.FOO).st_mode
|
||||
& (stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH),
|
||||
0,
|
||||
)
|
||||
self.assertEqual(
|
||||
os.stat(self.FOO).st_mode & (stat.S_IWRITE | stat.S_IWGRP | stat.S_IWOTH), 0
|
||||
os.stat(self.FOO).st_mode
|
||||
& (stat.S_IWRITE | stat.S_IWGRP | stat.S_IWOTH),
|
||||
0,
|
||||
)
|
||||
|
||||
ret = main(["unprotect", self.FOO])
|
||||
|
||||
+3
-1
@@ -21,7 +21,9 @@ class TestUtils(TestDvc):
|
||||
self.assertTrue(filecmp.cmp(src, dest, shallow=False))
|
||||
|
||||
utils.copyfile(src, dest_dir)
|
||||
self.assertTrue(filecmp.cmp(src, "{}/{}".format(dest_dir, src), shallow=False))
|
||||
self.assertTrue(
|
||||
filecmp.cmp(src, "{}/{}".format(dest_dir, src), shallow=False)
|
||||
)
|
||||
|
||||
shutil.rmtree(dest_dir)
|
||||
os.remove(src)
|
||||
|
||||
@@ -14,7 +14,8 @@ class TestRemoteAzure(TestCase):
|
||||
|
||||
def test_init_compat(self):
|
||||
url = "azure://ContainerName={container_name};{connection_string}".format(
|
||||
container_name=self.container_name, connection_string=self.connection_string
|
||||
container_name=self.container_name,
|
||||
connection_string=self.connection_string,
|
||||
)
|
||||
config = {"url": url}
|
||||
remote = RemoteAzure(None, config)
|
||||
|
||||
+3
-1
@@ -12,4 +12,6 @@ class TestStageChecksum(TestCase):
|
||||
d = {"md5": "123456", "cmd": "mycmd", "outs": outs, "deps": deps}
|
||||
|
||||
with mock.patch.object(stage, "dumpd", return_value=d):
|
||||
self.assertEqual(stage._compute_md5(), "e9521a22111493406ea64a88cda63e0b")
|
||||
self.assertEqual(
|
||||
stage._compute_md5(), "e9521a22111493406ea64a88cda63e0b"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user