From 4896f08ddf9cfa09a601b34043cf7503399b6f82 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Mon, 4 Feb 2019 00:10:36 +0200 Subject: [PATCH] dvc: comply with both flake8 and black flake8 provides some additional features like unused vars check, etc, so it makes total sense to use it as well. Signed-off-by: Ruslan Kuprieiev --- .flake8 | 4 ++ .pre-commit-config.yaml | 5 ++ dvc/analytics.py | 4 +- dvc/cache.py | 4 +- dvc/cli.py | 12 ++++- dvc/command/add.py | 4 +- dvc/command/cache.py | 11 +++- dvc/command/checkout.py | 4 +- dvc/command/config.py | 14 +++-- dvc/command/daemon.py | 5 +- dvc/command/data_sync.py | 27 +++++++--- dvc/command/destroy.py | 14 +++-- dvc/command/imp.py | 5 +- dvc/command/init.py | 4 +- dvc/command/install.py | 5 +- dvc/command/lock.py | 9 +++- dvc/command/metrics.py | 12 +++-- dvc/command/move.py | 8 ++- dvc/command/pipeline.py | 25 ++++++--- dvc/command/remote.py | 30 ++++++++--- dvc/command/remove.py | 18 +++++-- dvc/command/repro.py | 12 +++-- dvc/command/run.py | 12 ++++- dvc/command/unprotect.py | 4 +- dvc/config.py | 36 +++++++++---- dvc/dagascii.py | 9 +++- dvc/dependency/http.py | 4 +- dvc/exceptions.py | 15 ++++-- dvc/logger.py | 3 +- dvc/output/base.py | 17 +++++-- dvc/output/hdfs.py | 4 +- dvc/output/local.py | 16 ++++-- dvc/output/s3.py | 10 +++- dvc/output/ssh.py | 16 ++++-- dvc/project.py | 99 +++++++++++++++++++++++++++--------- dvc/remote/azure.py | 20 ++++++-- dvc/remote/base.py | 16 ++++-- dvc/remote/gs.py | 20 ++++++-- dvc/remote/hdfs.py | 20 ++++++-- dvc/remote/http.py | 8 ++- dvc/remote/local.py | 38 ++++++++++---- dvc/remote/s3.py | 30 ++++++++--- dvc/remote/ssh.py | 57 ++++++++++++++++----- dvc/scm/base.py | 4 +- dvc/scm/git.py | 4 +- dvc/stage.py | 20 ++++++-- dvc/state.py | 35 +++++++++---- dvc/system.py | 26 ++++++++-- dvc/updater.py | 3 +- pyproject.toml | 2 +- scripts/ci/before_install.sh | 6 +-- test-style-requirements.txt | 1 + tests/__main__.py | 4 +- tests/basic_env.py | 9 ++-- tests/test_cache.py | 20 ++++++-- tests/test_checkout.py | 12 +++-- tests/test_data_cloud.py | 39 ++++++++++---- tests/test_gc.py | 4 +- tests/test_metrics.py | 75 ++++++++++++++++++++++----- tests/test_move.py | 8 ++- tests/test_pipeline.py | 65 +++++++++++++++++++---- tests/test_remote.py | 19 +++++-- tests/test_repro.py | 58 ++++++++++++++++----- tests/test_run.py | 81 ++++++++++++++++++++++++----- tests/test_scm.py | 4 +- tests/test_unprotect.py | 8 ++- tests/test_utils.py | 4 +- tests/unit/remote/azure.py | 3 +- tests/unit/stage.py | 4 +- 69 files changed, 940 insertions(+), 268 deletions(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..50b8fe537 --- /dev/null +++ b/.flake8 @@ -0,0 +1,4 @@ +[flake8] +ignore = E203, E266, E501, W503, F403, F401 +max-line-length = 79 +select = B,C,E,F,W,T4,B9 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b9a75affd..3eff137df 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,3 +4,8 @@ repos: hooks: - id: black language_version: python3.6 +- repo: https://gitlab.com/pycqa/flake8 + rev: master + hooks: + - id: flake8 + language_version: python3.6 diff --git a/dvc/analytics.py b/dvc/analytics.py index 58ec7f3d4..a95dcbab3 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -228,7 +228,9 @@ class Analytics(object): core = config.config.get(Config.SECTION_CORE, {}) enabled = core.get(Config.SECTION_CORE_ANALYTICS, True) - logger.debug("Analytics is {}abled.".format("en" if enabled else "dis")) + logger.debug( + "Analytics is {}abled.".format("en" if enabled else "dis") + ) return enabled @staticmethod diff --git a/dvc/cache.py b/dvc/cache.py index 02ddd4871..f5945f317 100644 --- a/dvc/cache.py +++ b/dvc/cache.py @@ -35,7 +35,9 @@ class Cache(object): protected = config.get(Config.SECTION_CACHE_PROTECTED) sect = { - Config.PRIVATE_CWD: config.get(Config.PRIVATE_CWD, project.dvc_dir), + Config.PRIVATE_CWD: config.get( + Config.PRIVATE_CWD, project.dvc_dir + ), Config.SECTION_REMOTE_URL: cache_dir, Config.SECTION_CACHE_TYPE: cache_type, Config.SECTION_CACHE_PROTECTED: protected, diff --git a/dvc/cli.py b/dvc/cli.py index 8bf4e6e64..d81eb00a6 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -101,7 +101,11 @@ def parse_args(argv=None): "-q", "--quiet", action="store_true", default=False, help="Be quiet." ) log_level_group.add_argument( - "-v", "--verbose", action="store_true", default=False, help="Be verbose." + "-v", + "--verbose", + action="store_true", + default=False, + help="Be verbose.", ) # Main parser @@ -116,7 +120,11 @@ def parse_args(argv=None): # NOTE: On some python versions action='version' prints to stderr # instead of stdout https://bugs.python.org/issue18920 parser.add_argument( - "-V", "--version", action=VersionAction, nargs=0, help="Show program's version." + "-V", + "--version", + action=VersionAction, + nargs=0, + help="Show program's version.", ) # Sub commands diff --git a/dvc/command/add.py b/dvc/command/add.py index 1f8241a8b..bfb9c45e5 100644 --- a/dvc/command/add.py +++ b/dvc/command/add.py @@ -28,5 +28,7 @@ def add_parser(subparsers, parent_parser): default=False, help="Recursively add each file under the directory.", ) - add_parser.add_argument("targets", nargs="+", help="Input files/directories.") + add_parser.add_argument( + "targets", nargs="+", help="Input files/directories." + ) add_parser.set_defaults(func=CmdAdd) diff --git a/dvc/command/cache.py b/dvc/command/cache.py index 76bc1a547..3a3a974d9 100644 --- a/dvc/command/cache.py +++ b/dvc/command/cache.py @@ -22,7 +22,10 @@ def add_parser(subparsers, parent_parser): CACHE_HELP = "Manage cache settings." cache_parser = subparsers.add_parser( - "cache", parents=[parent_parser], description=CACHE_HELP, help=CACHE_HELP + "cache", + parents=[parent_parser], + description=CACHE_HELP, + help=CACHE_HELP, ) cache_subparsers = cache_parser.add_subparsers( @@ -42,7 +45,11 @@ def add_parser(subparsers, parent_parser): help=CACHE_DIR_HELP, ) cache_dir_parser.add_argument( - "-u", "--unset", default=False, action="store_true", help="Unset option." + "-u", + "--unset", + default=False, + action="store_true", + help="Unset option.", ) cache_dir_parser.add_argument( "value", diff --git a/dvc/command/checkout.py b/dvc/command/checkout.py index 790a5dfe7..1b0b87221 100644 --- a/dvc/command/checkout.py +++ b/dvc/command/checkout.py @@ -10,7 +10,9 @@ class CmdCheckout(CmdBase): else: for target in self.args.targets: self.project.checkout( - target=target, with_deps=self.args.with_deps, force=self.args.force + target=target, + with_deps=self.args.with_deps, + force=self.args.force, ) return 0 diff --git a/dvc/command/config.py b/dvc/command/config.py index cdbede8e1..b25087fa9 100644 --- a/dvc/command/config.py +++ b/dvc/command/config.py @@ -64,7 +64,9 @@ class CmdConfig(CmdBase): self.config.set(self.configobj, section, opt, value) self.config.save(self.configobj) except DvcException: - logger.error("failed to set '{}.{}' to '{}'".format(section, opt, value)) + logger.error( + "failed to set '{}.{}' to '{}'".format(section, opt, value) + ) return 1 return 0 @@ -104,8 +106,14 @@ def add_parser(subparsers, parent_parser): help=CONFIG_HELP, ) config_parser.add_argument( - "-u", "--unset", default=False, action="store_true", help="Unset option." + "-u", + "--unset", + default=False, + action="store_true", + help="Unset option.", ) config_parser.add_argument("name", help="Option name.") - config_parser.add_argument("value", nargs="?", default=None, help="Option value.") + config_parser.add_argument( + "value", nargs="?", default=None, help="Option value." + ) config_parser.set_defaults(func=CmdConfig) diff --git a/dvc/command/daemon.py b/dvc/command/daemon.py index 70e056977..43b89c96e 100644 --- a/dvc/command/daemon.py +++ b/dvc/command/daemon.py @@ -40,7 +40,10 @@ class CmdDaemonAnalytics(CmdDaemonBase): def add_parser(subparsers, parent_parser): DAEMON_HELP = "Service daemon." daemon_parser = subparsers.add_parser( - "daemon", parents=[parent_parser], description=DAEMON_HELP, help=DAEMON_HELP + "daemon", + parents=[parent_parser], + description=DAEMON_HELP, + help=DAEMON_HELP, ) daemon_subparsers = daemon_parser.add_subparsers( diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py index ac5037895..8d60b1b07 100644 --- a/dvc/command/data_sync.py +++ b/dvc/command/data_sync.py @@ -106,9 +106,14 @@ def add_parser(subparsers, parent_parser): # Pull PULL_HELP = "Pull data files from the cloud." pull_parser = subparsers.add_parser( - "pull", parents=[parent_cache_parser], description=PULL_HELP, help=PULL_HELP + "pull", + parents=[parent_cache_parser], + description=PULL_HELP, + help=PULL_HELP, + ) + pull_parser.add_argument( + "-r", "--remote", help="Remote repository to pull from." ) - pull_parser.add_argument("-r", "--remote", help="Remote repository to pull from.") pull_parser.add_argument( "-a", "--all-branches", @@ -149,9 +154,14 @@ def add_parser(subparsers, parent_parser): # Push PUSH_HELP = "Push data files to the cloud." push_parser = subparsers.add_parser( - "push", parents=[parent_cache_parser], description=PUSH_HELP, help=PUSH_HELP + "push", + parents=[parent_cache_parser], + description=PUSH_HELP, + help=PUSH_HELP, + ) + push_parser.add_argument( + "-r", "--remote", help="Remote repository to push to." ) - push_parser.add_argument("-r", "--remote", help="Remote repository to push to.") push_parser.add_argument( "-a", "--all-branches", @@ -185,9 +195,14 @@ def add_parser(subparsers, parent_parser): # Fetch FETCH_HELP = "Fetch data files from the cloud." fetch_parser = subparsers.add_parser( - "fetch", parents=[parent_cache_parser], description=FETCH_HELP, help=FETCH_HELP + "fetch", + parents=[parent_cache_parser], + description=FETCH_HELP, + help=FETCH_HELP, + ) + fetch_parser.add_argument( + "-r", "--remote", help="Remote repository to fetch from." ) - fetch_parser.add_argument("-r", "--remote", help="Remote repository to fetch from.") fetch_parser.add_argument( "-a", "--all-branches", diff --git a/dvc/command/destroy.py b/dvc/command/destroy.py index e9eea5998..7a95e922d 100644 --- a/dvc/command/destroy.py +++ b/dvc/command/destroy.py @@ -31,12 +31,20 @@ class CmdDestroy(CmdBase): def add_parser(subparsers, parent_parser): DESTROY_HELP = ( - "Destroy dvc. Will remove all project's information, " "data files and cache." + "Destroy dvc. Will remove all project's information, " + "data files and cache." ) destroy_parser = subparsers.add_parser( - "destroy", parents=[parent_parser], description=DESTROY_HELP, help=DESTROY_HELP + "destroy", + parents=[parent_parser], + description=DESTROY_HELP, + help=DESTROY_HELP, ) destroy_parser.add_argument( - "-f", "--force", action="store_true", default=False, help="Force destruction." + "-f", + "--force", + action="store_true", + default=False, + help="Force destruction.", ) destroy_parser.set_defaults(func=CmdDestroy) diff --git a/dvc/command/imp.py b/dvc/command/imp.py index 37611e56f..1e125ecd0 100644 --- a/dvc/command/imp.py +++ b/dvc/command/imp.py @@ -30,7 +30,10 @@ class CmdImport(CmdBase): def add_parser(subparsers, parent_parser): IMPORT_HELP = "Import files from URL." import_parser = subparsers.add_parser( - "import", parents=[parent_parser], description=IMPORT_HELP, help=IMPORT_HELP + "import", + parents=[parent_parser], + description=IMPORT_HELP, + help=IMPORT_HELP, ) import_parser.add_argument( "url", diff --git a/dvc/command/init.py b/dvc/command/init.py index 2eff8d0d1..bb311549b 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -23,7 +23,9 @@ class CmdInit(object): def add_parser(subparsers, parent_parser): """Setup parser for `dvc init`.""" - INIT_HELP = "Initialize dvc over a directory " "(should already be a git dir)." + INIT_HELP = ( + "Initialize dvc over a directory " "(should already be a git dir)." + ) init_parser = subparsers.add_parser( "init", parents=[parent_parser], description=INIT_HELP, help=INIT_HELP ) diff --git a/dvc/command/install.py b/dvc/command/install.py index 3fc5e554a..818ba5ea7 100644 --- a/dvc/command/install.py +++ b/dvc/command/install.py @@ -17,6 +17,9 @@ class CmdInstall(CmdBase): def add_parser(subparsers, parent_parser): INSTALL_HELP = "Install dvc hooks into the repository." install_parser = subparsers.add_parser( - "install", parents=[parent_parser], description=INSTALL_HELP, help=INSTALL_HELP + "install", + parents=[parent_parser], + description=INSTALL_HELP, + help=INSTALL_HELP, ) install_parser.set_defaults(func=CmdInstall) diff --git a/dvc/command/lock.py b/dvc/command/lock.py index 969cd9f26..79742e8d7 100644 --- a/dvc/command/lock.py +++ b/dvc/command/lock.py @@ -12,7 +12,9 @@ class CmdLockBase(CmdBase): self.project.lock_stage(target, unlock=unlock) except DvcException: logger.error( - "failed to {}lock '{}'".format("un" if unlock else "", target) + "failed to {}lock '{}'".format( + "un" if unlock else "", target + ) ) return 1 @@ -39,7 +41,10 @@ def add_parser(subparsers, parent_parser): UNLOCK_HELP = "Unlock DVC file." unlock_parser = subparsers.add_parser( - "unlock", parents=[parent_parser], description=UNLOCK_HELP, help=UNLOCK_HELP + "unlock", + parents=[parent_parser], + description=UNLOCK_HELP, + help=UNLOCK_HELP, ) unlock_parser.add_argument("targets", nargs="+", help="DVC files.") unlock_parser.set_defaults(func=CmdUnlock) diff --git a/dvc/command/metrics.py b/dvc/command/metrics.py index 5b56defee..53baa7214 100644 --- a/dvc/command/metrics.py +++ b/dvc/command/metrics.py @@ -59,7 +59,9 @@ class CmdMetricsModify(CmdBase): class CmdMetricsAdd(CmdBase): def run(self): try: - self.project.metrics_add(self.args.path, self.args.type, self.args.xpath) + self.project.metrics_add( + self.args.path, self.args.type, self.args.xpath + ) except DvcException: msg = "failed to add metric file '{}'".format(self.args.path) logger.error(msg) @@ -83,11 +85,15 @@ class CmdMetricsRemove(CmdBase): def add_parser(subparsers, parent_parser): METRICS_HELP = "Get metrics from all branches." metrics_parser = subparsers.add_parser( - "metrics", parents=[parent_parser], description=METRICS_HELP, help=METRICS_HELP + "metrics", + parents=[parent_parser], + description=METRICS_HELP, + help=METRICS_HELP, ) metrics_subparsers = metrics_parser.add_subparsers( - dest="cmd", help="Use dvc metrics CMD --help for command-specific help." + dest="cmd", + help="Use dvc metrics CMD --help for command-specific help.", ) fix_subparsers(metrics_subparsers) diff --git a/dvc/command/move.py b/dvc/command/move.py index 2715cac3a..dca6f58a7 100644 --- a/dvc/command/move.py +++ b/dvc/command/move.py @@ -10,7 +10,9 @@ class CmdMove(CmdBase): try: self.project.move(self.args.src, self.args.dst) except DvcException: - msg = "failed to move '{}' -> '{}'".format(self.args.src, self.args.dst) + msg = "failed to move '{}' -> '{}'".format( + self.args.src, self.args.dst + ) logger.error(msg) return 1 return 0 @@ -26,6 +28,8 @@ def add_parser(subparsers, parent_parser): move_parser = subparsers.add_parser( "move", parents=[parent_parser], description=description, help=help ) - move_parser.add_argument("src", help="Source path to a data file or directory.") + move_parser.add_argument( + "src", help="Source path to a data file or directory." + ) move_parser.add_argument("dst", help="Destination path.") move_parser.set_defaults(func=CmdMove) diff --git a/dvc/command/pipeline.py b/dvc/command/pipeline.py index 8927d1986..eeaf2607f 100644 --- a/dvc/command/pipeline.py +++ b/dvc/command/pipeline.py @@ -35,7 +35,9 @@ class CmdPipelineShow(CmdBase): stage = Stage.load(self.project, target) node = os.path.relpath(stage.path, self.project.root_dir) - pipelines = list(filter(lambda g: node in g.nodes(), self.project.pipelines())) + pipelines = list( + filter(lambda g: node in g.nodes(), self.project.pipelines()) + ) assert len(pipelines) == 1 G = pipelines[0] @@ -99,10 +101,15 @@ class CmdPipelineShow(CmdBase): for target in self.args.targets: try: if self.args.ascii: - self._show_ascii(target, self.args.commands, self.args.outs) + self._show_ascii( + target, self.args.commands, self.args.outs + ) elif self.args.dot: self.__write_dot( - target, self.args.commands, self.args.outs, self.args.dot + target, + self.args.commands, + self.args.outs, + self.args.dot, ) else: self._show(target, self.args.commands, self.args.outs) @@ -139,7 +146,8 @@ def add_parser(subparsers, parent_parser): ) pipeline_subparsers = pipeline_parser.add_subparsers( - dest="cmd", help="Use dvc pipeline CMD --help for command-specific help." + dest="cmd", + help="Use dvc pipeline CMD --help for command-specific help.", ) fix_subparsers(pipeline_subparsers) @@ -167,9 +175,14 @@ def add_parser(subparsers, parent_parser): help="Print output files instead of paths to DVC files.", ) pipeline_show_parser.add_argument( - "--ascii", action="store_true", default=False, help="Output DAG as ASCII." + "--ascii", + action="store_true", + default=False, + help="Output DAG as ASCII.", + ) + pipeline_show_parser.add_argument( + "--dot", help="Write DAG in .dot format." ) - pipeline_show_parser.add_argument("--dot", help="Write DAG in .dot format.") pipeline_show_parser.add_argument( "targets", nargs="*", help="DVC files. 'Dvcfile' by default." ) diff --git a/dvc/command/remote.py b/dvc/command/remote.py index 7e010ddf6..67a7f2452 100644 --- a/dvc/command/remote.py +++ b/dvc/command/remote.py @@ -32,7 +32,9 @@ class CmdRemoteAdd(CmdConfig): def run(self): remote = _get({Config.SECTION_REMOTE_URL: self.args.url}) if remote == RemoteLOCAL: - self.args.url = self.resolve_path(self.args.url, self.configobj.filename) + self.args.url = self.resolve_path( + self.args.url, self.configobj.filename + ) section = Config.SECTION_REMOTE_FMT.format(self.args.name) ret = self._set(section, Config.SECTION_REMOTE_URL, self.args.url) @@ -61,7 +63,9 @@ class CmdRemoteRemove(CmdConfig): if default == self.args.name: return self._unset( - Config.SECTION_CORE, opt=Config.SECTION_CORE_REMOTE, configobj=config + Config.SECTION_CORE, + opt=Config.SECTION_CORE_REMOTE, + configobj=config, ) def run(self): @@ -104,7 +108,9 @@ class CmdRemoteList(CmdConfig): r = re.match(Config.SECTION_REMOTE_REGEX, section) if r: name = r.group("name") - url = self.configobj[section].get(Config.SECTION_REMOTE_URL, "") + url = self.configobj[section].get( + Config.SECTION_REMOTE_URL, "" + ) logger.info("{}\t{}".format(name, url)) return 0 @@ -114,11 +120,15 @@ def add_parser(subparsers, parent_parser): REMOTE_HELP = "Manage set of tracked repositories." remote_parser = subparsers.add_parser( - "remote", parents=[parent_parser], description=REMOTE_HELP, help=REMOTE_HELP + "remote", + parents=[parent_parser], + description=REMOTE_HELP, + help=REMOTE_HELP, ) remote_subparsers = remote_parser.add_subparsers( - dest="cmd", help="Use dvc remote CMD --help for " "command-specific help." + dest="cmd", + help="Use dvc remote CMD --help for " "command-specific help.", ) fix_subparsers(remote_subparsers) @@ -152,7 +162,9 @@ def add_parser(subparsers, parent_parser): description=REMOTE_DEFAULT_HELP, help=REMOTE_DEFAULT_HELP, ) - remote_default_parser.add_argument("name", nargs="?", help="Name of the remote.") + remote_default_parser.add_argument( + "name", nargs="?", help="Name of the remote." + ) remote_default_parser.add_argument( "-u", "--unset", @@ -183,7 +195,11 @@ def add_parser(subparsers, parent_parser): remote_modify_parser.add_argument("option", help="Option.") remote_modify_parser.add_argument("value", nargs="?", help="Value.") remote_modify_parser.add_argument( - "-u", "--unset", default=False, action="store_true", help="Unset option." + "-u", + "--unset", + default=False, + action="store_true", + help="Unset option.", ) remote_modify_parser.set_defaults(func=CmdRemoteModify) diff --git a/dvc/command/remove.py b/dvc/command/remove.py index 18f220aec..67e17eb8f 100644 --- a/dvc/command/remove.py +++ b/dvc/command/remove.py @@ -14,13 +14,16 @@ class CmdRemove(CmdBase): if self.args.force: return False - msg = "Are you sure you want to remove {} with its outputs?".format(target) + msg = "Are you sure you want to remove {} with its outputs?".format( + target + ) if prompt.confirm(msg): return False raise DvcException( - "Cannot purge without a confirmation from the user." " Use '-f' to force." + "Cannot purge without a confirmation from the user." + " Use '-f' to force." ) def run(self): @@ -37,7 +40,10 @@ class CmdRemove(CmdBase): def add_parser(subparsers, parent_parser): REMOVE_HELP = "Remove outputs of DVC file." remove_parser = subparsers.add_parser( - "remove", parents=[parent_parser], description=REMOVE_HELP, help=REMOVE_HELP + "remove", + parents=[parent_parser], + description=REMOVE_HELP, + help=REMOVE_HELP, ) remove_parser_group = remove_parser.add_mutually_exclusive_group() remove_parser_group.add_argument( @@ -55,7 +61,11 @@ def add_parser(subparsers, parent_parser): help="Remove DVC file and all its outputs.", ) remove_parser.add_argument( - "-f", "--force", action="store_true", default=False, help="Force purge." + "-f", + "--force", + action="store_true", + default=False, + help="Force purge.", ) remove_parser.add_argument("targets", nargs="+", help="DVC files.") remove_parser.set_defaults(func=CmdRemove) diff --git a/dvc/command/repro.py b/dvc/command/repro.py index d2d32bbaf..4aeea1299 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -52,9 +52,14 @@ class CmdRepro(CmdBase): def add_parser(subparsers, parent_parser): REPRO_HELP = "Reproduce DVC file. Default file name - 'Dvcfile'." repro_parser = subparsers.add_parser( - "repro", parents=[parent_parser], description=REPRO_HELP, help=REPRO_HELP + "repro", + parents=[parent_parser], + description=REPRO_HELP, + help=REPRO_HELP, + ) + repro_parser.add_argument( + "targets", nargs="*", help="DVC file to reproduce." ) - repro_parser.add_argument("targets", nargs="*", help="DVC file to reproduce.") repro_parser.add_argument( "-f", "--force", @@ -67,7 +72,8 @@ def add_parser(subparsers, parent_parser): "--single-item", action="store_true", default=False, - help="Reproduce only single data item without recursive dependencies " "check.", + help="Reproduce only single data item without recursive dependencies " + "check.", ) repro_parser.add_argument( "-c", diff --git a/dvc/command/run.py b/dvc/command/run.py index 54eff2f9d..b516dba49 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -13,7 +13,12 @@ class CmdRun(CmdBase): overwrite = self.args.yes or self.args.overwrite_dvcfile if not any( - [self.args.deps, self.args.outs, self.args.outs_no_cache, self.args.command] + [ + self.args.deps, + self.args.outs, + self.args.outs_no_cache, + self.args.command, + ] ): # pragma: no cover logger.error( "too few arguments. Specify at least one: '-d', " @@ -64,7 +69,10 @@ class CmdRun(CmdBase): def add_parser(subparsers, parent_parser): - RUN_HELP = "Generate a stage file from a given " "command and execute the command." + RUN_HELP = ( + "Generate a stage file from a given " + "command and execute the command." + ) run_parser = subparsers.add_parser( "run", parents=[parent_parser], description=RUN_HELP, help=RUN_HELP ) diff --git a/dvc/command/unprotect.py b/dvc/command/unprotect.py index cf158d587..e603ad786 100644 --- a/dvc/command/unprotect.py +++ b/dvc/command/unprotect.py @@ -25,5 +25,7 @@ def add_parser(subparsers, parent_parser): description=UNPROTECT_HELP, help=UNPROTECT_HELP, ) - unprotect_parser.add_argument("targets", nargs="+", help="Data files/directory.") + unprotect_parser.add_argument( + "targets", nargs="+", help="Data files/directory." + ) unprotect_parser.set_defaults(func=CmdUnprotect) diff --git a/dvc/config.py b/dvc/config.py index c37a2d937..b726620a4 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -22,7 +22,9 @@ class ConfigError(DvcException): """ def __init__(self, msg, ex=None): - super(ConfigError, self).__init__("config file error: {}".format(msg), ex) + super(ConfigError, self).__init__( + "config file error: {}".format(msg), ex + ) def supported_cache_type(types): @@ -178,7 +180,9 @@ class Config(object): # pylint: disable=too-many-instance-attributes Optional( SECTION_CORE_INTERACTIVE, default=False ): SECTION_CORE_INTERACTIVE_SCHEMA, - Optional(SECTION_CORE_ANALYTICS, default=True): SECTION_CORE_ANALYTICS_SCHEMA, + Optional( + SECTION_CORE_ANALYTICS, default=True + ): SECTION_CORE_ANALYTICS_SCHEMA, # backward compatibility Optional(SECTION_CORE_CLOUD, default=""): SECTION_CORE_CLOUD_SCHEMA, Optional(SECTION_CORE_STORAGEPATH, default=""): str, @@ -198,7 +202,9 @@ class Config(object): # pylint: disable=too-many-instance-attributes Optional(SECTION_AWS_PROFILE): str, Optional(SECTION_AWS_CREDENTIALPATH): str, Optional(SECTION_AWS_ENDPOINT_URL): str, - Optional(SECTION_AWS_USE_SSL, default=True): And(str, is_bool, Use(to_bool)), + Optional(SECTION_AWS_USE_SSL, default=True): And( + str, is_bool, Use(to_bool) + ), } # backward compatibility @@ -232,7 +238,9 @@ class Config(object): # pylint: disable=too-many-instance-attributes Optional(SECTION_AWS_PROFILE): str, Optional(SECTION_AWS_CREDENTIALPATH): str, Optional(SECTION_AWS_ENDPOINT_URL): str, - Optional(SECTION_AWS_USE_SSL, default=True): And(str, is_bool, Use(to_bool)), + Optional(SECTION_AWS_USE_SSL, default=True): And( + str, is_bool, Use(to_bool) + ), Optional(SECTION_GCP_PROJECTNAME): str, Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA, Optional(SECTION_CACHE_PROTECTED, default=False): And( @@ -302,7 +310,9 @@ class Config(object): # pylint: disable=too-many-instance-attributes """ from appdirs import user_config_dir - return user_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR) + return user_config_dir( + appname=Config.APPNAME, appauthor=Config.APPAUTHOR + ) @staticmethod def get_system_config_dir(): @@ -313,7 +323,9 @@ class Config(object): # pylint: disable=too-many-instance-attributes """ from appdirs import site_config_dir - return site_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR) + return site_config_dir( + appname=Config.APPNAME, appauthor=Config.APPAUTHOR + ) @staticmethod def init(dvc_dir): @@ -403,7 +415,9 @@ class Config(object): # pylint: disable=too-many-instance-attributes self.config = Schema(self.SCHEMA).validate(self.config) # NOTE: now converting back to ConfigObj - self.config = configobj.ConfigObj(self.config, write_empty_values=True) + self.config = configobj.ConfigObj( + self.config, write_empty_values=True + ) self.config.filename = self.config_file self._resolve_paths(self.config, self.config_file) except Exception as ex: @@ -474,7 +488,9 @@ class Config(object): # pylint: disable=too-many-instance-attributes return if opt not in config[section].keys(): - raise ConfigError("option '{}.{}' doesn't exist".format(section, opt)) + raise ConfigError( + "option '{}.{}' doesn't exist".format(section, opt) + ) del config[section][opt] if not config[section]: @@ -508,7 +524,9 @@ class Config(object): # pylint: disable=too-many-instance-attributes raise ConfigError("section '{}' doesn't exist".format(section)) if opt not in config[section].keys(): - raise ConfigError("option '{}.{}' doesn't exist".format(section, opt)) + raise ConfigError( + "option '{}.{}' doesn't exist".format(section, opt) + ) logger.info(config[section][opt]) diff --git a/dvc/dagascii.py b/dvc/dagascii.py index e8107b8e0..e7620e675 100644 --- a/dvc/dagascii.py +++ b/dvc/dagascii.py @@ -357,9 +357,14 @@ def draw(vertexes, edges): y = vertex.view.xy[1] canvas.box( - int(round(x - minx)), int(round(y - miny)), vertex.view.w, vertex.view.h + int(round(x - minx)), + int(round(y - miny)), + vertex.view.w, + vertex.view.h, ) - canvas.text(int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data) + canvas.text( + int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data + ) canvas.draw() diff --git a/dvc/dependency/http.py b/dvc/dependency/http.py index c53db60c8..0814ccec3 100644 --- a/dvc/dependency/http.py +++ b/dvc/dependency/http.py @@ -10,7 +10,9 @@ class DependencyHTTP(DependencyBase, OutputBase): REMOTE = RemoteHTTP def __init__(self, stage, path, info=None, remote=None): - super(DependencyHTTP, self).__init__(stage, path, info=info, remote=remote) + super(DependencyHTTP, self).__init__( + stage, path, info=info, remote=remote + ) if path.startswith("remote"): path = urljoin(self.remote.cache_dir, urlparse(path).path) diff --git a/dvc/exceptions.py b/dvc/exceptions.py index 3176bf900..aecd8249b 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -41,10 +41,12 @@ class OutputDuplicationError(DvcException): assert isinstance(output, str) or isinstance(output, builtin_str) assert isinstance(stages, list) assert all( - isinstance(stage, str) or isinstance(stage, builtin_str) for stage in stages + isinstance(stage, str) or isinstance(stage, builtin_str) + for stage in stages ) msg = ( - "file/directory '{}' is specified as an output in more than one" "stage: {}" + "file/directory '{}' is specified as an output in more than one" + "stage: {}" ).format(output, "\n ".join(stages)) super(OutputDuplicationError, self).__init__(msg) @@ -80,9 +82,14 @@ class CircularDependencyError(DvcException): """ def __init__(self, dependency): - assert isinstance(dependency, str) or isinstance(dependency, builtin_str) + assert isinstance(dependency, str) or isinstance( + dependency, builtin_str + ) - msg = "file/directory '{}' is specified as an output and as a " "dependency." + msg = ( + "file/directory '{}' is specified as an output and as a " + "dependency." + ) super(CircularDependencyError, self).__init__(msg.format(dependency)) diff --git a/dvc/logger.py b/dvc/logger.py index 05b6b7a0a..824436a50 100644 --- a/dvc/logger.py +++ b/dvc/logger.py @@ -244,7 +244,8 @@ def _parse_exc(): if is_verbose(): stack_trace = "{line}\n{stack_trace}{line}\n".format( - line=colorize("-" * 60, color="red"), stack_trace="\n".join(tb_list) + line=colorize("-" * 60, color="red"), + stack_trace="\n".join(tb_list), ) else: stack_trace = "" diff --git a/dvc/output/base.py b/dvc/output/base.py index 833513f74..6303507e4 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -48,7 +48,9 @@ class OutputBase(object): DoesNotExistError = OutputDoesNotExistError IsNotFileOrDirError = OutputIsNotFileOrDirError - def __init__(self, stage, path, info=None, remote=None, cache=True, metric=False): + def __init__( + self, stage, path, info=None, remote=None, cache=True, metric=False + ): self.stage = stage self.project = stage.project self.url = path @@ -57,9 +59,14 @@ class OutputBase(object): self.use_cache = False if self.IS_DEPENDENCY else cache self.metric = False if self.IS_DEPENDENCY else metric - if self.use_cache and getattr(self.project.cache, self.REMOTE.scheme) is None: + if ( + self.use_cache + and getattr(self.project.cache, self.REMOTE.scheme) is None + ): raise DvcException( - "no cache location setup for '{}' outputs.".format(self.REMOTE.scheme) + "no cache location setup for '{}' outputs.".format( + self.REMOTE.scheme + ) ) def __repr__(self): @@ -133,7 +140,9 @@ class OutputBase(object): if not self.use_cache: self.info = self.remote.save_info(self.path_info) else: - self.info = getattr(self.project.cache, self.scheme).save(self.path_info) + self.info = getattr(self.project.cache, self.scheme).save( + self.path_info + ) def dumpd(self): ret = self.info.copy() diff --git a/dvc/output/hdfs.py b/dvc/output/hdfs.py index a162909d2..9bf8f6a4a 100644 --- a/dvc/output/hdfs.py +++ b/dvc/output/hdfs.py @@ -10,7 +10,9 @@ from dvc.remote.hdfs import RemoteHDFS class OutputHDFS(OutputBase): REMOTE = RemoteHDFS - def __init__(self, stage, path, info=None, remote=None, cache=True, metric=False): + def __init__( + self, stage, path, info=None, remote=None, cache=True, metric=False + ): super(OutputHDFS, self).__init__( stage, path, info=info, remote=remote, cache=cache, metric=metric ) diff --git a/dvc/output/local.py b/dvc/output/local.py index 8263680bd..756325afd 100644 --- a/dvc/output/local.py +++ b/dvc/output/local.py @@ -13,12 +13,16 @@ from dvc.output.base import OutputBase, OutputAlreadyTrackedError class OutputLOCAL(OutputBase): REMOTE = RemoteLOCAL - def __init__(self, stage, path, info=None, remote=None, cache=True, metric=False): + def __init__( + self, stage, path, info=None, remote=None, cache=True, metric=False + ): super(OutputLOCAL, self).__init__( stage, path, info, remote=remote, cache=cache, metric=metric ) if remote: - p = os.path.join(remote.prefix, urlparse(self.url).path.lstrip("/")) + p = os.path.join( + remote.prefix, urlparse(self.url).path.lstrip("/") + ) else: p = path @@ -34,7 +38,9 @@ class OutputLOCAL(OutputBase): @property def is_local(self): - return urlparse(self.url).scheme != "remote" and not os.path.isabs(self.url) + return urlparse(self.url).scheme != "remote" and not os.path.isabs( + self.url + ) @property def sep(self): @@ -51,7 +57,9 @@ class OutputLOCAL(OutputBase): def dumpd(self): ret = super(OutputLOCAL, self).dumpd() if self.is_local: - path = self.remote.unixpath(os.path.relpath(self.path, self.stage.cwd)) + path = self.remote.unixpath( + os.path.relpath(self.path, self.stage.cwd) + ) else: path = self.url diff --git a/dvc/output/s3.py b/dvc/output/s3.py index 300ba1bb6..48546b1a4 100644 --- a/dvc/output/s3.py +++ b/dvc/output/s3.py @@ -10,7 +10,9 @@ from dvc.output.base import OutputBase class OutputS3(OutputBase): REMOTE = RemoteS3 - def __init__(self, stage, path, info=None, remote=None, cache=True, metric=False): + def __init__( + self, stage, path, info=None, remote=None, cache=True, metric=False + ): super(OutputS3, self).__init__( stage, path, info=info, remote=remote, cache=cache, metric=metric ) @@ -18,4 +20,8 @@ class OutputS3(OutputBase): path = urlparse(path).path.lstrip("/") if remote: path = posixpath.join(remote.prefix, path) - self.path_info = {"scheme": self.scheme, "bucket": bucket, "path": path} + self.path_info = { + "scheme": self.scheme, + "bucket": bucket, + "path": path, + } diff --git a/dvc/output/ssh.py b/dvc/output/ssh.py index 2eac4c38b..fba970a59 100644 --- a/dvc/output/ssh.py +++ b/dvc/output/ssh.py @@ -11,17 +11,25 @@ from dvc.remote.ssh import RemoteSSH class OutputSSH(OutputBase): REMOTE = RemoteSSH - def __init__(self, stage, path, info=None, remote=None, cache=True, metric=False): + def __init__( + self, stage, path, info=None, remote=None, cache=True, metric=False + ): super(OutputSSH, self).__init__( stage, path, info=info, remote=remote, cache=cache, metric=metric ) parsed = urlparse(path) host = remote.host if remote else parsed.hostname - port = remote.port if remote else (parsed.port or RemoteSSH.DEFAULT_PORT) - user = remote.user if remote else (parsed.username or getpass.getuser()) + port = ( + remote.port if remote else (parsed.port or RemoteSSH.DEFAULT_PORT) + ) + user = ( + remote.user if remote else (parsed.username or getpass.getuser()) + ) if remote: - path = posixpath.join(remote.prefix, urlparse(path).path.lstrip("/")) + path = posixpath.join( + remote.prefix, urlparse(path).path.lstrip("/") + ) else: path = parsed.path diff --git a/dvc/project.py b/dvc/project.py index 4ba57fa3a..5a7c3b4aa 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -7,7 +7,11 @@ import os import dvc.prompt as prompt import dvc.logger as logger -from dvc.exceptions import DvcException, MoveNotDataSourceError, NotDvcProjectError +from dvc.exceptions import ( + DvcException, + MoveNotDataSourceError, + NotDvcProjectError, +) class InitError(DvcException): @@ -276,7 +280,9 @@ class Project(object): import dvc.output as Output from dvc.stage import Stage - from_out = Output.loads_from(Stage(self, cwd=os.curdir), [from_path])[0] + from_out = Output.loads_from(Stage(self, cwd=os.curdir), [from_path])[ + 0 + ] to_path = self._expand_target_path(from_path, to_path) @@ -289,7 +295,9 @@ class Project(object): ) except StopIteration: raise DvcException( - "unable to find stage file with output '{path}'".format(path=from_path) + "unable to find stage file with output '{path}'".format( + path=from_path + ) ) if not stage.is_data_source: @@ -349,7 +357,9 @@ class Project(object): def unprotect(self, path): if not os.path.exists(path): - raise DvcException("can't unprotect non-existing data '{}'".format(path)) + raise DvcException( + "can't unprotect non-existing data '{}'".format(path) + ) if os.path.isdir(path): self._unprotect_dir(path) @@ -553,7 +563,9 @@ class Project(object): used.append(out.path) self.state.remove_unused_links(used) - def checkout(self, target=None, with_deps=False, force=False, recursive=False): + def checkout( + self, target=None, with_deps=False, force=False, recursive=False + ): if target and not recursive: all_stages = self.active_stages() @@ -569,7 +581,9 @@ class Project(object): if stage.locked: logger.warning( "DVC file '{path}' is locked. Its dependencies are" - " not going to be checked out.".format(path=stage.relpath) + " not going to be checked out.".format( + path=stage.relpath + ) ) stage.checkout(force=force) @@ -597,7 +611,9 @@ class Project(object): return ret - def _collect_dir_cache(self, out, branch=None, remote=None, force=False, jobs=None): + def _collect_dir_cache( + self, out, branch=None, remote=None, force=False, jobs=None + ): info = out.dumpd() ret = [info] r = out.remote @@ -605,7 +621,9 @@ class Project(object): if self.cache.local.changed_cache_file(md5): try: - self.cloud.pull(ret, jobs=jobs, remote=remote, show_checksums=False) + self.cloud.pull( + ret, jobs=jobs, remote=remote, show_checksums=False + ) except DvcException as exc: msg = "Failed to pull cache for '{}': {}" logger.debug(msg.format(out, exc)) @@ -626,7 +644,9 @@ class Project(object): for i in self.cache.local.load_dir_cache(md5): i["branch"] = branch - i[r.PARAM_PATH] = os.path.join(info[r.PARAM_PATH], i[r.PARAM_RELPATH]) + i[r.PARAM_PATH] = os.path.join( + info[r.PARAM_PATH], i[r.PARAM_RELPATH] + ) ret.append(i) return ret @@ -680,7 +700,9 @@ class Project(object): cache["ssh"] = [] cache["azure"] = [] - for branch in self.scm.brancher(all_branches=all_branches, all_tags=all_tags): + for branch in self.scm.brancher( + all_branches=all_branches, all_tags=all_tags + ): if target: if recursive: stages = self.stages(target) @@ -703,7 +725,11 @@ class Project(object): for out in stage.outs: scheme = out.path_info["scheme"] cache[scheme] += self._collect_used_cache( - out, branch=branch, remote=remote, force=force, jobs=jobs + out, + branch=branch, + remote=remote, + force=force, + jobs=jobs, ) return cache @@ -809,7 +835,9 @@ class Project(object): self._do_gc("azure", self.cache.azure.gc, clist) if cloud: - self._do_gc("remote", self.cloud._get_cloud(remote, "gc -c").gc, clist) + self._do_gc( + "remote", self.cloud._get_cloud(remote, "gc -c").gc, clist + ) def push( self, @@ -833,7 +861,9 @@ class Project(object): jobs=jobs, recursive=recursive, )["local"] - self.cloud.push(used, jobs, remote=remote, show_checksums=show_checksums) + self.cloud.push( + used, jobs, remote=remote, show_checksums=show_checksums + ) def fetch( self, @@ -857,7 +887,9 @@ class Project(object): jobs=jobs, recursive=recursive, )["local"] - self.cloud.pull(used, jobs, remote=remote, show_checksums=show_checksums) + self.cloud.pull( + used, jobs, remote=remote, show_checksums=show_checksums + ) def pull( self, @@ -882,7 +914,10 @@ class Project(object): recursive=recursive, ) self.checkout( - target=target, with_deps=with_deps, force=force, recursive=recursive + target=target, + with_deps=with_deps, + force=force, + recursive=recursive, ) def _local_status(self, target=None, with_deps=False): @@ -938,7 +973,10 @@ class Project(object): if status == cloud.STATUS_OK: continue - prefix_map = {cloud.STATUS_DELETED: "deleted", cloud.STATUS_NEW: "new"} + prefix_map = { + cloud.STATUS_DELETED: "deleted", + cloud.STATUS_NEW: "new", + } ret[name] = prefix_map[status] @@ -1036,7 +1074,9 @@ class Project(object): abs_path = os.path.abspath(path) if os.path.isdir(abs_path) and recursive: matched = [ - out for out in outs if os.path.abspath(out.path).startswith(abs_path) + out + for out in outs + if os.path.abspath(out.path).startswith(abs_path) ] else: matched = [out for out in outs if out.path == abs_path] @@ -1056,17 +1096,23 @@ class Project(object): recursive=False, ): res = {} - for branch in self.scm.brancher(all_branches=all_branches, all_tags=all_tags): + for branch in self.scm.brancher( + all_branches=all_branches, all_tags=all_tags + ): astages = self.active_stages() outs = [out for stage in astages for out in stage.outs] if path: - outs = self._find_output_by_path(path, outs=outs, recursive=recursive) + outs = self._find_output_by_path( + path, outs=outs, recursive=recursive + ) stages = [out.stage.path for out in outs] entries = [] for out in outs: - if all([out.metric, not typ, isinstance(out.metric, dict)]): + if all( + [out.metric, not typ, isinstance(out.metric, dict)] + ): entries += [ ( out.path, @@ -1192,7 +1238,10 @@ class Project(object): def graph(self, stages=None, from_directory=None): import networkx as nx - from dvc.exceptions import OutputDuplicationError, WorkingDirectoryAsOutputError + from dvc.exceptions import ( + OutputDuplicationError, + WorkingDirectoryAsOutputError, + ) G = nx.DiGraph() G_active = nx.DiGraph() @@ -1217,7 +1266,9 @@ class Project(object): ) if overlaps: - raise WorkingDirectoryAsOutputError(stage.cwd, stage.relpath) + raise WorkingDirectoryAsOutputError( + stage.cwd, stage.relpath + ) # collect the whole DAG for stage in stages: @@ -1252,7 +1303,9 @@ class Project(object): G, G_active = self.graph(from_directory=from_directory) - return [G.subgraph(c).copy() for c in nx.weakly_connected_components(G)] + return [ + G.subgraph(c).copy() for c in nx.weakly_connected_components(G) + ] def stages(self, from_directory=None): """ diff --git a/dvc/remote/azure.py b/dvc/remote/azure.py index 0946cb235..16e69a7f8 100644 --- a/dvc/remote/azure.py +++ b/dvc/remote/azure.py @@ -84,7 +84,9 @@ class RemoteAzure(RemoteBase): raise NotImplementedError logger.debug( - "Removing azure://{}/{}".format(path_info["bucket"], path_info["path"]) + "Removing azure://{}/{}".format( + path_info["bucket"], path_info["path"] + ) ) self.blob_service.delete_blob(path_info["bucket"], path_info["path"]) @@ -93,7 +95,9 @@ class RemoteAzure(RemoteBase): blob_service = self.blob_service next_marker = None while True: - blobs = blob_service.list_blobs(bucket, prefix=prefix, marker=next_marker) + blobs = blob_service.list_blobs( + bucket, prefix=prefix, marker=next_marker + ) for blob in blobs: yield blob.name @@ -120,7 +124,9 @@ class RemoteAzure(RemoteBase): path = to_info["path"] logger.debug( - "Uploading '{}' to '{}/{}'".format(from_info["path"], bucket, path) + "Uploading '{}' to '{}/{}'".format( + from_info["path"], bucket, path + ) ) if not name: @@ -138,7 +144,9 @@ class RemoteAzure(RemoteBase): else: progress.finish_target(name) - def download(self, from_infos, to_infos, no_progress_bar=False, names=None): + def download( + self, from_infos, to_infos, no_progress_bar=False, names=None + ): names = self._verify_path_args(from_infos, to_infos, names) for to_info, from_info, name in zip(to_infos, from_infos, names): @@ -152,7 +160,9 @@ class RemoteAzure(RemoteBase): path = from_info["path"] logger.debug( - "Downloading '{}/{}' to '{}'".format(bucket, path, to_info["path"]) + "Downloading '{}/{}' to '{}'".format( + bucket, path, to_info["path"] + ) ) tmp_file = self.tmp_file(to_info["path"]) diff --git a/dvc/remote/base.py b/dvc/remote/base.py index ff1395c48..8af5cc3fd 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -56,7 +56,9 @@ class RemoteBase(object): deps_ok = all(self.REQUIRES.values()) if not deps_ok: missing = [k for k, v in self.REQUIRES.items() if v is None] - url = config.get(Config.SECTION_REMOTE_URL, "{}://".format(self.scheme)) + url = config.get( + Config.SECTION_REMOTE_URL, "{}://".format(self.scheme) + ) msg = ( "URL '{}' is supported but requires these missing " "dependencies: {}. If you have installed dvc using pip, " @@ -130,7 +132,9 @@ class RemoteBase(object): """ logger.debug( - "checking if '{}'('{}') has changed.".format(path_info, checksum_info) + "checking if '{}'('{}') has changed.".format( + path_info, checksum_info + ) ) if not self.exists(path_info): @@ -231,7 +235,9 @@ class RemoteBase(object): # is 32 bytes, so ~3200Mb list) and we don't really need all of it at # the same time, so it makes sense to use a generator to gradually # iterate over it, without keeping all of it in memory. - return (self.path_to_checksum(path) for path in self.list_cache_paths()) + return ( + self.path_to_checksum(path) for path in self.list_cache_paths() + ) def gc(self, cinfos): from dvc.remote.local import RemoteLOCAL @@ -255,7 +261,9 @@ class RemoteBase(object): expected = {self.PARAM_CHECKSUM: checksum} actual = self.save_info(cache) - logger.debug("Cache '{}' actual '{}'.".format(str(expected), str(actual))) + logger.debug( + "Cache '{}' actual '{}'.".format(str(expected), str(actual)) + ) if expected != actual: if self.exists(cache): diff --git a/dvc/remote/gs.py b/dvc/remote/gs.py index c4aa55c9a..756829848 100644 --- a/dvc/remote/gs.py +++ b/dvc/remote/gs.py @@ -63,7 +63,9 @@ class RemoteGS(RemoteBase): raise NotImplementedError return { - self.PARAM_CHECKSUM: self.get_md5(path_info["bucket"], path_info["path"]) + self.PARAM_CHECKSUM: self.get_md5( + path_info["bucket"], path_info["path"] + ) } def copy(self, from_info, to_info, gs=None): @@ -96,7 +98,9 @@ class RemoteGS(RemoteBase): raise NotImplementedError logger.debug( - "Removing gs://{}/{}".format(path_info["bucket"], path_info["path"]) + "Removing gs://{}/{}".format( + path_info["bucket"], path_info["path"] + ) ) blob = self.gs.bucket(path_info["bucket"]).get_blob(path_info["path"]) @@ -149,13 +153,17 @@ class RemoteGS(RemoteBase): except Exception: msg = "failed to upload '{}' to '{}/{}'" logger.error( - msg.format(from_info["path"], to_info["bucket"], to_info["path"]) + msg.format( + from_info["path"], to_info["bucket"], to_info["path"] + ) ) continue progress.finish_target(name) - def download(self, from_infos, to_infos, no_progress_bar=False, names=None): + def download( + self, from_infos, to_infos, no_progress_bar=False, names=None + ): names = self._verify_path_args(from_infos, to_infos, names) gs = self.gs @@ -194,7 +202,9 @@ class RemoteGS(RemoteBase): except Exception: msg = "failed to download '{}/{}' to '{}'" logger.error( - msg.format(from_info["bucket"], from_info["path"], to_info["path"]) + msg.format( + from_info["bucket"], from_info["path"], to_info["path"] + ) ) continue diff --git a/dvc/remote/hdfs.py b/dvc/remote/hdfs.py index 300e2e5c2..f62eede8a 100644 --- a/dvc/remote/hdfs.py +++ b/dvc/remote/hdfs.py @@ -27,7 +27,9 @@ class RemoteHDFS(RemoteBase): self.prefix = self.url self.user = self.group("user") if not self.user: - self.user = config.get(Config.SECTION_REMOTE_USER, getpass.getuser()) + self.user = config.get( + Config.SECTION_REMOTE_USER, getpass.getuser() + ) self.path_info = {"scheme": "hdfs", "user": self.user} @@ -78,7 +80,9 @@ class RemoteHDFS(RemoteBase): ) def rm(self, path_info): - self.hadoop_fs("rm {}".format(path_info["path"]), user=path_info["user"]) + self.hadoop_fs( + "rm {}".format(path_info["path"]), user=path_info["user"] + ) def save_info(self, path_info): if path_info["scheme"] != "hdfs": @@ -135,10 +139,14 @@ class RemoteHDFS(RemoteBase): cmd = "mkdir -p {}".format(posixpath.dirname(to_info["path"])) self.hadoop_fs(cmd, user=to_info["user"]) - cmd = "copyFromLocal {} {}".format(from_info["path"], to_info["path"]) + cmd = "copyFromLocal {} {}".format( + from_info["path"], to_info["path"] + ) self.hadoop_fs(cmd, user=to_info["user"]) - def download(self, from_infos, to_infos, no_progress_bar=False, names=None): + def download( + self, from_infos, to_infos, no_progress_bar=False, names=None + ): names = self._verify_path_args(from_infos, to_infos, names) for to_info, from_info, name in zip(to_infos, from_infos, names): @@ -156,7 +164,9 @@ class RemoteHDFS(RemoteBase): if not os.path.exists(dname): os.makedirs(dname) - cmd = "copyToLocal {} {}".format(from_info["path"], to_info["path"]) + cmd = "copyToLocal {} {}".format( + from_info["path"], to_info["path"] + ) self.hadoop_fs(cmd, user=from_info["user"]) def list_cache_paths(self): diff --git a/dvc/remote/http.py b/dvc/remote/http.py index c4f728596..7444ca87e 100644 --- a/dvc/remote/http.py +++ b/dvc/remote/http.py @@ -41,7 +41,9 @@ class RemoteHTTP(RemoteBase): def prefix(self): return self.cache_dir - def download(self, from_infos, to_infos, no_progress_bar=False, names=None): + def download( + self, from_infos, to_infos, no_progress_bar=False, names=None + ): names = self._verify_path_args(to_infos, from_infos, names) for to_info, from_info, name in zip(to_infos, from_infos, names): @@ -51,7 +53,9 @@ class RemoteHTTP(RemoteBase): if to_info["scheme"] != "local": raise NotImplementedError - msg = "Downloading '{}' to '{}'".format(from_info["path"], to_info["path"]) + msg = "Downloading '{}' to '{}'".format( + from_info["path"], to_info["path"] + ) logger.debug(msg) tmp_file = self.tmp_file(to_info["path"]) diff --git a/dvc/remote/local.py b/dvc/remote/local.py index 9521176bf..8ee326fcd 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -205,7 +205,8 @@ class RemoteLOCAL(RemoteBase): if len(files) > LARGE_DIR_SIZE: msg = ( - "Computing md5 for a large directory {}. " "This is only done once." + "Computing md5 for a large directory {}. " + "This is only done once." ) logger.info(msg.format(os.path.relpath(root))) bar = True @@ -223,7 +224,9 @@ class RemoteLOCAL(RemoteBase): processed += 1 md5 = self.state.update(path) - dir_info.append({self.PARAM_RELPATH: relpath, self.PARAM_CHECKSUM: md5}) + dir_info.append( + {self.PARAM_RELPATH: relpath, self.PARAM_CHECKSUM: md5} + ) if bar: progress.finish_target(title) @@ -349,7 +352,9 @@ class RemoteLOCAL(RemoteBase): for file in files ) - cached_files = set(os.path.join(path, file["relpath"]) for file in dir_info) + cached_files = set( + os.path.join(path, file["relpath"]) for file in dir_info + ) delta = working_dir_files - cached_files @@ -420,7 +425,9 @@ class RemoteLOCAL(RemoteBase): path = path_info["path"] msg = "Saving '{}' to cache '{}'." - logger.info(msg.format(os.path.relpath(path), os.path.relpath(self.cache_dir))) + logger.info( + msg.format(os.path.relpath(path), os.path.relpath(self.cache_dir)) + ) if os.path.isdir(path): return self._save_dir(path_info) @@ -460,7 +467,9 @@ class RemoteLOCAL(RemoteBase): raise NotImplementedError logger.debug( - "Uploading '{}' to '{}'".format(from_info["path"], to_info["path"]) + "Uploading '{}' to '{}'".format( + from_info["path"], to_info["path"] + ) ) if not name: @@ -477,7 +486,9 @@ class RemoteLOCAL(RemoteBase): ) ) - def download(self, from_infos, to_infos, no_progress_bar=False, names=None): + def download( + self, from_infos, to_infos, no_progress_bar=False, names=None + ): names = self._verify_path_args(from_infos, to_infos, names) for to_info, from_info, name in zip(to_infos, from_infos, names): @@ -488,7 +499,9 @@ class RemoteLOCAL(RemoteBase): raise NotImplementedError logger.debug( - "Downloading '{}' to '{}'".format(from_info["path"], to_info["path"]) + "Downloading '{}' to '{}'".format( + from_info["path"], to_info["path"] + ) ) if not name: @@ -561,7 +574,9 @@ class RemoteLOCAL(RemoteBase): progress.finish_target(title) for md5, info in ret.items(): - info["status"] = STATUS_MAP[(md5 in local_exists, md5 in remote_exists)] + info["status"] = STATUS_MAP[ + (md5 in local_exists, md5 in remote_exists) + ] return ret @@ -603,7 +618,12 @@ class RemoteLOCAL(RemoteBase): ) def _process( - self, checksum_infos, remote, jobs=None, show_checksums=False, download=False + self, + checksum_infos, + remote, + jobs=None, + show_checksums=False, + download=False, ): msg = "Preparing to {} data {} '{}'" logger.info( diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index dd3658537..fc0f88325 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -87,7 +87,9 @@ class RemoteS3(RemoteBase): try: obj = self.s3.head_object(Bucket=bucket, Key=path) except Exception: - raise DvcException("s3://{}/{} does not exist".format(bucket, path)) + raise DvcException( + "s3://{}/{} does not exist".format(bucket, path) + ) return obj["ETag"].strip('"') @@ -96,7 +98,9 @@ class RemoteS3(RemoteBase): raise NotImplementedError return { - self.PARAM_CHECKSUM: self.get_etag(path_info["bucket"], path_info["path"]) + self.PARAM_CHECKSUM: self.get_etag( + path_info["bucket"], path_info["path"] + ) } def copy(self, from_info, to_info, s3=None): @@ -122,10 +126,14 @@ class RemoteS3(RemoteBase): raise NotImplementedError logger.debug( - "Removing s3://{}/{}".format(path_info["bucket"], path_info["path"]) + "Removing s3://{}/{}".format( + path_info["bucket"], path_info["path"] + ) ) - self.s3.delete_object(Bucket=path_info["bucket"], Key=path_info["path"]) + self.s3.delete_object( + Bucket=path_info["bucket"], Key=path_info["path"] + ) def _list_paths(self, bucket, prefix): s3 = self.s3 @@ -186,7 +194,10 @@ class RemoteS3(RemoteBase): try: s3.upload_file( - from_info["path"], to_info["bucket"], to_info["path"], Callback=cb + from_info["path"], + to_info["bucket"], + to_info["path"], + Callback=cb, ) except Exception: msg = "failed to upload '{}'".format(from_info["path"]) @@ -195,7 +206,9 @@ class RemoteS3(RemoteBase): progress.finish_target(name) - def download(self, from_infos, to_infos, no_progress_bar=False, names=None): + def download( + self, from_infos, to_infos, no_progress_bar=False, names=None + ): names = self._verify_path_args(from_infos, to_infos, names) s3 = self.s3 @@ -232,7 +245,10 @@ class RemoteS3(RemoteBase): cb = Callback(name, total) s3.download_file( - from_info["bucket"], from_info["path"], tmp_file, Callback=cb + from_info["bucket"], + from_info["path"], + tmp_file, + Callback=cb, ) except Exception: msg = "failed to download '{}/{}'".format( diff --git a/dvc/remote/ssh.py b/dvc/remote/ssh.py index 8f3563510..f49199536 100644 --- a/dvc/remote/ssh.py +++ b/dvc/remote/ssh.py @@ -76,12 +76,16 @@ class RemoteSSH(RemoteBase): ) self.prefix = parsed.path self.port = ( - config.get(Config.SECTION_REMOTE_PORT) or parsed.port or self.DEFAULT_PORT + config.get(Config.SECTION_REMOTE_PORT) + or parsed.port + or self.DEFAULT_PORT ) self.keyfile = config.get(Config.SECTION_REMOTE_KEY_FILE, None) self.timeout = config.get(Config.SECTION_REMOTE_TIMEOUT, self.TIMEOUT) self.password = config.get(Config.SECTION_REMOTE_PASSWORD, None) - self.ask_password = config.get(Config.SECTION_REMOTE_ASK_PASSWORD, False) + self.ask_password = config.get( + Config.SECTION_REMOTE_ASK_PASSWORD, False + ) self.path_info = { "scheme": "ssh", @@ -91,7 +95,10 @@ class RemoteSSH(RemoteBase): } def ssh(self, host=None, user=None, port=None): - msg = "Establishing ssh connection with '{}' " "through port '{}' as user '{}'" + msg = ( + "Establishing ssh connection with '{}' " + "through port '{}' as user '{}'" + ) logger.debug(msg.format(host, port, user)) ssh = paramiko.SSHClient() @@ -121,7 +128,9 @@ class RemoteSSH(RemoteBase): assert not isinstance(path_info, list) assert path_info["scheme"] == "ssh" - with self.ssh(path_info["host"], path_info["user"], path_info["port"]) as ssh: + with self.ssh( + path_info["host"], path_info["user"], path_info["port"] + ) as ssh: try: self._exec(ssh, "test -e {}".format(path_info["path"])) exists = True @@ -139,7 +148,11 @@ class RemoteSSH(RemoteBase): stdout_chunks = [] stderr_chunks = [] - while not channel.closed or channel.recv_ready() or channel.recv_stderr_ready(): + while ( + not channel.closed + or channel.recv_ready() + or channel.recv_stderr_ready() + ): import select got_chunk = False @@ -180,7 +193,9 @@ class RemoteSSH(RemoteBase): raise NotImplementedError ssh = self.ssh( - host=path_info["host"], user=path_info["user"], port=path_info["port"] + host=path_info["host"], + user=path_info["user"], + port=path_info["port"], ) # Use different md5 commands depending on os @@ -214,7 +229,9 @@ class RemoteSSH(RemoteBase): ssh if ssh else self.ssh( - host=from_info["host"], user=from_info["user"], port=from_info["port"] + host=from_info["host"], + user=from_info["user"], + port=from_info["port"], ) ) @@ -254,12 +271,16 @@ class RemoteSSH(RemoteBase): ) ssh = self.ssh( - host=path_info["host"], user=path_info["user"], port=path_info["port"] + host=path_info["host"], + user=path_info["user"], + port=path_info["port"], ) ssh.open_sftp().remove(path_info["path"]) ssh.close() - def download(self, from_infos, to_infos, no_progress_bar=False, names=None): + def download( + self, from_infos, to_infos, no_progress_bar=False, names=None + ): names = self._verify_path_args(from_infos, to_infos, names) ssh = self.ssh( @@ -299,7 +320,9 @@ class RemoteSSH(RemoteBase): except Exception: msg = "failed to download '{}/{}' to '{}'" logger.error( - msg.format(from_info["host"], from_info["path"], to_info["path"]) + msg.format( + from_info["host"], from_info["path"], to_info["path"] + ) ) continue @@ -312,7 +335,9 @@ class RemoteSSH(RemoteBase): names = self._verify_path_args(to_infos, from_infos, names) ssh = self.ssh( - host=to_infos[0]["host"], user=to_infos[0]["user"], port=to_infos[0]["port"] + host=to_infos[0]["host"], + user=to_infos[0]["user"], + port=to_infos[0]["port"], ) sftp = ssh.open_sftp() @@ -336,11 +361,17 @@ class RemoteSSH(RemoteBase): self._exec(ssh, "mkdir -p {}".format(dname)) try: - sftp.put(from_info["path"], to_info["path"], callback=create_cb(name)) + sftp.put( + from_info["path"], + to_info["path"], + callback=create_cb(name), + ) except Exception: msg = "failed to upload '{}' to '{}/{}'" logger.error( - msg.format(from_info["path"], to_info["host"], to_info["path"]) + msg.format( + from_info["path"], to_info["host"], to_info["path"] + ) ) continue diff --git a/dvc/scm/base.py b/dvc/scm/base.py index f39f133cc..b02309fd1 100644 --- a/dvc/scm/base.py +++ b/dvc/scm/base.py @@ -76,7 +76,9 @@ class Base(object): def tag(self, tag): """Makes SCM create a tag with a specified name.""" - def brancher(self, branches=None, all_branches=False, tags=None, all_tags=False): + def brancher( + self, branches=None, all_branches=False, tags=None, all_tags=False + ): """Generator that iterates over specified revisions. Args: diff --git a/dvc/scm/git.py b/dvc/scm/git.py index cb161fb04..ad9abd8ae 100644 --- a/dvc/scm/git.py +++ b/dvc/scm/git.py @@ -76,7 +76,9 @@ class Git(Base): ignore_list = [] if os.path.exists(gitignore): ignore_list = open(gitignore, "r").readlines() - filtered = list(filter(lambda x: x.strip() == entry.strip(), ignore_list)) + filtered = list( + filter(lambda x: x.strip() == entry.strip(), ignore_list) + ) if filtered: return diff --git a/dvc/stage.py b/dvc/stage.py index c55b29e4c..5f7c9cdf8 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -131,7 +131,9 @@ class Stage(object): self.locked = locked def __repr__(self): - return "Stage: '{path}'".format(path=self.relpath if self.path else "No path") + return "Stage: '{path}'".format( + path=self.relpath if self.path else "No path" + ) @property def relpath(self): @@ -212,7 +214,9 @@ class Stage(object): return False def changed(self): - ret = any([self._changed_deps(), self._changed_outs(), self._changed_md5()]) + ret = any( + [self._changed_deps(), self._changed_outs(), self._changed_md5()] + ) if ret: msg = "Stage '{}' changed.".format(self.relpath) @@ -467,7 +471,9 @@ class Stage(object): self._check_dvc_filename(fname) logger.info( - "Saving information to '{file}'.".format(file=os.path.relpath(fname)) + "Saving information to '{file}'.".format( + file=os.path.relpath(fname) + ) ) with open(fname, "w") as fd: @@ -487,7 +493,9 @@ class Stage(object): # NOTE: excluding parameters that don't affect the state of the # pipeline. Not excluding `OutputLOCAL.PARAM_CACHE`, because if # it has changed, we might not have that output in our cache. - return dict_md5(d, exclude=[self.PARAM_LOCKED, OutputLOCAL.PARAM_METRIC]) + return dict_md5( + d, exclude=[self.PARAM_LOCKED, OutputLOCAL.PARAM_METRIC] + ) def save(self): for dep in self.deps: @@ -559,7 +567,9 @@ class Stage(object): def run(self, dry=False): if self.locked: logger.info( - "Verifying outputs in locked stage '{stage}'".format(stage=self.relpath) + "Verifying outputs in locked stage '{stage}'".format( + stage=self.relpath + ) ) if not dry: self.check_missing_outputs() diff --git a/dvc/state.py b/dvc/state.py index 97797226f..6c6aedc47 100644 --- a/dvc/state.py +++ b/dvc/state.py @@ -60,7 +60,9 @@ class State(object): # pylint: disable=too-many-instance-attributes LINK_STATE_TABLE = "link_state" LINK_STATE_TABLE_LAYOUT = ( - "path TEXT PRIMARY KEY, " "inode INTEGER NOT NULL, " "mtime TEXT NOT NULL" + "path TEXT PRIMARY KEY, " + "inode INTEGER NOT NULL, " + "mtime TEXT NOT NULL" ) STATE_ROW_LIMIT = 100000000 @@ -82,7 +84,8 @@ class State(object): # pylint: disable=too-many-instance-attributes Config.SECTION_STATE_ROW_LIMIT, self.STATE_ROW_LIMIT ) self.row_cleanup_quota = state_config.get( - Config.SECTION_STATE_ROW_CLEANUP_QUOTA, self.STATE_ROW_CLEANUP_QUOTA + Config.SECTION_STATE_ROW_CLEANUP_QUOTA, + self.STATE_ROW_CLEANUP_QUOTA, ) if not self.dvc_dir: @@ -92,7 +95,10 @@ class State(object): # pylint: disable=too-many-instance-attributes self.state_file = os.path.join(self.dvc_dir, self.STATE_FILE) # https://www.sqlite.org/tempfiles.html - self.temp_files = [self.state_file + "-journal", self.state_file + "-wal"] + self.temp_files = [ + self.state_file + "-journal", + self.state_file + "-wal", + ] self.database = None self.cursor = None @@ -186,8 +192,12 @@ class State(object): # pylint: disable=too-many-instance-attributes # Check that the state file is indeed a database cmd = "CREATE TABLE IF NOT EXISTS {} ({})" self._execute(cmd.format(self.STATE_TABLE, self.STATE_TABLE_LAYOUT)) - self._execute(cmd.format(self.STATE_INFO_TABLE, self.STATE_INFO_TABLE_LAYOUT)) - self._execute(cmd.format(self.LINK_STATE_TABLE, self.LINK_STATE_TABLE_LAYOUT)) + self._execute( + cmd.format(self.STATE_INFO_TABLE, self.STATE_INFO_TABLE_LAYOUT) + ) + self._execute( + cmd.format(self.LINK_STATE_TABLE, self.LINK_STATE_TABLE_LAYOUT) + ) cmd = ( "INSERT OR IGNORE INTO {} (count) SELECT 0 " @@ -253,7 +263,9 @@ class State(object): # pylint: disable=too-many-instance-attributes "DELETE FROM {} WHERE timestamp IN (" "SELECT timestamp FROM {} ORDER BY timestamp ASC LIMIT {});" ) - self._execute(cmd.format(self.STATE_TABLE, self.STATE_TABLE, delete)) + self._execute( + cmd.format(self.STATE_TABLE, self.STATE_TABLE, delete) + ) self._vacuum() @@ -268,7 +280,9 @@ class State(object): # pylint: disable=too-many-instance-attributes cmd = "UPDATE {} SET count = {} WHERE rowid = {}" self._execute( cmd.format( - self.STATE_INFO_TABLE, self._to_sqlite(count), self.STATE_INFO_ROW + self.STATE_INFO_TABLE, + self._to_sqlite(count), + self.STATE_INFO_ROW, ) ) @@ -423,8 +437,11 @@ class State(object): # pylint: disable=too-many-instance-attributes inode = self._inode(path) relpath = os.path.relpath(path, self.root_dir) - cmd = "REPLACE INTO {}(path, inode, mtime) " 'VALUES ("{}", {}, "{}")'.format( - self.LINK_STATE_TABLE, relpath, self._to_sqlite(inode), mtime + cmd = ( + "REPLACE INTO {}(path, inode, mtime) " + 'VALUES ("{}", {}, "{}")'.format( + self.LINK_STATE_TABLE, relpath, self._to_sqlite(inode), mtime + ) ) self._execute(cmd) diff --git a/dvc/system.py b/dvc/system.py index f27ea90bd..dbf261534 100644 --- a/dvc/system.py +++ b/dvc/system.py @@ -23,7 +23,11 @@ class System(object): raise DvcException("link", cause=exc) CreateHardLink = ctypes.windll.kernel32.CreateHardLinkW - CreateHardLink.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_void_p] + CreateHardLink.argtypes = [ + ctypes.c_wchar_p, + ctypes.c_wchar_p, + ctypes.c_void_p, + ] CreateHardLink.restype = ctypes.wintypes.BOOL res = CreateHardLink(link_name, source, None) @@ -158,10 +162,20 @@ class System(object): flags = FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT func = ctypes.windll.kernel32.CreateFileW - func.argtypes = [c_wchar_p, DWORD, DWORD, c_void_p, DWORD, DWORD, HANDLE] + func.argtypes = [ + c_wchar_p, + DWORD, + DWORD, + c_void_p, + DWORD, + DWORD, + HANDLE, + ] func.restype = HANDLE - hfile = func(path, 0, FILE_SHARE_READ, None, OPEN_EXISTING, flags, None) + hfile = func( + path, 0, FILE_SHARE_READ, None, OPEN_EXISTING, flags, None + ) if hfile is None: raise WinError() @@ -198,7 +212,11 @@ class System(object): info = System.getdirinfo(path) inode = abs( hash( - (info.dwVolumeSerialNumber, info.nFileIndexHigh, info.nFileIndexLow) + ( + info.dwVolumeSerialNumber, + info.nFileIndexHigh, + info.nFileIndexLow, + ) ) ) assert inode >= 0 diff --git a/dvc/updater.py b/dvc/updater.py index cefc6f14a..09b2032bf 100644 --- a/dvc/updater.py +++ b/dvc/updater.py @@ -128,7 +128,8 @@ class Updater(object): # pragma: no cover "formula": "Run {yellow}brew{reset} upgrade dvc", "cask": "Run {yellow}brew cask{reset} upgrade dvc", "apt": ( - "Run {yellow}apt-get{reset} install" " {blue}--only-upgrade{reset} dvc" + "Run {yellow}apt-get{reset} install" + " {blue}--only-upgrade{reset} dvc" ), "binary": ( "To upgrade follow this steps:\n" diff --git a/pyproject.toml b/pyproject.toml index b4f50b15e..a4fc8cbe3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [tool.black] -line-length = 88 +line-length = 79 include = '\.pyi?$' exclude = ''' /( diff --git a/scripts/ci/before_install.sh b/scripts/ci/before_install.sh index d02a5074f..83967a09d 100644 --- a/scripts/ci/before_install.sh +++ b/scripts/ci/before_install.sh @@ -20,9 +20,9 @@ pip install -r scripts/ci/requirements.txt # stop the build if there are any readme formatting errors python setup.py checkdocs -# stop the build if there are any black errors -black dvc --check -black tests --check +# stop the build if there are any black or flake8 errors +black ./ --check +flake8 ./ #NOTE: ssh keys for ssh test to be able to ssh to the localhost ls -la ~/.ssh/ diff --git a/test-style-requirements.txt b/test-style-requirements.txt index 0e5051ad0..75e6084cf 100644 --- a/test-style-requirements.txt +++ b/test-style-requirements.txt @@ -1,3 +1,4 @@ Pygments # required by collective.checkdocs collective.checkdocs black +flake8 diff --git a/tests/__main__.py b/tests/__main__.py index 279924954..4acdc549b 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -4,7 +4,9 @@ from subprocess import check_call REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) os.chdir(REPO_ROOT) -os.putenv("PATH", "{}:{}".format(os.path.join(REPO_ROOT, "bin"), os.getenv("PATH"))) +os.putenv( + "PATH", "{}:{}".format(os.path.join(REPO_ROOT, "bin"), os.getenv("PATH")) +) os.putenv("DVC_HOME", REPO_ROOT) os.putenv("DVC_TEST", "true") diff --git a/tests/basic_env.py b/tests/basic_env.py index 6726ef986..e308b9247 100644 --- a/tests/basic_env.py +++ b/tests/basic_env.py @@ -10,7 +10,9 @@ from dvc.project import Project class TestDir(TestCase): - GCP_CREDS_FILE = os.path.abspath(os.path.join("scripts", "ci", "gcp-creds.json")) + GCP_CREDS_FILE = os.path.abspath( + os.path.join("scripts", "ci", "gcp-creds.json") + ) DATA_DIR = "data_dir" DATA_SUB_DIR = os.path.join(DATA_DIR, "data_sub_dir") DATA = os.path.join(DATA_DIR, "data") @@ -32,7 +34,8 @@ class TestDir(TestCase): BAR_CONTENTS = BAR + "r" CODE = "code.py" CODE_CONTENTS = ( - "import sys\nimport shutil\n" "shutil.copyfile(sys.argv[1], sys.argv[2])" + "import sys\nimport shutil\n" + "shutil.copyfile(sys.argv[1], sys.argv[2])" ) def _pushd(self, d): @@ -87,7 +90,7 @@ class TestGit(TestDir): while retries: try: self.git = Repo.init() - except GitCommandNotFound as exc: + except GitCommandNotFound: retries -= 1 continue break diff --git a/tests/test_cache.py b/tests/test_cache.py index e4568c5c1..b3bab74f7 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -14,10 +14,14 @@ class TestCache(TestDvc): self.cache1_md5 = "123" self.cache2_md5 = "234" self.cache1 = os.path.join( - self.dvc.cache.local.cache_dir, self.cache1_md5[0:2], self.cache1_md5[2:] + self.dvc.cache.local.cache_dir, + self.cache1_md5[0:2], + self.cache1_md5[2:], ) self.cache2 = os.path.join( - self.dvc.cache.local.cache_dir, self.cache2_md5[0:2], self.cache2_md5[2:] + self.dvc.cache.local.cache_dir, + self.cache2_md5[0:2], + self.cache2_md5[2:], ) self.create(self.cache1, "1") self.create(self.cache2, "2") @@ -103,9 +107,15 @@ class TestSharedCacheDir(TestDir): ) ) self.assertEqual(len(subdirs), 3) - self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[0]))), 1) - self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[1]))), 1) - self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[2]))), 1) + self.assertEqual( + len(os.listdir(os.path.join(cache_dir, subdirs[0]))), 1 + ) + self.assertEqual( + len(os.listdir(os.path.join(cache_dir, subdirs[1]))), 1 + ) + self.assertEqual( + len(os.listdir(os.path.join(cache_dir, subdirs[2]))), 1 + ) class TestCacheLinkType(TestDvc): diff --git a/tests/test_checkout.py b/tests/test_checkout.py index 055287501..cc2ea77dd 100644 --- a/tests/test_checkout.py +++ b/tests/test_checkout.py @@ -113,7 +113,9 @@ class CheckoutBase(TestDvc): self.dvc.scm.commit("adding " + fname) def read_ignored(self): - return list(map(lambda s: s.strip("\n"), open(self.GIT_IGNORE).readlines())) + return list( + map(lambda s: s.strip("\n"), open(self.GIT_IGNORE).readlines()) + ) def outs_info(self, stage): FileInfo = collections.namedtuple("FileInfo", "path inode") @@ -125,7 +127,9 @@ class CheckoutBase(TestDvc): for file in files ] - return [FileInfo(path=path, inode=System.inode(path)) for path in paths] + return [ + FileInfo(path=path, inode=System.inode(path)) for path in paths + ] class TestRemoveFilesWhenCheckout(CheckoutBase): @@ -314,7 +318,9 @@ class TestCheckoutNotCachedFile(TestDvc): cmd = "python {} {} {}".format(self.CODE, self.FOO, "out") self.dvc.add(self.FOO) - stage = self.dvc.run(cmd=cmd, deps=[self.FOO, self.CODE], outs_no_cache=["out"]) + stage = self.dvc.run( + cmd=cmd, deps=[self.FOO, self.CODE], outs_no_cache=["out"] + ) self.assertTrue(stage is not None) self.dvc.checkout(force=True) diff --git a/tests/test_data_cloud.py b/tests/test_data_cloud.py index 7a892d894..aba9a48b5 100644 --- a/tests/test_data_cloud.py +++ b/tests/test_data_cloud.py @@ -70,7 +70,13 @@ def _should_test_gcp(): shutil.copyfile(TestDvc.GCP_CREDS_FILE, creds) try: check_output( - ["gcloud", "auth", "activate-service-account", "--key-file", creds] + [ + "gcloud", + "auth", + "activate-service-account", + "--key-file", + creds, + ] ) except (CalledProcessError, OSError): return False @@ -111,12 +117,16 @@ def _should_test_hdfs(): return False try: - check_output(["hadoop", "version"], shell=True, executable=os.getenv("SHELL")) + check_output( + ["hadoop", "version"], shell=True, executable=os.getenv("SHELL") + ) except (CalledProcessError, IOError): return False p = Popen( - "hadoop fs -ls hdfs://127.0.0.1/", shell=True, executable=os.getenv("SHELL") + "hadoop fs -ls hdfs://127.0.0.1/", + shell=True, + executable=os.getenv("SHELL"), ) p.communicate() if p.returncode != 0: @@ -134,11 +144,15 @@ def get_local_url(): def get_ssh_url(): - return "ssh://{}@127.0.0.1:{}".format(getpass.getuser(), get_local_storagepath()) + return "ssh://{}@127.0.0.1:{}".format( + getpass.getuser(), get_local_storagepath() + ) def get_hdfs_url(): - return "hdfs://{}@127.0.0.1{}".format(getpass.getuser(), get_local_storagepath()) + return "hdfs://{}@127.0.0.1{}".format( + getpass.getuser(), get_local_storagepath() + ) def get_aws_storagepath(): @@ -161,7 +175,9 @@ def get_azure_url_compat(): container_name = os.getenv("AZURE_STORAGE_CONTAINER_NAME") connection_string = os.getenv("AZURE_STORAGE_CONNECTION_STRING") assert container_name is not None - return "azure://ContainerName={};{}".format(container_name, connection_string) + return "azure://ContainerName={};{}".format( + container_name, connection_string + ) def get_azure_url(): @@ -205,7 +221,9 @@ class TestDataCloudBase(TestDvc): def _ensure_should_run(self): if not self._should_test(): - raise SkipTest("Test {} is disabled".format(self.__class__.__name__)) + raise SkipTest( + "Test {} is disabled".format(self.__class__.__name__) + ) def _setup_cloud(self): self._ensure_should_run() @@ -433,7 +451,8 @@ class TestDataCloudCLIBase(TestDvc): # NOTE: check if remote gc works correctly on directories self.main(["gc", "-c", "-f"] + args) shutil.move( - self.dvc.cache.local.cache_dir, self.dvc.cache.local.cache_dir + ".back" + self.dvc.cache.local.cache_dir, + self.dvc.cache.local.cache_dir + ".back", ) self.main(["fetch"] + args) @@ -625,7 +644,9 @@ class TestRecursiveSyncOperations(TestDataCloudBase): return RemoteLOCAL def _prepare_repo(self): - self.main(["remote", "add", "-d", TEST_REMOTE, self.cloud._cloud.cache_dir]) + self.main( + ["remote", "add", "-d", TEST_REMOTE, self.cloud._cloud.cache_dir] + ) self.create(self.DATA, self.DATA_CONTENTS) self.create(self.DATA_SUB, self.DATA_SUB_CONTENTS) diff --git a/tests/test_gc.py b/tests/test_gc.py index 7594ba816..cb7565852 100644 --- a/tests/test_gc.py +++ b/tests/test_gc.py @@ -118,7 +118,9 @@ class TestGCMultipleProjects(TestDvc): self.additional_dvc = Project.init(self.additional_path) cache_path = os.path.join(self._root_dir, ".dvc", "cache") - config_path = os.path.join(self.additional_path, ".dvc", "config.local") + config_path = os.path.join( + self.additional_path, ".dvc", "config.local" + ) cfg = configobj.ConfigObj() cfg.filename = config_path cfg["cache"] = {"dir": cache_path} diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 9f3628615..c76704518 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -142,13 +142,19 @@ class TestMetricsRecursive(TestDvc): def test(self): - ret = self.dvc.metrics_show("nested", all_branches=True, recursive=False) + ret = self.dvc.metrics_show( + "nested", all_branches=True, recursive=False + ) self.assertEqual(len(ret), 0) - ret = self.dvc.metrics_show("nested", all_branches=True, recursive=True) + ret = self.dvc.metrics_show( + "nested", all_branches=True, recursive=True + ) self.assertEqual(len(ret), 1) self.assertEqual( - ret["nested"][os.path.join("nested", "subnested", "metric_subnested")], + ret["nested"][ + os.path.join("nested", "subnested", "metric_subnested") + ], "subnested", ) self.assertEqual( @@ -214,7 +220,14 @@ class TestMetricsCLICompat(TestMetrics): self.assertEqual(ret, 0) ret = main( - ["metrics", "show", "--all-branches", "metric_tsv", "--tsv-path", "0,0"] + [ + "metrics", + "show", + "--all-branches", + "metric_tsv", + "--tsv-path", + "0,0", + ] ) self.assertEqual(ret, 0) @@ -231,7 +244,14 @@ class TestMetricsCLICompat(TestMetrics): self.assertEqual(ret, 0) ret = main( - ["metrics", "show", "--all-branches", "metric_csv", "--csv-path", "0,0"] + [ + "metrics", + "show", + "--all-branches", + "metric_csv", + "--csv-path", + "0,0", + ] ) self.assertEqual(ret, 0) @@ -255,23 +275,54 @@ class TestMetricsCLI(TestMetrics): self.assertEqual(ret, 0) ret = main( - ["metrics", "show", "-a", "metric_json", "-t", "json", "-x", "branch"] + [ + "metrics", + "show", + "-a", + "metric_json", + "-t", + "json", + "-x", + "branch", + ] ) self.assertEqual(ret, 0) - ret = main(["metrics", "show", "-a", "metric_tsv", "-t", "tsv", "-x", "0,0"]) - self.assertEqual(ret, 0) - ret = main( - ["metrics", "show", "-a", "metric_htsv", "-t", "htsv", "-x", "branch,0"] + ["metrics", "show", "-a", "metric_tsv", "-t", "tsv", "-x", "0,0"] ) self.assertEqual(ret, 0) - ret = main(["metrics", "show", "-a", "metric_csv", "-t", "csv", "-x", "0,0"]) + ret = main( + [ + "metrics", + "show", + "-a", + "metric_htsv", + "-t", + "htsv", + "-x", + "branch,0", + ] + ) self.assertEqual(ret, 0) ret = main( - ["metrics", "show", "-a", "metric_hcsv", "-t", "hcsv", "-x", "branch,0"] + ["metrics", "show", "-a", "metric_csv", "-t", "csv", "-x", "0,0"] + ) + self.assertEqual(ret, 0) + + ret = main( + [ + "metrics", + "show", + "-a", + "metric_hcsv", + "-t", + "hcsv", + "-x", + "branch,0", + ] ) self.assertEqual(ret, 0) diff --git a/tests/test_move.py b/tests/test_move.py index fb8c75c2e..9705c063b 100644 --- a/tests/test_move.py +++ b/tests/test_move.py @@ -132,7 +132,9 @@ class TestMoveDirectoryShouldNotOverwriteExisting(TestDvc): self.assertEqual(ret, 0) self.assertFalse(os.path.exists(self.DATA_DIR)) - self.assertFalse(os.path.exists(self.DATA_DIR + Stage.STAGE_FILE_SUFFIX)) + self.assertFalse( + os.path.exists(self.DATA_DIR + Stage.STAGE_FILE_SUFFIX) + ) self.assertTrue(os.path.exists(new_dir_name)) self.assertTrue(os.path.isfile(new_dir_name + Stage.STAGE_FILE_SUFFIX)) @@ -163,4 +165,6 @@ class TestMoveFileBetweenDirectories(TestDvc): self.assertTrue(os.path.exists(new_data_stage_file)) new_stage_file = load_stage_file(new_data_stage_file) - self.assertEqual(os.path.basename(self.DATA), new_stage_file["outs"][0]["path"]) + self.assertEqual( + os.path.basename(self.DATA), new_stage_file["outs"][0]["path"] + ) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index a95182b95..be2d0ccec 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -44,13 +44,22 @@ class TestPipelineShowSingle(TestDvc): def test_dot_commands(self): ret = main( - ["pipeline", "show", "--dot", self.dotFile, self.stage, "--commands"] + [ + "pipeline", + "show", + "--dot", + self.dotFile, + self.stage, + "--commands", + ] ) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.dotFile)) def test_dot_outs(self): - ret = main(["pipeline", "show", "--dot", self.dotFile, self.stage, "--outs"]) + ret = main( + ["pipeline", "show", "--dot", self.dotFile, self.stage, "--outs"] + ) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.dotFile)) @@ -85,12 +94,16 @@ class TestPipelineShow(TestRepro): self.assertEqual(ret, 0) def test_dot(self): - ret = main(["pipeline", "show", "--dot", self.dotFile, self.file1_stage]) + ret = main( + ["pipeline", "show", "--dot", self.dotFile, self.file1_stage] + ) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.dotFile)) def test_ascii_commands(self): - ret = main(["pipeline", "show", "--ascii", self.file1_stage, "--commands"]) + ret = main( + ["pipeline", "show", "--ascii", self.file1_stage, "--commands"] + ) self.assertEqual(ret, 0) def test_ascii_outs(self): @@ -99,14 +112,28 @@ class TestPipelineShow(TestRepro): def test_dot_commands(self): ret = main( - ["pipeline", "show", "--dot", self.dotFile, self.file1_stage, "--commands"] + [ + "pipeline", + "show", + "--dot", + self.dotFile, + self.file1_stage, + "--commands", + ] ) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.dotFile)) def test_dot_outs(self): ret = main( - ["pipeline", "show", "--dot", self.dotFile, self.file1_stage, "--outs"] + [ + "pipeline", + "show", + "--dot", + self.dotFile, + self.file1_stage, + "--outs", + ] ) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.dotFile)) @@ -142,12 +169,16 @@ class TestPipelineShowDeep(TestReproChangedDeepData): self.assertEqual(ret, 0) def test_dot(self): - ret = main(["pipeline", "show", "--dot", self.dotFile, self.file1_stage]) + ret = main( + ["pipeline", "show", "--dot", self.dotFile, self.file1_stage] + ) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.dotFile)) def test_ascii_commands(self): - ret = main(["pipeline", "show", "--ascii", self.file1_stage, "--commands"]) + ret = main( + ["pipeline", "show", "--ascii", self.file1_stage, "--commands"] + ) self.assertEqual(ret, 0) def test_ascii_outs(self): @@ -156,14 +187,28 @@ class TestPipelineShowDeep(TestReproChangedDeepData): def test_dot_commands(self): ret = main( - ["pipeline", "show", "--dot", self.dotFile, self.file1_stage, "--commands"] + [ + "pipeline", + "show", + "--dot", + self.dotFile, + self.file1_stage, + "--commands", + ] ) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.dotFile)) def test_dot_outs(self): ret = main( - ["pipeline", "show", "--dot", self.dotFile, self.file1_stage, "--outs"] + [ + "pipeline", + "show", + "--dot", + self.dotFile, + self.file1_stage, + "--outs", + ] ) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.dotFile)) diff --git a/tests/test_remote.py b/tests/test_remote.py index dea01f72f..cbc2f40ae 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -23,7 +23,9 @@ class TestRemote(TestDvc): self.assertEqual(main(["remote", "list"]), 0) self.assertEqual(main(["remote", "remove", remotes[0]]), 0) - self.assertEqual(main(["remote", "modify", remotes[0], "option", "value"]), 0) + self.assertEqual( + main(["remote", "modify", remotes[0], "option", "value"]), 0 + ) self.assertEqual(main(["remote", "list"]), 0) @@ -61,10 +63,14 @@ class TestRemoteRemoveDefault(TestDvc): self.assertEqual( main(["remote", "add", "--default", remote, "s3://bucket/name"]), 0 ) - self.assertEqual(main(["remote", "modify", remote, "profile", "default"]), 0) + self.assertEqual( + main(["remote", "modify", remote, "profile", "default"]), 0 + ) self.assertEqual(main(["config", "--local", "core.remote", remote]), 0) - config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, Config.CONFIG)) + config = configobj.ConfigObj( + os.path.join(self.dvc.dvc_dir, Config.CONFIG) + ) local_config = configobj.ConfigObj( os.path.join(self.dvc.dvc_dir, Config.CONFIG_LOCAL) ) @@ -72,11 +78,14 @@ class TestRemoteRemoveDefault(TestDvc): config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE], remote ) self.assertEqual( - local_config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE], remote + local_config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE], + remote, ) self.assertEqual(main(["remote", "remove", remote]), 0) - config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, Config.CONFIG)) + config = configobj.ConfigObj( + os.path.join(self.dvc.dvc_dir, Config.CONFIG) + ) local_config = configobj.ConfigObj( os.path.join(self.dvc.dvc_dir, Config.CONFIG_LOCAL) ) diff --git a/tests/test_repro.py b/tests/test_repro.py index c512d91de..9eee2f4b0 100644 --- a/tests/test_repro.py +++ b/tests/test_repro.py @@ -58,9 +58,13 @@ class TestReproFail(TestRepro): class TestReproCyclicGraph(TestDvc): def test(self): - self.dvc.run(deps=[self.FOO], outs=["bar.txt"], cmd="echo bar > bar.txt") + self.dvc.run( + deps=[self.FOO], outs=["bar.txt"], cmd="echo bar > bar.txt" + ) - self.dvc.run(deps=["bar.txt"], outs=["baz.txt"], cmd="echo baz > baz.txt") + self.dvc.run( + deps=["bar.txt"], outs=["baz.txt"], cmd="echo baz > baz.txt" + ) with open("cycle.dvc", "w") as fd: stage_dump = { @@ -257,7 +261,9 @@ class TestReproNoDeps(TestRepro): ) with open(code_file, "w+") as fd: fd.write(code) - self.dvc.run(fname=stage_file, outs=[out], cmd="python {}".format(code_file)) + self.dvc.run( + fname=stage_file, outs=[out], cmd="python {}".format(code_file) + ) stages = self.dvc.reproduce(stage_file) self.assertEqual(len(stages), 1) @@ -281,7 +287,9 @@ class TestReproChangedCode(TestRepro): def swap_code(self): os.unlink(self.CODE) new_contents = self.CODE_CONTENTS - new_contents += "\nshutil.copyfile('{}', " "sys.argv[2])\n".format(self.BAR) + new_contents += "\nshutil.copyfile('{}', " "sys.argv[2])\n".format( + self.BAR + ) self.create(self.CODE, new_contents) @@ -422,7 +430,9 @@ class TestReproIgnoreBuildCache(TestDvc): class TestReproPipeline(TestReproChangedDeepData): def test(self): - stages = self.dvc.reproduce(self.file1_stage, force=True, pipeline=True) + stages = self.dvc.reproduce( + self.file1_stage, force=True, pipeline=True + ) self.assertEqual(len(stages), 3) def test_cli(self): @@ -793,7 +803,9 @@ class TestReproExternalBase(TestDvc): def corrupted_cache(self): os.unlink("bar.dvc") - stage = self.dvc.run(deps=[self.FOO], outs=[self.BAR], cmd="echo bar > bar") + stage = self.dvc.run( + deps=[self.FOO], outs=[self.BAR], cmd="echo bar > bar" + ) with open(self.BAR, "w") as fd: fd.write("corrupting the cache") @@ -817,7 +829,11 @@ class TestReproExternalBase(TestDvc): return cache = ( - self.scheme + self.scheme_sep + self.bucket + self.sep + str(uuid.uuid4()) + self.scheme + + self.scheme_sep + + self.bucket + + self.sep + + str(uuid.uuid4()) ) ret = main(["config", "cache." + self.cache_scheme, "myrepo"]) @@ -827,7 +843,9 @@ class TestReproExternalBase(TestDvc): remote_name = "myremote" remote_key = str(uuid.uuid4()) - remote = self.scheme + self.scheme_sep + self.bucket + self.sep + remote_key + remote = ( + self.scheme + self.scheme_sep + self.bucket + self.sep + remote_key + ) ret = main(["remote", "add", remote_name, remote]) self.assertEqual(ret, 0) @@ -837,8 +855,12 @@ class TestReproExternalBase(TestDvc): foo_key = remote_key + self.sep + self.FOO bar_key = remote_key + self.sep + self.BAR - foo_path = self.scheme + self.scheme_sep + self.bucket + self.sep + foo_key - bar_path = self.scheme + self.scheme_sep + self.bucket + self.sep + bar_key + foo_path = ( + self.scheme + self.scheme_sep + self.bucket + self.sep + foo_key + ) + bar_path = ( + self.scheme + self.scheme_sep + self.bucket + self.sep + bar_key + ) # Using both plain and remote notation out_foo_path = "remote://" + remote_name + "/" + self.FOO @@ -853,11 +875,15 @@ class TestReproExternalBase(TestDvc): self.assertEqual(self.dvc.status(import_stage.path), {}) self.check_already_cached(import_stage) - import_remote_stage = self.dvc.imp(out_foo_path, out_foo_path + "_imported") + import_remote_stage = self.dvc.imp( + out_foo_path, out_foo_path + "_imported" + ) self.assertEqual(self.dvc.status(import_remote_stage.path), {}) cmd_stage = self.dvc.run( - outs=[out_bar_path], deps=[out_foo_path], cmd=self.cmd(foo_path, bar_path) + outs=[out_bar_path], + deps=[out_foo_path], + cmd=self.cmd(foo_path, bar_path), ) self.assertEqual(self.dvc.status(cmd_stage.path), {}) @@ -1161,7 +1187,9 @@ class TestReproExternalHTTP(TestReproExternalBase): fd.write(cmd) run_stage = self.dvc.run( - deps=[run_dependency], outs=[run_output], cmd="python create-output.py" + deps=[run_dependency], + outs=[run_output], + cmd="python create-output.py", ) self.assertTrue(run_stage is not None) @@ -1188,7 +1216,9 @@ class TestReproShell(TestDvc): fname = "shell.txt" stage = fname + ".dvc" - self.dvc.run(fname=stage, outs=[fname], cmd="echo $SHELL > {}".format(fname)) + self.dvc.run( + fname=stage, outs=[fname], cmd="echo $SHELL > {}".format(fname) + ) with open(fname, "r") as fd: self.assertEqual(os.getenv("SHELL"), fd.read().strip()) diff --git a/tests/test_run.py b/tests/test_run.py index 94d407c08..d68ce88de 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -63,7 +63,12 @@ class TestRun(TestDvc): class TestRunEmpty(TestDvc): def test(self): self.dvc.run( - cmd="", deps=[], outs=[], outs_no_cache=[], fname="empty.dvc", cwd=os.curdir + cmd="", + deps=[], + outs=[], + outs_no_cache=[], + fname="empty.dvc", + cwd=os.curdir, ) @@ -84,7 +89,12 @@ class TestRunBadStageFilename(TestDvc): def test(self): with self.assertRaises(StageFileBadNameError): self.dvc.run( - cmd="", deps=[], outs=[], outs_no_cache=[], fname="empty", cwd=os.curdir + cmd="", + deps=[], + outs=[], + outs_no_cache=[], + fname="empty", + cwd=os.curdir, ) with self.assertRaises(StageFileBadNameError): @@ -101,7 +111,8 @@ class TestRunBadStageFilename(TestDvc): class TestRunNoExec(TestDvc): def test(self): self.dvc.run( - cmd="python {} {} {}".format(self.CODE, self.FOO, "out"), no_exec=True + cmd="python {} {} {}".format(self.CODE, self.FOO, "out"), + no_exec=True, ) self.assertFalse(os.path.exists("out")) @@ -128,16 +139,25 @@ class TestRunCircularDependency(TestDvc): def test_non_normalized_paths(self): with self.assertRaises(CircularDependencyError): self.dvc.run( - cmd="", deps=["./foo"], outs=["foo"], fname="circular-dependency.dvc" + cmd="", + deps=["./foo"], + outs=["foo"], + fname="circular-dependency.dvc", ) def test_graph(self): - self.dvc.run(deps=[self.FOO], outs=["bar.txt"], cmd="echo bar > bar.txt") + self.dvc.run( + deps=[self.FOO], outs=["bar.txt"], cmd="echo bar > bar.txt" + ) - self.dvc.run(deps=["bar.txt"], outs=["baz.txt"], cmd="echo baz > baz.txt") + self.dvc.run( + deps=["bar.txt"], outs=["baz.txt"], cmd="echo baz > baz.txt" + ) with self.assertRaises(CyclicGraphError): - self.dvc.run(deps=["baz.txt"], outs=[self.FOO], cmd="echo baz > foo") + self.dvc.run( + deps=["baz.txt"], outs=[self.FOO], cmd="echo baz > foo" + ) class TestRunDuplicatedArguments(TestDvc): @@ -162,7 +182,10 @@ class TestRunDuplicatedArguments(TestDvc): def test_non_normalized_paths(self): with self.assertRaises(ArgumentDuplicationError): self.dvc.run( - cmd="", deps=[], outs=["foo", "./foo"], fname="circular-dependency.dvc" + cmd="", + deps=[], + outs=["foo", "./foo"], + fname="circular-dependency.dvc", ) @@ -172,7 +195,10 @@ class TestRunWorkingDirectoryAsOutput(TestDvc): with self.assertRaises(WorkingDirectoryAsOutputError): self.dvc.run( - cmd="", cwd=self.DATA_DIR, outs=[self.FOO], fname="inside-cwd.dvc" + cmd="", + cwd=self.DATA_DIR, + outs=[self.FOO], + fname="inside-cwd.dvc", ) @@ -271,7 +297,16 @@ class TestRunUnprotectOutsCopy(TestDvc): self.assertEqual(ret, 0) ret = main( - ["run", "-d", self.CODE, "-o", self.FOO, "python", self.CODE, self.FOO] + [ + "run", + "-d", + self.CODE, + "-o", + self.FOO, + "python", + self.CODE, + self.FOO, + ] ) self.assertEqual(ret, 0) self.assertFalse(os.access(self.FOO, os.W_OK)) @@ -313,7 +348,16 @@ class TestRunUnprotectOutsSymlink(TestDvc): self.assertEqual(ret, 0) ret = main( - ["run", "-d", self.CODE, "-o", self.FOO, "python", self.CODE, self.FOO] + [ + "run", + "-d", + self.CODE, + "-o", + self.FOO, + "python", + self.CODE, + self.FOO, + ] ) self.assertEqual(ret, 0) self.assertFalse(os.access(self.FOO, os.W_OK)) @@ -357,7 +401,16 @@ class TestRunUnprotectOutsHardlink(TestDvc): self.assertEqual(ret, 0) ret = main( - ["run", "-d", self.CODE, "-o", self.FOO, "python", self.CODE, self.FOO] + [ + "run", + "-d", + self.CODE, + "-o", + self.FOO, + "python", + self.CODE, + self.FOO, + ] ) self.assertEqual(ret, 0) self.assertFalse(os.access(self.FOO, os.W_OK)) @@ -468,7 +521,9 @@ class TestCmdRunOverwrite(TestDvc): time.sleep(1) - ret = main(["run", "--overwrite-dvcfile", "-f", "out.dvc", "-d", self.BAR]) + ret = main( + ["run", "--overwrite-dvcfile", "-f", "out.dvc", "-d", self.BAR] + ) self.assertEqual(ret, 0) # NOTE: check that dvcfile was overwritten diff --git a/tests/test_scm.py b/tests/test_scm.py index 1b92991b0..7c66cd596 100644 --- a/tests/test_scm.py +++ b/tests/test_scm.py @@ -45,7 +45,9 @@ class TestIgnore(TestGit): def _count_gitignore(self): with open(Git.GITIGNORE, "r") as fd: lines = fd.readlines() - return len(list(filter(lambda x: x.strip() == "/" + self.FOO, lines))) + return len( + list(filter(lambda x: x.strip() == "/" + self.FOO, lines)) + ) def test(self): git = Git(self._root_dir) diff --git a/tests/test_unprotect.py b/tests/test_unprotect.py index fdaa0ebd6..27780bddc 100644 --- a/tests/test_unprotect.py +++ b/tests/test_unprotect.py @@ -15,10 +15,14 @@ class TestUnprotect(TestDvc): self.assertEqual(ret, 0) self.assertNotEqual( - os.stat(self.FOO).st_mode & (stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH), 0 + os.stat(self.FOO).st_mode + & (stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH), + 0, ) self.assertEqual( - os.stat(self.FOO).st_mode & (stat.S_IWRITE | stat.S_IWGRP | stat.S_IWOTH), 0 + os.stat(self.FOO).st_mode + & (stat.S_IWRITE | stat.S_IWGRP | stat.S_IWOTH), + 0, ) ret = main(["unprotect", self.FOO]) diff --git a/tests/test_utils.py b/tests/test_utils.py index a613e44b7..637779c4f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -21,7 +21,9 @@ class TestUtils(TestDvc): self.assertTrue(filecmp.cmp(src, dest, shallow=False)) utils.copyfile(src, dest_dir) - self.assertTrue(filecmp.cmp(src, "{}/{}".format(dest_dir, src), shallow=False)) + self.assertTrue( + filecmp.cmp(src, "{}/{}".format(dest_dir, src), shallow=False) + ) shutil.rmtree(dest_dir) os.remove(src) diff --git a/tests/unit/remote/azure.py b/tests/unit/remote/azure.py index e089b8f86..79a6d48a0 100644 --- a/tests/unit/remote/azure.py +++ b/tests/unit/remote/azure.py @@ -14,7 +14,8 @@ class TestRemoteAzure(TestCase): def test_init_compat(self): url = "azure://ContainerName={container_name};{connection_string}".format( - container_name=self.container_name, connection_string=self.connection_string + container_name=self.container_name, + connection_string=self.connection_string, ) config = {"url": url} remote = RemoteAzure(None, config) diff --git a/tests/unit/stage.py b/tests/unit/stage.py index 92862fcd1..6c95a2aa4 100644 --- a/tests/unit/stage.py +++ b/tests/unit/stage.py @@ -12,4 +12,6 @@ class TestStageChecksum(TestCase): d = {"md5": "123456", "cmd": "mycmd", "outs": outs, "deps": deps} with mock.patch.object(stage, "dumpd", return_value=d): - self.assertEqual(stage._compute_md5(), "e9521a22111493406ea64a88cda63e0b") + self.assertEqual( + stage._compute_md5(), "e9521a22111493406ea64a88cda63e0b" + )