dvc: use black instead of flake8

Fixes #1546

Signed-off-by: Ruslan Kuprieiev <[email protected]>

.travis.yml

@@ -1,13 +1,13 @@
 language: python
 cache: pip
-sudo: required
-dist: trusty
-group: deprecated-2017Q4
-python:
-  - "2.7"
-  - "3.4"
-  - "3.5"
-  - "3.6"
+sudo: true
+dist: xenial
+python: "3.7"
+env:
+  - |
+    PYENV_ROOT="$HOME/.pyenv"
+    PATH="$PYENV_ROOT/bin:$PATH"
+    PYTHON_CONFIGURE_OPTS="--enable-framework"
 # Travis doesn't support testing python on Mac yet, so we need
 # to workaround it by installing it directly with brew.
 #
@@ -16,10 +16,22 @@ python:
 # version, that causes fpm to crash.
 matrix:
   include:
-  # 3.7 needs Xenial image because of https://github.com/travis-ci/travis-ci/issues/9069
-  - python: "3.7"
-    dist: xenial
-    sudo: true
+  - name: "2.7"
+    env:
+    - PYTHON_VER=2.7.13
+  - name: "3.4"
+    env:
+    - PYTHON_VER=3.4.6
+  - name: "3.5"
+    env:
+    - PYTHON_VER=3.5.3
+  - name: "3.6"
+    env:
+    - PYTHON_VER=3.6.2
+  - name: "3.7"
+    env:
+    # NOTE: using default python
+    - PYTHON_VER=
   - if: type = cron OR tag is present
     os: osx
     osx_image: xcode7.3
@@ -27,8 +39,6 @@ matrix:
     rvm: 2.4.0
     env:
     - PYTHON_VER=2.7.10
-    - PYENV_ROOT="$HOME/.pyenv"
-    - PATH="$PYENV_ROOT/bin:$PATH"
     - CPPFLAGS="-I/usr/local/opt/openssl/include $CPPFLAGS"
     - LDFLAGS="-L/usr/local/opt/openssl/lib $LDFLAGS"
   - if: type = cron OR tag is present
@@ -38,8 +48,6 @@ matrix:
     rvm: 2.4.0
     env:
     - PYTHON_VER=3.4.3
-    - PYENV_ROOT="$HOME/.pyenv"
-    - PATH="$PYENV_ROOT/bin:$PATH"
   - if: type = cron OR tag is present
     os: osx
     osx_image: xcode7.3
@@ -47,19 +55,23 @@ matrix:
     rvm: 2.4.0
     env:
     - PYTHON_VER=3.5.0
-    - PYENV_ROOT="$HOME/.pyenv"
-    - PATH="$PYENV_ROOT/bin:$PATH"
 before_install:
+  - pip install -r scripts/ci/requirements.txt
+  # stop the build if there are any readme formatting errors
+  - python setup.py checkdocs
+  # stop the build if there are any black errors
+  - black dvc --check
+  - black tests --check
   - bash ./scripts/ci/before_install.sh
-  - if  [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+  - if [[ -n "$PYTHON_VER" ]]; then
       eval "$(pyenv init -)";
-      PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing --keep --verbose "$PYTHON_VER";
+      pyenv install --skip-existing --keep --verbose "$PYTHON_VER";
       pyenv shell "$PYTHON_VER";
+    fi
+  - if  [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
       brew install openssl;
       brew cask install google-cloud-sdk;
     fi
-  - which python
-  - which pip
   - python --version
   - if [[ "$TRAVIS_OS_NAME" != "osx" ]]; then
       bash ./scripts/ci/install_hadoop.sh;
@@ -79,13 +91,6 @@ before_install:
     fi
 install:
   - bash ./scripts/ci/install.sh
-  - pip install flake8
-before_script:
-  # stop the build if there are any readme formatting errors
-  - python setup.py checkdocs
-  # stop the build if there are any flake8 errors
-  - flake8 dvc
-  - flake8 tests
 script:
   - bash ./scripts/ci/script.sh
 deploy:

dvc/__init__.py

@@ -10,12 +10,12 @@ import os
 import warnings
 
 
-VERSION_BASE = '0.24.3'
+VERSION_BASE = "0.24.3"
 __version__ = VERSION_BASE
 
 PACKAGEPATH = os.path.abspath(os.path.dirname(__file__))
 HOMEPATH = os.path.dirname(PACKAGEPATH)
-VERSIONPATH = os.path.join(PACKAGEPATH, 'version.py')
+VERSIONPATH = os.path.join(PACKAGEPATH, "version.py")
 
 
 def _update_version_file():
@@ -30,13 +30,13 @@ def _update_version_file():
 
     sha = repo.head.object.hexsha
     short_sha = repo.git.rev_parse(sha, short=6)
-    dirty = '.mod' if repo.is_dirty() else ''
-    ver = '{}+{}{}'.format(__version__, short_sha, dirty)
+    dirty = ".mod" if repo.is_dirty() else ""
+    ver = "{}+{}{}".format(__version__, short_sha, dirty)
 
     # Write a helper file, that will be installed with the package
     # and will provide a true version of the installed dvc
-    with open(VERSIONPATH, 'w+') as fobj:
-        fobj.write('# AUTOGENERATED by dvc/__init__.py\n')
+    with open(VERSIONPATH, "w+") as fobj:
+        fobj.write("# AUTOGENERATED by dvc/__init__.py\n")
         fobj.write('version = "{}"\n'.format(ver))
 
     return ver
@@ -48,11 +48,13 @@ def _remove_version_file():
         os.unlink(VERSIONPATH)
 
 
-if os.path.exists(os.path.join(HOMEPATH, 'setup.py')):
+if os.path.exists(os.path.join(HOMEPATH, "setup.py")):
     # dvc is run directly from source without installation or
     # __version__ is called from setup.py
-    if os.getenv('APPVEYOR_REPO_TAG', '').lower() != 'true' \
-       and os.getenv('TRAVIS_TAG', '') == '':
+    if (
+        os.getenv("APPVEYOR_REPO_TAG", "").lower() != "true"
+        and os.getenv("TRAVIS_TAG", "") == ""
+    ):
         __version__ = _update_version_file()
     else:  # pragma: no cover
         _remove_version_file()
@@ -61,6 +63,7 @@ else:  # pragma: no cover
     # auto-generated version.py to help us provide a true version
     try:
         from dvc.version import version
+
         __version__ = version
     except Exception:
         pass

dvc/__main__.py

@@ -6,5 +6,5 @@ import sys
 
 from dvc.main import main
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main(sys.argv[1:])

dvc/analytics.py

@@ -18,32 +18,33 @@ class Analytics(object):
     Args:
         info (dict): optional existing analytics report.
     """
-    URL = 'https://analytics.dvc.org'
+
+    URL = "https://analytics.dvc.org"
     TIMEOUT_POST = 5
 
-    USER_ID_FILE = 'user_id'
+    USER_ID_FILE = "user_id"
 
-    PARAM_DVC_VERSION = 'dvc_version'
-    PARAM_USER_ID = 'user_id'
-    PARAM_SYSTEM_INFO = 'system_info'
+    PARAM_DVC_VERSION = "dvc_version"
+    PARAM_USER_ID = "user_id"
+    PARAM_SYSTEM_INFO = "system_info"
 
-    PARAM_OS = 'os'
+    PARAM_OS = "os"
 
-    PARAM_WINDOWS_VERSION_MAJOR = 'windows_version_major'
-    PARAM_WINDOWS_VERSION_MINOR = 'windows_version_minor'
-    PARAM_WINDOWS_VERSION_BUILD = 'windows_version_build'
-    PARAM_WINDOWS_VERSION_SERVICE_PACK = 'windows_version_service_pack'
+    PARAM_WINDOWS_VERSION_MAJOR = "windows_version_major"
+    PARAM_WINDOWS_VERSION_MINOR = "windows_version_minor"
+    PARAM_WINDOWS_VERSION_BUILD = "windows_version_build"
+    PARAM_WINDOWS_VERSION_SERVICE_PACK = "windows_version_service_pack"
 
-    PARAM_MAC_VERSION = 'mac_version'
+    PARAM_MAC_VERSION = "mac_version"
 
-    PARAM_LINUX_DISTRO = 'linux_distro'
-    PARAM_LINUX_DISTRO_VERSION = 'linux_distro_version'
-    PARAM_LINUX_DISTRO_LIKE = 'linux_distro_like'
+    PARAM_LINUX_DISTRO = "linux_distro"
+    PARAM_LINUX_DISTRO_VERSION = "linux_distro_version"
+    PARAM_LINUX_DISTRO_LIKE = "linux_distro_like"
 
-    PARAM_SCM_CLASS = 'scm_class'
-    PARAM_IS_BINARY = 'is_binary'
-    PARAM_CMD_CLASS = 'cmd_class'
-    PARAM_CMD_RETURN_CODE = 'cmd_return_code'
+    PARAM_SCM_CLASS = "scm_class"
+    PARAM_IS_BINARY = "is_binary"
+    PARAM_CMD_CLASS = "cmd_class"
+    PARAM_CMD_RETURN_CODE = "cmd_return_code"
 
     def __init__(self, info=None):
         from dvc.config import Config
@@ -62,7 +63,7 @@ class Analytics(object):
                 raise
 
         self.user_id_file = os.path.join(cdir, self.USER_ID_FILE)
-        self.user_id_file_lock = Lock(cdir, self.USER_ID_FILE + '.lock')
+        self.user_id_file_lock = Lock(cdir, self.USER_ID_FILE + ".lock")
 
     @staticmethod
     def load(path):
@@ -71,7 +72,7 @@ class Analytics(object):
         Args:
             path (str): path to json file with analytics report.
         """
-        with open(path, 'r') as fobj:
+        with open(path, "r") as fobj:
             analytics = Analytics(info=json.load(fobj))
         os.unlink(path)
         return analytics
@@ -79,7 +80,7 @@ class Analytics(object):
     def _write_user_id(self):
         import uuid
 
-        with open(self.user_id_file, 'w+') as fobj:
+        with open(self.user_id_file, "w+") as fobj:
             user_id = str(uuid.uuid4())
             info = {self.PARAM_USER_ID: user_id}
             json.dump(info, fobj)
@@ -89,7 +90,7 @@ class Analytics(object):
         if not os.path.exists(self.user_id_file):
             return None
 
-        with open(self.user_id_file, 'r') as fobj:
+        with open(self.user_id_file, "r") as fobj:
             try:
                 info = json.load(fobj)
                 return info[self.PARAM_USER_ID]
@@ -112,9 +113,10 @@ class Analytics(object):
 
     def _collect_windows(self):
         import sys
+
         version = sys.getwindowsversion()  # pylint: disable=no-member
         info = {}
-        info[self.PARAM_OS] = 'windows'
+        info[self.PARAM_OS] = "windows"
         info[self.PARAM_WINDOWS_VERSION_MAJOR] = version.major
         info[self.PARAM_WINDOWS_VERSION_MINOR] = version.minor
         info[self.PARAM_WINDOWS_VERSION_BUILD] = version.build
@@ -123,15 +125,17 @@ class Analytics(object):
 
     def _collect_darwin(self):
         import platform
+
         info = {}
-        info[self.PARAM_OS] = 'mac'
+        info[self.PARAM_OS] = "mac"
         info[self.PARAM_MAC_VERSION] = platform.mac_ver()[0]
         return info
 
     def _collect_linux(self):
         import distro
+
         info = {}
-        info[self.PARAM_OS] = 'linux'
+        info[self.PARAM_OS] = "linux"
         info[self.PARAM_LINUX_DISTRO] = distro.id()
         info[self.PARAM_LINUX_DISTRO_VERSION] = distro.version()
         info[self.PARAM_LINUX_DISTRO_LIKE] = distro.like()
@@ -139,15 +143,16 @@ class Analytics(object):
 
     def _collect_system_info(self):
         import platform
+
         system = platform.system()
 
-        if system == 'Windows':
+        if system == "Windows":
             return self._collect_windows()
 
-        if system == 'Darwin':
+        if system == "Darwin":
             return self._collect_darwin()
 
-        if system == 'Linux':
+        if system == "Linux":
             return self._collect_linux()
 
         raise NotImplementedError
@@ -180,7 +185,7 @@ class Analytics(object):
         if ret is not None:
             self.info[self.PARAM_CMD_RETURN_CODE] = ret
 
-        if args is not None and hasattr(args, 'func'):
+        if args is not None and hasattr(args, "func"):
             assert args.func != CmdDaemonAnalytics
             self.info[self.PARAM_CMD_CLASS] = args.func.__name__
 
@@ -192,7 +197,7 @@ class Analytics(object):
         """
         import tempfile
 
-        with tempfile.NamedTemporaryFile(delete=False, mode='w') as fobj:
+        with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj:
             json.dump(self.info, fobj)
             return fobj.name
 
@@ -203,13 +208,13 @@ class Analytics(object):
         from dvc.exceptions import NotDvcProjectError
         from dvc.command.daemon import CmdDaemonBase
 
-        if os.getenv('DVC_TEST'):
+        if os.getenv("DVC_TEST"):
             return False
 
         if isinstance(cmd, CmdDaemonBase):
             return False
 
-        if cmd is None or not hasattr(cmd, 'config'):
+        if cmd is None or not hasattr(cmd, "config"):
             try:
                 dvc_dir = Project.find_dvc_dir()
                 config = Config(dvc_dir)
@@ -223,8 +228,7 @@ class Analytics(object):
 
         core = config.config.get(Config.SECTION_CORE, {})
         enabled = core.get(Config.SECTION_CORE_ANALYTICS, True)
-        logger.debug("Analytics is {}abled."
-                     .format('en' if enabled else 'dis'))
+        logger.debug("Analytics is {}abled.".format("en" if enabled else "dis"))
         return enabled
 
     @staticmethod
@@ -242,7 +246,7 @@ class Analytics(object):
 
         analytics = Analytics()
         analytics.collect_cmd(args, ret)
-        daemon(['analytics', analytics.dump()])
+        daemon(["analytics", analytics.dump()])
 
     def send(self):
         """Collect and send analytics."""

dvc/cache.py

@@ -14,7 +14,8 @@ class Cache(object):
         project (dvc.project.Project): project instance that this cache
             belongs to.
     """
-    CACHE_DIR = 'cache'
+
+    CACHE_DIR = "cache"
 
     def __init__(self, project):
         from dvc.remote import Remote
@@ -34,11 +35,10 @@ class Cache(object):
             protected = config.get(Config.SECTION_CACHE_PROTECTED)
 
             sect = {
-                Config.PRIVATE_CWD: config.get(Config.PRIVATE_CWD,
-                                               project.dvc_dir),
+                Config.PRIVATE_CWD: config.get(Config.PRIVATE_CWD, project.dvc_dir),
                 Config.SECTION_REMOTE_URL: cache_dir,
                 Config.SECTION_CACHE_TYPE: cache_type,
-                Config.SECTION_CACHE_PROTECTED: protected
+                Config.SECTION_CACHE_PROTECTED: protected,
             }
 
         self._local = Remote(project, sect)

dvc/cli.py

@@ -61,6 +61,7 @@ COMMANDS = [
 
 class DvcParser(argparse.ArgumentParser):
     """Custom parser class for dvc CLI."""
+
     def error(self, message):
         """Custom error method.
         Args:
@@ -77,6 +78,7 @@ class DvcParser(argparse.ArgumentParser):
 class VersionAction(argparse.Action):  # pragma: no cover
     # pylint: disable=too-few-public-methods
     """Shows dvc version and exits."""
+
     def __call__(self, parser, namespace, values, option_string=None):
         print(VERSION)
         sys.exit(0)
@@ -96,41 +98,34 @@ def parse_args(argv=None):
 
     log_level_group = parent_parser.add_mutually_exclusive_group()
     log_level_group.add_argument(
-        '-q',
-        '--quiet',
-        action='store_true',
-        default=False,
-        help='Be quiet.')
+        "-q", "--quiet", action="store_true", default=False, help="Be quiet."
+    )
     log_level_group.add_argument(
-        '-v',
-        '--verbose',
-        action='store_true',
-        default=False,
-        help='Be verbose.')
+        "-v", "--verbose", action="store_true", default=False, help="Be verbose."
+    )
 
     # Main parser
-    desc = 'Data Version Control'
+    desc = "Data Version Control"
     parser = DvcParser(
-        prog='dvc',
+        prog="dvc",
         description=desc,
         parents=[parent_parser],
-        formatter_class=argparse.RawTextHelpFormatter)
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
 
     # NOTE: On some python versions action='version' prints to stderr
     # instead of stdout https://bugs.python.org/issue18920
     parser.add_argument(
-        '-V',
-        '--version',
-        action=VersionAction,
-        nargs=0,
-        help="Show program's version.")
+        "-V", "--version", action=VersionAction, nargs=0, help="Show program's version."
+    )
 
     # Sub commands
     subparsers = parser.add_subparsers(
-        title='Available Commands',
-        metavar='COMMAND',
-        dest='cmd',
-        help='Use dvc COMMAND --help for command-specific help.')
+        title="Available Commands",
+        metavar="COMMAND",
+        dest="cmd",
+        help="Use dvc COMMAND --help for command-specific help.",
+    )
 
     fix_subparsers(subparsers)
 

dvc/command/add.py

@@ -11,26 +11,22 @@ class CmdAdd(CmdBase):
             try:
                 self.project.add(target, recursive=self.args.recursive)
             except DvcException:
-                logger.error('failed to add file')
+                logger.error("failed to add file")
                 return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    ADD_HELP = 'Add files/directories to dvc.'
+    ADD_HELP = "Add files/directories to dvc."
     add_parser = subparsers.add_parser(
-        'add',
-        parents=[parent_parser],
-        description=ADD_HELP,
-        help=ADD_HELP)
+        "add", parents=[parent_parser], description=ADD_HELP, help=ADD_HELP
+    )
     add_parser.add_argument(
-        '-R',
-        '--recursive',
-        action='store_true',
+        "-R",
+        "--recursive",
+        action="store_true",
         default=False,
-        help='Recursively add each file under the directory.')
-    add_parser.add_argument(
-        'targets',
-        nargs='+',
-        help='Input files/directories.')
+        help="Recursively add each file under the directory.",
+    )
+    add_parser.add_argument("targets", nargs="+", help="Input files/directories.")
     add_parser.set_defaults(func=CmdAdd)

dvc/command/base.py

@@ -15,7 +15,7 @@ def fix_subparsers(subparsers):
 
     if is_py3:  # pragma: no cover
         subparsers.required = True
-        subparsers.dest = 'cmd'
+        subparsers.dest = "cmd"
 
 
 class CmdBase(object):
@@ -31,6 +31,7 @@ class CmdBase(object):
     def default_targets(self):
         """Default targets for `dvc repro` and `dvc pipeline`."""
         from dvc.stage import Stage
+
         msg = "assuming default target '{}'.".format(Stage.STAGE_FILE)
         logger.warning(msg)
         return [Stage.STAGE_FILE]
@@ -51,7 +52,7 @@ class CmdBase(object):
             with self.project.lock:
                 return self.run()
         except LockError:
-            logger.error('failed to lock before running a command')
+            logger.error("failed to lock before running a command")
             return 1
 
     # Abstract methods that have to be implemented by any inheritance class

dvc/command/cache.py

@@ -9,9 +9,10 @@ from dvc.command.config import CmdConfig
 
 class CmdCacheDir(CmdConfig):
     def run(self):
-        self.args.name = 'cache.dir'
-        self.args.value = CmdRemoteAdd.resolve_path(self.args.value,
-                                                    self.configobj.filename)
+        self.args.name = "cache.dir"
+        self.args.value = CmdRemoteAdd.resolve_path(
+            self.args.value, self.configobj.filename
+        )
 
         return super(CmdCacheDir, self).run()
 
@@ -19,39 +20,36 @@ class CmdCacheDir(CmdConfig):
 def add_parser(subparsers, parent_parser):
     from dvc.command.config import parent_config_parser
 
-    CACHE_HELP = 'Manage cache settings.'
+    CACHE_HELP = "Manage cache settings."
     cache_parser = subparsers.add_parser(
-        'cache',
-        parents=[parent_parser],
-        description=CACHE_HELP,
-        help=CACHE_HELP)
+        "cache", parents=[parent_parser], description=CACHE_HELP, help=CACHE_HELP
+    )
 
     cache_subparsers = cache_parser.add_subparsers(
-        dest='cmd',
-        help='Use dvc cache CMD --help for command-specific help.')
+        dest="cmd", help="Use dvc cache CMD --help for command-specific help."
+    )
 
     fix_subparsers(cache_subparsers)
 
     parent_cache_config_parser = argparse.ArgumentParser(
-        add_help=False,
-        parents=[parent_config_parser])
-    CACHE_DIR_HELP = 'Configure cache directory location.'
+        add_help=False, parents=[parent_config_parser]
+    )
+    CACHE_DIR_HELP = "Configure cache directory location."
     cache_dir_parser = cache_subparsers.add_parser(
-        'dir',
+        "dir",
         parents=[parent_cache_config_parser],
         description=CACHE_DIR_HELP,
-        help=CACHE_DIR_HELP)
+        help=CACHE_DIR_HELP,
+    )
     cache_dir_parser.add_argument(
-        '-u',
-        '--unset',
-        default=False,
-        action='store_true',
-        help='Unset option.')
+        "-u", "--unset", default=False, action="store_true", help="Unset option."
+    )
     cache_dir_parser.add_argument(
-        'value',
-        nargs='?',
+        "value",
+        nargs="?",
         default=None,
         help="Path to cache directory. Relative paths are resolved relative "
-             "to the current directory and saved to config relative to the "
-             "config file location.")
+        "to the current directory and saved to config relative to the "
+        "config file location.",
+    )
     cache_dir_parser.set_defaults(func=CmdCacheDir)

dvc/command/checkout.py

@@ -9,33 +9,33 @@ class CmdCheckout(CmdBase):
             self.project.checkout(force=self.args.force)
         else:
             for target in self.args.targets:
-                self.project.checkout(target=target,
-                                      with_deps=self.args.with_deps,
-                                      force=self.args.force)
+                self.project.checkout(
+                    target=target, with_deps=self.args.with_deps, force=self.args.force
+                )
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    CHECKOUT_HELP = 'Checkout data files from cache.'
+    CHECKOUT_HELP = "Checkout data files from cache."
     checkout_parser = subparsers.add_parser(
-        'checkout',
+        "checkout",
         parents=[parent_parser],
         description=CHECKOUT_HELP,
-        help=CHECKOUT_HELP)
+        help=CHECKOUT_HELP,
+    )
     checkout_parser.add_argument(
-        '-d',
-        '--with-deps',
-        action='store_true',
+        "-d",
+        "--with-deps",
+        action="store_true",
         default=False,
-        help='Checkout all dependencies of the specified target.')
+        help="Checkout all dependencies of the specified target.",
+    )
     checkout_parser.add_argument(
-        '-f',
-        '--force',
-        action='store_true',
+        "-f",
+        "--force",
+        action="store_true",
         default=False,
-        help='Do not prompt when removing working directory files.')
-    checkout_parser.add_argument(
-        'targets',
-        nargs='*',
-        help='DVC files.')
+        help="Do not prompt when removing working directory files.",
+    )
+    checkout_parser.add_argument("targets", nargs="*", help="DVC files.")
     checkout_parser.set_defaults(func=CmdCheckout)

dvc/command/config.py

@@ -64,13 +64,12 @@ class CmdConfig(CmdBase):
             self.config.set(self.configobj, section, opt, value)
             self.config.save(self.configobj)
         except DvcException:
-            logger.error("failed to set '{}.{}' to '{}'"
-                         .format(section, opt, value))
+            logger.error("failed to set '{}.{}' to '{}'".format(section, opt, value))
             return 1
         return 0
 
     def run(self):
-        section, opt = self.args.name.lower().strip().split('.', 1)
+        section, opt = self.args.name.lower().strip().split(".", 1)
 
         if self.args.unset:
             return self._unset(section, opt)
@@ -82,42 +81,31 @@ class CmdConfig(CmdBase):
 
 parent_config_parser = argparse.ArgumentParser(add_help=False)
 parent_config_parser.add_argument(
-   '--global',
-   dest='glob',
-   action='store_true',
-   default=False,
-   help='Use global config.')
+    "--global",
+    dest="glob",
+    action="store_true",
+    default=False,
+    help="Use global config.",
+)
 parent_config_parser.add_argument(
-   '--system',
-   action='store_true',
-   default=False,
-   help='Use system config.')
+    "--system", action="store_true", default=False, help="Use system config."
+)
 parent_config_parser.add_argument(
-   '--local',
-   action='store_true',
-   default=False,
-   help='Use local config.')
+    "--local", action="store_true", default=False, help="Use local config."
+)
 
 
 def add_parser(subparsers, parent_parser):
-    CONFIG_HELP = 'Get or set config options.'
+    CONFIG_HELP = "Get or set config options."
     config_parser = subparsers.add_parser(
-        'config',
+        "config",
         parents=[parent_config_parser, parent_parser],
         description=CONFIG_HELP,
-        help=CONFIG_HELP)
+        help=CONFIG_HELP,
+    )
     config_parser.add_argument(
-        '-u',
-        '--unset',
-        default=False,
-        action='store_true',
-        help='Unset option.')
-    config_parser.add_argument(
-        'name',
-        help='Option name.')
-    config_parser.add_argument(
-        'value',
-        nargs='?',
-        default=None,
-        help='Option value.')
+        "-u", "--unset", default=False, action="store_true", help="Unset option."
+    )
+    config_parser.add_argument("name", help="Option name.")
+    config_parser.add_argument("value", nargs="?", default=None, help="Option value.")
     config_parser.set_defaults(func=CmdConfig)

dvc/command/daemon.py

@@ -38,34 +38,32 @@ class CmdDaemonAnalytics(CmdDaemonBase):
 
 
 def add_parser(subparsers, parent_parser):
-    DAEMON_HELP = 'Service daemon.'
+    DAEMON_HELP = "Service daemon."
     daemon_parser = subparsers.add_parser(
-        'daemon',
-        parents=[parent_parser],
-        description=DAEMON_HELP,
-        help=DAEMON_HELP)
+        "daemon", parents=[parent_parser], description=DAEMON_HELP, help=DAEMON_HELP
+    )
 
     daemon_subparsers = daemon_parser.add_subparsers(
-        dest='cmd',
-        help='Use dvc daemon CMD --help for command-specific help.')
+        dest="cmd", help="Use dvc daemon CMD --help for command-specific help."
+    )
 
     fix_subparsers(daemon_subparsers)
 
-    DAEMON_UPDATER_HELP = 'Fetch latest available version.'
+    DAEMON_UPDATER_HELP = "Fetch latest available version."
     daemon_updater_parser = daemon_subparsers.add_parser(
-        'updater',
+        "updater",
         parents=[parent_parser],
         description=DAEMON_UPDATER_HELP,
-        help=DAEMON_UPDATER_HELP)
+        help=DAEMON_UPDATER_HELP,
+    )
     daemon_updater_parser.set_defaults(func=CmdDaemonUpdater)
 
-    DAEMON_ANALYTICS_HELP = 'Send dvc usage analytics.'
+    DAEMON_ANALYTICS_HELP = "Send dvc usage analytics."
     daemon_analytics_parser = daemon_subparsers.add_parser(
-        'analytics',
+        "analytics",
         parents=[parent_parser],
         description=DAEMON_ANALYTICS_HELP,
-        help=DAEMON_ANALYTICS_HELP)
-    daemon_analytics_parser.add_argument(
-        'target',
-        help="Analytics file.")
+        help=DAEMON_ANALYTICS_HELP,
+    )
+    daemon_analytics_parser.add_argument("target", help="Analytics file.")
     daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics)

dvc/command/data_sync.py

@@ -24,17 +24,19 @@ class CmdDataBase(CmdBase):
 class CmdDataPull(CmdDataBase):
     def do_run(self, target=None):
         try:
-            self.project.pull(target=target,
-                              jobs=self.args.jobs,
-                              remote=self.args.remote,
-                              show_checksums=self.args.show_checksums,
-                              all_branches=self.args.all_branches,
-                              all_tags=self.args.all_tags,
-                              with_deps=self.args.with_deps,
-                              force=self.args.force,
-                              recursive=self.args.recursive)
+            self.project.pull(
+                target=target,
+                jobs=self.args.jobs,
+                remote=self.args.remote,
+                show_checksums=self.args.show_checksums,
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+                with_deps=self.args.with_deps,
+                force=self.args.force,
+                recursive=self.args.recursive,
+            )
         except Exception:
-            logger.error('failed to pull data from the cloud')
+            logger.error("failed to pull data from the cloud")
             return 1
         return 0
 
@@ -42,16 +44,18 @@ class CmdDataPull(CmdDataBase):
 class CmdDataPush(CmdDataBase):
     def do_run(self, target=None):
         try:
-            self.project.push(target=target,
-                              jobs=self.args.jobs,
-                              remote=self.args.remote,
-                              show_checksums=self.args.show_checksums,
-                              all_branches=self.args.all_branches,
-                              all_tags=self.args.all_tags,
-                              with_deps=self.args.with_deps,
-                              recursive=self.args.recursive)
+            self.project.push(
+                target=target,
+                jobs=self.args.jobs,
+                remote=self.args.remote,
+                show_checksums=self.args.show_checksums,
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+                with_deps=self.args.with_deps,
+                recursive=self.args.recursive,
+            )
         except Exception:
-            logger.error('failed to push data to the cloud')
+            logger.error("failed to push data to the cloud")
             return 1
         return 0
 
@@ -59,16 +63,18 @@ class CmdDataPush(CmdDataBase):
 class CmdDataFetch(CmdDataBase):
     def do_run(self, target=None):
         try:
-            self.project.fetch(target=target,
-                               jobs=self.args.jobs,
-                               remote=self.args.remote,
-                               show_checksums=self.args.show_checksums,
-                               all_branches=self.args.all_branches,
-                               all_tags=self.args.all_tags,
-                               with_deps=self.args.with_deps,
-                               recursive=self.args.recursive)
+            self.project.fetch(
+                target=target,
+                jobs=self.args.jobs,
+                remote=self.args.remote,
+                show_checksums=self.args.show_checksums,
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+                with_deps=self.args.with_deps,
+                recursive=self.args.recursive,
+            )
         except Exception:
-            logger.error('failed to fetch data from the cloud')
+            logger.error("failed to fetch data from the cloud")
             return 1
         return 0
 
@@ -78,178 +84,179 @@ def add_parser(subparsers, parent_parser):
 
     # Parent parser used in pull/push/status
     parent_cache_parser = argparse.ArgumentParser(
-        add_help=False,
-        parents=[parent_parser])
+        add_help=False, parents=[parent_parser]
+    )
     parent_cache_parser.add_argument(
-        '-j',
-        '--jobs',
+        "-j",
+        "--jobs",
         type=int,
         default=None,
-        help='Number of jobs to run simultaneously.')
+        help="Number of jobs to run simultaneously.",
+    )
     parent_cache_parser.add_argument(
-        '--show-checksums',
-        action='store_true',
+        "--show-checksums",
+        action="store_true",
         default=False,
-        help='Show checksums instead of file names.')
+        help="Show checksums instead of file names.",
+    )
     parent_cache_parser.add_argument(
-        'targets',
-        nargs='*',
-        default=None,
-        help='DVC files.')
+        "targets", nargs="*", default=None, help="DVC files."
+    )
 
     # Pull
-    PULL_HELP = 'Pull data files from the cloud.'
+    PULL_HELP = "Pull data files from the cloud."
     pull_parser = subparsers.add_parser(
-        'pull',
-        parents=[parent_cache_parser],
-        description=PULL_HELP,
-        help=PULL_HELP)
+        "pull", parents=[parent_cache_parser], description=PULL_HELP, help=PULL_HELP
+    )
+    pull_parser.add_argument("-r", "--remote", help="Remote repository to pull from.")
     pull_parser.add_argument(
-        '-r',
-        '--remote',
-        help='Remote repository to pull from.')
-    pull_parser.add_argument(
-        '-a',
-        '--all-branches',
-        action='store_true',
+        "-a",
+        "--all-branches",
+        action="store_true",
         default=False,
-        help='Fetch cache for all branches.')
+        help="Fetch cache for all branches.",
+    )
     pull_parser.add_argument(
-        '-T',
-        '--all-tags',
-        action='store_true',
+        "-T",
+        "--all-tags",
+        action="store_true",
         default=False,
-        help='Fetch cache for all tags.')
+        help="Fetch cache for all tags.",
+    )
     pull_parser.add_argument(
-        '-d',
-        '--with-deps',
-        action='store_true',
+        "-d",
+        "--with-deps",
+        action="store_true",
         default=False,
-        help='Fetch cache for all dependencies of the specified target.')
+        help="Fetch cache for all dependencies of the specified target.",
+    )
     pull_parser.add_argument(
-        '-f',
-        '--force',
-        action='store_true',
+        "-f",
+        "--force",
+        action="store_true",
         default=False,
-        help='Do not prompt when removing working directory files.')
+        help="Do not prompt when removing working directory files.",
+    )
     pull_parser.add_argument(
-        '-R',
-        '--recursive',
-        action='store_true',
+        "-R",
+        "--recursive",
+        action="store_true",
         default=False,
-        help='Pull cache for subdirectories of the specified directory.')
+        help="Pull cache for subdirectories of the specified directory.",
+    )
     pull_parser.set_defaults(func=CmdDataPull)
 
     # Push
-    PUSH_HELP = 'Push data files to the cloud.'
+    PUSH_HELP = "Push data files to the cloud."
     push_parser = subparsers.add_parser(
-        'push',
-        parents=[parent_cache_parser],
-        description=PUSH_HELP,
-        help=PUSH_HELP)
+        "push", parents=[parent_cache_parser], description=PUSH_HELP, help=PUSH_HELP
+    )
+    push_parser.add_argument("-r", "--remote", help="Remote repository to push to.")
     push_parser.add_argument(
-        '-r',
-        '--remote',
-        help='Remote repository to push to.')
-    push_parser.add_argument(
-        '-a',
-        '--all-branches',
-        action='store_true',
+        "-a",
+        "--all-branches",
+        action="store_true",
         default=False,
-        help='Push cache for all branches.')
+        help="Push cache for all branches.",
+    )
     push_parser.add_argument(
-        '-T',
-        '--all-tags',
-        action='store_true',
+        "-T",
+        "--all-tags",
+        action="store_true",
         default=False,
-        help='Push cache for all tags.')
+        help="Push cache for all tags.",
+    )
     push_parser.add_argument(
-        '-d',
-        '--with-deps',
-        action='store_true',
+        "-d",
+        "--with-deps",
+        action="store_true",
         default=False,
-        help='Push cache for all dependencies of the specified target.')
+        help="Push cache for all dependencies of the specified target.",
+    )
     push_parser.add_argument(
-        '-R',
-        '--recursive',
-        action='store_true',
+        "-R",
+        "--recursive",
+        action="store_true",
         default=False,
-        help='Push cache from subdirectories of specified directory.')
+        help="Push cache from subdirectories of specified directory.",
+    )
     push_parser.set_defaults(func=CmdDataPush)
 
     # Fetch
-    FETCH_HELP = 'Fetch data files from the cloud.'
+    FETCH_HELP = "Fetch data files from the cloud."
     fetch_parser = subparsers.add_parser(
-        'fetch',
-        parents=[parent_cache_parser],
-        description=FETCH_HELP,
-        help=FETCH_HELP)
+        "fetch", parents=[parent_cache_parser], description=FETCH_HELP, help=FETCH_HELP
+    )
+    fetch_parser.add_argument("-r", "--remote", help="Remote repository to fetch from.")
     fetch_parser.add_argument(
-        '-r',
-        '--remote',
-        help='Remote repository to fetch from.')
-    fetch_parser.add_argument(
-        '-a',
-        '--all-branches',
-        action='store_true',
+        "-a",
+        "--all-branches",
+        action="store_true",
         default=False,
-        help='Fetch cache for all branches.')
+        help="Fetch cache for all branches.",
+    )
     fetch_parser.add_argument(
-        '-T',
-        '--all-tags',
-        action='store_true',
+        "-T",
+        "--all-tags",
+        action="store_true",
         default=False,
-        help='Fetch cache for all tags.')
+        help="Fetch cache for all tags.",
+    )
     fetch_parser.add_argument(
-        '-d',
-        '--with-deps',
-        action='store_true',
+        "-d",
+        "--with-deps",
+        action="store_true",
         default=False,
-        help='Fetch cache for all dependencies of the '
-        'specified target.')
+        help="Fetch cache for all dependencies of the " "specified target.",
+    )
     fetch_parser.add_argument(
-        '-R',
-        '--recursive',
-        action='store_true',
+        "-R",
+        "--recursive",
+        action="store_true",
         default=False,
-        help='Fetch cache for subdirectories of specified directory.')
+        help="Fetch cache for subdirectories of specified directory.",
+    )
     fetch_parser.set_defaults(func=CmdDataFetch)
 
     # Status
-    STATUS_HELP = 'Show the project status.'
+    STATUS_HELP = "Show the project status."
     status_parser = subparsers.add_parser(
-        'status',
+        "status",
         parents=[parent_cache_parser],
         description=STATUS_HELP,
-        help=STATUS_HELP)
+        help=STATUS_HELP,
+    )
     status_parser.add_argument(
-        '-c',
-        '--cloud',
-        action='store_true',
+        "-c",
+        "--cloud",
+        action="store_true",
         default=False,
-        help='Show status of a local cache compared to a remote repository.')
+        help="Show status of a local cache compared to a remote repository.",
+    )
     status_parser.add_argument(
-        '-r',
-        '--remote',
-        help='Remote repository to compare local cache to.')
+        "-r", "--remote", help="Remote repository to compare local cache to."
+    )
     status_parser.add_argument(
-        '-a',
-        '--all-branches',
-        action='store_true',
+        "-a",
+        "--all-branches",
+        action="store_true",
         default=False,
-        help='Show status of a local cache compared to a remote repository '
-             'for all branches.')
+        help="Show status of a local cache compared to a remote repository "
+        "for all branches.",
+    )
     status_parser.add_argument(
-        '-T',
-        '--all-tags',
-        action='store_true',
+        "-T",
+        "--all-tags",
+        action="store_true",
         default=False,
-        help='Show status of a local cache compared to a remote repository '
-             'for all tags.')
+        help="Show status of a local cache compared to a remote repository "
+        "for all tags.",
+    )
     status_parser.add_argument(
-        '-d',
-        '--with-deps',
-        action='store_true',
+        "-d",
+        "--with-deps",
+        action="store_true",
         default=False,
-        help='Show status for all dependencies of the specified target.')
+        help="Show status for all dependencies of the specified target.",
+    )
     status_parser.set_defaults(func=CmdDataStatus)

dvc/command/destroy.py

@@ -10,10 +10,10 @@ class CmdDestroy(CmdBase):
     def run_cmd(self):
         try:
             statement = (
-                'This will destroy all information about your pipelines,'
-                ' all data files, as well as cache in .dvc/cache.'
-                '\n'
-                'Are you sure you want to continue?'
+                "This will destroy all information about your pipelines,"
+                " all data files, as well as cache in .dvc/cache."
+                "\n"
+                "Are you sure you want to continue?"
             )
 
             if not self.args.force and not prompt.confirm(statement):
@@ -24,23 +24,19 @@ class CmdDestroy(CmdBase):
 
             self.project.destroy()
         except Exception:
-            logger.error('failed to destroy DVC')
+            logger.error("failed to destroy DVC")
             return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    DESTROY_HELP = "Destroy dvc. Will remove all project's information, " \
-                   "data files and cache."
+    DESTROY_HELP = (
+        "Destroy dvc. Will remove all project's information, " "data files and cache."
+    )
     destroy_parser = subparsers.add_parser(
-        'destroy',
-        parents=[parent_parser],
-        description=DESTROY_HELP,
-        help=DESTROY_HELP)
+        "destroy", parents=[parent_parser], description=DESTROY_HELP, help=DESTROY_HELP
+    )
     destroy_parser.add_argument(
-        '-f',
-        '--force',
-        action='store_true',
-        default=False,
-        help='Force destruction.')
+        "-f", "--force", action="store_true", default=False, help="Force destruction."
+    )
     destroy_parser.set_defaults(func=CmdDestroy)

dvc/command/gc.py

@@ -9,86 +9,91 @@ from dvc.command.base import CmdBase
 
 class CmdGC(CmdBase):
     def run(self):
-        msg = 'this will remove all cache except the cache that is used in '
+        msg = "this will remove all cache except the cache that is used in "
         if not self.args.all_branches and not self.args.all_tags:
-            msg += 'the current git branch'
+            msg += "the current git branch"
         elif self.args.all_branches and not self.args.all_tags:
-            msg += 'all git branches'
+            msg += "all git branches"
         elif not self.args.all_branches and self.args.all_tags:
-            msg += 'all git tags'
+            msg += "all git tags"
         else:
-            msg += 'all git branches and all git tags'
+            msg += "all git branches and all git tags"
 
         if self.args.projects is not None and len(self.args.projects) > 0:
-            msg += ' of the current and the following projects:'
+            msg += " of the current and the following projects:"
 
             for project_path in self.args.projects:
-                msg += '\n  - %s' % os.path.abspath(project_path)
+                msg += "\n  - %s" % os.path.abspath(project_path)
         else:
-            msg += ' of the current project.'
+            msg += " of the current project."
 
         logger.warning(msg)
 
-        msg = 'Are you sure you want to proceed?'
+        msg = "Are you sure you want to proceed?"
         if not self.args.force and not prompt.confirm(msg):
             return 1
 
-        self.project.gc(all_branches=self.args.all_branches,
-                        all_tags=self.args.all_tags,
-                        cloud=self.args.cloud,
-                        remote=self.args.remote,
-                        force=self.args.force,
-                        jobs=self.args.jobs,
-                        projects=self.args.projects)
+        self.project.gc(
+            all_branches=self.args.all_branches,
+            all_tags=self.args.all_tags,
+            cloud=self.args.cloud,
+            remote=self.args.remote,
+            force=self.args.force,
+            jobs=self.args.jobs,
+            projects=self.args.projects,
+        )
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    GC_HELP = 'Collect garbage.'
+    GC_HELP = "Collect garbage."
     gc_parser = subparsers.add_parser(
-        'gc',
-        parents=[parent_parser],
-        description=GC_HELP,
-        help=GC_HELP)
+        "gc", parents=[parent_parser], description=GC_HELP, help=GC_HELP
+    )
     gc_parser.add_argument(
-        '-a',
-        '--all-branches',
-        action='store_true',
+        "-a",
+        "--all-branches",
+        action="store_true",
         default=False,
-        help='Collect garbage for all branches.')
+        help="Collect garbage for all branches.",
+    )
     gc_parser.add_argument(
-        '-T',
-        '--all-tags',
-        action='store_true',
+        "-T",
+        "--all-tags",
+        action="store_true",
         default=False,
-        help='Collect garbage for all tags.')
+        help="Collect garbage for all tags.",
+    )
     gc_parser.add_argument(
-        '-c',
-        '--cloud',
-        action='store_true',
+        "-c",
+        "--cloud",
+        action="store_true",
         default=False,
-        help='Collect garbage in remote repository.')
+        help="Collect garbage in remote repository.",
+    )
     gc_parser.add_argument(
-        '-r',
-        '--remote',
-        help='Remote repository to collect garbage in.')
+        "-r", "--remote", help="Remote repository to collect garbage in."
+    )
     gc_parser.add_argument(
-        '-f',
-        '--force',
-        action='store_true',
+        "-f",
+        "--force",
+        action="store_true",
         default=False,
-        help='Force garbage collection.')
+        help="Force garbage collection.",
+    )
     gc_parser.add_argument(
-        '-j',
-        '--jobs',
+        "-j",
+        "--jobs",
         type=int,
         default=None,
-        help='Number of jobs to run simultaneously.')
+        help="Number of jobs to run simultaneously.",
+    )
     gc_parser.add_argument(
-        '-p',
-        '--projects',
+        "-p",
+        "--projects",
         type=str,
-        nargs='*',
+        nargs="*",
         default=None,
-        help='Collect garbage for all given projects.')
+        help="Collect garbage for all given projects.",
+    )
     gc_parser.set_defaults(func=CmdGC)

dvc/command/imp.py

@@ -11,30 +11,29 @@ from dvc.command.base import CmdBase
 class CmdImport(CmdBase):
     def run(self):
         try:
-            default_out = os.path.basename(
-                urlparse(self.args.url).path
-            )
+            default_out = os.path.basename(urlparse(self.args.url).path)
 
             out = self.args.out or default_out
 
             self.project.imp(self.args.url, out)
         except DvcException:
-            logger.error('failed to import {}. You could also try downloading '
-                         'it manually and adding it with `dvc add` command.'
-                         .format(self.args.url))
+            logger.error(
+                "failed to import {}. You could also try downloading "
+                "it manually and adding it with `dvc add` command.".format(
+                    self.args.url
+                )
+            )
             return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    IMPORT_HELP = 'Import files from URL.'
+    IMPORT_HELP = "Import files from URL."
     import_parser = subparsers.add_parser(
-        'import',
-        parents=[parent_parser],
-        description=IMPORT_HELP,
-        help=IMPORT_HELP)
+        "import", parents=[parent_parser], description=IMPORT_HELP, help=IMPORT_HELP
+    )
     import_parser.add_argument(
-        'url',
+        "url",
         help="URL. Supported urls: "
         "'/path/to/file', "
         "'C:\\\\path\\to\\file', "
@@ -44,9 +43,7 @@ def add_parser(subparsers, parent_parser):
         "'hdfs://example.com/path/to/file', "
         "'ssh://example.com:/path/to/file', "
         "'remote://myremote/path/to/file'(see "
-        "`dvc remote` commands). ")
-    import_parser.add_argument(
-        'out',
-        nargs='?',
-        help='Output.')
+        "`dvc remote` commands). ",
+    )
+    import_parser.add_argument("out", nargs="?", help="Output.")
     import_parser.set_defaults(func=CmdImport)

dvc/command/init.py

@@ -11,35 +11,34 @@ class CmdInit(object):
         from dvc.project import Project, InitError
 
         try:
-            self.project = Project.init('.',
-                                        no_scm=self.args.no_scm,
-                                        force=self.args.force)
+            self.project = Project.init(
+                ".", no_scm=self.args.no_scm, force=self.args.force
+            )
             self.config = self.project.config
         except InitError:
-            logger.error('failed to initiate dvc')
+            logger.error("failed to initiate dvc")
             return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
     """Setup parser for `dvc init`."""
-    INIT_HELP = 'Initialize dvc over a directory ' \
-                '(should already be a git dir).'
+    INIT_HELP = "Initialize dvc over a directory " "(should already be a git dir)."
     init_parser = subparsers.add_parser(
-        'init',
-        parents=[parent_parser],
-        description=INIT_HELP,
-        help=INIT_HELP)
+        "init", parents=[parent_parser], description=INIT_HELP, help=INIT_HELP
+    )
     init_parser.add_argument(
-        '--no-scm',
-        action='store_true',
+        "--no-scm",
+        action="store_true",
         default=False,
         help="Initiate dvc in directory that is "
-        "not tracked by any scm tool (e.g. git).")
+        "not tracked by any scm tool (e.g. git).",
+    )
     init_parser.add_argument(
-        '-f',
-        '--force',
-        action='store_true',
+        "-f",
+        "--force",
+        action="store_true",
         default=False,
-        help="Overwrite '.dvc' if it exists. Will remove all local cache.")
+        help="Overwrite '.dvc' if it exists. Will remove all local cache.",
+    )
     init_parser.set_defaults(func=CmdInit)

dvc/command/install.py

@@ -9,16 +9,14 @@ class CmdInstall(CmdBase):
         try:
             self.project.install()
         except Exception:
-            logger.error('failed to install dvc hooks')
+            logger.error("failed to install dvc hooks")
             return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    INSTALL_HELP = 'Install dvc hooks into the repository.'
+    INSTALL_HELP = "Install dvc hooks into the repository."
     install_parser = subparsers.add_parser(
-        'install',
-        parents=[parent_parser],
-        description=INSTALL_HELP,
-        help=INSTALL_HELP)
+        "install", parents=[parent_parser], description=INSTALL_HELP, help=INSTALL_HELP
+    )
     install_parser.set_defaults(func=CmdInstall)

dvc/command/lock.py

@@ -12,8 +12,7 @@ class CmdLockBase(CmdBase):
                 self.project.lock_stage(target, unlock=unlock)
             except DvcException:
                 logger.error(
-                    "failed to {}lock '{}'"
-                    .format('un' if unlock else '', target)
+                    "failed to {}lock '{}'".format("un" if unlock else "", target)
                 )
 
                 return 1
@@ -31,26 +30,16 @@ class CmdUnlock(CmdLockBase):
 
 
 def add_parser(subparsers, parent_parser):
-    LOCK_HELP = 'Lock DVC file.'
+    LOCK_HELP = "Lock DVC file."
     lock_parser = subparsers.add_parser(
-        'lock',
-        parents=[parent_parser],
-        description=LOCK_HELP,
-        help=LOCK_HELP)
-    lock_parser.add_argument(
-        'targets',
-        nargs='+',
-        help='DVC files.')
+        "lock", parents=[parent_parser], description=LOCK_HELP, help=LOCK_HELP
+    )
+    lock_parser.add_argument("targets", nargs="+", help="DVC files.")
     lock_parser.set_defaults(func=CmdLock)
 
-    UNLOCK_HELP = 'Unlock DVC file.'
+    UNLOCK_HELP = "Unlock DVC file."
     unlock_parser = subparsers.add_parser(
-        'unlock',
-        parents=[parent_parser],
-        description=UNLOCK_HELP,
-        help=UNLOCK_HELP)
-    unlock_parser.add_argument(
-        'targets',
-        nargs='+',
-        help='DVC files.')
+        "unlock", parents=[parent_parser], description=UNLOCK_HELP, help=UNLOCK_HELP
+    )
+    unlock_parser.add_argument("targets", nargs="+", help="DVC files.")
     unlock_parser.set_defaults(func=CmdUnlock)

dvc/command/metrics.py

@@ -10,31 +10,33 @@ class CmdMetricsShow(CmdBase):
         try:
             # backward compatibility
             if self.args.json_path:
-                typ = 'json'
+                typ = "json"
                 xpath = self.args.json_path
             elif self.args.tsv_path:
-                typ = 'tsv'
+                typ = "tsv"
                 xpath = self.args.tsv_path
             elif self.args.htsv_path:
-                typ = 'htsv'
+                typ = "htsv"
                 xpath = self.args.htsv_path
             elif self.args.csv_path:
-                typ = 'csv'
+                typ = "csv"
                 xpath = self.args.csv_path
             elif self.args.hcsv_path:
-                typ = 'hcsv'
+                typ = "hcsv"
                 xpath = self.args.hcsv_path
             else:
                 typ = self.args.type
                 xpath = self.args.xpath
 
-            self.project.metrics_show(self.args.path,
-                                      typ=typ,
-                                      xpath=xpath,
-                                      all_branches=self.args.all_branches,
-                                      all_tags=self.args.all_tags)
+            self.project.metrics_show(
+                self.args.path,
+                typ=typ,
+                xpath=xpath,
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+            )
         except DvcException:
-            logger.error('failed to show metrics')
+            logger.error("failed to show metrics")
             return 1
 
         return 0
@@ -43,11 +45,11 @@ class CmdMetricsShow(CmdBase):
 class CmdMetricsModify(CmdBase):
     def run(self):
         try:
-            self.project.metrics_modify(self.args.path,
-                                        typ=self.args.type,
-                                        xpath=self.args.xpath)
+            self.project.metrics_modify(
+                self.args.path, typ=self.args.type, xpath=self.args.xpath
+            )
         except DvcException:
-            logger.error('failed to modify metrics')
+            logger.error("failed to modify metrics")
             return 1
 
         return 0
@@ -56,9 +58,7 @@ class CmdMetricsModify(CmdBase):
 class CmdMetricsAdd(CmdBase):
     def run(self):
         try:
-            self.project.metrics_add(self.args.path,
-                                     self.args.type,
-                                     self.args.xpath)
+            self.project.metrics_add(self.args.path, self.args.type, self.args.xpath)
         except DvcException:
             msg = "failed to add metric file '{}'".format(self.args.path)
             logger.error(msg)
@@ -80,112 +80,99 @@ class CmdMetricsRemove(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    METRICS_HELP = 'Get metrics from all branches.'
+    METRICS_HELP = "Get metrics from all branches."
     metrics_parser = subparsers.add_parser(
-        'metrics',
-        parents=[parent_parser],
-        description=METRICS_HELP,
-        help=METRICS_HELP)
+        "metrics", parents=[parent_parser], description=METRICS_HELP, help=METRICS_HELP
+    )
 
     metrics_subparsers = metrics_parser.add_subparsers(
-        dest='cmd',
-        help="Use dvc metrics CMD --help for command-specific help.")
+        dest="cmd", help="Use dvc metrics CMD --help for command-specific help."
+    )
 
     fix_subparsers(metrics_subparsers)
 
-    METRICS_SHOW_HELP = 'Show metrics.'
+    METRICS_SHOW_HELP = "Show metrics."
     metrics_show_parser = metrics_subparsers.add_parser(
-        'show',
+        "show",
         parents=[parent_parser],
         description=METRICS_SHOW_HELP,
-        help=METRICS_SHOW_HELP)
+        help=METRICS_SHOW_HELP,
+    )
+    metrics_show_parser.add_argument("path", nargs="?", help="Path to metrics file.")
     metrics_show_parser.add_argument(
-        'path',
-        nargs='?',
-        help='Path to metrics file.')
+        "-t", "--type", help="Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV)."
+    )
     metrics_show_parser.add_argument(
-        '-t',
-        '--type',
-        help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV).')
-    metrics_show_parser.add_argument(
-        '-x',
-        '--xpath',
-        help='JSON/TSV/HTSV/CSV/HCSV path.')
+        "-x", "--xpath", help="JSON/TSV/HTSV/CSV/HCSV path."
+    )
     metrics_show_group = metrics_show_parser.add_mutually_exclusive_group()
+    metrics_show_group.add_argument("--json-path", help="JSON path.")
     metrics_show_group.add_argument(
-        '--json-path',
-        help='JSON path.')
+        "--tsv-path", help="TSV path 'row,column' (e.g. '1,2')."
+    )
     metrics_show_group.add_argument(
-        '--tsv-path',
-        help="TSV path 'row,column' (e.g. '1,2').")
+        "--htsv-path", help="Headed TSV path 'row,column (e.g. 'Name,3')."
+    )
     metrics_show_group.add_argument(
-        '--htsv-path',
-        help="Headed TSV path 'row,column (e.g. 'Name,3').")
+        "--csv-path", help="CSV path 'row,column' (e.g. '1,2')."
+    )
     metrics_show_group.add_argument(
-        '--csv-path',
-        help="CSV path 'row,column' (e.g. '1,2').")
-    metrics_show_group.add_argument(
-        '--hcsv-path',
-        help="Headed CSV path 'row,column' (e.g. 'Name,3').")
+        "--hcsv-path", help="Headed CSV path 'row,column' (e.g. 'Name,3')."
+    )
     metrics_show_parser.add_argument(
-        '-a',
-        '--all-branches',
-        action='store_true',
+        "-a",
+        "--all-branches",
+        action="store_true",
         default=False,
-        help='Show metrics for all branches.')
+        help="Show metrics for all branches.",
+    )
     metrics_show_parser.add_argument(
-        '-T',
-        '--all-tags',
-        action='store_true',
+        "-T",
+        "--all-tags",
+        action="store_true",
         default=False,
-        help='Show metrics for all tags.')
+        help="Show metrics for all tags.",
+    )
     metrics_show_parser.set_defaults(func=CmdMetricsShow)
 
-    METRICS_ADD_HELP = 'Add metrics.'
+    METRICS_ADD_HELP = "Add metrics."
     metrics_add_parser = metrics_subparsers.add_parser(
-        'add',
+        "add",
         parents=[parent_parser],
         description=METRICS_ADD_HELP,
-        help=METRICS_ADD_HELP)
+        help=METRICS_ADD_HELP,
+    )
     metrics_add_parser.add_argument(
-        '-t',
-        '--type',
-        help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV).')
+        "-t", "--type", help="Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV)."
+    )
     metrics_add_parser.add_argument(
-        '-x',
-        '--xpath',
-        help='JSON/TSV/HTSV/CSV/HCSV path.')
-    metrics_add_parser.add_argument(
-        'path',
-        help='Path to metrics file.')
+        "-x", "--xpath", help="JSON/TSV/HTSV/CSV/HCSV path."
+    )
+    metrics_add_parser.add_argument("path", help="Path to metrics file.")
     metrics_add_parser.set_defaults(func=CmdMetricsAdd)
 
-    METRICS_MODIFY_HELP = 'Modify metrics.'
+    METRICS_MODIFY_HELP = "Modify metrics."
     metrics_modify_parser = metrics_subparsers.add_parser(
-        'modify',
+        "modify",
         parents=[parent_parser],
         description=METRICS_MODIFY_HELP,
-        help=METRICS_MODIFY_HELP)
+        help=METRICS_MODIFY_HELP,
+    )
     metrics_modify_parser.add_argument(
-        '-t',
-        '--type',
-        help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV).')
+        "-t", "--type", help="Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV)."
+    )
     metrics_modify_parser.add_argument(
-        '-x',
-        '--xpath',
-        help='JSON/TSV/HTSV/CSV/HCSV path.')
-    metrics_modify_parser.add_argument(
-        'path',
-        help='Metrics file.')
+        "-x", "--xpath", help="JSON/TSV/HTSV/CSV/HCSV path."
+    )
+    metrics_modify_parser.add_argument("path", help="Metrics file.")
     metrics_modify_parser.set_defaults(func=CmdMetricsModify)
 
-    METRICS_REMOVE_HELP = 'Remove metrics.'
+    METRICS_REMOVE_HELP = "Remove metrics."
     metrics_remove_parser = metrics_subparsers.add_parser(
-        'remove',
+        "remove",
         parents=[parent_parser],
         description=METRICS_REMOVE_HELP,
-        help=METRICS_REMOVE_HELP)
-    metrics_remove_parser.add_argument(
-        'path',
-        help='Path to metrics file.')
+        help=METRICS_REMOVE_HELP,
+    )
+    metrics_remove_parser.add_argument("path", help="Path to metrics file.")
     metrics_remove_parser.set_defaults(func=CmdMetricsRemove)

dvc/command/move.py

@@ -10,24 +10,22 @@ class CmdMove(CmdBase):
         try:
             self.project.move(self.args.src, self.args.dst)
         except DvcException:
-            msg = "failed to move '{}' -> '{}'".format(self.args.src,
-                                                       self.args.dst)
+            msg = "failed to move '{}' -> '{}'".format(self.args.src, self.args.dst)
             logger.error(msg)
             return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    description = "Rename or move a data file or a directory that "\
-           "is under DVC control. It renames and modifies "\
-           "the corresponding DVC file to reflect the changes."
+    description = (
+        "Rename or move a data file or a directory that "
+        "is under DVC control. It renames and modifies "
+        "the corresponding DVC file to reflect the changes."
+    )
     help = "Rename or move a DVC controlled data file or a directory."
     move_parser = subparsers.add_parser(
-        'move',
-        parents=[parent_parser],
-        description=description,
-        help=help)
-    move_parser.add_argument('src',
-                             help='Source path to a data file or directory.')
-    move_parser.add_argument('dst', help='Destination path.')
+        "move", parents=[parent_parser], description=description, help=help
+    )
+    move_parser.add_argument("src", help="Source path to a data file or directory.")
+    move_parser.add_argument("dst", help="Destination path.")
     move_parser.set_defaults(func=CmdMove)

dvc/command/pipeline.py

@@ -16,7 +16,7 @@ class CmdPipelineShow(CmdBase):
 
         stage = Stage.load(self.project, target)
         G = self.project.graph()[0]
-        stages = networkx.get_node_attributes(G, 'stage')
+        stages = networkx.get_node_attributes(G, "stage")
         node = os.path.relpath(stage.path, self.project.root_dir)
 
         for n in networkx.dfs_postorder_nodes(G, node):
@@ -35,12 +35,11 @@ class CmdPipelineShow(CmdBase):
         stage = Stage.load(self.project, target)
         node = os.path.relpath(stage.path, self.project.root_dir)
 
-        pipelines = list(filter(lambda g: node in g.nodes(),
-                                self.project.pipelines()))
+        pipelines = list(filter(lambda g: node in g.nodes(), self.project.pipelines()))
 
         assert len(pipelines) == 1
         G = pipelines[0]
-        stages = networkx.get_node_attributes(G, 'stage')
+        stages = networkx.get_node_attributes(G, "stage")
 
         nodes = []
         for n in G.nodes():
@@ -66,8 +65,7 @@ class CmdPipelineShow(CmdBase):
             elif outs:
                 for from_out in from_stage.outs:
                     for to_out in to_stage.outs:
-                        edges.append((str(from_out),
-                                      str(to_out)))
+                        edges.append((str(from_out), str(to_out)))
             else:
                 edges.append((from_stage.relpath, to_stage.relpath))
 
@@ -101,18 +99,13 @@ class CmdPipelineShow(CmdBase):
         for target in self.args.targets:
             try:
                 if self.args.ascii:
-                    self._show_ascii(target,
-                                     self.args.commands,
-                                     self.args.outs)
+                    self._show_ascii(target, self.args.commands, self.args.outs)
                 elif self.args.dot:
-                    self.__write_dot(target,
-                                     self.args.commands,
-                                     self.args.outs,
-                                     self.args.dot)
+                    self.__write_dot(
+                        target, self.args.commands, self.args.outs, self.args.dot
+                    )
                 else:
-                    self._show(target,
-                               self.args.commands,
-                               self.args.outs)
+                    self._show(target, self.args.commands, self.args.outs)
             except DvcException:
                 msg = "failed to show pipeline for '{}'".format(target)
                 logger.error(msg)
@@ -126,67 +119,67 @@ class CmdPipelineList(CmdBase):
 
         pipelines = self.project.pipelines()
         for p in pipelines:
-            stages = networkx.get_node_attributes(p, 'stage')
+            stages = networkx.get_node_attributes(p, "stage")
             for stage in stages:
                 logger.info(stage)
             if len(stages) != 0:
-                logger.info('=' * 80)
-        logger.info('{} pipeline(s) total'.format(len(pipelines)))
+                logger.info("=" * 80)
+        logger.info("{} pipeline(s) total".format(len(pipelines)))
 
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    PIPELINE_HELP = 'Manage pipeline.'
+    PIPELINE_HELP = "Manage pipeline."
     pipeline_parser = subparsers.add_parser(
-        'pipeline',
+        "pipeline",
         parents=[parent_parser],
         description=PIPELINE_HELP,
-        help=PIPELINE_HELP)
+        help=PIPELINE_HELP,
+    )
 
     pipeline_subparsers = pipeline_parser.add_subparsers(
-        dest='cmd',
-        help="Use dvc pipeline CMD --help for command-specific help.")
+        dest="cmd", help="Use dvc pipeline CMD --help for command-specific help."
+    )
 
     fix_subparsers(pipeline_subparsers)
 
-    PIPELINE_SHOW_HELP = 'Show pipeline.'
+    PIPELINE_SHOW_HELP = "Show pipeline."
     pipeline_show_parser = pipeline_subparsers.add_parser(
-        'show',
+        "show",
         parents=[parent_parser],
         description=PIPELINE_SHOW_HELP,
-        help=PIPELINE_SHOW_HELP)
+        help=PIPELINE_SHOW_HELP,
+    )
     pipeline_show_group = pipeline_show_parser.add_mutually_exclusive_group()
     pipeline_show_group.add_argument(
-        '-c',
-        '--commands',
-        action='store_true',
+        "-c",
+        "--commands",
+        action="store_true",
         default=False,
-        help='Print commands instead of paths to DVC files.')
+        help="Print commands instead of paths to DVC files.",
+    )
     pipeline_show_group.add_argument(
-        '-o',
-        '--outs',
-        action='store_true',
+        "-o",
+        "--outs",
+        action="store_true",
         default=False,
-        help='Print output files instead of paths to DVC files.')
+        help="Print output files instead of paths to DVC files.",
+    )
     pipeline_show_parser.add_argument(
-        '--ascii',
-        action='store_true',
-        default=False,
-        help='Output DAG as ASCII.')
+        "--ascii", action="store_true", default=False, help="Output DAG as ASCII."
+    )
+    pipeline_show_parser.add_argument("--dot", help="Write DAG in .dot format.")
     pipeline_show_parser.add_argument(
-        '--dot',
-        help='Write DAG in .dot format.')
-    pipeline_show_parser.add_argument(
-        'targets',
-        nargs='*',
-        help="DVC files. 'Dvcfile' by default.")
+        "targets", nargs="*", help="DVC files. 'Dvcfile' by default."
+    )
     pipeline_show_parser.set_defaults(func=CmdPipelineShow)
 
-    PIPELINE_LIST_HELP = 'List pipelines.'
+    PIPELINE_LIST_HELP = "List pipelines."
     pipeline_list_parser = pipeline_subparsers.add_parser(
-        'list',
+        "list",
         parents=[parent_parser],
         description=PIPELINE_LIST_HELP,
-        help=PIPELINE_LIST_HELP)
+        help=PIPELINE_LIST_HELP,
+    )
     pipeline_list_parser.set_defaults(func=CmdPipelineList)

dvc/command/remote.py

@@ -32,8 +32,7 @@ class CmdRemoteAdd(CmdConfig):
     def run(self):
         remote = _get({Config.SECTION_REMOTE_URL: self.args.url})
         if remote == RemoteLOCAL:
-            self.args.url = self.resolve_path(self.args.url,
-                                              self.configobj.filename)
+            self.args.url = self.resolve_path(self.args.url, self.configobj.filename)
 
         section = Config.SECTION_REMOTE_FMT.format(self.args.name)
         ret = self._set(section, Config.SECTION_REMOTE_URL, self.args.url)
@@ -43,9 +42,9 @@ class CmdRemoteAdd(CmdConfig):
         if self.args.default:
             msg = "Setting '{}' as a default remote.".format(self.args.name)
             logger.info(msg)
-            ret = self._set(Config.SECTION_CORE,
-                            Config.SECTION_CORE_REMOTE,
-                            self.args.name)
+            ret = self._set(
+                Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, self.args.name
+            )
 
         return ret
 
@@ -61,9 +60,9 @@ class CmdRemoteRemove(CmdConfig):
             return 0
 
         if default == self.args.name:
-            return self._unset(Config.SECTION_CORE,
-                               opt=Config.SECTION_CORE_REMOTE,
-                               configobj=config)
+            return self._unset(
+                Config.SECTION_CORE, opt=Config.SECTION_CORE_REMOTE, configobj=config
+            )
 
     def run(self):
         section = Config.SECTION_REMOTE_FMT.format(self.args.name)
@@ -71,10 +70,12 @@ class CmdRemoteRemove(CmdConfig):
         if ret != 0:
             return ret
 
-        for configobj in [self.config._local_config,
-                          self.config._project_config,
-                          self.config._global_config,
-                          self.config._system_config]:
+        for configobj in [
+            self.config._local_config,
+            self.config._project_config,
+            self.config._global_config,
+            self.config._system_config,
+        ]:
             self._remove_default(configobj)
             self.config.save(configobj)
             if configobj == self.configobj:
@@ -86,14 +87,14 @@ class CmdRemoteRemove(CmdConfig):
 class CmdRemoteModify(CmdConfig):
     def run(self):
         section = Config.SECTION_REMOTE_FMT.format(self.args.name)
-        self.args.name = '{}.{}'.format(section, self.args.option)
+        self.args.name = "{}.{}".format(section, self.args.option)
         return super(CmdRemoteModify, self).run()
 
 
 class CmdRemoteDefault(CmdConfig):
     def run(self):
         self.args.value = self.args.name
-        self.args.name = 'core.remote'
+        self.args.name = "core.remote"
         return super(CmdRemoteDefault, self).run()
 
 
@@ -102,108 +103,95 @@ class CmdRemoteList(CmdConfig):
         for section in self.configobj.keys():
             r = re.match(Config.SECTION_REMOTE_REGEX, section)
             if r:
-                name = r.group('name')
-                url = self.configobj[section].get(Config.SECTION_REMOTE_URL,
-                                                  '')
-                logger.info('{}\t{}'.format(name, url))
+                name = r.group("name")
+                url = self.configobj[section].get(Config.SECTION_REMOTE_URL, "")
+                logger.info("{}\t{}".format(name, url))
         return 0
 
 
 def add_parser(subparsers, parent_parser):
     from dvc.command.config import parent_config_parser
 
-    REMOTE_HELP = 'Manage set of tracked repositories.'
+    REMOTE_HELP = "Manage set of tracked repositories."
     remote_parser = subparsers.add_parser(
-        'remote',
-        parents=[parent_parser],
-        description=REMOTE_HELP,
-        help=REMOTE_HELP)
+        "remote", parents=[parent_parser], description=REMOTE_HELP, help=REMOTE_HELP
+    )
 
     remote_subparsers = remote_parser.add_subparsers(
-        dest='cmd',
-        help='Use dvc remote CMD --help for '
-        'command-specific help.')
+        dest="cmd", help="Use dvc remote CMD --help for " "command-specific help."
+    )
 
     fix_subparsers(remote_subparsers)
 
-    REMOTE_ADD_HELP = 'Add remote.'
+    REMOTE_ADD_HELP = "Add remote."
     remote_add_parser = remote_subparsers.add_parser(
-        'add',
+        "add",
         parents=[parent_config_parser, parent_parser],
         description=REMOTE_ADD_HELP,
-        help=REMOTE_ADD_HELP)
+        help=REMOTE_ADD_HELP,
+    )
+    remote_add_parser.add_argument("name", help="Name.")
     remote_add_parser.add_argument(
-        'name',
-        help='Name.')
+        "url",
+        help="URL. See full list of supported urls at "
+        "https://dvc.org/doc/commands-reference/remote",
+    )
     remote_add_parser.add_argument(
-        'url',
-        help='URL. See full list of supported urls at '
-        'https://dvc.org/doc/commands-reference/remote')
-    remote_add_parser.add_argument(
-        '-d',
-        '--default',
-        action='store_true',
+        "-d",
+        "--default",
+        action="store_true",
         default=False,
-        help='Set as default remote.')
+        help="Set as default remote.",
+    )
     remote_add_parser.set_defaults(func=CmdRemoteAdd)
 
-    REMOTE_DEFAULT_HELP = 'Set/unset default remote.'
+    REMOTE_DEFAULT_HELP = "Set/unset default remote."
     remote_default_parser = remote_subparsers.add_parser(
-        'default',
+        "default",
         parents=[parent_config_parser, parent_parser],
         description=REMOTE_DEFAULT_HELP,
-        help=REMOTE_DEFAULT_HELP)
+        help=REMOTE_DEFAULT_HELP,
+    )
+    remote_default_parser.add_argument("name", nargs="?", help="Name of the remote.")
     remote_default_parser.add_argument(
-        'name',
-        nargs='?',
-        help='Name of the remote.')
-    remote_default_parser.add_argument(
-        '-u',
-        '--unset',
-        action='store_true',
+        "-u",
+        "--unset",
+        action="store_true",
         default=False,
-        help='Unset default remote.')
+        help="Unset default remote.",
+    )
     remote_default_parser.set_defaults(func=CmdRemoteDefault)
 
-    REMOTE_REMOVE_HELP = 'Remove remote.'
+    REMOTE_REMOVE_HELP = "Remove remote."
     remote_remove_parser = remote_subparsers.add_parser(
-        'remove',
+        "remove",
         parents=[parent_config_parser, parent_parser],
         description=REMOTE_REMOVE_HELP,
-        help=REMOTE_REMOVE_HELP)
-    remote_remove_parser.add_argument(
-        'name',
-        help='Name')
+        help=REMOTE_REMOVE_HELP,
+    )
+    remote_remove_parser.add_argument("name", help="Name")
     remote_remove_parser.set_defaults(func=CmdRemoteRemove)
 
-    REMOTE_MODIFY_HELP = 'Modify remote.'
+    REMOTE_MODIFY_HELP = "Modify remote."
     remote_modify_parser = remote_subparsers.add_parser(
-        'modify',
+        "modify",
         parents=[parent_config_parser, parent_parser],
         description=REMOTE_MODIFY_HELP,
-        help=REMOTE_MODIFY_HELP)
+        help=REMOTE_MODIFY_HELP,
+    )
+    remote_modify_parser.add_argument("name", help="Name.")
+    remote_modify_parser.add_argument("option", help="Option.")
+    remote_modify_parser.add_argument("value", nargs="?", help="Value.")
     remote_modify_parser.add_argument(
-        'name',
-        help='Name.')
-    remote_modify_parser.add_argument(
-        'option',
-        help='Option.')
-    remote_modify_parser.add_argument(
-        'value',
-        nargs='?',
-        help='Value.')
-    remote_modify_parser.add_argument(
-        '-u',
-        '--unset',
-        default=False,
-        action='store_true',
-        help='Unset option.')
+        "-u", "--unset", default=False, action="store_true", help="Unset option."
+    )
     remote_modify_parser.set_defaults(func=CmdRemoteModify)
 
-    REMOTE_LIST_HELP = 'List remotes.'
+    REMOTE_LIST_HELP = "List remotes."
     remote_list_parser = remote_subparsers.add_parser(
-        'list',
+        "list",
         parents=[parent_config_parser, parent_parser],
         description=REMOTE_LIST_HELP,
-        help=REMOTE_LIST_HELP)
+        help=REMOTE_LIST_HELP,
+    )
     remote_list_parser.set_defaults(func=CmdRemoteList)

dvc/command/remove.py

@@ -14,17 +14,13 @@ class CmdRemove(CmdBase):
         if self.args.force:
             return False
 
-        msg = (
-            'Are you sure you want to remove {} with its outputs?'
-            .format(target)
-        )
+        msg = "Are you sure you want to remove {} with its outputs?".format(target)
 
         if prompt.confirm(msg):
             return False
 
         raise DvcException(
-            'Cannot purge without a confirmation from the user.'
-            " Use '-f' to force."
+            "Cannot purge without a confirmation from the user." " Use '-f' to force."
         )
 
     def run(self):
@@ -33,39 +29,33 @@ class CmdRemove(CmdBase):
                 outs_only = self._is_outs_only(target)
                 self.project.remove(target, outs_only=outs_only)
             except DvcException:
-                logger.error('failed to remove {}'.format(target))
+                logger.error("failed to remove {}".format(target))
                 return 1
         return 0
 
 
 def add_parser(subparsers, parent_parser):
-    REMOVE_HELP = 'Remove outputs of DVC file.'
+    REMOVE_HELP = "Remove outputs of DVC file."
     remove_parser = subparsers.add_parser(
-        'remove',
-        parents=[parent_parser],
-        description=REMOVE_HELP,
-        help=REMOVE_HELP)
+        "remove", parents=[parent_parser], description=REMOVE_HELP, help=REMOVE_HELP
+    )
     remove_parser_group = remove_parser.add_mutually_exclusive_group()
     remove_parser_group.add_argument(
-        '-o',
-        '--outs',
-        action='store_true',
+        "-o",
+        "--outs",
+        action="store_true",
         default=True,
-        help='Only remove DVC file outputs.(default)')
+        help="Only remove DVC file outputs.(default)",
+    )
     remove_parser_group.add_argument(
-        '-p',
-        '--purge',
-        action='store_true',
+        "-p",
+        "--purge",
+        action="store_true",
         default=False,
-        help='Remove DVC file and all its outputs.')
+        help="Remove DVC file and all its outputs.",
+    )
     remove_parser.add_argument(
-        '-f',
-        '--force',
-        action='store_true',
-        default=False,
-        help='Force purge.')
-    remove_parser.add_argument(
-        'targets',
-        nargs='+',
-        help='DVC files.')
+        "-f", "--force", action="store_true", default=False, help="Force purge."
+    )
+    remove_parser.add_argument("targets", nargs="+", help="DVC files.")
     remove_parser.set_defaults(func=CmdRemove)

dvc/command/repro.py

@@ -25,14 +25,15 @@ class CmdRepro(CmdBase):
         for target in self.args.targets:
             try:
                 stages = self.project.reproduce(
-                               target,
-                               recursive=recursive,
-                               force=self.args.force,
-                               dry=self.args.dry,
-                               interactive=self.args.interactive,
-                               pipeline=self.args.pipeline,
-                               all_pipelines=self.args.all_pipelines,
-                               ignore_build_cache=self.args.ignore_build_cache)
+                    target,
+                    recursive=recursive,
+                    force=self.args.force,
+                    dry=self.args.dry,
+                    interactive=self.args.interactive,
+                    pipeline=self.args.pipeline,
+                    all_pipelines=self.args.all_pipelines,
+                    ignore_build_cache=self.args.ignore_build_cache,
+                )
 
                 if len(stages) == 0:
                     logger.info(CmdDataStatus.UP_TO_DATE_MSG)
@@ -51,67 +52,70 @@ class CmdRepro(CmdBase):
 def add_parser(subparsers, parent_parser):
     REPRO_HELP = "Reproduce DVC file. Default file name - 'Dvcfile'."
     repro_parser = subparsers.add_parser(
-        'repro',
-        parents=[parent_parser],
-        description=REPRO_HELP,
-        help=REPRO_HELP)
+        "repro", parents=[parent_parser], description=REPRO_HELP, help=REPRO_HELP
+    )
+    repro_parser.add_argument("targets", nargs="*", help="DVC file to reproduce.")
     repro_parser.add_argument(
-        'targets',
-        nargs='*',
-        help='DVC file to reproduce.')
-    repro_parser.add_argument(
-        '-f',
-        '--force',
-        action='store_true',
+        "-f",
+        "--force",
+        action="store_true",
         default=False,
-        help='Reproduce even if dependencies were not changed.')
+        help="Reproduce even if dependencies were not changed.",
+    )
     repro_parser.add_argument(
-        '-s',
-        '--single-item',
-        action='store_true',
+        "-s",
+        "--single-item",
+        action="store_true",
         default=False,
-        help='Reproduce only single data item without recursive dependencies '
-             'check.')
+        help="Reproduce only single data item without recursive dependencies " "check.",
+    )
     repro_parser.add_argument(
-        '-c',
-        '--cwd',
+        "-c",
+        "--cwd",
         default=os.path.curdir,
-        help='Directory within your project to reroduce from.')
+        help="Directory within your project to reroduce from.",
+    )
     repro_parser.add_argument(
-        '-m',
-        '--metrics',
-        action='store_true',
+        "-m",
+        "--metrics",
+        action="store_true",
         default=False,
-        help='Show metrics after reproduction.')
+        help="Show metrics after reproduction.",
+    )
     repro_parser.add_argument(
-        '--dry',
-        action='store_true',
+        "--dry",
+        action="store_true",
         default=False,
-        help='Only print the commands that would be executed without '
-             'actually executing.')
+        help="Only print the commands that would be executed without "
+        "actually executing.",
+    )
     repro_parser.add_argument(
-        '-i',
-        '--interactive',
-        action='store_true',
+        "-i",
+        "--interactive",
+        action="store_true",
         default=False,
-        help='Ask for confirmation before reproducing each stage.')
+        help="Ask for confirmation before reproducing each stage.",
+    )
     repro_parser.add_argument(
-        '-p',
-        '--pipeline',
-        action='store_true',
+        "-p",
+        "--pipeline",
+        action="store_true",
         default=False,
-        help='Reproduce the whole pipeline that the specified stage file '
-             'belongs to.')
+        help="Reproduce the whole pipeline that the specified stage file "
+        "belongs to.",
+    )
     repro_parser.add_argument(
-        '-P',
-        '--all-pipelines',
-        action='store_true',
+        "-P",
+        "--all-pipelines",
+        action="store_true",
         default=False,
-        help='Reproduce all pipelines in the project.')
+        help="Reproduce all pipelines in the project.",
+    )
     repro_parser.add_argument(
-        '--ignore-build-cache',
-        action='store_true',
+        "--ignore-build-cache",
+        action="store_true",
         default=False,
         help="Reproduce all descendants of a changed stage even if their "
-             "direct dependencies didn't change.")
+        "direct dependencies didn't change.",
+    )
     repro_parser.set_defaults(func=CmdRepro)

dvc/command/root.py

@@ -18,8 +18,6 @@ class CmdRoot(CmdBase):
 def add_parser(subparsers, parent_parser):
     ROOT_HELP = "Relative path to project's directory."
     root_parser = subparsers.add_parser(
-        'root',
-        parents=[parent_parser],
-        description=ROOT_HELP,
-        help=ROOT_HELP)
+        "root", parents=[parent_parser], description=ROOT_HELP, help=ROOT_HELP
+    )
     root_parser.set_defaults(func=CmdRoot)

dvc/command/run.py

@@ -10,30 +10,33 @@ from dvc.exceptions import DvcException
 
 class CmdRun(CmdBase):
     def run(self):
-        overwrite = (self.args.yes or self.args.overwrite_dvcfile)
+        overwrite = self.args.yes or self.args.overwrite_dvcfile
 
-        if not any([self.args.deps,
-                    self.args.outs,
-                    self.args.outs_no_cache,
-                    self.args.command]):  # pragma: no cover
-            logger.error("too few arguments. Specify at least one: '-d', "
-                         "'-o', '-O', 'command'.")
+        if not any(
+            [self.args.deps, self.args.outs, self.args.outs_no_cache, self.args.command]
+        ):  # pragma: no cover
+            logger.error(
+                "too few arguments. Specify at least one: '-d', "
+                "'-o', '-O', 'command'."
+            )
             return 1
 
         try:
-            self.project.run(cmd=self._parsed_cmd(),
-                             outs=self.args.outs,
-                             outs_no_cache=self.args.outs_no_cache,
-                             metrics_no_cache=self.args.metrics_no_cache,
-                             deps=self.args.deps,
-                             fname=self.args.file,
-                             cwd=self.args.cwd,
-                             no_exec=self.args.no_exec,
-                             overwrite=overwrite,
-                             ignore_build_cache=self.args.ignore_build_cache,
-                             remove_outs=self.args.remove_outs)
+            self.project.run(
+                cmd=self._parsed_cmd(),
+                outs=self.args.outs,
+                outs_no_cache=self.args.outs_no_cache,
+                metrics_no_cache=self.args.metrics_no_cache,
+                deps=self.args.deps,
+                fname=self.args.file,
+                cwd=self.args.cwd,
+                no_exec=self.args.no_exec,
+                overwrite=overwrite,
+                ignore_build_cache=self.args.ignore_build_cache,
+                remove_outs=self.args.remove_outs,
+            )
         except DvcException:
-            logger.error('failed to run command')
+            logger.error("failed to run command")
             return 1
 
         return 0
@@ -49,97 +52,103 @@ class CmdRun(CmdBase):
                 dvc run echo "foo bar"
         """
         if len(self.args.command) < 2:
-            return ' '.join(self.args.command)
+            return " ".join(self.args.command)
 
-        return ' '.join(self._quote_argument(arg) for arg in self.args.command)
+        return " ".join(self._quote_argument(arg) for arg in self.args.command)
 
     def _quote_argument(self, argument):
-        if ' ' not in argument or '"' in argument:
+        if " " not in argument or '"' in argument:
             return argument
 
         return '"{}"'.format(argument)
 
 
 def add_parser(subparsers, parent_parser):
-    RUN_HELP = 'Generate a stage file from a given ' \
-               'command and execute the command.'
+    RUN_HELP = "Generate a stage file from a given " "command and execute the command."
     run_parser = subparsers.add_parser(
-        'run',
-        parents=[parent_parser],
-        description=RUN_HELP,
-        help=RUN_HELP)
+        "run", parents=[parent_parser], description=RUN_HELP, help=RUN_HELP
+    )
     run_parser.add_argument(
-        '-d',
-        '--deps',
-        action='append',
+        "-d",
+        "--deps",
+        action="append",
         default=[],
-        help='Declare dependencies for reproducible cmd.')
+        help="Declare dependencies for reproducible cmd.",
+    )
     run_parser.add_argument(
-        '-o',
-        '--outs',
-        action='append',
+        "-o",
+        "--outs",
+        action="append",
         default=[],
-        help='Declare output data file or data directory.')
+        help="Declare output data file or data directory.",
+    )
     run_parser.add_argument(
-        '-O',
-        '--outs-no-cache',
-        action='append',
+        "-O",
+        "--outs-no-cache",
+        action="append",
         default=[],
-        help='Declare output regular file or directory '
-             '(sync to Git, not DVC cache).')
+        help="Declare output regular file or directory "
+        "(sync to Git, not DVC cache).",
+    )
     run_parser.add_argument(
-        '-M',
-        '--metrics-no-cache',
-        action='append',
+        "-M",
+        "--metrics-no-cache",
+        action="append",
         default=[],
-        help='Declare output metric file or '
-             'directory (not cached by DVC).')
+        help="Declare output metric file or " "directory (not cached by DVC).",
+    )
     run_parser.add_argument(
-        '-f',
-        '--file',
+        "-f",
+        "--file",
         help="Specify name of the stage file. It should be "
-             "either 'Dvcfile' or have a '.dvc' suffix (e.g. "
-             "'prepare.dvc', 'clean.dvc', etc) in order for "
-             "dvc to be able to find it later. By default "
-             "the first output basename + .dvc is used as "
-             "a stage filename.")
+        "either 'Dvcfile' or have a '.dvc' suffix (e.g. "
+        "'prepare.dvc', 'clean.dvc', etc) in order for "
+        "dvc to be able to find it later. By default "
+        "the first output basename + .dvc is used as "
+        "a stage filename.",
+    )
     run_parser.add_argument(
-        '-c',
-        '--cwd',
+        "-c",
+        "--cwd",
         default=os.path.curdir,
         help="Directory within your project to run your command and place "
-             "stage file in.")
+        "stage file in.",
+    )
     run_parser.add_argument(
-        '--no-exec',
-        action='store_true',
+        "--no-exec",
+        action="store_true",
         default=False,
-        help="Only create stage file without actually running it.")
+        help="Only create stage file without actually running it.",
+    )
     run_parser.add_argument(
-        '-y',
-        '--yes',
-        action='store_true',
+        "-y",
+        "--yes",
+        action="store_true",
         default=False,
         help="(OBSOLETED, use --overwrite-dvcfile instead) Automatic 'yes' "
-             "answer to all prompts. E.g. when '.dvc' file exists and dvc "
-             "asks if you want to overwrite it.")
+        "answer to all prompts. E.g. when '.dvc' file exists and dvc "
+        "asks if you want to overwrite it.",
+    )
     run_parser.add_argument(
-        '--overwrite-dvcfile',
-        action='store_true',
+        "--overwrite-dvcfile",
+        action="store_true",
         default=False,
-        help="Overwrite existing dvc file without asking for confirmation.")
+        help="Overwrite existing dvc file without asking for confirmation.",
+    )
     run_parser.add_argument(
-        '--ignore-build-cache',
-        action='store_true',
+        "--ignore-build-cache",
+        action="store_true",
         default=False,
         help="Run this stage even if it has been already ran with the same "
-             "command/dependencies/outputs/etc before.")
+        "command/dependencies/outputs/etc before.",
+    )
     run_parser.add_argument(
-        '--remove-outs',
-        action='store_true',
+        "--remove-outs",
+        action="store_true",
         default=False,
-        help="Remove outputs before running the command.")
+        help="Remove outputs before running the command.",
+    )
     run_parser.add_argument(
-        'command',
-        nargs=argparse.REMAINDER,
-        help='Command to execute.')
+        "command", nargs=argparse.REMAINDER, help="Command to execute."
+    )
     run_parser.set_defaults(func=CmdRun)

dvc/command/status.py

@@ -6,36 +6,38 @@ from dvc.command.data_sync import CmdDataBase
 
 class CmdDataStatus(CmdDataBase):
     STATUS_LEN = 10
-    STATUS_INDENT = '\t'
+    STATUS_INDENT = "\t"
     UP_TO_DATE_MSG = "Pipeline is up to date. Nothing to reproduce."
 
     def _normalize(self, s):
-        s += ':'
+        s += ":"
         assert len(s) < self.STATUS_LEN
-        return s + (self.STATUS_LEN - len(s))*' '
+        return s + (self.STATUS_LEN - len(s)) * " "
 
     def _show(self, status, indent=0):
         ind = indent * self.STATUS_INDENT
 
         for key, value in status.items():
             if isinstance(value, dict):
-                logger.info('{}{}'.format(ind, key))
-                self._show(value, indent+1)
+                logger.info("{}{}".format(ind, key))
+                self._show(value, indent + 1)
             else:
-                msg = '{}{}{}'.format(ind, self._normalize(value), key)
+                msg = "{}{}{}".format(ind, self._normalize(value), key)
                 logger.info(msg)
 
     def do_run(self, target=None):
         indent = 1 if self.args.cloud else 0
         try:
-            st = self.project.status(target=target,
-                                     jobs=self.args.jobs,
-                                     cloud=self.args.cloud,
-                                     show_checksums=self.args.show_checksums,
-                                     remote=self.args.remote,
-                                     all_branches=self.args.all_branches,
-                                     all_tags=self.args.all_tags,
-                                     with_deps=self.args.with_deps)
+            st = self.project.status(
+                target=target,
+                jobs=self.args.jobs,
+                cloud=self.args.cloud,
+                show_checksums=self.args.show_checksums,
+                remote=self.args.remote,
+                all_branches=self.args.all_branches,
+                all_tags=self.args.all_tags,
+                with_deps=self.args.with_deps,
+            )
             if st:
                 if self.args.quiet:
                     return 1
@@ -45,6 +47,6 @@ class CmdDataStatus(CmdDataBase):
                 logger.info(self.UP_TO_DATE_MSG)
 
         except Exception:
-            logger.error('failed to obtain data status')
+            logger.error("failed to obtain data status")
             return 1
         return 0

dvc/command/unprotect.py

@@ -18,14 +18,12 @@ class CmdUnprotect(CmdBase):
 
 
 def add_parser(subparsers, parent_parser):
-    UNPROTECT_HELP = 'Unprotect data file/directory.'
+    UNPROTECT_HELP = "Unprotect data file/directory."
     unprotect_parser = subparsers.add_parser(
-        'unprotect',
+        "unprotect",
         parents=[parent_parser],
         description=UNPROTECT_HELP,
-        help=UNPROTECT_HELP)
-    unprotect_parser.add_argument(
-        'targets',
-        nargs='+',
-        help='Data files/directory.')
+        help=UNPROTECT_HELP,
+    )
+    unprotect_parser.add_argument("targets", nargs="+", help="Data files/directory.")
     unprotect_parser.set_defaults(func=CmdUnprotect)

dvc/config.py

@@ -20,10 +20,9 @@ class ConfigError(DvcException):
         msg (str): error message.
         ex (Exception): optional exception that has caused this error.
     """
+
     def __init__(self, msg, ex=None):
-        super(ConfigError, self).__init__(
-            'config file error: {}'.format(msg), ex
-        )
+        super(ConfigError, self).__init__("config file error: {}".format(msg), ex)
 
 
 def supported_cache_type(types):
@@ -33,9 +32,9 @@ def supported_cache_type(types):
         types (list/string): type(s) of links that dvc should try out.
     """
     if isinstance(types, str):
-        types = [typ.strip() for typ in types.split(',')]
+        types = [typ.strip() for typ in types.split(",")]
     for typ in types:
-        if typ not in ['reflink', 'hardlink', 'symlink', 'copy']:
+        if typ not in ["reflink", "hardlink", "symlink", "copy"]:
             return False
     return True
 
@@ -46,7 +45,7 @@ def supported_loglevel(level):
     Args:
         level (str): log level name.
     """
-    return level in ['info', 'debug', 'warning', 'error']
+    return level in ["info", "debug", "warning", "error"]
 
 
 def supported_cloud(cloud):
@@ -55,7 +54,7 @@ def supported_cloud(cloud):
     Args:
         cloud (str): cloud type name.
     """
-    return cloud in ['aws', 'gcp', 'local', '']
+    return cloud in ["aws", "gcp", "local", ""]
 
 
 def is_bool(val):
@@ -67,7 +66,7 @@ def is_bool(val):
     Returns:
         bool: True if value stands for boolean, False otherwise.
     """
-    return val.lower() in ['true', 'false']
+    return val.lower() in ["true", "false"]
 
 
 def to_bool(val):
@@ -79,7 +78,7 @@ def to_bool(val):
     Returns:
         bool: True if value.lower() == 'true', False otherwise.
     """
-    return val.lower() == 'true'
+    return val.lower() == "true"
 
 
 def is_whole(val):
@@ -120,36 +119,37 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     Raises:
         ConfigError: thrown when config has an invalid format.
     """
-    APPNAME = 'dvc'
-    APPAUTHOR = 'iterative'
+
+    APPNAME = "dvc"
+    APPAUTHOR = "iterative"
 
     # NOTE: used internally in RemoteLOCAL to know config
     # location, that url should resolved relative to.
-    PRIVATE_CWD = '_cwd'
+    PRIVATE_CWD = "_cwd"
 
-    CONFIG = 'config'
-    CONFIG_LOCAL = 'config.local'
+    CONFIG = "config"
+    CONFIG_LOCAL = "config.local"
 
-    SECTION_CORE = 'core'
-    SECTION_CORE_LOGLEVEL = 'loglevel'
+    SECTION_CORE = "core"
+    SECTION_CORE_LOGLEVEL = "loglevel"
     SECTION_CORE_LOGLEVEL_SCHEMA = And(Use(str.lower), supported_loglevel)
-    SECTION_CORE_REMOTE = 'remote'
+    SECTION_CORE_REMOTE = "remote"
     SECTION_CORE_INTERACTIVE_SCHEMA = And(str, is_bool, Use(to_bool))
-    SECTION_CORE_INTERACTIVE = 'interactive'
-    SECTION_CORE_ANALYTICS = 'analytics'
+    SECTION_CORE_INTERACTIVE = "interactive"
+    SECTION_CORE_ANALYTICS = "analytics"
     SECTION_CORE_ANALYTICS_SCHEMA = And(str, is_bool, Use(to_bool))
 
-    SECTION_CACHE = 'cache'
-    SECTION_CACHE_DIR = 'dir'
-    SECTION_CACHE_TYPE = 'type'
+    SECTION_CACHE = "cache"
+    SECTION_CACHE_DIR = "dir"
+    SECTION_CACHE_TYPE = "type"
     SECTION_CACHE_TYPE_SCHEMA = supported_cache_type
-    SECTION_CACHE_PROTECTED = 'protected'
-    SECTION_CACHE_LOCAL = 'local'
-    SECTION_CACHE_S3 = 's3'
-    SECTION_CACHE_GS = 'gs'
-    SECTION_CACHE_SSH = 'ssh'
-    SECTION_CACHE_HDFS = 'hdfs'
-    SECTION_CACHE_AZURE = 'azure'
+    SECTION_CACHE_PROTECTED = "protected"
+    SECTION_CACHE_LOCAL = "local"
+    SECTION_CACHE_S3 = "s3"
+    SECTION_CACHE_GS = "gs"
+    SECTION_CACHE_SSH = "ssh"
+    SECTION_CACHE_HDFS = "hdfs"
+    SECTION_CACHE_AZURE = "azure"
     SECTION_CACHE_SCHEMA = {
         Optional(SECTION_CACHE_LOCAL): str,
         Optional(SECTION_CACHE_S3): str,
@@ -157,92 +157,87 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         Optional(SECTION_CACHE_HDFS): str,
         Optional(SECTION_CACHE_SSH): str,
         Optional(SECTION_CACHE_AZURE): str,
-
         Optional(SECTION_CACHE_DIR): str,
         Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA,
-        Optional(SECTION_CACHE_PROTECTED,
-                 default=False): And(str, is_bool, Use(to_bool)),
-
+        Optional(SECTION_CACHE_PROTECTED, default=False): And(
+            str, is_bool, Use(to_bool)
+        ),
         Optional(PRIVATE_CWD): str,
     }
 
     # backward compatibility
-    SECTION_CORE_CLOUD = 'cloud'
+    SECTION_CORE_CLOUD = "cloud"
     SECTION_CORE_CLOUD_SCHEMA = And(Use(str.lower), supported_cloud)
-    SECTION_CORE_STORAGEPATH = 'storagepath'
+    SECTION_CORE_STORAGEPATH = "storagepath"
 
     SECTION_CORE_SCHEMA = {
-        Optional(SECTION_CORE_LOGLEVEL,
-                 default='info'): And(str, Use(str.lower),
-                                      SECTION_CORE_LOGLEVEL_SCHEMA),
-        Optional(SECTION_CORE_REMOTE, default=''): And(str, Use(str.lower)),
-        Optional(SECTION_CORE_INTERACTIVE,
-                 default=False): SECTION_CORE_INTERACTIVE_SCHEMA,
-        Optional(SECTION_CORE_ANALYTICS,
-                 default=True): SECTION_CORE_ANALYTICS_SCHEMA,
-
+        Optional(SECTION_CORE_LOGLEVEL, default="info"): And(
+            str, Use(str.lower), SECTION_CORE_LOGLEVEL_SCHEMA
+        ),
+        Optional(SECTION_CORE_REMOTE, default=""): And(str, Use(str.lower)),
+        Optional(
+            SECTION_CORE_INTERACTIVE, default=False
+        ): SECTION_CORE_INTERACTIVE_SCHEMA,
+        Optional(SECTION_CORE_ANALYTICS, default=True): SECTION_CORE_ANALYTICS_SCHEMA,
         # backward compatibility
-        Optional(SECTION_CORE_CLOUD, default=''): SECTION_CORE_CLOUD_SCHEMA,
-        Optional(SECTION_CORE_STORAGEPATH, default=''): str,
+        Optional(SECTION_CORE_CLOUD, default=""): SECTION_CORE_CLOUD_SCHEMA,
+        Optional(SECTION_CORE_STORAGEPATH, default=""): str,
     }
 
     # backward compatibility
-    SECTION_AWS = 'aws'
-    SECTION_AWS_STORAGEPATH = 'storagepath'
-    SECTION_AWS_CREDENTIALPATH = 'credentialpath'
-    SECTION_AWS_ENDPOINT_URL = 'endpointurl'
-    SECTION_AWS_REGION = 'region'
-    SECTION_AWS_PROFILE = 'profile'
-    SECTION_AWS_USE_SSL = 'use_ssl'
+    SECTION_AWS = "aws"
+    SECTION_AWS_STORAGEPATH = "storagepath"
+    SECTION_AWS_CREDENTIALPATH = "credentialpath"
+    SECTION_AWS_ENDPOINT_URL = "endpointurl"
+    SECTION_AWS_REGION = "region"
+    SECTION_AWS_PROFILE = "profile"
+    SECTION_AWS_USE_SSL = "use_ssl"
     SECTION_AWS_SCHEMA = {
         SECTION_AWS_STORAGEPATH: str,
         Optional(SECTION_AWS_REGION): str,
         Optional(SECTION_AWS_PROFILE): str,
         Optional(SECTION_AWS_CREDENTIALPATH): str,
         Optional(SECTION_AWS_ENDPOINT_URL): str,
-        Optional(SECTION_AWS_USE_SSL,
-                 default=True): And(str, is_bool, Use(to_bool)),
+        Optional(SECTION_AWS_USE_SSL, default=True): And(str, is_bool, Use(to_bool)),
     }
 
     # backward compatibility
-    SECTION_GCP = 'gcp'
+    SECTION_GCP = "gcp"
     SECTION_GCP_STORAGEPATH = SECTION_AWS_STORAGEPATH
-    SECTION_GCP_PROJECTNAME = 'projectname'
+    SECTION_GCP_PROJECTNAME = "projectname"
     SECTION_GCP_SCHEMA = {
         SECTION_GCP_STORAGEPATH: str,
         Optional(SECTION_GCP_PROJECTNAME): str,
     }
 
     # backward compatibility
-    SECTION_LOCAL = 'local'
+    SECTION_LOCAL = "local"
     SECTION_LOCAL_STORAGEPATH = SECTION_AWS_STORAGEPATH
-    SECTION_LOCAL_SCHEMA = {
-        SECTION_LOCAL_STORAGEPATH: str,
-    }
+    SECTION_LOCAL_SCHEMA = {SECTION_LOCAL_STORAGEPATH: str}
 
-    SECTION_AZURE_CONNECTION_STRING = 'connection_string'
+    SECTION_AZURE_CONNECTION_STRING = "connection_string"
 
     SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P<name>.*)"\s*$'
     SECTION_REMOTE_FMT = 'remote "{}"'
-    SECTION_REMOTE_URL = 'url'
-    SECTION_REMOTE_USER = 'user'
-    SECTION_REMOTE_PORT = 'port'
-    SECTION_REMOTE_KEY_FILE = 'keyfile'
-    SECTION_REMOTE_TIMEOUT = 'timeout'
-    SECTION_REMOTE_PASSWORD = 'password'
-    SECTION_REMOTE_ASK_PASSWORD = 'ask_password'
+    SECTION_REMOTE_URL = "url"
+    SECTION_REMOTE_USER = "user"
+    SECTION_REMOTE_PORT = "port"
+    SECTION_REMOTE_KEY_FILE = "keyfile"
+    SECTION_REMOTE_TIMEOUT = "timeout"
+    SECTION_REMOTE_PASSWORD = "password"
+    SECTION_REMOTE_ASK_PASSWORD = "ask_password"
     SECTION_REMOTE_SCHEMA = {
         SECTION_REMOTE_URL: str,
         Optional(SECTION_AWS_REGION): str,
         Optional(SECTION_AWS_PROFILE): str,
         Optional(SECTION_AWS_CREDENTIALPATH): str,
         Optional(SECTION_AWS_ENDPOINT_URL): str,
-        Optional(SECTION_AWS_USE_SSL,
-                 default=True): And(str, is_bool, Use(to_bool)),
+        Optional(SECTION_AWS_USE_SSL, default=True): And(str, is_bool, Use(to_bool)),
         Optional(SECTION_GCP_PROJECTNAME): str,
         Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA,
-        Optional(SECTION_CACHE_PROTECTED,
-                 default=False): And(str, is_bool, Use(to_bool)),
+        Optional(SECTION_CACHE_PROTECTED, default=False): And(
+            str, is_bool, Use(to_bool)
+        ),
         Optional(SECTION_REMOTE_USER): str,
         Optional(SECTION_REMOTE_PORT): Use(int),
         Optional(SECTION_REMOTE_KEY_FILE): str,
@@ -250,13 +245,12 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         Optional(SECTION_REMOTE_PASSWORD): str,
         Optional(SECTION_REMOTE_ASK_PASSWORD): And(str, is_bool, Use(to_bool)),
         Optional(SECTION_AZURE_CONNECTION_STRING): str,
-
         Optional(PRIVATE_CWD): str,
     }
 
-    SECTION_STATE = 'state'
-    SECTION_STATE_ROW_LIMIT = 'row_limit'
-    SECTION_STATE_ROW_CLEANUP_QUOTA = 'row_cleanup_quota'
+    SECTION_STATE = "state"
+    SECTION_STATE_ROW_LIMIT = "row_limit"
+    SECTION_STATE_ROW_CLEANUP_QUOTA = "row_cleanup_quota"
     SECTION_STATE_SCHEMA = {
         Optional(SECTION_STATE_ROW_LIMIT): And(Use(int), is_whole),
         Optional(SECTION_STATE_ROW_CLEANUP_QUOTA): And(Use(int), is_percent),
@@ -267,7 +261,6 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         Optional(Regex(SECTION_REMOTE_REGEX)): SECTION_REMOTE_SCHEMA,
         Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA,
         Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA,
-
         # backward compatibility
         Optional(SECTION_AWS, default={}): SECTION_AWS_SCHEMA,
         Optional(SECTION_GCP, default={}): SECTION_GCP_SCHEMA,
@@ -275,10 +268,12 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
     }
 
     def __init__(self, dvc_dir=None, validate=True):
-        self.system_config_file = os.path.join(self.get_system_config_dir(),
-                                               self.CONFIG)
-        self.global_config_file = os.path.join(self.get_global_config_dir(),
-                                               self.CONFIG)
+        self.system_config_file = os.path.join(
+            self.get_system_config_dir(), self.CONFIG
+        )
+        self.global_config_file = os.path.join(
+            self.get_global_config_dir(), self.CONFIG
+        )
 
         if dvc_dir is not None:
             self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
@@ -306,8 +301,8 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
             str: path to the global config directory.
         """
         from appdirs import user_config_dir
-        return user_config_dir(appname=Config.APPNAME,
-                               appauthor=Config.APPAUTHOR)
+
+        return user_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR)
 
     @staticmethod
     def get_system_config_dir():
@@ -317,8 +312,8 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
             str: path to the system config directory.
         """
         from appdirs import site_config_dir
-        return site_config_dir(appname=Config.APPNAME,
-                               appauthor=Config.APPAUTHOR)
+
+        return site_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR)
 
     @staticmethod
     def init(dvc_dir):
@@ -331,7 +326,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
             dvc.config.Config: config object.
         """
         config_file = os.path.join(dvc_dir, Config.CONFIG)
-        open(config_file, 'w+').close()
+        open(config_file, "w+").close()
         return Config(dvc_dir)
 
     def _load(self):
@@ -408,8 +403,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
                 self.config = Schema(self.SCHEMA).validate(self.config)
 
             # NOTE: now converting back to ConfigObj
-            self.config = configobj.ConfigObj(self.config,
-                                              write_empty_values=True)
+            self.config = configobj.ConfigObj(self.config, write_empty_values=True)
             self.config.filename = self.config_file
             self._resolve_paths(self.config, self.config_file)
         except Exception as ex:
@@ -439,10 +433,12 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
         if config is not None:
             clist = [config]
         else:
-            clist = [self._system_config,
-                     self._global_config,
-                     self._project_config,
-                     self._local_config]
+            clist = [
+                self._system_config,
+                self._global_config,
+                self._project_config,
+                self._local_config,
+            ]
 
         for conf in clist:
             if conf.filename is None:
@@ -478,8 +474,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
             return
 
         if opt not in config[section].keys():
-            raise ConfigError("option '{}.{}' doesn't exist".format(section,
-                                                                    opt))
+            raise ConfigError("option '{}.{}' doesn't exist".format(section, opt))
         del config[section][opt]
 
         if not config[section]:
@@ -513,8 +508,7 @@ class Config(object):  # pylint: disable=too-many-instance-attributes
             raise ConfigError("section '{}' doesn't exist".format(section))
 
         if opt not in config[section].keys():
-            raise ConfigError("option '{}.{}' doesn't exist".format(section,
-                                                                    opt))
+            raise ConfigError("option '{}.{}' doesn't exist".format(section, opt))
 
         logger.info(config[section][opt])
 

dvc/daemon.py

@@ -23,12 +23,14 @@ def _spawn_windows(cmd, env):
     startupinfo = STARTUPINFO()
     startupinfo.dwFlags |= STARTF_USESHOWWINDOW
 
-    Popen(cmd,
-          env=env,
-          close_fds=True,
-          shell=False,
-          creationflags=creationflags,
-          startupinfo=startupinfo).communicate()
+    Popen(
+        cmd,
+        env=env,
+        close_fds=True,
+        shell=False,
+        creationflags=creationflags,
+        startupinfo=startupinfo,
+    ).communicate()
 
 
 def _spawn_posix(cmd, env):
@@ -71,18 +73,18 @@ def daemon(args):
     """
     cmd = [sys.executable]
     if not is_binary():
-        cmd += ['-m', 'dvc']
-    cmd += ['daemon', '-q'] + args
+        cmd += ["-m", "dvc"]
+    cmd += ["daemon", "-q"] + args
 
     env = fix_env()
     file_path = os.path.abspath(inspect.stack()[0][1])
-    env['PYTHONPATH'] = os.path.dirname(os.path.dirname(file_path))
+    env["PYTHONPATH"] = os.path.dirname(os.path.dirname(file_path))
 
     logger.debug("Trying to spawn '{}' with env '{}'".format(cmd, env))
 
-    if os.name == 'nt':
+    if os.name == "nt":
         _spawn_windows(cmd, env)
-    elif os.name == 'posix':
+    elif os.name == "posix":
         _spawn_posix(cmd, env)
     else:
         raise NotImplementedError

dvc/dagascii.py

@@ -18,6 +18,7 @@ class VertexViewer(object):
     Args:
         name (str): name of the vertex.
     """
+
     HEIGHT = 3  # top and bottom box edges + text
 
     def __init__(self, name):
@@ -43,6 +44,7 @@ class AsciiCanvas(object):
         cols (int): number of columns in the canvas. Should be > 1.
         lines (int): number of lines in the canvas. Should be > 1.
     """
+
     TIMEOUT = 10
 
     def __init__(self, cols, lines):
@@ -52,16 +54,17 @@ class AsciiCanvas(object):
         self.cols = cols
         self.lines = lines
 
-        self.canvas = [[' '] * cols for l in range(lines)]
+        self.canvas = [[" "] * cols for l in range(lines)]
 
     def draw(self):
         """Draws ASCII canvas on the screen."""
         if sys.stdout.isatty():  # pragma: no cover
             from asciimatics.screen import Screen
+
             Screen.wrapper(self._do_draw)
         else:
             for line in self.canvas:
-                print(''.join(line))
+                print("".join(line))
 
     def _do_draw(self, screen):  # pragma: no cover
         # pylint: disable=too-many-locals
@@ -93,13 +96,15 @@ class AsciiCanvas(object):
                 line = []
                 for x in range(smaxcol + 1):
                     x_index = offset_x + x
-                    if len(self.canvas) > y_index \
-                       and len(self.canvas[y_index]) > x_index:
+                    if (
+                        len(self.canvas) > y_index
+                        and len(self.canvas[y_index]) > x_index
+                    ):
                         line.append(self.canvas[y_index][x_index])
                     else:
-                        line.append(' ')
+                        line.append(" ")
                 assert len(line) == (smaxcol + 1)
-                screen.print_at(''.join(line), 0, y)
+                screen.print_at("".join(line), 0, y)
 
             screen.refresh()
 
@@ -118,23 +123,23 @@ class AsciiCanvas(object):
                 continue
 
             k = event.key_code
-            if k == screen.KEY_DOWN or k == ord('s'):
+            if k == screen.KEY_DOWN or k == ord("s"):
                 offset_y += 1
-            elif k == screen.KEY_PAGE_DOWN or k == ord('S'):
+            elif k == screen.KEY_PAGE_DOWN or k == ord("S"):
                 offset_y += smaxrow
-            elif k == screen.KEY_UP or k == ord('w'):
+            elif k == screen.KEY_UP or k == ord("w"):
                 offset_y -= 1
-            elif k == screen.KEY_PAGE_UP or k == ord('W'):
+            elif k == screen.KEY_PAGE_UP or k == ord("W"):
                 offset_y -= smaxrow
-            elif k == screen.KEY_RIGHT or k == ord('d'):
+            elif k == screen.KEY_RIGHT or k == ord("d"):
                 offset_x += 1
-            elif k == ord('D'):
+            elif k == ord("D"):
                 offset_x += smaxcol
-            elif k == screen.KEY_LEFT or k == ord('a'):
+            elif k == screen.KEY_LEFT or k == ord("a"):
                 offset_x -= 1
-            elif k == ord('A'):
+            elif k == ord("A"):
                 offset_x -= smaxcol
-            elif k == ord('q') or k == ord('Q'):
+            elif k == ord("q") or k == ord("Q"):
                 break
 
             if offset_y > max_y:
@@ -235,17 +240,17 @@ class AsciiCanvas(object):
         height -= 1
 
         for x in range(x0, x0 + width):
-            self.point(x, y0, '-')
-            self.point(x, y0 + height, '-')
+            self.point(x, y0, "-")
+            self.point(x, y0 + height, "-")
 
         for y in range(y0, y0 + height):
-            self.point(x0, y, '|')
-            self.point(x0 + width, y, '|')
+            self.point(x0, y, "|")
+            self.point(x0 + width, y, "|")
 
-        self.point(x0, y0, '+')
-        self.point(x0 + width, y0, '+')
-        self.point(x0, y0 + height, '+')
-        self.point(x0 + width, y0 + height, '+')
+        self.point(x0, y0, "+")
+        self.point(x0 + width, y0, "+")
+        self.point(x0, y0 + height, "+")
+        self.point(x0 + width, y0 + height, "+")
 
 
 def _build_sugiyama_layout(vertexes, edges):
@@ -306,8 +311,8 @@ def draw(vertexes, edges):
 
     for vertex in sug.g.sV:
         # NOTE: moving boxes w/2 to the left
-        Xs.append(vertex.view.xy[0] - vertex.view.w/2.0)
-        Xs.append(vertex.view.xy[0] + vertex.view.w/2.0)
+        Xs.append(vertex.view.xy[0] - vertex.view.w / 2.0)
+        Xs.append(vertex.view.xy[0] + vertex.view.w / 2.0)
         Ys.append(vertex.view.xy[1])
         Ys.append(vertex.view.xy[1] + vertex.view.h)
 
@@ -344,22 +349,17 @@ def draw(vertexes, edges):
             assert end_x >= 0
             assert end_y >= 0
 
-            canvas.line(start_x, start_y,
-                        end_x, end_y,
-                        '*')
+            canvas.line(start_x, start_y, end_x, end_y, "*")
 
     for vertex in sug.g.sV:
         # NOTE: moving boxes w/2 to the left
-        x = vertex.view.xy[0] - vertex.view.w/2.0
+        x = vertex.view.xy[0] - vertex.view.w / 2.0
         y = vertex.view.xy[1]
 
-        canvas.box(int(round(x - minx)),
-                   int(round(y - miny)),
-                   vertex.view.w,
-                   vertex.view.h)
+        canvas.box(
+            int(round(x - minx)), int(round(y - miny)), vertex.view.w, vertex.view.h
+        )
 
-        canvas.text(int(round(x - minx)) + 1,
-                    int(round(y - miny)) + 1,
-                    vertex.data)
+        canvas.text(int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data)
 
     canvas.draw()

dvc/data_cloud.py

@@ -26,15 +26,16 @@ class DataCloud(object):
     Raises:
         config.ConfigError: thrown when config has invalid format.
     """
+
     CLOUD_MAP = {
-        'aws': RemoteS3,
-        'gcp': RemoteGS,
-        'azure': RemoteAzure,
-        'ssh': RemoteSSH,
-        'hdfs': RemoteHDFS,
-        'local': RemoteLOCAL,
-        'http': RemoteHTTP,
-        'https': RemoteHTTP,
+        "aws": RemoteS3,
+        "gcp": RemoteGS,
+        "azure": RemoteAzure,
+        "ssh": RemoteSSH,
+        "hdfs": RemoteHDFS,
+        "local": RemoteLOCAL,
+        "http": RemoteHTTP,
+        "https": RemoteHTTP,
     }
 
     def __init__(self, project, config=None):
@@ -44,13 +45,13 @@ class DataCloud(object):
 
     @property
     def _cloud(self):
-        remote = self._core.get(Config.SECTION_CORE_REMOTE, '')
-        if remote != '':
+        remote = self._core.get(Config.SECTION_CORE_REMOTE, "")
+        if remote != "":
             return self._init_remote(remote)
 
         if self._core.get(Config.SECTION_CORE_CLOUD, None):
             # backward compatibility
-            msg = 'using obsoleted config format. Consider updating.'
+            msg = "using obsoleted config format. Consider updating."
             logger.warning(msg)
             return self._init_compat()
 
@@ -66,8 +67,8 @@ class DataCloud(object):
         return Remote(self.project, cloud_config)
 
     def _init_compat(self):
-        name = self._core.get(Config.SECTION_CORE_CLOUD, '').strip().lower()
-        if name == '':
+        name = self._core.get(Config.SECTION_CORE_CLOUD, "").strip().lower()
+        if name == "":
             self._cloud = None
             return None
 
@@ -91,7 +92,7 @@ class DataCloud(object):
     def _init_cloud(self, cloud_config, cloud_type):
         global_storage_path = self._core.get(Config.SECTION_CORE_STORAGEPATH)
         if global_storage_path:
-            logger.warning('using obsoleted config format. Consider updating.')
+            logger.warning("using obsoleted config format. Consider updating.")
 
         cloud = cloud_type(self.project, cloud_config)
         return cloud
@@ -104,11 +105,10 @@ class DataCloud(object):
             return self._cloud
 
         raise ConfigError(
-            'No remote repository specified. Setup default repository with\n'
-            '    dvc config core.remote <name>\n'
-            'or use:\n'
-            '    dvc {} -r <name>\n'
-            .format(cmd)
+            "No remote repository specified. Setup default repository with\n"
+            "    dvc config core.remote <name>\n"
+            "or use:\n"
+            "    dvc {} -r <name>\n".format(cmd)
         )
 
     def push(self, targets, jobs=None, remote=None, show_checksums=False):
@@ -122,11 +122,12 @@ class DataCloud(object):
             show_checksums (bool): show checksums instead of file names in
                 information messages.
         """
-        return self.project.cache.local.push(targets,
-                                             jobs=jobs,
-                                             remote=self._get_cloud(remote,
-                                                                    'push'),
-                                             show_checksums=show_checksums)
+        return self.project.cache.local.push(
+            targets,
+            jobs=jobs,
+            remote=self._get_cloud(remote, "push"),
+            show_checksums=show_checksums,
+        )
 
     def pull(self, targets, jobs=None, remote=None, show_checksums=False):
         """Pull data items in a cloud-agnostic way.
@@ -139,11 +140,12 @@ class DataCloud(object):
             show_checksums (bool): show checksums instead of file names in
                 information messages.
         """
-        return self.project.cache.local.pull(targets,
-                                             jobs=jobs,
-                                             remote=self._get_cloud(remote,
-                                                                    'pull'),
-                                             show_checksums=show_checksums)
+        return self.project.cache.local.pull(
+            targets,
+            jobs=jobs,
+            remote=self._get_cloud(remote, "pull"),
+            show_checksums=show_checksums,
+        )
 
     def status(self, targets, jobs=None, remote=None, show_checksums=False):
         """Check status of data items in a cloud-agnostic way.
@@ -157,8 +159,7 @@ class DataCloud(object):
             show_checksums (bool): show checksums instead of file names in
                 information messages.
         """
-        cloud = self._get_cloud(remote, 'status')
-        return self.project.cache.local.status(targets,
-                                               jobs=jobs,
-                                               remote=cloud,
-                                               show_checksums=show_checksums)
+        cloud = self._get_cloud(remote, "status")
+        return self.project.cache.local.status(
+            targets, jobs=jobs, remote=cloud, show_checksums=show_checksums
+        )

dvc/dependency/__init__.py

@@ -26,13 +26,13 @@ DEPS = [
 ]
 
 DEP_MAP = {
-    'local': DependencyLOCAL,
-    'ssh': DependencySSH,
-    's3': DependencyS3,
-    'gs': DependencyGS,
-    'hdfs': DependencyHDFS,
-    'http': DependencyHTTP,
-    'https': DependencyHTTP,
+    "local": DependencyLOCAL,
+    "ssh": DependencySSH,
+    "s3": DependencyS3,
+    "gs": DependencyGS,
+    "hdfs": DependencyHDFS,
+    "http": DependencyHTTP,
+    "https": DependencyHTTP,
 }
 
 
@@ -47,7 +47,7 @@ del SCHEMA[schema.Optional(OutputBase.PARAM_METRIC)]
 
 def _get(stage, p, info):
     parsed = urlparse(p)
-    if parsed.scheme == 'remote':
+    if parsed.scheme == "remote":
         name = Config.SECTION_REMOTE_FMT.format(parsed.netloc)
         sect = stage.project.config.config[name]
         remote = Remote(stage.project, sect)

dvc/dependency/http.py

@@ -10,14 +10,8 @@ class DependencyHTTP(DependencyBase, OutputBase):
     REMOTE = RemoteHTTP
 
     def __init__(self, stage, path, info=None, remote=None):
-        super(DependencyHTTP, self).__init__(stage,
-                                             path,
-                                             info=info,
-                                             remote=remote)
-        if path.startswith('remote'):
+        super(DependencyHTTP, self).__init__(stage, path, info=info, remote=remote)
+        if path.startswith("remote"):
             path = urljoin(self.remote.cache_dir, urlparse(path).path)
 
-        self.path_info = {
-            'scheme': urlparse(path).scheme,
-            'path': path,
-        }
+        self.path_info = {"scheme": urlparse(path).scheme, "path": path}

dvc/exceptions.py

@@ -14,6 +14,7 @@ class DvcException(Exception):
         msg (str): message for this exception.
         cause (Exception): optional cause exception.
     """
+
     def __init__(self, msg, cause=None):
         # NOTE: unlike python 3, python 2 doesn't have built-in support
         # for chained exceptions, so we are using our own implementation.
@@ -35,15 +36,16 @@ class OutputDuplicationError(DvcException):
         output (str): path to the file/directory.
         stages (list): list of paths to stages.
     """
+
     def __init__(self, output, stages):
         assert isinstance(output, str) or isinstance(output, builtin_str)
         assert isinstance(stages, list)
         assert all(
-            isinstance(stage, str) or isinstance(stage, builtin_str)
-            for stage in stages
+            isinstance(stage, str) or isinstance(stage, builtin_str) for stage in stages
         )
-        msg = ("file/directory '{}' is specified as an output in more than one"
-               "stage: {}").format(output, "\n    ".join(stages))
+        msg = (
+            "file/directory '{}' is specified as an output in more than one" "stage: {}"
+        ).format(output, "\n    ".join(stages))
         super(OutputDuplicationError, self).__init__(msg)
 
 
@@ -56,13 +58,15 @@ class WorkingDirectoryAsOutputError(DvcException):
         fname (str): path to the stage file that has cwd specified as an
             output.
     """
+
     def __init__(self, cwd, fname):
         assert isinstance(cwd, str) or isinstance(cwd, builtin_str)
         assert isinstance(fname, str) or isinstance(fname, builtin_str)
         msg = (
             "current working directory '{cwd}' is specified as an output in"
-            " '{fname}'. Use another CWD to prevent any data removal."
-            .format(cwd=cwd, fname=fname)
+            " '{fname}'. Use another CWD to prevent any data removal.".format(
+                cwd=cwd, fname=fname
+            )
         )
         super(WorkingDirectoryAsOutputError, self).__init__(msg)
 
@@ -74,12 +78,11 @@ class CircularDependencyError(DvcException):
     Args:
         dependency (str): path to the dependency.
     """
-    def __init__(self, dependency):
-        assert (isinstance(dependency, str) or
-                isinstance(dependency, builtin_str))
 
-        msg = ("file/directory '{}' is specified as an output and as a "
-               "dependency.")
+    def __init__(self, dependency):
+        assert isinstance(dependency, str) or isinstance(dependency, builtin_str)
+
+        msg = "file/directory '{}' is specified as an output and as a " "dependency."
         super(CircularDependencyError, self).__init__(msg.format(dependency))
 
 
@@ -90,6 +93,7 @@ class ArgumentDuplicationError(DvcException):
     Args:
         path (str): path to the file/directory.
     """
+
     def __init__(self, path):
         assert isinstance(path, str) or isinstance(path, builtin_str)
         msg = "file '{}' is specified more than once."
@@ -103,11 +107,14 @@ class MoveNotDataSourceError(DvcException):
     Args:
         path (str): path to the file/directory.
     """
+
     def __init__(self, path):
-        msg = "move is not permitted for stages that are not data sources. " \
-              "You need to either move '{path}' to a new location and edit " \
-              "it by hand, or remove '{path}' and create a new one at the " \
-              "desired location."
+        msg = (
+            "move is not permitted for stages that are not data sources. "
+            "You need to either move '{path}' to a new location and edit "
+            "it by hand, or remove '{path}' and create a new one at the "
+            "desired location."
+        )
         super(MoveNotDataSourceError, self).__init__(msg.format(path=path))
 
 
@@ -117,6 +124,7 @@ class NotDvcProjectError(DvcException):
     Args:
         root (str): path to the directory.
     """
+
     def __init__(self, root):
         msg = "not a dvc repository (checked up to mount point '{}')"
         super(NotDvcProjectError, self).__init__(msg.format(root))
@@ -124,6 +132,7 @@ class NotDvcProjectError(DvcException):
 
 class DvcParserError(DvcException):
     """Base class for CLI parser errors."""
+
     def __init__(self):
         super(DvcParserError, self).__init__("parser error")
 
@@ -131,12 +140,11 @@ class DvcParserError(DvcException):
 class CyclicGraphError(DvcException):
     def __init__(self, stages):
         assert isinstance(stages, list)
-        stages = '\n'.join('\t- {}'.format(stage) for stage in stages)
+        stages = "\n".join("\t- {}".format(stage) for stage in stages)
         msg = (
             "you've introduced a cycle in your pipeline that involves"
-            ' the following stages:'
-            '\n'
-            '{stages}'
-            .format(stages=stages)
+            " the following stages:"
+            "\n"
+            "{stages}".format(stages=stages)
         )
         super(CyclicGraphError, self).__init__(msg)

dvc/istextfile.py

@@ -18,7 +18,7 @@ def _int2byte(i):
     return chr(i)
 
 
-TEXT_CHARS = (b''.join(_int2byte(i) for i in range(32, 127)) + b'\n\r\t\f\b')
+TEXT_CHARS = b"".join(_int2byte(i) for i in range(32, 127)) + b"\n\r\t\f\b"
 
 
 def istextfile(fname, blocksize=512):
@@ -27,14 +27,14 @@ def istextfile(fname, blocksize=512):
         If more than 30% of the chars in the block are non-text, or there
         are NUL ('\x00') bytes in the block, assume this is a binary file.
     """
-    with open(fname, 'rb') as fobj:
+    with open(fname, "rb") as fobj:
         block = fobj.read(blocksize)
 
     if not block:
         # An empty file is considered a valid text file
         return True
 
-    if b'\x00' in block:
+    if b"\x00" in block:
         # Files with null bytes are binary
         return False
 

dvc/lock.py

@@ -21,7 +21,8 @@ class Lock(object):
             in.
         name (str): name of the lock file.
     """
-    LOCK_FILE = 'lock'
+
+    LOCK_FILE = "lock"
     TIMEOUT = 5
 
     def __init__(self, dvc_dir, name=LOCK_FILE):
@@ -32,8 +33,10 @@ class Lock(object):
         try:
             self._lock = zc.lockfile.LockFile(self.lock_file)
         except zc.lockfile.LockError:
-            raise LockError('cannot perform the cmd since DVC is busy and '
-                            'locked. Please retry the cmd later.')
+            raise LockError(
+                "cannot perform the cmd since DVC is busy and "
+                "locked. Please retry the cmd later."
+            )
 
     def lock(self):
         """Acquire lock for dvc project."""

dvc/logger.py

@@ -21,34 +21,34 @@ def info(message):
 
 def debug(message):
     """Prints a debug message."""
-    prefix = colorize('Debug', color='blue')
+    prefix = colorize("Debug", color="blue")
 
-    out = '{prefix}: {message}'.format(prefix=prefix, message=message)
+    out = "{prefix}: {message}".format(prefix=prefix, message=message)
 
     logger.debug(out)
 
 
 def warning(message):
     """Prints a warning message."""
-    prefix = colorize('Warning', color='yellow')
+    prefix = colorize("Warning", color="yellow")
 
-    out = '{prefix}: {message}'.format(prefix=prefix, message=message)
+    out = "{prefix}: {message}".format(prefix=prefix, message=message)
 
     logger.warning(out)
 
 
 def error(message=None):
     """Prints an error message."""
-    prefix = colorize('Error', color='red')
+    prefix = colorize("Error", color="red")
 
     exception, stack_trace = _parse_exc()
 
     out = (
-        '{prefix}: {description}'
-        '\n'
-        '{stack_trace}'
-        '\n'
-        '{footer}'.format(
+        "{prefix}: {description}"
+        "\n"
+        "{stack_trace}"
+        "\n"
+        "{footer}".format(
             prefix=prefix,
             description=_description(message, exception),
             stack_trace=stack_trace,
@@ -66,45 +66,41 @@ def box(message, border_color=None):
         message (str): message to print.
         border_color (str): name of a color to outline the box with.
     """
-    lines = message.split('\n')
+    lines = message.split("\n")
     max_width = max(_visual_width(line) for line in lines)
 
     padding_horizontal = 5
     padding_vertical = 1
 
-    box_size_horizontal = (max_width + (padding_horizontal * 2))
+    box_size_horizontal = max_width + (padding_horizontal * 2)
 
-    chars = {
-        'corner':     '+',
-        'horizontal': '-',
-        'vertical':   '|',
-        'empty':      ' ',
-    }
+    chars = {"corner": "+", "horizontal": "-", "vertical": "|", "empty": " "}
 
     margin = "{corner}{line}{corner}\n".format(
-        corner=chars['corner'],
-        line=chars['horizontal'] * box_size_horizontal,
+        corner=chars["corner"], line=chars["horizontal"] * box_size_horizontal
     )
 
     padding_lines = [
         "{border}{space}{border}\n".format(
-            border=colorize(chars['vertical'], color=border_color),
-            space=chars['empty'] * box_size_horizontal,
-        ) * padding_vertical
+            border=colorize(chars["vertical"], color=border_color),
+            space=chars["empty"] * box_size_horizontal,
+        )
+        * padding_vertical
     ]
 
     content_lines = [
         "{border}{space}{content}{space}{border}\n".format(
-            border=colorize(chars['vertical'], color=border_color),
-            space=chars['empty'] * padding_horizontal,
+            border=colorize(chars["vertical"], color=border_color),
+            space=chars["empty"] * padding_horizontal,
             content=_visual_center(line, max_width),
-        ) for line in lines
+        )
+        for line in lines
     ]
 
     box_str = "{margin}{padding}{content}{padding}{margin}".format(
         margin=colorize(margin, color=border_color),
-        padding=''.join(padding_lines),
-        content=''.join(content_lines),
+        padding="".join(padding_lines),
+        content="".join(content_lines),
     )
 
     logger.info(box_str)
@@ -126,11 +122,11 @@ def set_level(level_name):
         return
 
     levels = {
-        'info': logging.INFO,
-        'debug': logging.DEBUG,
-        'warning': logging.WARNING,
-        'error': logging.ERROR,
-        'critical': logging.CRITICAL,
+        "info": logging.INFO,
+        "debug": logging.DEBUG,
+        "warning": logging.WARNING,
+        "error": logging.ERROR,
+        "critical": logging.CRITICAL,
     }
 
     logger.setLevel(levels.get(level_name))
@@ -182,16 +178,14 @@ def colorize(message, color=None):
         return message
 
     colors = {
-        'green': colorama.Fore.GREEN,
-        'yellow': colorama.Fore.YELLOW,
-        'blue': colorama.Fore.BLUE,
-        'red': colorama.Fore.RED,
+        "green": colorama.Fore.GREEN,
+        "yellow": colorama.Fore.YELLOW,
+        "blue": colorama.Fore.BLUE,
+        "red": colorama.Fore.RED,
     }
 
-    return '{color}{message}{nc}'.format(
-        color=colors.get(color, ''),
-        message=message,
-        nc=colorama.Fore.RESET,
+    return "{color}{message}{nc}".format(
+        color=colors.get(color, ""), message=message, nc=colorama.Fore.RESET
     )
 
 
@@ -205,7 +199,7 @@ def set_default_level():
 
 
 def _add_handlers():
-    formatter = '%(message)s'
+    formatter = "%(message)s"
 
     class _LogLevelFilter(logging.Filter):
         # pylint: disable=too-few-public-methods
@@ -230,9 +224,9 @@ def _walk_exc(exc):
     tb_list = [traceback.format_exc()]
 
     # NOTE: parsing chained exceptions. See dvc/exceptions.py for more info.
-    while hasattr(exc, 'cause') and exc.cause is not None:
+    while hasattr(exc, "cause") and exc.cause is not None:
         exc_list.append(str(exc.cause))
-        if hasattr(exc, 'cause_tb') and exc.cause_tb is not None:
+        if hasattr(exc, "cause_tb") and exc.cause_tb is not None:
             tb_list.insert(0, str(exc.cause_tb))
         exc = exc.cause
 
@@ -242,45 +236,44 @@ def _walk_exc(exc):
 def _parse_exc():
     exc = sys.exc_info()[1]
     if not exc:
-        return (None, '')
+        return (None, "")
 
     exc_list, tb_list = _walk_exc(exc)
 
-    exception = ': '.join(exc_list)
+    exception = ": ".join(exc_list)
 
     if is_verbose():
-        stack_trace = '{line}\n{stack_trace}{line}\n'.format(
-            line=colorize('-' * 60, color='red'),
-            stack_trace='\n'.join(tb_list),
+        stack_trace = "{line}\n{stack_trace}{line}\n".format(
+            line=colorize("-" * 60, color="red"), stack_trace="\n".join(tb_list)
         )
     else:
-        stack_trace = ''
+        stack_trace = ""
 
     return (exception, stack_trace)
 
 
 def _description(message, exception):
     if exception and message:
-        description = '{message} - {exception}'
+        description = "{message} - {exception}"
     elif exception:
-        description = '{exception}'
+        description = "{exception}"
     elif message:
-        description = '{message}'
+        description = "{message}"
 
     return description.format(message=message, exception=exception)
 
 
 def _footer():
-    return '{phrase} Hit us up at {url}, we are always happy to help!'.format(
-        phrase=colorize('Having any troubles?', 'yellow'),
-        url=colorize('https://dvc.org/support', 'blue'),
+    return "{phrase} Hit us up at {url}, we are always happy to help!".format(
+        phrase=colorize("Having any troubles?", "yellow"),
+        url=colorize("https://dvc.org/support", "blue"),
     )
 
 
 def _visual_width(line):
     """Get the the number of columns required to display a string"""
 
-    return len(re.sub(colorama.ansitowin32.AnsiToWin32.ANSI_CSI_RE, '', line))
+    return len(re.sub(colorama.ansitowin32.AnsiToWin32.ANSI_CSI_RE, "", line))
 
 
 def _visual_center(line, width):
@@ -288,12 +281,12 @@ def _visual_center(line, width):
 
     spaces = max(width - _visual_width(line), 0)
     left_padding = int(spaces / 2)
-    right_padding = (spaces - left_padding)
+    right_padding = spaces - left_padding
 
-    return (left_padding * ' ') + line + (right_padding * ' ')
+    return (left_padding * " ") + line + (right_padding * " ")
 
 
-logger = logging.getLogger('dvc')  # pylint: disable=invalid-name
+logger = logging.getLogger("dvc")  # pylint: disable=invalid-name
 
 set_default_level()
 _add_handlers()

dvc/main.py

@@ -39,7 +39,7 @@ def main(argv=None):
     except DvcParserError:
         ret = 254
     except Exception:  # pylint: disable=broad-except
-        logger.error('unexpected error')
+        logger.error("unexpected error")
         ret = 255
 
     Analytics().send_cmd(cmd, args, ret)

dvc/output/__init__.py

@@ -25,44 +25,41 @@ OUTS = [
     # NOTE: OutputLOCAL is the default choice
 ]
 
-OUTS_MAP = {'hdfs': OutputHDFS,
-            's3': OutputS3,
-            'gs': OutputGS,
-            'ssh': OutputSSH,
-            'local': OutputLOCAL}
+OUTS_MAP = {
+    "hdfs": OutputHDFS,
+    "s3": OutputS3,
+    "gs": OutputGS,
+    "ssh": OutputSSH,
+    "local": OutputLOCAL,
+}
 
 SCHEMA = {
-        OutputBase.PARAM_PATH: str,
-
-        # NOTE: currently there are only 3 possible checksum names:
-        #
-        #    1) md5 (LOCAL, SSH, GS);
-        #    2) etag (S3);
-        #    3) checksum (HDFS);
-        #
-        # so when a few types of outputs share the same name, we only need
-        # specify it once.
-        schema.Optional(RemoteLOCAL.PARAM_CHECKSUM): schema.Or(str, None),
-        schema.Optional(RemoteS3.PARAM_CHECKSUM): schema.Or(str, None),
-        schema.Optional(RemoteHDFS.PARAM_CHECKSUM): schema.Or(str, None),
-
-        schema.Optional(OutputBase.PARAM_CACHE): bool,
-        schema.Optional(OutputBase.PARAM_METRIC): OutputBase.METRIC_SCHEMA,
+    OutputBase.PARAM_PATH: str,
+    # NOTE: currently there are only 3 possible checksum names:
+    #
+    #    1) md5 (LOCAL, SSH, GS);
+    #    2) etag (S3);
+    #    3) checksum (HDFS);
+    #
+    # so when a few types of outputs share the same name, we only need
+    # specify it once.
+    schema.Optional(RemoteLOCAL.PARAM_CHECKSUM): schema.Or(str, None),
+    schema.Optional(RemoteS3.PARAM_CHECKSUM): schema.Or(str, None),
+    schema.Optional(RemoteHDFS.PARAM_CHECKSUM): schema.Or(str, None),
+    schema.Optional(OutputBase.PARAM_CACHE): bool,
+    schema.Optional(OutputBase.PARAM_METRIC): OutputBase.METRIC_SCHEMA,
 }
 
 
 def _get(stage, p, info, cache, metric):
     parsed = urlparse(p)
-    if parsed.scheme == 'remote':
+    if parsed.scheme == "remote":
         name = Config.SECTION_REMOTE_FMT.format(parsed.netloc)
         sect = stage.project.config.config[name]
         remote = Remote(stage.project, sect)
-        return OUTS_MAP[remote.scheme](stage,
-                                       p,
-                                       info,
-                                       cache=cache,
-                                       remote=remote,
-                                       metric=metric)
+        return OUTS_MAP[remote.scheme](
+            stage, p, info, cache=cache, remote=remote, metric=metric
+        )
 
     for o in OUTS:
         if o.supported(p):

dvc/output/base.py

@@ -30,26 +30,25 @@ class OutputBase(object):
 
     REMOTE = None
 
-    PARAM_PATH = 'path'
-    PARAM_CACHE = 'cache'
-    PARAM_METRIC = 'metric'
-    PARAM_METRIC_TYPE = 'type'
-    PARAM_METRIC_XPATH = 'xpath'
+    PARAM_PATH = "path"
+    PARAM_CACHE = "cache"
+    PARAM_METRIC = "metric"
+    PARAM_METRIC_TYPE = "type"
+    PARAM_METRIC_XPATH = "xpath"
 
-    METRIC_SCHEMA = Or(None, bool,
-                       {Optional(PARAM_METRIC_TYPE): Or(str, None),
-                        Optional(PARAM_METRIC_XPATH): Or(str, None)})
+    METRIC_SCHEMA = Or(
+        None,
+        bool,
+        {
+            Optional(PARAM_METRIC_TYPE): Or(str, None),
+            Optional(PARAM_METRIC_XPATH): Or(str, None),
+        },
+    )
 
     DoesNotExistError = OutputDoesNotExistError
     IsNotFileOrDirError = OutputIsNotFileOrDirError
 
-    def __init__(self,
-                 stage,
-                 path,
-                 info=None,
-                 remote=None,
-                 cache=True,
-                 metric=False):
+    def __init__(self, stage, path, info=None, remote=None, cache=True, metric=False):
         self.stage = stage
         self.project = stage.project
         self.url = path
@@ -58,15 +57,14 @@ class OutputBase(object):
         self.use_cache = False if self.IS_DEPENDENCY else cache
         self.metric = False if self.IS_DEPENDENCY else metric
 
-        if self.use_cache and getattr(self.project.cache,
-                                      self.REMOTE.scheme) is None:
-            raise DvcException("no cache location setup for '{}' outputs."
-                               .format(self.REMOTE.scheme))
+        if self.use_cache and getattr(self.project.cache, self.REMOTE.scheme) is None:
+            raise DvcException(
+                "no cache location setup for '{}' outputs.".format(self.REMOTE.scheme)
+            )
 
     def __repr__(self):
         return "{class_name}: '{url}'".format(
-            class_name=type(self).__name__,
-            url=(self.url or 'No url')
+            class_name=type(self).__name__, url=(self.url or "No url")
         )
 
     def __str__(self):
@@ -92,11 +90,11 @@ class OutputBase(object):
 
     @property
     def path(self):
-        return self.path_info['path']
+        return self.path_info["path"]
 
     @property
     def sep(self):
-        return '/'
+        return "/"
 
     @property
     def checksum(self):
@@ -113,8 +111,9 @@ class OutputBase(object):
         if not self.use_cache:
             return self.info != self.remote.save_info(self.path_info)
 
-        return getattr(self.project.cache, self.scheme).changed(self.path_info,
-                                                                self.info)
+        return getattr(self.project.cache, self.scheme).changed(
+            self.path_info, self.info
+        )
 
     def changed_cache(self):
         if not self.use_cache or not self.checksum:
@@ -127,15 +126,14 @@ class OutputBase(object):
     def status(self):
         if self.changed():
             # FIXME better msgs
-            return {str(self): 'changed'}
+            return {str(self): "changed"}
         return {}
 
     def save(self):
         if not self.use_cache:
             self.info = self.remote.save_info(self.path_info)
         else:
-            self.info = getattr(self.project.cache,
-                                self.scheme).save(self.path_info)
+            self.info = getattr(self.project.cache, self.scheme).save(self.path_info)
 
     def dumpd(self):
         ret = self.info.copy()
@@ -147,8 +145,10 @@ class OutputBase(object):
         ret[self.PARAM_CACHE] = self.use_cache
 
         if isinstance(self.metric, dict):
-            if self.PARAM_METRIC_XPATH in self.metric \
-               and not self.metric[self.PARAM_METRIC_XPATH]:
+            if (
+                self.PARAM_METRIC_XPATH in self.metric
+                and not self.metric[self.PARAM_METRIC_XPATH]
+            ):
                 del self.metric[self.PARAM_METRIC_XPATH]
 
         ret[self.PARAM_METRIC] = self.metric
@@ -162,20 +162,20 @@ class OutputBase(object):
         if not self.use_cache:
             return
 
-        getattr(self.project.cache, self.scheme).checkout(self.path_info,
-                                                          self.info,
-                                                          force=force)
+        getattr(self.project.cache, self.scheme).checkout(
+            self.path_info, self.info, force=force
+        )
 
     def remove(self, ignore_remove=False):
         self.remote.remove(self.path_info)
-        if self.scheme != 'local':
+        if self.scheme != "local":
             return
 
         if ignore_remove and self.use_cache and self.is_local:
             self.project.scm.ignore_remove(self.path)
 
     def move(self, out):
-        if self.scheme == 'local' and self.use_cache and self.is_local:
+        if self.scheme == "local" and self.use_cache and self.is_local:
             self.project.scm.ignore_remove(self.path)
 
         self.remote.move(self.path_info, out.path_info)
@@ -183,5 +183,5 @@ class OutputBase(object):
         self.path_info = out.path_info
         self.save()
 
-        if self.scheme == 'local' and self.use_cache and self.is_local:
+        if self.scheme == "local" and self.use_cache and self.is_local:
             self.project.scm.ignore(self.path)

dvc/output/hdfs.py

@@ -10,22 +10,11 @@ from dvc.remote.hdfs import RemoteHDFS
 class OutputHDFS(OutputBase):
     REMOTE = RemoteHDFS
 
-    def __init__(self,
-                 stage,
-                 path,
-                 info=None,
-                 remote=None,
-                 cache=True,
-                 metric=False):
-        super(OutputHDFS, self).__init__(stage,
-                                         path,
-                                         info=info,
-                                         remote=remote,
-                                         cache=cache,
-                                         metric=metric)
+    def __init__(self, stage, path, info=None, remote=None, cache=True, metric=False):
+        super(OutputHDFS, self).__init__(
+            stage, path, info=info, remote=remote, cache=cache, metric=metric
+        )
         if remote:
-            path = posixpath.join(remote.url, urlparse(path).path.lstrip('/'))
-        user = remote.user if remote else self.group('user')
-        self.path_info = {'scheme': 'hdfs',
-                          'user': user,
-                          'path': path}
+            path = posixpath.join(remote.url, urlparse(path).path.lstrip("/"))
+        user = remote.user if remote else self.group("user")
+        self.path_info = {"scheme": "hdfs", "user": user, "path": path}

dvc/output/local.py

@@ -13,22 +13,12 @@ from dvc.output.base import OutputBase, OutputAlreadyTrackedError
 class OutputLOCAL(OutputBase):
     REMOTE = RemoteLOCAL
 
-    def __init__(self,
-                 stage,
-                 path,
-                 info=None,
-                 remote=None,
-                 cache=True,
-                 metric=False):
-        super(OutputLOCAL, self).__init__(stage,
-                                          path,
-                                          info,
-                                          remote=remote,
-                                          cache=cache,
-                                          metric=metric)
+    def __init__(self, stage, path, info=None, remote=None, cache=True, metric=False):
+        super(OutputLOCAL, self).__init__(
+            stage, path, info, remote=remote, cache=cache, metric=metric
+        )
         if remote:
-            p = os.path.join(remote.prefix,
-                             urlparse(self.url).path.lstrip('/'))
+            p = os.path.join(remote.prefix, urlparse(self.url).path.lstrip("/"))
         else:
             p = path
 
@@ -37,16 +27,14 @@ class OutputLOCAL(OutputBase):
             p = os.path.join(stage.cwd, p)
         p = os.path.abspath(os.path.normpath(p))
 
-        self.path_info = {'scheme': 'local',
-                          'path': p}
+        self.path_info = {"scheme": "local", "path": p}
 
     def __str__(self):
         return self.rel_path
 
     @property
     def is_local(self):
-        return (urlparse(self.url).scheme != 'remote'
-                and not os.path.isabs(self.url))
+        return urlparse(self.url).scheme != "remote" and not os.path.isabs(self.url)
 
     @property
     def sep(self):
@@ -63,8 +51,7 @@ class OutputLOCAL(OutputBase):
     def dumpd(self):
         ret = super(OutputLOCAL, self).dumpd()
         if self.is_local:
-            path = self.remote.unixpath(os.path.relpath(self.path,
-                                                        self.stage.cwd))
+            path = self.remote.unixpath(os.path.relpath(self.path, self.stage.cwd))
         else:
             path = self.url
 
@@ -91,11 +78,12 @@ class OutputLOCAL(OutputBase):
         if not os.path.exists(self.path):
             raise self.DoesNotExistError(self.rel_path)
 
-        if (not os.path.isfile(self.path) and not os.path.isdir(self.path)):
+        if not os.path.isfile(self.path) and not os.path.isdir(self.path):
             raise self.IsNotFileOrDirError(self.rel_path)
 
-        if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \
-           (os.path.isdir(self.path) and len(os.listdir(self.path)) == 0):
+        if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or (
+            os.path.isdir(self.path) and len(os.listdir(self.path)) == 0
+        ):
             msg = "file/directory '{}' is empty.".format(self.rel_path)
             logger.warning(msg)
 

dvc/output/s3.py

@@ -10,23 +10,12 @@ from dvc.output.base import OutputBase
 class OutputS3(OutputBase):
     REMOTE = RemoteS3
 
-    def __init__(self,
-                 stage,
-                 path,
-                 info=None,
-                 remote=None,
-                 cache=True,
-                 metric=False):
-        super(OutputS3, self).__init__(stage,
-                                       path,
-                                       info=info,
-                                       remote=remote,
-                                       cache=cache,
-                                       metric=metric)
+    def __init__(self, stage, path, info=None, remote=None, cache=True, metric=False):
+        super(OutputS3, self).__init__(
+            stage, path, info=info, remote=remote, cache=cache, metric=metric
+        )
         bucket = remote.bucket if remote else urlparse(path).netloc
-        path = urlparse(path).path.lstrip('/')
+        path = urlparse(path).path.lstrip("/")
         if remote:
             path = posixpath.join(remote.prefix, path)
-        self.path_info = {'scheme': self.scheme,
-                          'bucket': bucket,
-                          'path': path}
+        self.path_info = {"scheme": self.scheme, "bucket": bucket, "path": path}

dvc/output/ssh.py

@@ -11,34 +11,24 @@ from dvc.remote.ssh import RemoteSSH
 class OutputSSH(OutputBase):
     REMOTE = RemoteSSH
 
-    def __init__(self,
-                 stage,
-                 path,
-                 info=None,
-                 remote=None,
-                 cache=True,
-                 metric=False):
-        super(OutputSSH, self).__init__(stage,
-                                        path,
-                                        info=info,
-                                        remote=remote,
-                                        cache=cache,
-                                        metric=metric)
+    def __init__(self, stage, path, info=None, remote=None, cache=True, metric=False):
+        super(OutputSSH, self).__init__(
+            stage, path, info=info, remote=remote, cache=cache, metric=metric
+        )
         parsed = urlparse(path)
         host = remote.host if remote else parsed.hostname
-        port = remote.port if remote else (parsed.port
-                                           or RemoteSSH.DEFAULT_PORT)
-        user = remote.user if remote else (parsed.username
-                                           or getpass.getuser())
+        port = remote.port if remote else (parsed.port or RemoteSSH.DEFAULT_PORT)
+        user = remote.user if remote else (parsed.username or getpass.getuser())
 
         if remote:
-            path = posixpath.join(remote.prefix,
-                                  urlparse(path).path.lstrip('/'))
+            path = posixpath.join(remote.prefix, urlparse(path).path.lstrip("/"))
         else:
             path = parsed.path
 
-        self.path_info = {'scheme': 'ssh',
-                          'host': host,
-                          'port': port,
-                          'user': user,
-                          'path': path}
+        self.path_info = {
+            "scheme": "ssh",
+            "host": host,
+            "port": port,
+            "user": user,
+            "path": path,
+        }

dvc/progress.py

@@ -14,6 +14,7 @@ class Progress(object):
     """
     Simple multi-target progress bar.
     """
+
     def __init__(self):
         self._n_total = 0
         self._n_finished = 0
@@ -31,11 +32,11 @@ class Progress(object):
         return self._n_total == self._n_finished
 
     def _clearln(self):
-        self._print('\r\x1b[K', end='')
+        self._print("\r\x1b[K", end="")
 
     def _writeln(self, line):
         self._clearln()
-        self._print(line, end='')
+        self._print(line, end="")
         sys.stdout.flush()
 
     def refresh(self, line=None):
@@ -83,7 +84,7 @@ class Progress(object):
             percent = "?% "
         else:
             total = int(total)
-            state = int((100 * current)/total) if current < total else 100
+            state = int((100 * current) / total) if current < total else 100
             percent = str(state) + "% "
 
         if self._n_total > 1:
@@ -91,9 +92,9 @@ class Progress(object):
         else:
             num = ""
 
-        n_sh = int((state * bar_len)/100)
+        n_sh = int((state * bar_len) / 100)
         n_sp = bar_len - n_sh
-        pbar = "[" + '#'*n_sh + ' '*n_sp + "] "
+        pbar = "[" + "#" * n_sh + " " * n_sp + "] "
 
         return num + pbar + percent + target_name
 

dvc/prompt.py

@@ -14,7 +14,7 @@ def _ask(prompt, limited_to=None):
         return None
 
     while True:
-        answer = input(prompt + ' ').lower()
+        answer = input(prompt + " ").lower()
 
         if not limited_to:
             return answer
@@ -22,8 +22,11 @@ def _ask(prompt, limited_to=None):
         if answer in limited_to:
             return answer
 
-        print("Your response must be one of: {options}. Please try again."
-              .format(options=limited_to))
+        print(
+            "Your response must be one of: {options}. Please try again.".format(
+                options=limited_to
+            )
+        )
 
 
 def confirm(statement):
@@ -35,9 +38,9 @@ def confirm(statement):
     Returns:
         bool: whether or not specified statement was confirmed.
     """
-    prompt = '{statement} [y/n]'.format(statement=statement)
-    answer = _ask(prompt, limited_to=['yes', 'no', 'y', 'n'])
-    return answer and answer.startswith('y')
+    prompt = "{statement} [y/n]".format(statement=statement)
+    answer = _ask(prompt, limited_to=["yes", "no", "y", "n"])
+    return answer and answer.startswith("y")
 
 
 def password(statement):
@@ -49,5 +52,5 @@ def password(statement):
     Returns:
         str: password entered by the user.
     """
-    prompt = '{statement}: '.format(statement=statement)
+    prompt = "{statement}: ".format(statement=statement)
     return getpass(prompt)

dvc/remote/azure.py

@@ -25,71 +25,75 @@ class Callback(object):
 
 
 class RemoteAzure(RemoteBase):
-    scheme = 'azure'
-    REGEX = (r'azure://((?P<path>[^=;]*)?|('
-             # backward compatibility
-             r'(ContainerName=(?P<container_name>[^;]+);?)?'
-             r'(?P<connection_string>.+)?)?)$')
-    REQUIRES = {'azure-storage-blob': BlockBlobService}
-    PARAM_CHECKSUM = 'etag'
+    scheme = "azure"
+    REGEX = (
+        r"azure://((?P<path>[^=;]*)?|("
+        # backward compatibility
+        r"(ContainerName=(?P<container_name>[^;]+);?)?"
+        r"(?P<connection_string>.+)?)?)$"
+    )
+    REQUIRES = {"azure-storage-blob": BlockBlobService}
+    PARAM_CHECKSUM = "etag"
     COPY_POLL_SECONDS = 5
 
     def __init__(self, project, config):
         super(RemoteAzure, self).__init__(project, config)
 
-        self.url = config.get(Config.SECTION_REMOTE_URL, 'azure://')
+        self.url = config.get(Config.SECTION_REMOTE_URL, "azure://")
         match = re.match(self.REGEX, self.url)  # backward compatibility
 
-        path = match.group('path')
+        path = match.group("path")
         self.bucket = (
-            urlparse(self.url if path else '').netloc
-            or match.group('container_name')  # backward compatibility
-            or os.getenv('AZURE_STORAGE_CONTAINER_NAME'))
+            urlparse(self.url if path else "").netloc
+            or match.group("container_name")  # backward compatibility
+            or os.getenv("AZURE_STORAGE_CONTAINER_NAME")
+        )
 
-        self.prefix = urlparse(self.url).path.lstrip('/') if path else ''
+        self.prefix = urlparse(self.url).path.lstrip("/") if path else ""
 
         self.connection_string = (
             config.get(Config.SECTION_AZURE_CONNECTION_STRING)
-            or match.group('connection_string')  # backward compatibility
-            or os.getenv('AZURE_STORAGE_CONNECTION_STRING'))
+            or match.group("connection_string")  # backward compatibility
+            or os.getenv("AZURE_STORAGE_CONNECTION_STRING")
+        )
 
         if not self.bucket:
-            raise ValueError('azure storage container name missing')
+            raise ValueError("azure storage container name missing")
 
         if not self.connection_string:
-            raise ValueError('azure storage connection string missing')
+            raise ValueError("azure storage connection string missing")
 
         self.__blob_service = None
 
-        self.path_info = {'scheme': self.scheme, 'bucket': self.bucket}
+        self.path_info = {"scheme": self.scheme, "bucket": self.bucket}
 
     @property
     def blob_service(self):
         if self.__blob_service is None:
-            logger.debug('URL {}'.format(self.url))
-            logger.debug('Connection string {}'.format(self.connection_string))
+            logger.debug("URL {}".format(self.url))
+            logger.debug("Connection string {}".format(self.connection_string))
             self.__blob_service = BlockBlobService(
-                connection_string=self.connection_string)
-            logger.debug('Container name {}'.format(self.bucket))
+                connection_string=self.connection_string
+            )
+            logger.debug("Container name {}".format(self.bucket))
             self.__blob_service.create_container(self.bucket)
         return self.__blob_service
 
     def remove(self, path_info):
-        if path_info['scheme'] != self.scheme:
+        if path_info["scheme"] != self.scheme:
             raise NotImplementedError
 
-        logger.debug('Removing azure://{}/{}'.format(path_info['bucket'],
-                                                     path_info['path']))
+        logger.debug(
+            "Removing azure://{}/{}".format(path_info["bucket"], path_info["path"])
+        )
 
-        self.blob_service.delete_blob(path_info['bucket'], path_info['path'])
+        self.blob_service.delete_blob(path_info["bucket"], path_info["path"])
 
     def _list_paths(self, bucket, prefix):
         blob_service = self.blob_service
         next_marker = None
         while True:
-            blobs = blob_service.list_blobs(bucket,
-                                            prefix=prefix,
-                                            marker=next_marker)
+            blobs = blob_service.list_blobs(bucket, prefix=prefix, marker=next_marker)
 
             for blob in blobs:
                 yield blob.name
@@ -106,68 +110,68 @@ class RemoteAzure(RemoteBase):
         names = self._verify_path_args(to_infos, from_infos, names)
 
         for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info['scheme'] != self.scheme:
+            if to_info["scheme"] != self.scheme:
                 raise NotImplementedError
 
-            if from_info['scheme'] != 'local':
+            if from_info["scheme"] != "local":
                 raise NotImplementedError
 
-            bucket = to_info['bucket']
-            path = to_info['path']
+            bucket = to_info["bucket"]
+            path = to_info["path"]
 
-            logger.debug("Uploading '{}' to '{}/{}'".format(
-                from_info['path'], bucket, path))
+            logger.debug(
+                "Uploading '{}' to '{}/{}'".format(from_info["path"], bucket, path)
+            )
 
             if not name:
-                name = os.path.basename(from_info['path'])
+                name = os.path.basename(from_info["path"])
 
             cb = Callback(name)
 
             try:
                 self.blob_service.create_blob_from_path(
-                    bucket, path, from_info['path'], progress_callback=cb)
+                    bucket, path, from_info["path"], progress_callback=cb
+                )
             except Exception:
-                msg = "failed to upload '{}'".format(from_info['path'])
+                msg = "failed to upload '{}'".format(from_info["path"])
                 logger.warning(msg)
             else:
                 progress.finish_target(name)
 
-    def download(self,
-                 from_infos,
-                 to_infos,
-                 no_progress_bar=False,
-                 names=None):
+    def download(self, from_infos, to_infos, no_progress_bar=False, names=None):
         names = self._verify_path_args(from_infos, to_infos, names)
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info['scheme'] != self.scheme:
+            if from_info["scheme"] != self.scheme:
                 raise NotImplementedError
 
-            if to_info['scheme'] != 'local':
+            if to_info["scheme"] != "local":
                 raise NotImplementedError
 
-            bucket = from_info['bucket']
-            path = from_info['path']
+            bucket = from_info["bucket"]
+            path = from_info["path"]
 
-            logger.debug("Downloading '{}/{}' to '{}'".format(
-                bucket, path, to_info['path']))
+            logger.debug(
+                "Downloading '{}/{}' to '{}'".format(bucket, path, to_info["path"])
+            )
 
-            tmp_file = self.tmp_file(to_info['path'])
+            tmp_file = self.tmp_file(to_info["path"])
             if not name:
-                name = os.path.basename(to_info['path'])
+                name = os.path.basename(to_info["path"])
 
             cb = None if no_progress_bar else Callback(name)
 
-            self._makedirs(to_info['path'])
+            self._makedirs(to_info["path"])
 
             try:
                 self.blob_service.get_blob_to_path(
-                    bucket, path, tmp_file, progress_callback=cb)
+                    bucket, path, tmp_file, progress_callback=cb
+                )
             except Exception:
                 msg = "failed to download '{}/{}'".format(bucket, path)
                 logger.warning(msg)
             else:
-                os.rename(tmp_file, to_info['path'])
+                os.rename(tmp_file, to_info["path"])
 
                 if not no_progress_bar:
                     progress.finish_target(name)

dvc/remote/base.py

@@ -30,8 +30,9 @@ STATUS_MAP = {
 
 class DataCloudError(DvcException):
     """ Data Cloud exception """
+
     def __init__(self, msg):
-        super(DataCloudError, self).__init__('Data sync error: {}'.format(msg))
+        super(DataCloudError, self).__init__("Data sync error: {}".format(msg))
 
 
 class RemoteBaseCmdError(DvcException):
@@ -55,8 +56,7 @@ class RemoteBase(object):
         deps_ok = all(self.REQUIRES.values())
         if not deps_ok:
             missing = [k for k, v in self.REQUIRES.items() if v is None]
-            url = config.get(Config.SECTION_REMOTE_URL,
-                             '{}://'.format(self.scheme))
+            url = config.get(Config.SECTION_REMOTE_URL, "{}://".format(self.scheme))
             msg = (
                 "URL '{}' is supported but requires these missing "
                 "dependencies: {}. If you have installed dvc using pip, "
@@ -73,15 +73,15 @@ class RemoteBase(object):
                 "\n"
                 "If you have installed dvc from a binary package and you "
                 "are still seeing this message, please report it to us "
-                "using https://github.com/iterative/dvc/issues. Thank you!"
-                .format(url, missing, " ".join(missing), self.scheme)
+                "using https://github.com/iterative/dvc/issues. Thank you!".format(
+                    url, missing, " ".join(missing), self.scheme
+                )
             )
             raise RemoteMissingDepsError(msg)
 
     def __repr__(self):
         return "{class_name}: '{url}'".format(
-            class_name=type(self).__name__,
-            url=(self.url or 'No url')
+            class_name=type(self).__name__, url=(self.url or "No url")
         )
 
     def compat_config(config):
@@ -106,7 +106,8 @@ class RemoteBase(object):
     def tmp_file(fname):
         """ Temporary name for a partial download """
         import uuid
-        return fname + '.' + str(uuid.uuid4())
+
+        return fname + "." + str(uuid.uuid4())
 
     def save_info(self, path_info):
         raise NotImplementedError
@@ -128,8 +129,9 @@ class RemoteBase(object):
             bool: True if data has changed, False otherwise.
         """
 
-        logger.debug("checking if '{}'('{}') has changed."
-                     .format(path_info, checksum_info))
+        logger.debug(
+            "checking if '{}'('{}') has changed.".format(path_info, checksum_info)
+        )
 
         if not self.exists(path_info):
             logger.debug("'{}' doesn't exist.".format(path_info))
@@ -141,14 +143,18 @@ class RemoteBase(object):
             return True
 
         if self.changed_cache(checksum):
-            logger.debug("cache for '{}'('{}') has changed."
-                         .format(path_info, checksum))
+            logger.debug(
+                "cache for '{}'('{}') has changed.".format(path_info, checksum)
+            )
             return True
 
         actual = self.save_info(path_info)[self.PARAM_CHECKSUM]
         if checksum != actual:
-            logger.debug("checksum '{}'(actual '{}') for '{}' has changed."
-                         .format(checksum, actual, path_info))
+            logger.debug(
+                "checksum '{}'(actual '{}') for '{}' has changed.".format(
+                    checksum, actual, path_info
+                )
+            )
             return True
 
         logger.debug("'{}' hasn't changed.".format(path_info))
@@ -211,14 +217,11 @@ class RemoteBase(object):
 
     def checksum_to_path_info(self, checksum):
         path_info = self.path_info.copy()
-        path_info['path'] = self.checksum_to_path(checksum)
+        path_info["path"] = self.checksum_to_path(checksum)
         return path_info
 
     def md5s_to_path_infos(self, md5s):
-        return [
-            self.checksum_to_path_info(md5)
-            for md5 in md5s
-        ]
+        return [self.checksum_to_path_info(md5) for md5 in md5s]
 
     def list_cache_paths(self):
         raise NotImplementedError
@@ -228,17 +231,14 @@ class RemoteBase(object):
         # is 32 bytes, so ~3200Mb list) and we don't really need all of it at
         # the same time, so it makes sense to use a generator to gradually
         # iterate over it, without keeping all of it in memory.
-        return (
-            self.path_to_checksum(path)
-            for path in self.list_cache_paths()
-        )
+        return (self.path_to_checksum(path) for path in self.list_cache_paths())
 
     def gc(self, cinfos):
         from dvc.remote.local import RemoteLOCAL
 
-        used = [info[RemoteLOCAL.PARAM_CHECKSUM] for info in cinfos['local']]
+        used = [info[RemoteLOCAL.PARAM_CHECKSUM] for info in cinfos["local"]]
 
-        if self.scheme != '':
+        if self.scheme != "":
             used += [info[self.PARAM_CHECKSUM] for info in cinfos[self.scheme]]
 
         removed = False
@@ -255,12 +255,11 @@ class RemoteBase(object):
         expected = {self.PARAM_CHECKSUM: checksum}
         actual = self.save_info(cache)
 
-        logger.debug("Cache '{}' actual '{}'.".format(str(expected),
-                                                      str(actual)))
+        logger.debug("Cache '{}' actual '{}'.".format(str(expected), str(actual)))
 
         if expected != actual:
             if self.exists(cache):
-                msg = 'corrupted cache file {}'
+                msg = "corrupted cache file {}"
                 logger.warning(msg.format(str(cache)))
                 self.remove(cache)
             return True
@@ -293,15 +292,13 @@ class RemoteBase(object):
         if not force and not self.already_cached(path_info):
             msg = (
                 "file '{}' is going to be removed."
-                ' Are you sure you want to proceed?'
-                .format(str(path_info))
+                " Are you sure you want to proceed?".format(str(path_info))
             )
 
             if not prompt.confirm(msg):
                 raise DvcException(
                     "unable to remove '{}' without a confirmation"
-                    " from the user. Use '-f' to force."
-                    .format(str(path_info))
+                    " from the user. Use '-f' to force.".format(str(path_info))
                 )
 
         self.remove(path_info)
@@ -316,8 +313,8 @@ class RemoteBase(object):
         self.copy(from_info, path_info)
 
     def checkout(self, path_info, checksum_info, force=False):
-        scheme = path_info['scheme']
-        if scheme not in ['', 'local'] and scheme != self.scheme:
+        scheme = path_info["scheme"]
+        if scheme not in ["", "local"] and scheme != self.scheme:
             raise NotImplementedError
 
         checksum = checksum_info.get(self.PARAM_CHECKSUM)

dvc/remote/gs.py

@@ -16,29 +16,29 @@ from dvc.exceptions import DvcException
 
 
 class RemoteGS(RemoteBase):
-    scheme = 'gs'
-    REGEX = r'^gs://(?P<path>.*)$'
-    REQUIRES = {'google.cloud.storage': storage}
-    PARAM_CHECKSUM = 'md5'
+    scheme = "gs"
+    REGEX = r"^gs://(?P<path>.*)$"
+    REQUIRES = {"google.cloud.storage": storage}
+    PARAM_CHECKSUM = "md5"
 
     def __init__(self, project, config):
         super(RemoteGS, self).__init__(project, config)
-        storagepath = 'gs://'
-        storagepath += config.get(Config.SECTION_AWS_STORAGEPATH, '/')
-        storagepath.lstrip('/')
+        storagepath = "gs://"
+        storagepath += config.get(Config.SECTION_AWS_STORAGEPATH, "/")
+        storagepath.lstrip("/")
         self.url = config.get(Config.SECTION_REMOTE_URL, storagepath)
         self.projectname = config.get(Config.SECTION_GCP_PROJECTNAME, None)
 
         parsed = urlparse(self.url)
         self.bucket = parsed.netloc
-        self.prefix = parsed.path.lstrip('/')
+        self.prefix = parsed.path.lstrip("/")
 
-        self.path_info = {'scheme': 'gs', 'bucket': self.bucket}
+        self.path_info = {"scheme": "gs", "bucket": self.bucket}
 
     @staticmethod
     def compat_config(config):
         ret = config.copy()
-        url = 'gs://' + ret.pop(Config.SECTION_AWS_STORAGEPATH, '').lstrip('/')
+        url = "gs://" + ret.pop(Config.SECTION_AWS_STORAGEPATH, "").lstrip("/")
         ret[Config.SECTION_REMOTE_URL] = url
         return ret
 
@@ -56,48 +56,50 @@ class RemoteGS(RemoteBase):
 
         b64_md5 = blob.md5_hash
         md5 = base64.b64decode(b64_md5)
-        return codecs.getencoder('hex')(md5)[0].decode('utf-8')
+        return codecs.getencoder("hex")(md5)[0].decode("utf-8")
 
     def save_info(self, path_info):
-        if path_info['scheme'] != 'gs':
+        if path_info["scheme"] != "gs":
             raise NotImplementedError
 
-        return {self.PARAM_CHECKSUM: self.get_md5(path_info['bucket'],
-                                                  path_info['path'])}
+        return {
+            self.PARAM_CHECKSUM: self.get_md5(path_info["bucket"], path_info["path"])
+        }
 
     def copy(self, from_info, to_info, gs=None):
         gs = gs if gs else self.gs
 
-        blob = gs.bucket(from_info['bucket']).get_blob(from_info['path'])
+        blob = gs.bucket(from_info["bucket"]).get_blob(from_info["path"])
         if not blob:
-            msg = "'{}' doesn't exist in the cloud".format(from_info['path'])
+            msg = "'{}' doesn't exist in the cloud".format(from_info["path"])
             raise DvcException(msg)
 
-        bucket = self.gs.bucket(to_info['bucket'])
-        bucket.copy_blob(blob,
-                         self.gs.bucket(to_info['bucket']),
-                         new_name=to_info['path'])
+        bucket = self.gs.bucket(to_info["bucket"])
+        bucket.copy_blob(
+            blob, self.gs.bucket(to_info["bucket"]), new_name=to_info["path"]
+        )
 
     def save(self, path_info):
-        if path_info['scheme'] != 'gs':
+        if path_info["scheme"] != "gs":
             raise NotImplementedError
 
-        md5 = self.get_md5(path_info['bucket'], path_info['path'])
+        md5 = self.get_md5(path_info["bucket"], path_info["path"])
         path = self.checksum_to_path(md5)
-        to_info = {'scheme': 'gs', 'bucket': self.bucket, 'path': path}
+        to_info = {"scheme": "gs", "bucket": self.bucket, "path": path}
 
         self.copy(path_info, to_info)
 
         return {self.PARAM_CHECKSUM: md5}
 
     def remove(self, path_info):
-        if path_info['scheme'] != 'gs':
+        if path_info["scheme"] != "gs":
             raise NotImplementedError
 
-        logger.debug("Removing gs://{}/{}".format(path_info['bucket'],
-                                                  path_info['path']))
+        logger.debug(
+            "Removing gs://{}/{}".format(path_info["bucket"], path_info["path"])
+        )
 
-        blob = self.gs.bucket(path_info['bucket']).get_blob(path_info['path'])
+        blob = self.gs.bucket(path_info["bucket"]).get_blob(path_info["path"])
         if not blob:
             return
 
@@ -112,10 +114,10 @@ class RemoteGS(RemoteBase):
 
     def exists(self, path_info):
         assert not isinstance(path_info, list)
-        assert path_info['scheme'] == 'gs'
+        assert path_info["scheme"] == "gs"
 
-        paths = self._list_paths(path_info['bucket'], path_info['path'])
-        return any(path_info['path'] == path for path in paths)
+        paths = self._list_paths(path_info["bucket"], path_info["path"])
+        return any(path_info["path"] == path for path in paths)
 
     def upload(self, from_infos, to_infos, names=None):
         names = self._verify_path_args(to_infos, from_infos, names)
@@ -123,82 +125,80 @@ class RemoteGS(RemoteBase):
         gs = self.gs
 
         for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info['scheme'] != 'gs':
+            if to_info["scheme"] != "gs":
                 raise NotImplementedError
 
-            if from_info['scheme'] != 'local':
+            if from_info["scheme"] != "local":
                 raise NotImplementedError
 
-            logger.debug("Uploading '{}' to '{}/{}'".format(from_info['path'],
-                                                            to_info['bucket'],
-                                                            to_info['path']))
+            logger.debug(
+                "Uploading '{}' to '{}/{}'".format(
+                    from_info["path"], to_info["bucket"], to_info["path"]
+                )
+            )
 
             if not name:
-                name = os.path.basename(from_info['path'])
+                name = os.path.basename(from_info["path"])
 
             progress.update_target(name, 0, None)
 
             try:
-                bucket = gs.bucket(to_info['bucket'])
-                blob = bucket.blob(to_info['path'])
-                blob.upload_from_filename(from_info['path'])
+                bucket = gs.bucket(to_info["bucket"])
+                blob = bucket.blob(to_info["path"])
+                blob.upload_from_filename(from_info["path"])
             except Exception:
                 msg = "failed to upload '{}' to '{}/{}'"
-                logger.error(msg.format(from_info['path'],
-                                        to_info['bucket'],
-                                        to_info['path']))
+                logger.error(
+                    msg.format(from_info["path"], to_info["bucket"], to_info["path"])
+                )
                 continue
 
             progress.finish_target(name)
 
-    def download(self,
-                 from_infos,
-                 to_infos,
-                 no_progress_bar=False,
-                 names=None):
+    def download(self, from_infos, to_infos, no_progress_bar=False, names=None):
         names = self._verify_path_args(from_infos, to_infos, names)
 
         gs = self.gs
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info['scheme'] != 'gs':
+            if from_info["scheme"] != "gs":
                 raise NotImplementedError
 
-            if to_info['scheme'] == 'gs':
+            if to_info["scheme"] == "gs":
                 self.copy(from_info, to_info, gs=gs)
                 continue
 
-            if to_info['scheme'] != 'local':
+            if to_info["scheme"] != "local":
                 raise NotImplementedError
 
-            msg = "Downloading '{}/{}' to '{}'".format(from_info['bucket'],
-                                                       from_info['path'],
-                                                       to_info['path'])
+            msg = "Downloading '{}/{}' to '{}'".format(
+                from_info["bucket"], from_info["path"], to_info["path"]
+            )
             logger.debug(msg)
 
-            tmp_file = self.tmp_file(to_info['path'])
+            tmp_file = self.tmp_file(to_info["path"])
             if not name:
-                name = os.path.basename(to_info['path'])
+                name = os.path.basename(to_info["path"])
 
             if not no_progress_bar:
                 # percent_cb is not available for download_to_filename, so
                 # lets at least update progress at pathpoints(start, finish)
                 progress.update_target(name, 0, None)
 
-            self._makedirs(to_info['path'])
+            self._makedirs(to_info["path"])
 
             try:
-                bucket = gs.bucket(from_info['bucket'])
-                blob = bucket.get_blob(from_info['path'])
+                bucket = gs.bucket(from_info["bucket"])
+                blob = bucket.get_blob(from_info["path"])
                 blob.download_to_filename(tmp_file)
             except Exception:
                 msg = "failed to download '{}/{}' to '{}'"
-                logger.error(msg.format(from_info['bucket'],
-                                        from_info['path'],
-                                        to_info['path']))
+                logger.error(
+                    msg.format(from_info["bucket"], from_info["path"], to_info["path"])
+                )
                 continue
 
-            os.rename(tmp_file, to_info['path'])
+            os.rename(tmp_file, to_info["path"])
 
             if not no_progress_bar:
                 progress.finish_target(name)

dvc/remote/hdfs.py

@@ -17,43 +17,44 @@ class RemoteHDFSCmdError(RemoteBaseCmdError):
 
 
 class RemoteHDFS(RemoteBase):
-    scheme = 'hdfs'
-    REGEX = r'^hdfs://((?P<user>.*)@)?.*$'
-    PARAM_CHECKSUM = 'checksum'
+    scheme = "hdfs"
+    REGEX = r"^hdfs://((?P<user>.*)@)?.*$"
+    PARAM_CHECKSUM = "checksum"
 
     def __init__(self, project, config):
         super(RemoteHDFS, self).__init__(project, config)
-        self.url = config.get(Config.SECTION_REMOTE_URL, '/')
+        self.url = config.get(Config.SECTION_REMOTE_URL, "/")
         self.prefix = self.url
-        self.user = self.group('user')
+        self.user = self.group("user")
         if not self.user:
-            self.user = config.get(Config.SECTION_REMOTE_USER,
-                                   getpass.getuser())
+            self.user = config.get(Config.SECTION_REMOTE_USER, getpass.getuser())
 
-        self.path_info = {'scheme': 'hdfs', 'user': self.user}
+        self.path_info = {"scheme": "hdfs", "user": self.user}
 
     def hadoop_fs(self, cmd, user=None):
-        cmd = 'hadoop fs -' + cmd
+        cmd = "hadoop fs -" + cmd
         if user:
-            cmd = 'HADOOP_USER_NAME={} '.format(user) + cmd
+            cmd = "HADOOP_USER_NAME={} ".format(user) + cmd
 
         # NOTE: close_fds doesn't work with redirected stdin/stdout/stderr.
         # See https://github.com/iterative/dvc/issues/1197.
-        close_fds = (os.name != 'nt')
+        close_fds = os.name != "nt"
 
-        executable = os.getenv('SHELL') if os.name != 'nt' else None
-        p = Popen(cmd,
-                  shell=True,
-                  close_fds=close_fds,
-                  executable=executable,
-                  env=fix_env(os.environ),
-                  stdin=PIPE,
-                  stdout=PIPE,
-                  stderr=PIPE)
+        executable = os.getenv("SHELL") if os.name != "nt" else None
+        p = Popen(
+            cmd,
+            shell=True,
+            close_fds=close_fds,
+            executable=executable,
+            env=fix_env(os.environ),
+            stdin=PIPE,
+            stdout=PIPE,
+            stderr=PIPE,
+        )
         out, err = p.communicate()
         if p.returncode != 0:
             raise RemoteHDFSCmdError(cmd, p.returncode, err)
-        return out.decode('utf-8')
+        return out.decode("utf-8")
 
     @staticmethod
     def _group(regex, s, gname):
@@ -62,60 +63,61 @@ class RemoteHDFS(RemoteBase):
         return match.group(gname)
 
     def checksum(self, path_info):
-        regex = r'.*\t.*\t(?P<checksum>.*)'
-        stdout = self.hadoop_fs('checksum {}'.format(path_info['path']),
-                                user=path_info['user'])
-        return self._group(regex, stdout, 'checksum')
+        regex = r".*\t.*\t(?P<checksum>.*)"
+        stdout = self.hadoop_fs(
+            "checksum {}".format(path_info["path"]), user=path_info["user"]
+        )
+        return self._group(regex, stdout, "checksum")
 
     def copy(self, from_info, to_info):
-        dname = posixpath.dirname(to_info['path'])
-        self.hadoop_fs('mkdir -p {}'.format(dname), user=to_info['user'])
-        self.hadoop_fs('cp -f {} {}'.format(from_info['path'],
-                                            to_info['path']),
-                       user=to_info['user'])
+        dname = posixpath.dirname(to_info["path"])
+        self.hadoop_fs("mkdir -p {}".format(dname), user=to_info["user"])
+        self.hadoop_fs(
+            "cp -f {} {}".format(from_info["path"], to_info["path"]),
+            user=to_info["user"],
+        )
 
     def rm(self, path_info):
-        self.hadoop_fs('rm {}'.format(path_info['path']),
-                       user=path_info['user'])
+        self.hadoop_fs("rm {}".format(path_info["path"]), user=path_info["user"])
 
     def save_info(self, path_info):
-        if path_info['scheme'] != 'hdfs':
+        if path_info["scheme"] != "hdfs":
             raise NotImplementedError
 
-        assert path_info.get('path')
+        assert path_info.get("path")
 
         return {self.PARAM_CHECKSUM: self.checksum(path_info)}
 
     def save(self, path_info):
-        if path_info['scheme'] != 'hdfs':
+        if path_info["scheme"] != "hdfs":
             raise NotImplementedError
 
-        assert path_info.get('path')
+        assert path_info.get("path")
 
         checksum = self.checksum(path_info)
         dest = path_info.copy()
-        dest['path'] = self.checksum_to_path(checksum)
+        dest["path"] = self.checksum_to_path(checksum)
 
         self.copy(path_info, dest)
 
         return {self.PARAM_CHECKSUM: checksum}
 
     def remove(self, path_info):
-        if path_info['scheme'] != 'hdfs':
+        if path_info["scheme"] != "hdfs":
             raise NotImplementedError
 
-        assert path_info.get('path')
+        assert path_info.get("path")
 
-        logger.debug('Removing {}'.format(path_info['path']))
+        logger.debug("Removing {}".format(path_info["path"]))
 
         self.rm(path_info)
 
     def exists(self, path_info):
         assert not isinstance(path_info, list)
-        assert path_info['scheme'] == 'hdfs'
+        assert path_info["scheme"] == "hdfs"
 
         try:
-            self.hadoop_fs('test -e {}'.format(path_info['path']))
+            self.hadoop_fs("test -e {}".format(path_info["path"]))
             return True
         except RemoteHDFSCmdError:
             return False
@@ -124,56 +126,50 @@ class RemoteHDFS(RemoteBase):
         names = self._verify_path_args(to_infos, from_infos, names)
 
         for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info['scheme'] != 'hdfs':
+            if to_info["scheme"] != "hdfs":
                 raise NotImplementedError
 
-            if from_info['scheme'] != 'local':
+            if from_info["scheme"] != "local":
                 raise NotImplementedError
 
-            cmd = 'mkdir -p {}'.format(posixpath.dirname(to_info['path']))
-            self.hadoop_fs(cmd, user=to_info['user'])
+            cmd = "mkdir -p {}".format(posixpath.dirname(to_info["path"]))
+            self.hadoop_fs(cmd, user=to_info["user"])
 
-            cmd = 'copyFromLocal {} {}'.format(from_info['path'],
-                                               to_info['path'])
-            self.hadoop_fs(cmd, user=to_info['user'])
+            cmd = "copyFromLocal {} {}".format(from_info["path"], to_info["path"])
+            self.hadoop_fs(cmd, user=to_info["user"])
 
-    def download(self,
-                 from_infos,
-                 to_infos,
-                 no_progress_bar=False,
-                 names=None):
+    def download(self, from_infos, to_infos, no_progress_bar=False, names=None):
         names = self._verify_path_args(from_infos, to_infos, names)
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info['scheme'] != 'hdfs':
+            if from_info["scheme"] != "hdfs":
                 raise NotImplementedError
 
-            if to_info['scheme'] == 'hdfs':
+            if to_info["scheme"] == "hdfs":
                 self.copy(from_info, to_info)
                 continue
 
-            if to_info['scheme'] != 'local':
+            if to_info["scheme"] != "local":
                 raise NotImplementedError
 
-            dname = os.path.dirname(to_info['path'])
+            dname = os.path.dirname(to_info["path"])
             if not os.path.exists(dname):
                 os.makedirs(dname)
 
-            cmd = 'copyToLocal {} {}'.format(from_info['path'],
-                                             to_info['path'])
-            self.hadoop_fs(cmd, user=from_info['user'])
+            cmd = "copyToLocal {} {}".format(from_info["path"], to_info["path"])
+            self.hadoop_fs(cmd, user=from_info["user"])
 
     def list_cache_paths(self):
         try:
-            self.hadoop_fs('test -e {}'.format(self.prefix))
+            self.hadoop_fs("test -e {}".format(self.prefix))
         except RemoteHDFSCmdError:
             return []
 
-        stdout = self.hadoop_fs('ls -R {}'.format(self.prefix))
-        lines = stdout.split('\n')
+        stdout = self.hadoop_fs("ls -R {}".format(self.prefix))
+        lines = stdout.split("\n")
         flist = []
         for line in lines:
-            if not line.startswith('-'):
+            if not line.startswith("-"):
                 continue
             flist.append(line.split()[-1])
         return flist

dvc/remote/http.py

@@ -25,47 +25,42 @@ class ProgressBarCallback(object):
 
 
 class RemoteHTTP(RemoteBase):
-    scheme = 'http'
-    REGEX = r'^https?://.*$'
+    scheme = "http"
+    REGEX = r"^https?://.*$"
     REQUEST_TIMEOUT = 10
     CHUNK_SIZE = 1000000  # Megabyte
-    PARAM_CHECKSUM = 'etag'
+    PARAM_CHECKSUM = "etag"
 
     def __init__(self, project, config):
         super(RemoteHTTP, self).__init__(project, config)
         self.cache_dir = config.get(Config.SECTION_REMOTE_URL)
         self.url = self.cache_dir
-        self.path_info = {'scheme': 'http'}
+        self.path_info = {"scheme": "http"}
 
     @property
     def prefix(self):
         return self.cache_dir
 
-    def download(self,
-                 from_infos,
-                 to_infos,
-                 no_progress_bar=False,
-                 names=None):
+    def download(self, from_infos, to_infos, no_progress_bar=False, names=None):
         names = self._verify_path_args(to_infos, from_infos, names)
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info['scheme'] not in ['http', 'https']:
+            if from_info["scheme"] not in ["http", "https"]:
                 raise NotImplementedError
 
-            if to_info['scheme'] != 'local':
+            if to_info["scheme"] != "local":
                 raise NotImplementedError
 
-            msg = "Downloading '{}' to '{}'".format(from_info['path'],
-                                                    to_info['path'])
+            msg = "Downloading '{}' to '{}'".format(from_info["path"], to_info["path"])
             logger.debug(msg)
 
-            tmp_file = self.tmp_file(to_info['path'])
+            tmp_file = self.tmp_file(to_info["path"])
             if not name:
-                name = os.path.basename(to_info['path'])
+                name = os.path.basename(to_info["path"])
 
-            self._makedirs(to_info['path'])
+            self._makedirs(to_info["path"])
 
-            total = self._content_length(from_info['path'])
+            total = self._content_length(from_info["path"])
 
             if no_progress_bar or not total:
                 cb = None
@@ -73,50 +68,52 @@ class RemoteHTTP(RemoteBase):
                 cb = ProgressBarCallback(name, total)
 
             try:
-                self._download_to(from_info['path'], tmp_file, callback=cb)
+                self._download_to(from_info["path"], tmp_file, callback=cb)
             except Exception:
-                msg = "failed to download '{}'".format(from_info['path'])
+                msg = "failed to download '{}'".format(from_info["path"])
                 logger.error(msg)
                 continue
 
-            os.rename(tmp_file, to_info['path'])
+            os.rename(tmp_file, to_info["path"])
 
             if not no_progress_bar:
                 progress.finish_target(name)
 
     def exists(self, path_info):
         assert not isinstance(path_info, list)
-        assert path_info['scheme'] in ['http', 'https']
-        return bool(self._request('HEAD', path_info.get('path')))
+        assert path_info["scheme"] in ["http", "https"]
+        return bool(self._request("HEAD", path_info.get("path")))
 
     def cache_exists(self, md5s):
         assert isinstance(md5s, list)
 
         def func(md5):
-            return bool(self._request('HEAD', self.checksum_to_path(md5)))
+            return bool(self._request("HEAD", self.checksum_to_path(md5)))
 
         return list(filter(func, md5s))
 
     def save_info(self, path_info):
-        if path_info['scheme'] not in ['http', 'https']:
+        if path_info["scheme"] not in ["http", "https"]:
             raise NotImplementedError
 
-        return {self.PARAM_CHECKSUM: self._etag(path_info['path'])}
+        return {self.PARAM_CHECKSUM: self._etag(path_info["path"])}
 
     def _content_length(self, url):
-        return self._request('HEAD', url).headers.get('Content-Length')
+        return self._request("HEAD", url).headers.get("Content-Length")
 
     def _etag(self, url):
-        etag = (self._request('HEAD', url).headers.get('ETag')
-                or self._request('HEAD', url).headers.get('Content-MD5'))
+        etag = self._request("HEAD", url).headers.get("ETag") or self._request(
+            "HEAD", url
+        ).headers.get("Content-MD5")
 
         if not etag:
             raise DvcException(
-                "could not find an ETag or Content-MD5 header for '{url}'"
-                .format(url=url)
+                "could not find an ETag or Content-MD5 header for '{url}'".format(
+                    url=url
+                )
             )
 
-        if etag.startswith('W/'):
+        if etag.startswith("W/"):
             raise DvcException(
                 "Weak ETags are not supported."
                 " (Etag: '{etag}', URL: '{url}')".format(etag=etag, url=url)
@@ -125,9 +122,9 @@ class RemoteHTTP(RemoteBase):
         return etag
 
     def _download_to(self, url, file, callback=None):
-        r = self._request('GET', url, stream=True)
+        r = self._request("GET", url, stream=True)
 
-        with open(file, 'wb') as fd:
+        with open(file, "wb") as fd:
             bytes_transfered = 0
 
             for chunk in r.iter_content(chunk_size=self.CHUNK_SIZE):
@@ -138,13 +135,13 @@ class RemoteHTTP(RemoteBase):
                     callback(bytes_transfered)
 
     def _request(self, method, url, **kwargs):
-        kwargs.setdefault('allow_redirects', True)
-        kwargs.setdefault('timeout', self.REQUEST_TIMEOUT)
+        kwargs.setdefault("allow_redirects", True)
+        kwargs.setdefault("timeout", self.REQUEST_TIMEOUT)
 
         try:
             return requests.request(method, url, **kwargs)
         except requests.exceptions.RequestException:
-            raise DvcException('could not perform a {} request'.format(method))
+            raise DvcException("could not perform a {} request".format(method))
 
     def gc(self):
         raise NotImplementedError

dvc/remote/local.py

@@ -23,19 +23,19 @@ from concurrent.futures import ThreadPoolExecutor
 
 
 class RemoteLOCAL(RemoteBase):
-    scheme = 'local'
-    REGEX = r'^(?P<path>.*)$'
-    PARAM_CHECKSUM = 'md5'
-    PARAM_PATH = 'path'
-    PARAM_RELPATH = 'relpath'
-    MD5_DIR_SUFFIX = '.dir'
+    scheme = "local"
+    REGEX = r"^(?P<path>.*)$"
+    PARAM_CHECKSUM = "md5"
+    PARAM_PATH = "path"
+    PARAM_RELPATH = "relpath"
+    MD5_DIR_SUFFIX = ".dir"
 
-    CACHE_TYPES = ['reflink', 'hardlink', 'symlink', 'copy']
+    CACHE_TYPES = ["reflink", "hardlink", "symlink", "copy"]
     CACHE_TYPE_MAP = {
-        'copy': shutil.copyfile,
-        'symlink': System.symlink,
-        'hardlink': System.hardlink,
-        'reflink': System.reflink,
+        "copy": shutil.copyfile,
+        "symlink": System.symlink,
+        "hardlink": System.hardlink,
+        "reflink": System.reflink,
     }
 
     def __init__(self, project, config):
@@ -52,7 +52,7 @@ class RemoteLOCAL(RemoteBase):
         types = config.get(Config.SECTION_CACHE_TYPE, None)
         if types:
             if isinstance(types, str):
-                types = [t.strip() for t in types.split(',')]
+                types = [t.strip() for t in types.split(",")]
             self.cache_types = types
         else:
             self.cache_types = self.CACHE_TYPES
@@ -60,12 +60,12 @@ class RemoteLOCAL(RemoteBase):
         if self.cache_dir is not None and not os.path.exists(self.cache_dir):
             os.mkdir(self.cache_dir)
 
-        self.path_info = {'scheme': 'local'}
+        self.path_info = {"scheme": "local"}
 
     @staticmethod
     def compat_config(config):
         ret = config.copy()
-        url = ret.pop(Config.SECTION_AWS_STORAGEPATH, '')
+        url = ret.pop(Config.SECTION_AWS_STORAGEPATH, "")
         ret[Config.SECTION_REMOTE_URL] = url
         return ret
 
@@ -99,7 +99,7 @@ class RemoteLOCAL(RemoteBase):
         cache = self.get(md5)
         if self.state.changed(cache, md5=md5):
             if os.path.exists(cache):
-                msg = 'Corrupted cache file {}.'
+                msg = "Corrupted cache file {}."
                 logger.warning(msg.format(os.path.relpath(cache)))
                 remove(cache)
             return True
@@ -107,8 +107,8 @@ class RemoteLOCAL(RemoteBase):
 
     def exists(self, path_info):
         assert not isinstance(path_info, list)
-        assert path_info['scheme'] == 'local'
-        return os.path.exists(path_info['path'])
+        assert path_info["scheme"] == "local"
+        return os.path.exists(path_info["path"])
 
     def changed_cache(self, md5):
         cache = self.get(md5)
@@ -141,7 +141,7 @@ class RemoteLOCAL(RemoteBase):
 
         # NOTE: just create an empty file for an empty cache
         if os.path.getsize(cache) == 0:
-            open(path, 'w+').close()
+            open(path, "w+").close()
 
             msg = "Created empty file: {} -> {}".format(cache, path)
             logger.debug(msg)
@@ -156,8 +156,11 @@ class RemoteLOCAL(RemoteBase):
                     os.chmod(path, stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH)
 
                 msg = "Created {}'{}': {} -> {}".format(
-                    'protected ' if self.protected else '',
-                    self.cache_types[0], cache, path)
+                    "protected " if self.protected else "",
+                    self.cache_types[0],
+                    cache,
+                    path,
+                )
 
                 logger.debug(msg)
                 return
@@ -168,17 +171,17 @@ class RemoteLOCAL(RemoteBase):
                 del self.cache_types[0]
                 i -= 1
 
-        raise DvcException('no possible cache types left to try out.')
+        raise DvcException("no possible cache types left to try out.")
 
     @property
     def ospath(self):
-        if os.name == 'nt':
+        if os.name == "nt":
             return ntpath
         return posixpath
 
     @classmethod
     def to_ospath(cls, path):
-        if os.name == 'nt':
+        if os.name == "nt":
             return cls.ntpath(path)
         return cls.unixpath(path)
 
@@ -186,13 +189,13 @@ class RemoteLOCAL(RemoteBase):
     def unixpath(path):
         assert not ntpath.isabs(path)
         assert not posixpath.isabs(path)
-        return path.replace('\\', '/')
+        return path.replace("\\", "/")
 
     @staticmethod
     def ntpath(path):
         assert not ntpath.isabs(path)
         assert not posixpath.isabs(path)
-        return path.replace('/', '\\')
+        return path.replace("/", "\\")
 
     def collect_dir_cache(self, dname):
         dir_info = []
@@ -201,8 +204,9 @@ class RemoteLOCAL(RemoteBase):
             bar = False
 
             if len(files) > LARGE_DIR_SIZE:
-                msg = "Computing md5 for a large directory {}. " \
-                      "This is only done once."
+                msg = (
+                    "Computing md5 for a large directory {}. " "This is only done once."
+                )
                 logger.info(msg.format(os.path.relpath(root)))
                 bar = True
                 title = os.path.relpath(root)
@@ -219,8 +223,7 @@ class RemoteLOCAL(RemoteBase):
                     processed += 1
 
                 md5 = self.state.update(path)
-                dir_info.append({self.PARAM_RELPATH: relpath,
-                                 self.PARAM_CHECKSUM: md5})
+                dir_info.append({self.PARAM_RELPATH: relpath, self.PARAM_CHECKSUM: md5})
 
             if bar:
                 progress.finish_target(title)
@@ -240,7 +243,7 @@ class RemoteLOCAL(RemoteBase):
         assert self.is_dir_cache(path)
 
         try:
-            with open(path, 'r') as fd:
+            with open(path, "r") as fd:
                 d = json.load(fd)
         except Exception:
             msg = "Failed to load dir cache '{}'"
@@ -253,7 +256,7 @@ class RemoteLOCAL(RemoteBase):
             return []
 
         for info in d:
-            info['relpath'] = self.to_ospath(info['relpath'])
+            info["relpath"] = self.to_ospath(info["relpath"])
 
         return d
 
@@ -269,8 +272,8 @@ class RemoteLOCAL(RemoteBase):
 
         # NOTE: Writing first and renaming after that
         # to make sure that the operation is atomic.
-        tmp = '{}.{}'.format(path, str(uuid.uuid4()))
-        with open(tmp, 'w+') as fd:
+        tmp = "{}.{}".format(path, str(uuid.uuid4()))
+        with open(tmp, "w+") as fd:
             json.dump(dir_info, fd, sort_keys=True)
         move(tmp, path)
 
@@ -279,7 +282,7 @@ class RemoteLOCAL(RemoteBase):
         return cache.endswith(cls.MD5_DIR_SUFFIX)
 
     def do_checkout(self, path_info, checksum, force=False):
-        path = path_info['path']
+        path = path_info["path"]
         md5 = checksum
         cache = self.get(md5)
 
@@ -309,7 +312,7 @@ class RemoteLOCAL(RemoteBase):
             p = os.path.join(path, relpath)
             c = self.get(m)
 
-            entry_info = {'scheme': path_info['scheme'], self.PARAM_PATH: p}
+            entry_info = {"scheme": path_info["scheme"], self.PARAM_PATH: p}
 
             entry_checksum_info = {self.PARAM_CHECKSUM: m}
 
@@ -330,9 +333,9 @@ class RemoteLOCAL(RemoteBase):
             progress.finish_target(dir_relpath)
 
     def already_cached(self, path_info):
-        assert path_info['scheme'] in ['', 'local']
+        assert path_info["scheme"] in ["", "local"]
 
-        current_md5 = self.state.update(path_info['path'])
+        current_md5 = self.state.update(path_info["path"])
 
         if not current_md5:
             return False
@@ -346,26 +349,23 @@ class RemoteLOCAL(RemoteBase):
             for file in files
         )
 
-        cached_files = set(
-            os.path.join(path, file['relpath'])
-            for file in dir_info
-        )
+        cached_files = set(os.path.join(path, file["relpath"]) for file in dir_info)
 
         delta = working_dir_files - cached_files
 
         for file in delta:
-            self.safe_remove({'scheme': 'local', 'path': file}, force=force)
+            self.safe_remove({"scheme": "local", "path": file}, force=force)
 
     def _move(self, inp, outp):
         # moving in two stages to make the whole operation atomic in
         # case inp and outp are in different filesystems and actual
         # physical copying of data is happening
-        tmp = '{}.{}'.format(outp, str(uuid.uuid4()))
+        tmp = "{}.{}".format(outp, str(uuid.uuid4()))
         move(inp, tmp)
         move(tmp, outp)
 
     def _save_file(self, path_info):
-        path = path_info['path']
+        path = path_info["path"]
         md5 = self.state.update(path)
         assert md5 is not None
 
@@ -382,7 +382,7 @@ class RemoteLOCAL(RemoteBase):
         return {self.PARAM_CHECKSUM: md5}
 
     def _save_dir(self, path_info):
-        path = path_info['path']
+        path = path_info["path"]
         md5, dir_info = self.state.update_info(path)
         dir_relpath = os.path.relpath(path)
         dir_size = len(dir_info)
@@ -414,14 +414,13 @@ class RemoteLOCAL(RemoteBase):
         return {self.PARAM_CHECKSUM: md5}
 
     def save(self, path_info):
-        if path_info['scheme'] != 'local':
+        if path_info["scheme"] != "local":
             raise NotImplementedError
 
-        path = path_info['path']
+        path = path_info["path"]
 
         msg = "Saving '{}' to cache '{}'."
-        logger.info(msg.format(os.path.relpath(path),
-                               os.path.relpath(self.cache_dir)))
+        logger.info(msg.format(os.path.relpath(path), os.path.relpath(self.cache_dir)))
 
         if os.path.isdir(path):
             return self._save_dir(path_info)
@@ -429,22 +428,22 @@ class RemoteLOCAL(RemoteBase):
             return self._save_file(path_info)
 
     def save_info(self, path_info):
-        if path_info['scheme'] != 'local':
+        if path_info["scheme"] != "local":
             raise NotImplementedError
 
-        return {self.PARAM_CHECKSUM: self.state.update(path_info['path'])}
+        return {self.PARAM_CHECKSUM: self.state.update(path_info["path"])}
 
     def remove(self, path_info):
-        if path_info['scheme'] != 'local':
+        if path_info["scheme"] != "local":
             raise NotImplementedError
 
-        remove(path_info['path'])
+        remove(path_info["path"])
 
     def move(self, from_info, to_info):
-        if from_info['scheme'] != 'local' or to_info['scheme'] != 'local':
+        if from_info["scheme"] != "local" or to_info["scheme"] != "local":
             raise NotImplementedError
 
-        move(from_info['path'], to_info['path'])
+        move(from_info["path"], to_info["path"])
 
     def cache_exists(self, md5s):
         assert isinstance(md5s, list)
@@ -454,64 +453,66 @@ class RemoteLOCAL(RemoteBase):
         names = self._verify_path_args(to_infos, from_infos, names)
 
         for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info['scheme'] != 'local':
+            if to_info["scheme"] != "local":
                 raise NotImplementedError
 
-            if from_info['scheme'] != 'local':
+            if from_info["scheme"] != "local":
                 raise NotImplementedError
 
-            logger.debug("Uploading '{}' to '{}'".format(from_info['path'],
-                                                         to_info['path']))
+            logger.debug(
+                "Uploading '{}' to '{}'".format(from_info["path"], to_info["path"])
+            )
 
             if not name:
-                name = os.path.basename(from_info['path'])
+                name = os.path.basename(from_info["path"])
 
-            self._makedirs(to_info['path'])
+            self._makedirs(to_info["path"])
 
             try:
-                copyfile(from_info['path'], to_info['path'], name=name)
+                copyfile(from_info["path"], to_info["path"], name=name)
             except Exception:
                 logger.error(
-                    "failed to upload '{}' to '{}'"
-                    .format(from_info['path'], to_info['path'])
+                    "failed to upload '{}' to '{}'".format(
+                        from_info["path"], to_info["path"]
+                    )
                 )
 
-    def download(self,
-                 from_infos,
-                 to_infos,
-                 no_progress_bar=False,
-                 names=None):
+    def download(self, from_infos, to_infos, no_progress_bar=False, names=None):
         names = self._verify_path_args(from_infos, to_infos, names)
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info['scheme'] != 'local':
+            if from_info["scheme"] != "local":
                 raise NotImplementedError
 
-            if to_info['scheme'] != 'local':
+            if to_info["scheme"] != "local":
                 raise NotImplementedError
 
-            logger.debug("Downloading '{}' to '{}'".format(from_info['path'],
-                                                           to_info['path']))
+            logger.debug(
+                "Downloading '{}' to '{}'".format(from_info["path"], to_info["path"])
+            )
 
             if not name:
-                name = os.path.basename(to_info['path'])
+                name = os.path.basename(to_info["path"])
 
-            self._makedirs(to_info['path'])
-            tmp_file = self.tmp_file(to_info['path'])
+            self._makedirs(to_info["path"])
+            tmp_file = self.tmp_file(to_info["path"])
             try:
-                copyfile(from_info['path'],
-                         tmp_file,
-                         no_progress_bar=no_progress_bar,
-                         name=name)
+                copyfile(
+                    from_info["path"],
+                    tmp_file,
+                    no_progress_bar=no_progress_bar,
+                    name=name,
+                )
             except Exception:
                 logger.error(
-                    "failed to download '{}' to '{}'"
-                    .format(from_info['path'], to_info['path'])
+                    "failed to download '{}' to '{}'".format(
+                        from_info["path"], to_info["path"]
+                    )
                 )
 
                 continue
 
-            os.rename(tmp_file, to_info['path'])
+            os.rename(tmp_file, to_info["path"])
 
     def _group(self, checksum_infos, show_checksums=False):
         by_md5 = {}
@@ -520,18 +521,18 @@ class RemoteLOCAL(RemoteBase):
             md5 = info[self.PARAM_CHECKSUM]
 
             if show_checksums:
-                by_md5[md5] = {'name': md5}
+                by_md5[md5] = {"name": md5}
                 continue
 
             name = info[self.PARAM_PATH]
-            branch = info.get('branch')
+            branch = info.get("branch")
             if branch:
-                name += '({})'.format(branch)
+                name += "({})".format(branch)
 
             if md5 not in by_md5.keys():
-                by_md5[md5] = {'name': name}
+                by_md5[md5] = {"name": name}
             else:
-                by_md5[md5]['name'] += ' ' + name
+                by_md5[md5]["name"] += " " + name
 
         return by_md5
 
@@ -560,8 +561,7 @@ class RemoteLOCAL(RemoteBase):
         progress.finish_target(title)
 
         for md5, info in ret.items():
-            info['status'] = STATUS_MAP[(md5 in local_exists,
-                                         md5 in remote_exists)]
+            info["status"] = STATUS_MAP[(md5 in local_exists, md5 in remote_exists)]
 
         return ret
 
@@ -576,10 +576,10 @@ class RemoteLOCAL(RemoteBase):
         path_infos = []
         names = []
         for md5, info in status_info.items():
-            if info['status'] == status:
+            if info["status"] == status:
                 cache.append(self.checksum_to_path_info(md5))
                 path_infos.append(remote.checksum_to_path_info(md5))
-                names.append(info['name'])
+                names.append(info["name"])
             current += 1
             progress.update_target(title, current, total)
 
@@ -594,20 +594,25 @@ class RemoteLOCAL(RemoteBase):
             to_infos = path_infos
             from_infos = cache
 
-        return list(zip(to_chunks(from_infos, jobs),
-                        to_chunks(to_infos, jobs),
-                        to_chunks(names, jobs)))
+        return list(
+            zip(
+                to_chunks(from_infos, jobs),
+                to_chunks(to_infos, jobs),
+                to_chunks(names, jobs),
+            )
+        )
 
-    def _process(self,
-                 checksum_infos,
-                 remote,
-                 jobs=None,
-                 show_checksums=False,
-                 download=False):
+    def _process(
+        self, checksum_infos, remote, jobs=None, show_checksums=False, download=False
+    ):
         msg = "Preparing to {} data {} '{}'"
-        logger.info(msg.format('download' if download else 'upload',
-                               'from' if download else 'to',
-                               remote.url))
+        logger.info(
+            msg.format(
+                "download" if download else "upload",
+                "from" if download else "to",
+                remote.url,
+            )
+        )
 
         if download:
             func = remote.download
@@ -619,10 +624,9 @@ class RemoteLOCAL(RemoteBase):
         if jobs is None:
             jobs = remote.JOBS
 
-        status_info = self.status(checksum_infos,
-                                  remote,
-                                  jobs=jobs,
-                                  show_checksums=show_checksums)
+        status_info = self.status(
+            checksum_infos, remote, jobs=jobs, show_checksums=show_checksums
+        )
 
         chunks = self._get_chunks(download, remote, status_info, status, jobs)
 
@@ -632,33 +636,26 @@ class RemoteLOCAL(RemoteBase):
         futures = []
         with ThreadPoolExecutor(max_workers=jobs) as executor:
             for from_infos, to_infos, names in chunks:
-                res = executor.submit(func,
-                                      from_infos,
-                                      to_infos,
-                                      names=names)
+                res = executor.submit(func, from_infos, to_infos, names=names)
                 futures.append(res)
 
         for f in futures:
             f.result()
 
-    def push(self,
-             checksum_infos,
-             remote,
-             jobs=None,
-             show_checksums=False):
-        self._process(checksum_infos,
-                      remote,
-                      jobs=jobs,
-                      show_checksums=show_checksums,
-                      download=False)
+    def push(self, checksum_infos, remote, jobs=None, show_checksums=False):
+        self._process(
+            checksum_infos,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=False,
+        )
 
-    def pull(self,
-             checksum_infos,
-             remote,
-             jobs=None,
-             show_checksums=False):
-        self._process(checksum_infos,
-                      remote,
-                      jobs=jobs,
-                      show_checksums=show_checksums,
-                      download=True)
+    def pull(self, checksum_infos, remote, jobs=None, show_checksums=False):
+        self._process(
+            checksum_infos,
+            remote,
+            jobs=jobs,
+            show_checksums=show_checksums,
+            download=True,
+        )

dvc/remote/s3.py

@@ -30,28 +30,26 @@ class Callback(object):
 
 
 class RemoteS3(RemoteBase):
-    scheme = 's3'
-    REGEX = r'^s3://(?P<path>.*)$'
-    REQUIRES = {'boto3': boto3}
-    PARAM_CHECKSUM = 'etag'
+    scheme = "s3"
+    REGEX = r"^s3://(?P<path>.*)$"
+    REQUIRES = {"boto3": boto3}
+    PARAM_CHECKSUM = "etag"
 
     def __init__(self, project, config):
         super(RemoteS3, self).__init__(project, config)
 
-        storagepath = 's3://{}'.format(
-            config.get(Config.SECTION_AWS_STORAGEPATH, '').lstrip('/')
+        storagepath = "s3://{}".format(
+            config.get(Config.SECTION_AWS_STORAGEPATH, "").lstrip("/")
         )
 
         self.url = config.get(Config.SECTION_REMOTE_URL, storagepath)
 
-        self.region = (
-            os.environ.get('AWS_DEFAULT_REGION') or
-            config.get(Config.SECTION_AWS_REGION)
+        self.region = os.environ.get("AWS_DEFAULT_REGION") or config.get(
+            Config.SECTION_AWS_REGION
         )
 
-        self.profile = (
-            os.environ.get('AWS_PROFILE') or
-            config.get(Config.SECTION_AWS_PROFILE)
+        self.profile = os.environ.get("AWS_PROFILE") or config.get(
+            Config.SECTION_AWS_PROFILE
         )
 
         self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL)
@@ -60,108 +58,107 @@ class RemoteS3(RemoteBase):
 
         shared_creds = config.get(Config.SECTION_AWS_CREDENTIALPATH)
         if shared_creds:
-            os.environ.setdefault('AWS_SHARED_CREDENTIALS_FILE', shared_creds)
+            os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds)
 
         parsed = urlparse(self.url)
         self.bucket = parsed.netloc
-        self.prefix = parsed.path.lstrip('/')
+        self.prefix = parsed.path.lstrip("/")
 
-        self.path_info = {'scheme': self.scheme, 'bucket': self.bucket}
+        self.path_info = {"scheme": self.scheme, "bucket": self.bucket}
 
     @staticmethod
     def compat_config(config):
         ret = config.copy()
-        url = 's3://' + ret.pop(Config.SECTION_AWS_STORAGEPATH, '').lstrip('/')
+        url = "s3://" + ret.pop(Config.SECTION_AWS_STORAGEPATH, "").lstrip("/")
         ret[Config.SECTION_REMOTE_URL] = url
         return ret
 
     @property
     def s3(self):
-        session = boto3.session.Session(profile_name=self.profile,
-                                        region_name=self.region)
+        session = boto3.session.Session(
+            profile_name=self.profile, region_name=self.region
+        )
 
-        return session.client('s3',
-                              endpoint_url=self.endpoint_url,
-                              use_ssl=self.use_ssl)
+        return session.client(
+            "s3", endpoint_url=self.endpoint_url, use_ssl=self.use_ssl
+        )
 
     def get_etag(self, bucket, path):
         try:
             obj = self.s3.head_object(Bucket=bucket, Key=path)
         except Exception:
-            raise DvcException('s3://{}/{} does not exist'.format(bucket,
-                                                                  path))
+            raise DvcException("s3://{}/{} does not exist".format(bucket, path))
 
-        return obj['ETag'].strip('"')
+        return obj["ETag"].strip('"')
 
     def save_info(self, path_info):
-        if path_info['scheme'] != 's3':
+        if path_info["scheme"] != "s3":
             raise NotImplementedError
 
-        return {self.PARAM_CHECKSUM: self.get_etag(path_info['bucket'],
-                                                   path_info['path'])}
+        return {
+            self.PARAM_CHECKSUM: self.get_etag(path_info["bucket"], path_info["path"])
+        }
 
     def copy(self, from_info, to_info, s3=None):
         s3 = s3 if s3 else self.s3
 
-        source = {'Bucket': from_info['bucket'],
-                  'Key': from_info['path']}
-        self.s3.copy(source, to_info['bucket'], to_info['path'])
+        source = {"Bucket": from_info["bucket"], "Key": from_info["path"]}
+        self.s3.copy(source, to_info["bucket"], to_info["path"])
 
     def save(self, path_info):
-        if path_info['scheme'] != 's3':
+        if path_info["scheme"] != "s3":
             raise NotImplementedError
 
-        etag = self.get_etag(path_info['bucket'], path_info['path'])
+        etag = self.get_etag(path_info["bucket"], path_info["path"])
         path = self.checksum_to_path(etag)
-        to_info = {'scheme': 's3', 'bucket': self.bucket, 'path': path}
+        to_info = {"scheme": "s3", "bucket": self.bucket, "path": path}
 
         self.copy(path_info, to_info)
 
         return {self.PARAM_CHECKSUM: etag}
 
     def remove(self, path_info):
-        if path_info['scheme'] != 's3':
+        if path_info["scheme"] != "s3":
             raise NotImplementedError
 
-        logger.debug('Removing s3://{}/{}'.format(path_info['bucket'],
-                                                  path_info['path']))
+        logger.debug(
+            "Removing s3://{}/{}".format(path_info["bucket"], path_info["path"])
+        )
 
-        self.s3.delete_object(Bucket=path_info['bucket'],
-                              Key=path_info['path'])
+        self.s3.delete_object(Bucket=path_info["bucket"], Key=path_info["path"])
 
     def _list_paths(self, bucket, prefix):
         s3 = self.s3
 
-        kwargs = {'Bucket': bucket,
-                  'Prefix': prefix}
+        kwargs = {"Bucket": bucket, "Prefix": prefix}
         while True:
             # NOTE: list_objects_v2() is 90% faster than head_object [1]
             #
             # [1] https://www.peterbe.com/plog/
             #     fastest-way-to-find-out-if-a-file-exists-in-s3
             resp = s3.list_objects_v2(**kwargs)
-            contents = resp.get('Contents', None)
+            contents = resp.get("Contents", None)
             if not contents:
                 break
 
             for obj in contents:
-                yield obj['Key']
+                yield obj["Key"]
 
-            token = resp.get('NextContinuationToken', None)
+            token = resp.get("NextContinuationToken", None)
             if not token:
                 break
 
-            kwargs['ContinuationToken'] = token
+            kwargs["ContinuationToken"] = token
 
     def list_cache_paths(self):
         return self._list_paths(self.bucket, self.prefix)
 
     def exists(self, path_info):
         assert not isinstance(path_info, list)
-        assert path_info['scheme'] == 's3'
+        assert path_info["scheme"] == "s3"
 
-        paths = self._list_paths(path_info['bucket'], path_info['path'])
-        return any(path_info['path'] == path for path in paths)
+        paths = self._list_paths(path_info["bucket"], path_info["path"])
+        return any(path_info["path"] == path for path in paths)
 
     def upload(self, from_infos, to_infos, names=None):
         names = self._verify_path_args(to_infos, from_infos, names)
@@ -169,85 +166,82 @@ class RemoteS3(RemoteBase):
         s3 = self.s3
 
         for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info['scheme'] != 's3':
+            if to_info["scheme"] != "s3":
                 raise NotImplementedError
 
-            if from_info['scheme'] != 'local':
+            if from_info["scheme"] != "local":
                 raise NotImplementedError
 
-            logger.debug("Uploading '{}' to '{}/{}'".format(from_info['path'],
-                                                            to_info['bucket'],
-                                                            to_info['path']))
+            logger.debug(
+                "Uploading '{}' to '{}/{}'".format(
+                    from_info["path"], to_info["bucket"], to_info["path"]
+                )
+            )
 
             if not name:
-                name = os.path.basename(from_info['path'])
+                name = os.path.basename(from_info["path"])
 
-            total = os.path.getsize(from_info['path'])
+            total = os.path.getsize(from_info["path"])
             cb = Callback(name, total)
 
             try:
-                s3.upload_file(from_info['path'],
-                               to_info['bucket'],
-                               to_info['path'],
-                               Callback=cb)
+                s3.upload_file(
+                    from_info["path"], to_info["bucket"], to_info["path"], Callback=cb
+                )
             except Exception:
-                msg = "failed to upload '{}'".format(from_info['path'])
+                msg = "failed to upload '{}'".format(from_info["path"])
                 logger.error(msg)
                 continue
 
             progress.finish_target(name)
 
-    def download(self,
-                 from_infos,
-                 to_infos,
-                 no_progress_bar=False,
-                 names=None):
+    def download(self, from_infos, to_infos, no_progress_bar=False, names=None):
         names = self._verify_path_args(from_infos, to_infos, names)
 
         s3 = self.s3
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info['scheme'] != 's3':
+            if from_info["scheme"] != "s3":
                 raise NotImplementedError
 
-            if to_info['scheme'] == 's3':
+            if to_info["scheme"] == "s3":
                 self.copy(from_info, to_info, s3=s3)
                 continue
 
-            if to_info['scheme'] != 'local':
+            if to_info["scheme"] != "local":
                 raise NotImplementedError
 
-            msg = "Downloading '{}/{}' to '{}'".format(from_info['bucket'],
-                                                       from_info['path'],
-                                                       to_info['path'])
+            msg = "Downloading '{}/{}' to '{}'".format(
+                from_info["bucket"], from_info["path"], to_info["path"]
+            )
             logger.debug(msg)
 
-            tmp_file = self.tmp_file(to_info['path'])
+            tmp_file = self.tmp_file(to_info["path"])
             if not name:
-                name = os.path.basename(to_info['path'])
+                name = os.path.basename(to_info["path"])
 
-            self._makedirs(to_info['path'])
+            self._makedirs(to_info["path"])
 
             try:
                 if no_progress_bar:
                     cb = None
                 else:
                     total = s3.head_object(
-                                    Bucket=from_info['bucket'],
-                                    Key=from_info['path'])['ContentLength']
+                        Bucket=from_info["bucket"], Key=from_info["path"]
+                    )["ContentLength"]
                     cb = Callback(name, total)
 
-                s3.download_file(from_info['bucket'],
-                                 from_info['path'],
-                                 tmp_file,
-                                 Callback=cb)
+                s3.download_file(
+                    from_info["bucket"], from_info["path"], tmp_file, Callback=cb
+                )
             except Exception:
-                msg = "failed to download '{}/{}'".format(from_info['bucket'],
-                                                          from_info['path'])
+                msg = "failed to download '{}/{}'".format(
+                    from_info["bucket"], from_info["path"]
+                )
                 logger.error(msg)
                 continue
 
-            os.rename(tmp_file, to_info['path'])
+            os.rename(tmp_file, to_info["path"])
 
             if not no_progress_bar:
                 progress.finish_target(name)

dvc/remote/ssh.py

@@ -19,26 +19,28 @@ from dvc.exceptions import DvcException
 from dvc.remote.base import RemoteBase, RemoteBaseCmdError
 
 
-def sizeof_fmt(num, suffix='B'):
+def sizeof_fmt(num, suffix="B"):
     """ Convert number of bytes to human-readable string """
-    for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
+    for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]:
         if abs(num) < 1024.0:
             return "%3.1f%s%s" % (num, unit, suffix)
         num /= 1024.0
-    return "%.1f%s%s" % (num, 'Y', suffix)
+    return "%.1f%s%s" % (num, "Y", suffix)
 
 
 def percent_cb(name, complete, total):
     """ Callback for updating target progress """
-    logger.debug('{}: {} transferred out of {}'.format(name,
-                                                       sizeof_fmt(complete),
-                                                       sizeof_fmt(total)))
+    logger.debug(
+        "{}: {} transferred out of {}".format(
+            name, sizeof_fmt(complete), sizeof_fmt(total)
+        )
+    )
     progress.update_target(name, complete, total)
 
 
 def create_cb(name):
     """ Create callback function for multipart object """
-    return (lambda cur, tot: percent_cb(name, cur, tot))
+    return lambda cur, tot: percent_cb(name, cur, tot)
 
 
 class RemoteSSHCmdError(RemoteBaseCmdError):
@@ -46,48 +48,50 @@ class RemoteSSHCmdError(RemoteBaseCmdError):
 
 
 class RemoteSSH(RemoteBase):
-    scheme = 'ssh'
+    scheme = "ssh"
 
     # NOTE: we support both URL-like (ssh://[user@]host.xz[:port]/path) and
     # SCP-like (ssh://[user@]host.xz:path) urls.
-    REGEX = r'^ssh://.*$'
+    REGEX = r"^ssh://.*$"
 
-    REQUIRES = {'paramiko': paramiko}
+    REQUIRES = {"paramiko": paramiko}
 
     JOBS = 4
 
-    PARAM_CHECKSUM = 'md5'
+    PARAM_CHECKSUM = "md5"
 
     DEFAULT_PORT = 22
     TIMEOUT = 1800
 
     def __init__(self, project, config):
         super(RemoteSSH, self).__init__(project, config)
-        self.url = config.get(Config.SECTION_REMOTE_URL, 'ssh://')
+        self.url = config.get(Config.SECTION_REMOTE_URL, "ssh://")
 
         parsed = urlparse(self.url)
         self.host = parsed.hostname
-        self.user = (config.get(Config.SECTION_REMOTE_USER)
-                     or parsed.username
-                     or getpass.getuser())
+        self.user = (
+            config.get(Config.SECTION_REMOTE_USER)
+            or parsed.username
+            or getpass.getuser()
+        )
         self.prefix = parsed.path
-        self.port = (config.get(Config.SECTION_REMOTE_PORT)
-                     or parsed.port
-                     or self.DEFAULT_PORT)
+        self.port = (
+            config.get(Config.SECTION_REMOTE_PORT) or parsed.port or self.DEFAULT_PORT
+        )
         self.keyfile = config.get(Config.SECTION_REMOTE_KEY_FILE, None)
         self.timeout = config.get(Config.SECTION_REMOTE_TIMEOUT, self.TIMEOUT)
         self.password = config.get(Config.SECTION_REMOTE_PASSWORD, None)
-        self.ask_password = config.get(Config.SECTION_REMOTE_ASK_PASSWORD,
-                                       False)
+        self.ask_password = config.get(Config.SECTION_REMOTE_ASK_PASSWORD, False)
 
-        self.path_info = {'scheme': 'ssh',
-                          'host': self.host,
-                          'user': self.user,
-                          'port': self.port}
+        self.path_info = {
+            "scheme": "ssh",
+            "host": self.host,
+            "user": self.user,
+            "port": self.port,
+        }
 
     def ssh(self, host=None, user=None, port=None):
-        msg = "Establishing ssh connection with '{}' " \
-              "through port '{}' as user '{}'"
+        msg = "Establishing ssh connection with '{}' " "through port '{}' as user '{}'"
         logger.debug(msg.format(host, port, user))
 
         ssh = paramiko.SSHClient()
@@ -96,28 +100,30 @@ class RemoteSSH(RemoteBase):
         ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 
         if self.ask_password and self.password is None:
-            msg = ("Enter a private key passphrase or a password for "
-                   "host '{}' port '{}' user '{}'").format(host, port, user)
+            msg = (
+                "Enter a private key passphrase or a password for "
+                "host '{}' port '{}' user '{}'"
+            ).format(host, port, user)
             self.password = prompt.password(msg)
 
-        ssh.connect(host,
-                    username=user,
-                    port=port,
-                    key_filename=self.keyfile,
-                    timeout=self.timeout,
-                    password=self.password)
+        ssh.connect(
+            host,
+            username=user,
+            port=port,
+            key_filename=self.keyfile,
+            timeout=self.timeout,
+            password=self.password,
+        )
 
         return ssh
 
     def exists(self, path_info):
         assert not isinstance(path_info, list)
-        assert path_info['scheme'] == 'ssh'
+        assert path_info["scheme"] == "ssh"
 
-        with self.ssh(path_info['host'],
-                      path_info['user'],
-                      path_info['port']) as ssh:
+        with self.ssh(path_info["host"], path_info["user"], path_info["port"]) as ssh:
             try:
-                self._exec(ssh, 'test -e {}'.format(path_info['path']))
+                self._exec(ssh, "test -e {}".format(path_info["path"]))
                 exists = True
             except RemoteSSHCmdError:
                 exists = False
@@ -133,10 +139,9 @@ class RemoteSSH(RemoteBase):
 
         stdout_chunks = []
         stderr_chunks = []
-        while (not channel.closed
-               or channel.recv_ready()
-               or channel.recv_stderr_ready()):
+        while not channel.closed or channel.recv_ready() or channel.recv_stderr_ready():
             import select
+
             got_chunk = False
             readq, _, _ = select.select([stdout.channel], [], [], self.timeout)
             for c in readq:
@@ -150,10 +155,12 @@ class RemoteSSH(RemoteBase):
                     stderr_chunks.append(s)
                     got_chunk = True
 
-            if not got_chunk \
-               and stdout.channel.exit_status_ready() \
-               and not stderr.channel.recv_stderr_ready() \
-               and not stdout.channel.recv_ready():
+            if (
+                not got_chunk
+                and stdout.channel.exit_status_ready()
+                and not stderr.channel.recv_stderr_ready()
+                and not stdout.channel.recv_ready()
+            ):
                 stdout.channel.shutdown_read()
                 stdout.channel.close()
                 break
@@ -163,32 +170,32 @@ class RemoteSSH(RemoteBase):
 
         ret = stdout.channel.recv_exit_status()
         if ret != 0:
-            err = b''.join(stderr_chunks).decode('utf-8')
+            err = b"".join(stderr_chunks).decode("utf-8")
             raise RemoteSSHCmdError(cmd, ret, err)
 
-        return b''.join(stdout_chunks).decode('utf-8')
+        return b"".join(stdout_chunks).decode("utf-8")
 
     def md5(self, path_info):
-        if path_info['scheme'] != 'ssh':
+        if path_info["scheme"] != "ssh":
             raise NotImplementedError
 
-        ssh = self.ssh(host=path_info['host'],
-                       user=path_info['user'],
-                       port=path_info['port'])
+        ssh = self.ssh(
+            host=path_info["host"], user=path_info["user"], port=path_info["port"]
+        )
 
         # Use different md5 commands depending on os
-        stdout = self._exec(ssh, 'uname').strip()
-        if stdout == 'Darwin':
-            md5cmd = 'md5'
+        stdout = self._exec(ssh, "uname").strip()
+        if stdout == "Darwin":
+            md5cmd = "md5"
             index = -1
-        elif stdout == 'Linux':
-            md5cmd = 'md5sum'
+        elif stdout == "Linux":
+            md5cmd = "md5sum"
             index = 0
         else:
             msg = "'{}' is not supported as a remote".format(stdout)
             raise DvcException(msg)
 
-        stdout = self._exec(ssh, '{} {}'.format(md5cmd, path_info['path']))
+        stdout = self._exec(ssh, "{} {}".format(md5cmd, path_info["path"]))
         md5 = stdout.split()[index]
         ssh.close()
 
@@ -197,102 +204,106 @@ class RemoteSSH(RemoteBase):
         return md5
 
     def copy(self, from_info, to_info, ssh=None):
-        if from_info['scheme'] != 'ssh' or to_info['scheme'] != 'ssh':
+        if from_info["scheme"] != "ssh" or to_info["scheme"] != "ssh":
             raise NotImplementedError
 
-        assert from_info['host'] == to_info['host']
-        assert from_info['user'] == to_info['user']
+        assert from_info["host"] == to_info["host"]
+        assert from_info["user"] == to_info["user"]
 
-        s = ssh if ssh else self.ssh(host=from_info['host'],
-                                     user=from_info['user'],
-                                     port=from_info['port'])
+        s = (
+            ssh
+            if ssh
+            else self.ssh(
+                host=from_info["host"], user=from_info["user"], port=from_info["port"]
+            )
+        )
 
-        dname = posixpath.dirname(to_info['path'])
-        self._exec(s, 'mkdir -p {}'.format(dname))
-        self._exec(s, 'cp {} {}'.format(from_info['path'], to_info['path']))
+        dname = posixpath.dirname(to_info["path"])
+        self._exec(s, "mkdir -p {}".format(dname))
+        self._exec(s, "cp {} {}".format(from_info["path"], to_info["path"]))
 
         if not ssh:
             s.close()
 
     def save_info(self, path_info):
-        if path_info['scheme'] != 'ssh':
+        if path_info["scheme"] != "ssh":
             raise NotImplementedError
 
         return {self.PARAM_CHECKSUM: self.md5(path_info)}
 
     def save(self, path_info):
-        if path_info['scheme'] != 'ssh':
+        if path_info["scheme"] != "ssh":
             raise NotImplementedError
 
         md5 = self.md5(path_info)
         dest = path_info.copy()
-        dest['path'] = self.checksum_to_path(md5)
+        dest["path"] = self.checksum_to_path(md5)
 
         self.copy(path_info, dest)
 
         return {self.PARAM_CHECKSUM: md5}
 
     def remove(self, path_info):
-        if path_info['scheme'] != 'ssh':
+        if path_info["scheme"] != "ssh":
             raise NotImplementedError
 
-        logger.debug('Removing ssh://{}@{}/{}'.format(path_info['user'],
-                                                      path_info['host'],
-                                                      path_info['path']))
+        logger.debug(
+            "Removing ssh://{}@{}/{}".format(
+                path_info["user"], path_info["host"], path_info["path"]
+            )
+        )
 
-        ssh = self.ssh(host=path_info['host'],
-                       user=path_info['user'],
-                       port=path_info['port'])
-        ssh.open_sftp().remove(path_info['path'])
+        ssh = self.ssh(
+            host=path_info["host"], user=path_info["user"], port=path_info["port"]
+        )
+        ssh.open_sftp().remove(path_info["path"])
         ssh.close()
 
-    def download(self,
-                 from_infos,
-                 to_infos,
-                 no_progress_bar=False,
-                 names=None):
+    def download(self, from_infos, to_infos, no_progress_bar=False, names=None):
         names = self._verify_path_args(from_infos, to_infos, names)
 
-        ssh = self.ssh(host=from_infos[0]['host'],
-                       user=from_infos[0]['user'],
-                       port=from_infos[0]['port'])
+        ssh = self.ssh(
+            host=from_infos[0]["host"],
+            user=from_infos[0]["user"],
+            port=from_infos[0]["port"],
+        )
 
         for to_info, from_info, name in zip(to_infos, from_infos, names):
-            if from_info['scheme'] != 'ssh':
+            if from_info["scheme"] != "ssh":
                 raise NotImplementedError
 
-            if to_info['scheme'] == 'ssh':
-                assert from_info['host'] == to_info['host']
-                assert from_info['port'] == to_info['port']
-                assert from_info['user'] == to_info['user']
+            if to_info["scheme"] == "ssh":
+                assert from_info["host"] == to_info["host"]
+                assert from_info["port"] == to_info["port"]
+                assert from_info["user"] == to_info["user"]
                 self.copy(from_info, to_info, ssh=ssh)
                 continue
 
-            if to_info['scheme'] != 'local':
+            if to_info["scheme"] != "local":
                 raise NotImplementedError
 
-            msg = "Downloading '{}/{}' to '{}'".format(from_info['host'],
-                                                       from_info['path'],
-                                                       to_info['path'])
+            msg = "Downloading '{}/{}' to '{}'".format(
+                from_info["host"], from_info["path"], to_info["path"]
+            )
             logger.debug(msg)
 
             if not name:
-                name = os.path.basename(to_info['path'])
+                name = os.path.basename(to_info["path"])
 
-            self._makedirs(to_info['path'])
-            tmp_file = self.tmp_file(to_info['path'])
+            self._makedirs(to_info["path"])
+            tmp_file = self.tmp_file(to_info["path"])
             try:
-                ssh.open_sftp().get(from_info['path'],
-                                    tmp_file,
-                                    callback=create_cb(name))
+                ssh.open_sftp().get(
+                    from_info["path"], tmp_file, callback=create_cb(name)
+                )
             except Exception:
                 msg = "failed to download '{}/{}' to '{}'"
-                logger.error(msg.format(from_info['host'],
-                                        from_info['path'],
-                                        to_info['path']))
+                logger.error(
+                    msg.format(from_info["host"], from_info["path"], to_info["path"])
+                )
                 continue
 
-            os.rename(tmp_file, to_info['path'])
+            os.rename(tmp_file, to_info["path"])
             progress.finish_target(name)
 
         ssh.close()
@@ -300,37 +311,37 @@ class RemoteSSH(RemoteBase):
     def upload(self, from_infos, to_infos, names=None):
         names = self._verify_path_args(to_infos, from_infos, names)
 
-        ssh = self.ssh(host=to_infos[0]['host'],
-                       user=to_infos[0]['user'],
-                       port=to_infos[0]['port'])
+        ssh = self.ssh(
+            host=to_infos[0]["host"], user=to_infos[0]["user"], port=to_infos[0]["port"]
+        )
         sftp = ssh.open_sftp()
 
         for from_info, to_info, name in zip(from_infos, to_infos, names):
-            if to_info['scheme'] != 'ssh':
+            if to_info["scheme"] != "ssh":
                 raise NotImplementedError
 
-            if from_info['scheme'] != 'local':
+            if from_info["scheme"] != "local":
                 raise NotImplementedError
 
-            logger.debug("Uploading '{}' to '{}/{}'".format(from_info['path'],
-                                                            to_info['host'],
-                                                            to_info['path']))
+            logger.debug(
+                "Uploading '{}' to '{}/{}'".format(
+                    from_info["path"], to_info["host"], to_info["path"]
+                )
+            )
 
             if not name:
-                name = os.path.basename(from_info['path'])
+                name = os.path.basename(from_info["path"])
 
-            dname = posixpath.dirname(to_info['path'])
-            self._exec(ssh, 'mkdir -p {}'.format(dname))
+            dname = posixpath.dirname(to_info["path"])
+            self._exec(ssh, "mkdir -p {}".format(dname))
 
             try:
-                sftp.put(from_info['path'],
-                         to_info['path'],
-                         callback=create_cb(name))
+                sftp.put(from_info["path"], to_info["path"], callback=create_cb(name))
             except Exception:
                 msg = "failed to upload '{}' to '{}/{}'"
-                logger.error(msg.format(from_info['path'],
-                                        to_info['host'],
-                                        to_info['path']))
+                logger.error(
+                    msg.format(from_info["path"], to_info["host"], to_info["path"])
+                )
                 continue
 
             progress.finish_target(name)
@@ -339,10 +350,8 @@ class RemoteSSH(RemoteBase):
         ssh.close()
 
     def list_cache_paths(self):
-        ssh = self.ssh(host=self.host,
-                       user=self.user,
-                       port=self.port)
-        cmd = 'find {} -type f -follow -print'.format(self.prefix)
+        ssh = self.ssh(host=self.host, user=self.user, port=self.port)
+        cmd = "find {} -type f -follow -print".format(self.prefix)
         stdout = self._exec(ssh, cmd)
         flist = stdout.split()
         ssh.close()

dvc/scm/base.py

@@ -19,14 +19,14 @@ class FileNotInRepoError(DvcException):
 
 class Base(object):
     """Base class for source control management driver implementations."""
+
     def __init__(self, root_dir=os.curdir, project=None):
         self.project = project
         self.root_dir = root_dir
 
     def __repr__(self):
         return "{class_name}: '{directory}'".format(
-            class_name=type(self).__name__,
-            directory=self.dir
+            class_name=type(self).__name__, directory=self.dir
         )
 
     @property
@@ -76,11 +76,7 @@ class Base(object):
     def tag(self, tag):
         """Makes SCM create a tag with a specified name."""
 
-    def brancher(self,
-                 branches=None,
-                 all_branches=False,
-                 tags=None,
-                 all_tags=False):
+    def brancher(self, branches=None, all_branches=False, tags=None, all_tags=False):
         """Generator that iterates over specified revisions.
 
         Args:
@@ -92,9 +88,8 @@ class Base(object):
         Yields:
             str: the current revision.
         """
-        if not branches and not all_branches \
-           and not tags and not all_tags:
-            yield ''
+        if not branches and not all_branches and not tags and not all_tags:
+            yield ""
             return
 
         saved = self.active_branch()
@@ -130,7 +125,7 @@ class Base(object):
 
     def active_branch(self):  # pylint: disable=no-self-use
         """Returns current branch in the project."""
-        return ''
+        return ""
 
     def list_branches(self):  # pylint: disable=no-self-use
         """Returns a list of available branches in the project."""

dvc/scm/git.py

@@ -17,24 +17,26 @@ from dvc.scm.base import Base, SCMError, FileNotInRepoError
 
 class Git(Base):
     """Class for managing Git."""
-    GITIGNORE = '.gitignore'
-    GIT_DIR = '.git'
+
+    GITIGNORE = ".gitignore"
+    GIT_DIR = ".git"
 
     def __init__(self, root_dir=os.curdir, project=None):
         super(Git, self).__init__(root_dir, project=project)
 
         import git
         from git.exc import InvalidGitRepositoryError
+
         try:
             self.repo = git.Repo(root_dir)
         except InvalidGitRepositoryError:
-            msg = '{} is not a git repository'
+            msg = "{} is not a git repository"
             raise SCMError(msg.format(root_dir))
 
         # NOTE: fixing LD_LIBRARY_PATH for binary built by PyInstaller.
         # http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
         env = fix_env(None)
-        libpath = env.get('LD_LIBRARY_PATH', None)
+        libpath = env.get("LD_LIBRARY_PATH", None)
         self.repo.git.update_environment(LD_LIBRARY_PATH=libpath)
 
     @staticmethod
@@ -60,7 +62,7 @@ class Git(Base):
     def _get_gitignore(self, path):
         assert os.path.isabs(path)
         # NOTE: using '/' prefix to make path unambiguous
-        entry = '/' + os.path.basename(path)
+        entry = "/" + os.path.basename(path)
         gitignore = os.path.join(os.path.dirname(path), self.GITIGNORE)
 
         if not gitignore.startswith(self.root_dir):
@@ -73,21 +75,21 @@ class Git(Base):
 
         ignore_list = []
         if os.path.exists(gitignore):
-            ignore_list = open(gitignore, 'r').readlines()
-            filtered = list(filter(lambda x: x.strip() == entry.strip(),
-                                   ignore_list))
+            ignore_list = open(gitignore, "r").readlines()
+            filtered = list(filter(lambda x: x.strip() == entry.strip(), ignore_list))
             if filtered:
                 return
 
-        msg = "Adding '{}' to '{}'.".format(os.path.relpath(path),
-                                            os.path.relpath(gitignore))
+        msg = "Adding '{}' to '{}'.".format(
+            os.path.relpath(path), os.path.relpath(gitignore)
+        )
         logger.info(msg)
 
         content = entry
         if ignore_list:
-            content = '\n' + content
+            content = "\n" + content
 
-        with open(gitignore, 'a') as fobj:
+        with open(gitignore, "a") as fobj:
             fobj.write(content)
 
         if self.project is not None:
@@ -99,12 +101,12 @@ class Git(Base):
         if not os.path.exists(gitignore):
             return
 
-        with open(gitignore, 'r') as fobj:
+        with open(gitignore, "r") as fobj:
             lines = fobj.readlines()
 
         filtered = list(filter(lambda x: x.strip() != entry.strip(), lines))
 
-        with open(gitignore, 'w') as fobj:
+        with open(gitignore, "w") as fobj:
             fobj.writelines(filtered)
 
         if self.project is not None:
@@ -120,8 +122,7 @@ class Git(Base):
                 "failed to add '{}' to git. You can add those files"
                 " manually using 'git add'."
                 " See 'https://github.com/iterative/dvc/issues/610'"
-                " for more details."
-                .format(str(paths))
+                " for more details.".format(str(paths))
             )
 
             logger.error(msg)
@@ -131,7 +132,7 @@ class Git(Base):
 
     def checkout(self, branch, create_new=False):
         if create_new:
-            self.repo.git.checkout('HEAD', b=branch)
+            self.repo.git.checkout("HEAD", b=branch)
         else:
             self.repo.git.checkout(branch)
 
@@ -158,17 +159,14 @@ class Git(Base):
         return [t.name for t in self.repo.tags]
 
     def _install_hook(self, name, cmd):
-        hook = os.path.join(self.root_dir,
-                            self.GIT_DIR,
-                            'hooks',
-                            name)
+        hook = os.path.join(self.root_dir, self.GIT_DIR, "hooks", name)
         if os.path.isfile(hook):
             msg = "git hook '{}' already exists."
             raise SCMError(msg.format(os.path.relpath(hook)))
-        with open(hook, 'w+') as fobj:
-            fobj.write('#!/bin/sh\nexec dvc {}\n'.format(cmd))
+        with open(hook, "w+") as fobj:
+            fobj.write("#!/bin/sh\nexec dvc {}\n".format(cmd))
         os.chmod(hook, 0o777)
 
     def install(self):
-        self._install_hook('post-checkout', 'checkout')
-        self._install_hook('pre-commit', 'status')
+        self._install_hook("post-checkout", "checkout")
+        self._install_hook("pre-commit", "status")

dvc/stage.py

@@ -72,11 +72,11 @@ class MissingDep(DvcException):
         assert len(deps) > 0
 
         if len(deps) > 1:
-            dep = 'dependencies'
+            dep = "dependencies"
         else:
-            dep = 'dependency'
+            dep = "dependency"
 
-        msg = 'missing {}: {}'.format(dep, ', '.join(map(str, deps)))
+        msg = "missing {}: {}".format(dep, ", ".join(map(str, deps)))
         super(MissingDep, self).__init__(msg)
 
 
@@ -84,23 +84,23 @@ class MissingDataSource(DvcException):
     def __init__(self, missing_files):
         assert len(missing_files) > 0
 
-        source = 'source'
+        source = "source"
         if len(missing_files) > 1:
-            source += 's'
+            source += "s"
 
-        msg = 'missing data {}: {}'.format(source, ', '.join(missing_files))
+        msg = "missing data {}: {}".format(source, ", ".join(missing_files))
         super(MissingDataSource, self).__init__(msg)
 
 
 class Stage(object):
-    STAGE_FILE = 'Dvcfile'
-    STAGE_FILE_SUFFIX = '.dvc'
+    STAGE_FILE = "Dvcfile"
+    STAGE_FILE_SUFFIX = ".dvc"
 
-    PARAM_MD5 = 'md5'
-    PARAM_CMD = 'cmd'
-    PARAM_DEPS = 'deps'
-    PARAM_OUTS = 'outs'
-    PARAM_LOCKED = 'locked'
+    PARAM_MD5 = "md5"
+    PARAM_CMD = "cmd"
+    PARAM_DEPS = "deps"
+    PARAM_OUTS = "outs"
+    PARAM_LOCKED = "locked"
 
     SCHEMA = {
         Optional(PARAM_MD5): Or(str, None),
@@ -110,15 +110,17 @@ class Stage(object):
         Optional(PARAM_LOCKED): bool,
     }
 
-    def __init__(self,
-                 project,
-                 path=None,
-                 cmd=None,
-                 cwd=os.curdir,
-                 deps=[],
-                 outs=[],
-                 md5=None,
-                 locked=False):
+    def __init__(
+        self,
+        project,
+        path=None,
+        cmd=None,
+        cwd=os.curdir,
+        deps=[],
+        outs=[],
+        md5=None,
+        locked=False,
+    ):
         self.project = project
         self.path = path
         self.cmd = cmd
@@ -129,9 +131,7 @@ class Stage(object):
         self.locked = locked
 
     def __repr__(self):
-        return "Stage: '{path}'".format(
-            path=self.relpath if self.path else 'No path'
-        )
+        return "Stage: '{path}'".format(path=self.relpath if self.path else "No path")
 
     @property
     def relpath(self):
@@ -144,8 +144,10 @@ class Stage(object):
 
     @staticmethod
     def is_valid_filename(path):
-        return (path.endswith(Stage.STAGE_FILE_SUFFIX)
-                or os.path.basename(path) == Stage.STAGE_FILE)
+        return (
+            path.endswith(Stage.STAGE_FILE_SUFFIX)
+            or os.path.basename(path) == Stage.STAGE_FILE
+        )
 
     @staticmethod
     def is_stage_file(path):
@@ -165,9 +167,7 @@ class Stage(object):
     @property
     def is_import(self):
         """Whether the stage file was created with `dvc import`."""
-        return (not self.cmd
-                and len(self.deps) == 1
-                and len(self.outs) == 1)
+        return not self.cmd and len(self.deps) == 1 and len(self.outs) == 1
 
     def _changed_deps(self):
         if self.locked:
@@ -176,15 +176,19 @@ class Stage(object):
         if self.is_callback:
             logger.warning(
                 "Dvc file '{fname}' is a 'callback' stage (has a command and"
-                " no dependencies) and thus always considered as changed."
-                .format(fname=self.relpath)
+                " no dependencies) and thus always considered as changed.".format(
+                    fname=self.relpath
+                )
             )
             return True
 
         for dep in self.deps:
             if dep.changed():
-                logger.warning("Dependency '{dep}' of '{stage}' changed."
-                               .format(dep=dep, stage=self.relpath))
+                logger.warning(
+                    "Dependency '{dep}' of '{stage}' changed.".format(
+                        dep=dep, stage=self.relpath
+                    )
+                )
                 return True
 
         return False
@@ -193,8 +197,9 @@ class Stage(object):
         for out in self.outs:
             if out.changed():
                 logger.warning(
-                    "Output '{out}' of '{stage}' changed."
-                    .format(out=out, stage=self.relpath)
+                    "Output '{out}' of '{stage}' changed.".format(
+                        out=out, stage=self.relpath
+                    )
                 )
                 return True
 
@@ -207,16 +212,14 @@ class Stage(object):
         return False
 
     def changed(self):
-        ret = any([self._changed_deps(),
-                   self._changed_outs(),
-                   self._changed_md5()])
+        ret = any([self._changed_deps(), self._changed_outs(), self._changed_md5()])
 
         if ret:
             msg = "Stage '{}' changed.".format(self.relpath)
-            color = 'yellow'
+            color = "yellow"
         else:
             msg = "Stage '{}' didn't change.".format(self.relpath)
-            color = 'green'
+            color = "green"
 
         logger.info(logger.colorize(msg, color))
 
@@ -231,7 +234,7 @@ class Stage(object):
 
     def unprotect_outs(self):
         for out in self.outs:
-            if out.scheme != 'local' or not out.exists:
+            if out.scheme != "local" or not out.exists:
                 continue
             self.project.unprotect(out.path)
 
@@ -247,13 +250,12 @@ class Stage(object):
             # Removing outputs only if we actually have command to reproduce
             self.remove_outs(ignore_remove=False)
 
-        msg = (
-            "Going to reproduce '{stage}'. Are you sure you want to continue?"
-            .format(stage=self.relpath)
+        msg = "Going to reproduce '{stage}'. Are you sure you want to continue?".format(
+            stage=self.relpath
         )
 
         if interactive and not prompt.confirm(msg):
-            raise DvcException('reproduction aborted by the user')
+            raise DvcException("reproduction aborted by the user")
 
         logger.info("Reproducing '{stage}'".format(stage=self.relpath))
 
@@ -281,7 +283,7 @@ class Stage(object):
             return (cls.STAGE_FILE, cwd if cwd else os.getcwd())
 
         out = outs[0]
-        if out.scheme == 'local':
+        if out.scheme == "local":
             path = os.path
         else:
             path = posixpath
@@ -333,29 +335,29 @@ class Stage(object):
         return old_d == new_d
 
     @staticmethod
-    def create(project=None,
-               cmd=None,
-               deps=[],
-               outs=[],
-               outs_no_cache=[],
-               metrics_no_cache=[],
-               fname=None,
-               cwd=os.curdir,
-               locked=False,
-               add=False,
-               overwrite=True,
-               ignore_build_cache=False,
-               remove_outs=False):
+    def create(
+        project=None,
+        cmd=None,
+        deps=[],
+        outs=[],
+        outs_no_cache=[],
+        metrics_no_cache=[],
+        fname=None,
+        cwd=os.curdir,
+        locked=False,
+        add=False,
+        overwrite=True,
+        ignore_build_cache=False,
+        remove_outs=False,
+    ):
 
-        stage = Stage(project=project,
-                      cwd=cwd,
-                      cmd=cmd,
-                      locked=locked)
+        stage = Stage(project=project, cwd=cwd, cmd=cmd, locked=locked)
 
         stage.outs = output.loads_from(stage, outs, use_cache=True)
         stage.outs += output.loads_from(stage, outs_no_cache, use_cache=False)
-        stage.outs += output.loads_from(stage, metrics_no_cache,
-                                        use_cache=False, metric=True)
+        stage.outs += output.loads_from(
+            stage, metrics_no_cache, use_cache=False, metric=True
+        )
         stage.deps = dependency.loads_from(stage, deps)
 
         stage._check_circular_dependency()
@@ -364,8 +366,9 @@ class Stage(object):
         if fname is not None and os.path.basename(fname) != fname:
             raise StageFileBadNameError(
                 "stage file name '{fname}' should not contain subdirectories."
-                " Use '-c|--cwd' to change location of the stage file."
-                .format(fname=fname)
+                " Use '-c|--cwd' to change location of the stage file.".format(
+                    fname=fname
+                )
             )
 
         fname, cwd = Stage._stage_fname_cwd(fname, cwd, stage.outs, add=add)
@@ -388,11 +391,13 @@ class Stage(object):
 
         if os.path.exists(path):
             if not ignore_build_cache and stage.is_cached:
-                logger.info('Stage is cached, skipping.')
+                logger.info("Stage is cached, skipping.")
                 return None
 
-            msg = "'{}' already exists. Do you wish to run the command and " \
-                  "overwrite it?".format(stage.relpath)
+            msg = (
+                "'{}' already exists. Do you wish to run the command and "
+                "overwrite it?".format(stage.relpath)
+            )
 
             if not overwrite and not prompt.confirm(msg):
                 raise StageFileAlreadyExistsError(stage.relpath)
@@ -406,8 +411,9 @@ class Stage(object):
         if not Stage.is_valid_filename(fname):
             raise StageFileBadNameError(
                 "bad stage filename '{}'. Stage files should be named"
-                " 'Dvcfile' or have a '.dvc' suffix(e.g. '{}.dvc')."
-                .format(os.path.relpath(fname), os.path.basename(fname))
+                " 'Dvcfile' or have a '.dvc' suffix(e.g. '{}.dvc').".format(
+                    os.path.relpath(fname), os.path.basename(fname)
+                )
             )
 
     @staticmethod
@@ -423,17 +429,19 @@ class Stage(object):
         if not Stage.is_stage_file(fname):
             raise StageFileIsNotDvcFileError(fname)
 
-        with open(fname, 'r') as fd:
+        with open(fname, "r") as fd:
             d = yaml.safe_load(fd) or {}
 
         Stage.validate(d, fname=os.path.relpath(fname))
 
-        stage = Stage(project=project,
-                      path=os.path.abspath(fname),
-                      cwd=os.path.dirname(os.path.abspath(fname)),
-                      cmd=d.get(Stage.PARAM_CMD),
-                      md5=d.get(Stage.PARAM_MD5),
-                      locked=d.get(Stage.PARAM_LOCKED, False))
+        stage = Stage(
+            project=project,
+            path=os.path.abspath(fname),
+            cwd=os.path.dirname(os.path.abspath(fname)),
+            cmd=d.get(Stage.PARAM_CMD),
+            md5=d.get(Stage.PARAM_MD5),
+            locked=d.get(Stage.PARAM_LOCKED, False),
+        )
 
         stage.deps = dependency.loadd_from(stage, d.get(Stage.PARAM_DEPS, []))
         stage.outs = output.loadd_from(stage, d.get(Stage.PARAM_OUTS, []))
@@ -458,10 +466,11 @@ class Stage(object):
 
         self._check_dvc_filename(fname)
 
-        logger.info("Saving information to '{file}'."
-                    .format(file=os.path.relpath(fname)))
+        logger.info(
+            "Saving information to '{file}'.".format(file=os.path.relpath(fname))
+        )
 
-        with open(fname, 'w') as fd:
+        with open(fname, "w") as fd:
             yaml.safe_dump(self.dumpd(), fd, default_flow_style=False)
 
         self.project.files_to_git_add.append(os.path.relpath(fname))
@@ -478,8 +487,7 @@ class Stage(object):
         # NOTE: excluding parameters that don't affect the state of the
         # pipeline. Not excluding `OutputLOCAL.PARAM_CACHE`, because if
         # it has changed, we might not have that output in our cache.
-        return dict_md5(d, exclude=[self.PARAM_LOCKED,
-                                    OutputLOCAL.PARAM_METRIC])
+        return dict_md5(d, exclude=[self.PARAM_LOCKED, OutputLOCAL.PARAM_METRIC])
 
     def save(self):
         for dep in self.deps:
@@ -497,8 +505,10 @@ class Stage(object):
             raise MissingDep(missing)
 
     def _warn_if_fish(self, executable):  # pragma: no cover
-        if executable is None \
-           or os.path.basename(os.path.realpath(executable)) != 'fish':
+        if (
+            executable is None
+            or os.path.basename(os.path.realpath(executable)) != "fish"
+        ):
             return
 
         logger.warning(
@@ -512,9 +522,8 @@ class Stage(object):
     def _check_circular_dependency(self):
         from dvc.exceptions import CircularDependencyError
 
-        circular_dependencies = (
-            set(file.path for file in self.deps) &
-            set(file.path for file in self.outs)
+        circular_dependencies = set(file.path for file in self.deps) & set(
+            file.path for file in self.outs
         )
 
         if circular_dependencies:
@@ -532,14 +541,16 @@ class Stage(object):
 
     def _run(self):
         self._check_missing_deps()
-        executable = os.getenv('SHELL') if os.name != 'nt' else None
+        executable = os.getenv("SHELL") if os.name != "nt" else None
         self._warn_if_fish(executable)
 
-        p = subprocess.Popen(self.cmd,
-                             cwd=self.cwd,
-                             shell=True,
-                             env=fix_env(os.environ),
-                             executable=executable)
+        p = subprocess.Popen(
+            self.cmd,
+            cwd=self.cwd,
+            shell=True,
+            env=fix_env(os.environ),
+            executable=executable,
+        )
         p.communicate()
 
         if p.returncode != 0:
@@ -547,14 +558,18 @@ class Stage(object):
 
     def run(self, dry=False):
         if self.locked:
-            logger.info("Verifying outputs in locked stage '{stage}'"
-                        .format(stage=self.relpath))
+            logger.info(
+                "Verifying outputs in locked stage '{stage}'".format(stage=self.relpath)
+            )
             if not dry:
                 self.check_missing_outputs()
 
         elif self.is_import:
-            logger.info("Importing '{dep}' -> '{out}'"
-                        .format(dep=self.deps[0].path, out=self.outs[0].path))
+            logger.info(
+                "Importing '{dep}' -> '{out}'".format(
+                    dep=self.deps[0].path, out=self.outs[0].path
+                )
+            )
             if not dry:
                 if self._already_cached():
                     self.outs[0].checkout()
@@ -568,7 +583,7 @@ class Stage(object):
                 self.check_missing_outputs()
 
         else:
-            logger.info('Running command:\n\t{}'.format(self.cmd))
+            logger.info("Running command:\n\t{}".format(self.cmd))
             if not dry:
                 if self._already_cached():
                     self.checkout()
@@ -580,7 +595,7 @@ class Stage(object):
 
     def check_missing_outputs(self):
         paths = [
-            out.path if out.scheme != 'local' else out.rel_path
+            out.path if out.scheme != "local" else out.rel_path
             for out in self.outs
             if not out.exists
         ]
@@ -607,9 +622,9 @@ class Stage(object):
         ret = {}
 
         if not self.locked:
-            ret.update(self._status(self.deps, 'deps'))
+            ret.update(self._status(self.deps, "deps"))
 
-        ret.update(self._status(self.outs, 'outs'))
+        ret.update(self._status(self.outs, "outs"))
 
         if ret or self.changed_md5() or self.is_callback:
             return {self.relpath: ret}
@@ -617,9 +632,11 @@ class Stage(object):
         return {}
 
     def _already_cached(self):
-        return (not self.changed_md5()
-                and all(not dep.changed() for dep in self.deps)
-                and all(not out.changed_cache()
-                        if out.use_cache
-                        else not out.changed()
-                        for out in self.outs))
+        return (
+            not self.changed_md5()
+            and all(not dep.changed() for dep in self.deps)
+            and all(
+                not out.changed_cache() if out.use_cache else not out.changed()
+                for out in self.outs
+            )
+        )

dvc/state.py

@@ -18,15 +18,16 @@ from dvc.exceptions import DvcException
 
 class StateVersionTooNewError(DvcException):
     """Thrown when dvc version is older than the state database version."""
+
     def __init__(self, dvc_version, expected, actual):
         super(StateVersionTooNewError, self).__init__(
             "you are using an old version '{dvc_version}' of dvc that is "
             "using state file version '{expected}' which is not compatible "
             "with the state file version '{actual}' that is used in this "
-            "projet. Please upgrade right now!"
-            .format(dvc_version=dvc_version,
-                    expected=expected,
-                    actual=actual))
+            "projet. Please upgrade right now!".format(
+                dvc_version=dvc_version, expected=expected, actual=actual
+            )
+        )
 
 
 class State(object):  # pylint: disable=too-many-instance-attributes
@@ -41,29 +42,32 @@ class State(object):  # pylint: disable=too-many-instance-attributes
         StateVersionTooNewError: thrown when dvc version is older than the
             state database version.
     """
-    VERSION = 3
-    STATE_FILE = 'state'
-    STATE_TABLE = 'state'
-    STATE_TABLE_LAYOUT = ("inode INTEGER PRIMARY KEY, "
-                          "mtime TEXT NOT NULL, "
-                          "size TEXT NOT NULL, "
-                          "md5 TEXT NOT NULL, "
-                          "timestamp TEXT NOT NULL")
 
-    STATE_INFO_TABLE = 'state_info'
-    STATE_INFO_TABLE_LAYOUT = 'count INTEGER'
+    VERSION = 3
+    STATE_FILE = "state"
+    STATE_TABLE = "state"
+    STATE_TABLE_LAYOUT = (
+        "inode INTEGER PRIMARY KEY, "
+        "mtime TEXT NOT NULL, "
+        "size TEXT NOT NULL, "
+        "md5 TEXT NOT NULL, "
+        "timestamp TEXT NOT NULL"
+    )
+
+    STATE_INFO_TABLE = "state_info"
+    STATE_INFO_TABLE_LAYOUT = "count INTEGER"
     STATE_INFO_ROW = 1
 
-    LINK_STATE_TABLE = 'link_state'
-    LINK_STATE_TABLE_LAYOUT = ("path TEXT PRIMARY KEY, "
-                               "inode INTEGER NOT NULL, "
-                               "mtime TEXT NOT NULL")
+    LINK_STATE_TABLE = "link_state"
+    LINK_STATE_TABLE_LAYOUT = (
+        "path TEXT PRIMARY KEY, " "inode INTEGER NOT NULL, " "mtime TEXT NOT NULL"
+    )
 
     STATE_ROW_LIMIT = 100000000
     STATE_ROW_CLEANUP_QUOTA = 50
 
-    MAX_INT = 2**63 - 1
-    MAX_UINT = 2**64 - 2
+    MAX_INT = 2 ** 63 - 1
+    MAX_UINT = 2 ** 64 - 2
 
     def __init__(self, project, config):
         self.project = project
@@ -74,11 +78,12 @@ class State(object):  # pylint: disable=too-many-instance-attributes
         self.row_cleanup_quota = 50
 
         state_config = config.get(Config.SECTION_STATE, {})
-        self.row_limit = state_config.get(Config.SECTION_STATE_ROW_LIMIT,
-                                          self.STATE_ROW_LIMIT)
+        self.row_limit = state_config.get(
+            Config.SECTION_STATE_ROW_LIMIT, self.STATE_ROW_LIMIT
+        )
         self.row_cleanup_quota = state_config.get(
-            Config.SECTION_STATE_ROW_CLEANUP_QUOTA,
-            self.STATE_ROW_CLEANUP_QUOTA)
+            Config.SECTION_STATE_ROW_CLEANUP_QUOTA, self.STATE_ROW_CLEANUP_QUOTA
+        )
 
         if not self.dvc_dir:
             self.state_file = None
@@ -87,10 +92,7 @@ class State(object):  # pylint: disable=too-many-instance-attributes
         self.state_file = os.path.join(self.dvc_dir, self.STATE_FILE)
 
         # https://www.sqlite.org/tempfiles.html
-        self.temp_files = [
-            self.state_file + '-journal',
-            self.state_file + '-wal',
-        ]
+        self.temp_files = [self.state_file + "-journal", self.state_file + "-wal"]
 
         self.database = None
         self.cursor = None
@@ -125,7 +127,7 @@ class State(object):  # pylint: disable=too-many-instance-attributes
         if not md5 or not actual:
             return True
 
-        return actual.split('.')[0] != md5.split('.')[0]
+        return actual.split(".")[0] != md5.split(".")[0]
 
     def _execute(self, cmd):
         logger.debug(cmd)
@@ -171,8 +173,10 @@ class State(object):  # pylint: disable=too-many-instance-attributes
             if version > self.VERSION:
                 raise StateVersionTooNewError(VERSION, self.VERSION, version)
             elif version < self.VERSION:
-                msg = "State file version '{}' is too old. " \
-                      "Reformatting to the current version '{}'."
+                msg = (
+                    "State file version '{}' is too old. "
+                    "Reformatting to the current version '{}'."
+                )
                 logger.warning(msg.format(version, self.VERSION))
                 cmd = "DROP TABLE IF EXISTS {};"
                 self._execute(cmd.format(self.STATE_TABLE))
@@ -181,17 +185,15 @@ class State(object):  # pylint: disable=too-many-instance-attributes
 
         # Check that the state file is indeed a database
         cmd = "CREATE TABLE IF NOT EXISTS {} ({})"
-        self._execute(cmd.format(self.STATE_TABLE,
-                                 self.STATE_TABLE_LAYOUT))
-        self._execute(cmd.format(self.STATE_INFO_TABLE,
-                                 self.STATE_INFO_TABLE_LAYOUT))
-        self._execute(cmd.format(self.LINK_STATE_TABLE,
-                                 self.LINK_STATE_TABLE_LAYOUT))
+        self._execute(cmd.format(self.STATE_TABLE, self.STATE_TABLE_LAYOUT))
+        self._execute(cmd.format(self.STATE_INFO_TABLE, self.STATE_INFO_TABLE_LAYOUT))
+        self._execute(cmd.format(self.LINK_STATE_TABLE, self.LINK_STATE_TABLE_LAYOUT))
 
-        cmd = "INSERT OR IGNORE INTO {} (count) SELECT 0 " \
-              "WHERE NOT EXISTS (SELECT * FROM {})"
-        self._execute(cmd.format(self.STATE_INFO_TABLE,
-                                 self.STATE_INFO_TABLE))
+        cmd = (
+            "INSERT OR IGNORE INTO {} (count) SELECT 0 "
+            "WHERE NOT EXISTS (SELECT * FROM {})"
+        )
+        self._execute(cmd.format(self.STATE_INFO_TABLE, self.STATE_INFO_TABLE))
 
         cmd = "PRAGMA user_version = {};"
         self._execute(cmd.format(self.VERSION))
@@ -228,15 +230,14 @@ class State(object):  # pylint: disable=too-many-instance-attributes
         # NOTE: see https://bugs.python.org/issue28518
         self.database.isolation_level = None
         self._execute("VACUUM")
-        self.database.isolation_level = ''
+        self.database.isolation_level = ""
 
     def dump(self):
         """Saves state database."""
         assert self.database is not None
 
         cmd = "SELECT count from {} WHERE rowid={}"
-        self._execute(cmd.format(self.STATE_INFO_TABLE,
-                                 self.STATE_INFO_ROW))
+        self._execute(cmd.format(self.STATE_INFO_TABLE, self.STATE_INFO_ROW))
         ret = self._fetchall()
         assert len(ret) == 1
         assert len(ret[0]) == 1
@@ -246,13 +247,13 @@ class State(object):  # pylint: disable=too-many-instance-attributes
             msg = "cleaning up state, this might take a while."
             logger.warning(msg)
 
-            delete = (count - self.row_limit)
-            delete += int(self.row_limit * (self.row_cleanup_quota/100.))
-            cmd = "DELETE FROM {} WHERE timestamp IN (" \
-                  "SELECT timestamp FROM {} ORDER BY timestamp ASC LIMIT {});"
-            self._execute(cmd.format(self.STATE_TABLE,
-                                     self.STATE_TABLE,
-                                     delete))
+            delete = count - self.row_limit
+            delete += int(self.row_limit * (self.row_cleanup_quota / 100.0))
+            cmd = (
+                "DELETE FROM {} WHERE timestamp IN ("
+                "SELECT timestamp FROM {} ORDER BY timestamp ASC LIMIT {});"
+            )
+            self._execute(cmd.format(self.STATE_TABLE, self.STATE_TABLE, delete))
 
             self._vacuum()
 
@@ -265,9 +266,11 @@ class State(object):  # pylint: disable=too-many-instance-attributes
             count = ret[0][0]
 
         cmd = "UPDATE {} SET count = {} WHERE rowid = {}"
-        self._execute(cmd.format(self.STATE_INFO_TABLE,
-                                 self._to_sqlite(count),
-                                 self.STATE_INFO_ROW))
+        self._execute(
+            cmd.format(
+                self.STATE_INFO_TABLE, self._to_sqlite(count), self.STATE_INFO_ROW
+            )
+        )
 
         self.database.commit()
         self.cursor.close()
@@ -297,7 +300,7 @@ class State(object):  # pylint: disable=too-many-instance-attributes
 
     @staticmethod
     def _inode(path):
-        logger.debug('Path {} inode {}'.format(path, System.inode(path)))
+        logger.debug("Path {} inode {}".format(path, System.inode(path)))
         return System.inode(path)
 
     def _do_update(self, path):
@@ -310,21 +313,28 @@ class State(object):  # pylint: disable=too-many-instance-attributes
         actual_mtime, actual_size = self._mtime_and_size(path)
         actual_inode = self._inode(path)
 
-        cmd = ('SELECT * from {} WHERE inode={}'
-               .format(self.STATE_TABLE, self._to_sqlite(actual_inode)))
+        cmd = "SELECT * from {} WHERE inode={}".format(
+            self.STATE_TABLE, self._to_sqlite(actual_inode)
+        )
 
         self._execute(cmd)
         ret = self._fetchall()
         if not ret:
             md5, info = self._collect(path)
-            cmd = 'INSERT INTO {}(inode, mtime, size, md5, timestamp) ' \
-                  'VALUES ({}, "{}", "{}", "{}", "{}")'
-            self._execute(cmd.format(self.STATE_TABLE,
-                                     self._to_sqlite(actual_inode),
-                                     actual_mtime,
-                                     actual_size,
-                                     md5,
-                                     int(nanotime.timestamp(time.time()))))
+            cmd = (
+                "INSERT INTO {}(inode, mtime, size, md5, timestamp) "
+                'VALUES ({}, "{}", "{}", "{}", "{}")'
+            )
+            self._execute(
+                cmd.format(
+                    self.STATE_TABLE,
+                    self._to_sqlite(actual_inode),
+                    actual_mtime,
+                    actual_size,
+                    md5,
+                    int(nanotime.timestamp(time.time())),
+                )
+            )
             self.inserts += 1
         else:
             assert len(ret) == 1
@@ -334,26 +344,38 @@ class State(object):  # pylint: disable=too-many-instance-attributes
             assert inode == actual_inode
             logger.debug(
                 "Inode '{}', mtime '{}', actual mtime '{}', size '{}', "
-                "actual size '{}'."
-                .format(inode, mtime, actual_mtime, size, actual_size))
+                "actual size '{}'.".format(
+                    inode, mtime, actual_mtime, size, actual_size
+                )
+            )
             if actual_mtime != mtime or actual_size != size:
                 md5, info = self._collect(path)
-                cmd = ('UPDATE {} SET '
-                       'mtime = "{}", size = "{}", '
-                       'md5 = "{}", timestamp = "{}" '
-                       'WHERE inode = {}')
-                self._execute(cmd.format(self.STATE_TABLE,
-                                         actual_mtime,
-                                         actual_size,
-                                         md5,
-                                         int(nanotime.timestamp(time.time())),
-                                         self._to_sqlite(actual_inode)))
+                cmd = (
+                    "UPDATE {} SET "
+                    'mtime = "{}", size = "{}", '
+                    'md5 = "{}", timestamp = "{}" '
+                    "WHERE inode = {}"
+                )
+                self._execute(
+                    cmd.format(
+                        self.STATE_TABLE,
+                        actual_mtime,
+                        actual_size,
+                        md5,
+                        int(nanotime.timestamp(time.time())),
+                        self._to_sqlite(actual_inode),
+                    )
+                )
             else:
                 info = None
                 cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}'
-                self._execute(cmd.format(self.STATE_TABLE,
-                                         int(nanotime.timestamp(time.time())),
-                                         self._to_sqlite(actual_inode)))
+                self._execute(
+                    cmd.format(
+                        self.STATE_TABLE,
+                        int(nanotime.timestamp(time.time())),
+                        self._to_sqlite(actual_inode),
+                    )
+                )
 
         return (md5, info)
 
@@ -401,11 +423,9 @@ class State(object):  # pylint: disable=too-many-instance-attributes
         inode = self._inode(path)
         relpath = os.path.relpath(path, self.root_dir)
 
-        cmd = 'REPLACE INTO {}(path, inode, mtime) ' \
-              'VALUES ("{}", {}, "{}")'.format(self.LINK_STATE_TABLE,
-                                               relpath,
-                                               self._to_sqlite(inode),
-                                               mtime)
+        cmd = "REPLACE INTO {}(path, inode, mtime) " 'VALUES ("{}", {}, "{}")'.format(
+            self.LINK_STATE_TABLE, relpath, self._to_sqlite(inode), mtime
+        )
         self._execute(cmd)
 
     def remove_unused_links(self, used):
@@ -416,7 +436,7 @@ class State(object):  # pylint: disable=too-many-instance-attributes
         """
         unused = []
 
-        self._execute('SELECT * FROM {}'.format(self.LINK_STATE_TABLE))
+        self._execute("SELECT * FROM {}".format(self.LINK_STATE_TABLE))
         for row in self.cursor:
             relpath, inode, mtime = row
             inode = self._from_sqlite(inode)

dvc/system.py

@@ -8,7 +8,7 @@ import os
 class System(object):
     @staticmethod
     def is_unix():
-        return os.name != 'nt'
+        return os.name != "nt"
 
     @staticmethod
     def hardlink(source, link_name):
@@ -20,17 +20,15 @@ class System(object):
                 os.link(source, link_name)
                 return
             except Exception as exc:
-                raise DvcException('link', cause=exc)
+                raise DvcException("link", cause=exc)
 
         CreateHardLink = ctypes.windll.kernel32.CreateHardLinkW
-        CreateHardLink.argtypes = [ctypes.c_wchar_p,
-                                   ctypes.c_wchar_p,
-                                   ctypes.c_void_p]
+        CreateHardLink.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_void_p]
         CreateHardLink.restype = ctypes.wintypes.BOOL
 
         res = CreateHardLink(link_name, source, None)
         if res == 0:
-            raise DvcException('CreateHardLinkW', cause=ctypes.WinError())
+            raise DvcException("CreateHardLinkW", cause=ctypes.WinError())
 
     @staticmethod
     def symlink(source, link_name):
@@ -54,25 +52,25 @@ class System(object):
         func.restype = ctypes.c_ubyte
 
         if func(link_name, source, flags) == 0:
-            raise DvcException('CreateSymbolicLinkW', cause=ctypes.WinError())
+            raise DvcException("CreateSymbolicLinkW", cause=ctypes.WinError())
 
     @staticmethod
     def _reflink_darwin(src, dst):
         import ctypes
 
-        clib = ctypes.CDLL('libc.dylib')
-        if not hasattr(clib, 'clonefile'):
+        clib = ctypes.CDLL("libc.dylib")
+        if not hasattr(clib, "clonefile"):
             return -1
 
         clonefile = clib.clonefile
-        clonefile.argtypes = [ctypes.c_char_p,
-                              ctypes.c_char_p,
-                              ctypes.c_int]
+        clonefile.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_int]
         clonefile.restype = ctypes.c_int
 
-        return clonefile(ctypes.c_char_p(src.encode('utf-8')),
-                         ctypes.c_char_p(dst.encode('utf-8')),
-                         ctypes.c_int(0))
+        return clonefile(
+            ctypes.c_char_p(src.encode("utf-8")),
+            ctypes.c_char_p(dst.encode("utf-8")),
+            ctypes.c_int(0),
+        )
 
     @staticmethod
     def _reflink_windows(src, dst):
@@ -85,8 +83,8 @@ class System(object):
 
         FICLONE = 0x40049409
 
-        s = open(src, 'r')
-        d = open(dst, 'w+')
+        s = open(src, "r")
+        d = open(dst, "w+")
 
         try:
             ret = fcntl.ioctl(d.fileno(), FICLONE, s.fileno())
@@ -111,11 +109,11 @@ class System(object):
 
         system = platform.system()
         try:
-            if system == 'Windows':
+            if system == "Windows":
                 ret = System._reflink_windows(source, link_name)
-            elif system == 'Darwin':
+            elif system == "Darwin":
                 ret = System._reflink_darwin(source, link_name)
-            elif system == 'Linux':
+            elif system == "Linux":
                 ret = System._reflink_linux(source, link_name)
             else:
                 ret = -1
@@ -123,7 +121,7 @@ class System(object):
             ret = -1
 
         if ret != 0:
-            raise DvcException('reflink is not supported')
+            raise DvcException("reflink is not supported")
 
     @staticmethod
     def getdirinfo(path):
@@ -141,40 +139,29 @@ class System(object):
         OPEN_EXISTING = 3
 
         class FILETIME(Structure):
-            _fields_ = [("dwLowDateTime", DWORD),
-                        ("dwHighDateTime", DWORD)]
+            _fields_ = [("dwLowDateTime", DWORD), ("dwHighDateTime", DWORD)]
 
         class BY_HANDLE_FILE_INFORMATION(Structure):
-            _fields_ = [("dwFileAttributes", DWORD),
-                        ("ftCreationTime", FILETIME),
-                        ("ftLastAccessTime", FILETIME),
-                        ("ftLastWriteTime", FILETIME),
-                        ("dwVolumeSerialNumber", DWORD),
-                        ("nFileSizeHigh", DWORD),
-                        ("nFileSizeLow", DWORD),
-                        ("nNumberOfLinks", DWORD),
-                        ("nFileIndexHigh", DWORD),
-                        ("nFileIndexLow", DWORD)]
+            _fields_ = [
+                ("dwFileAttributes", DWORD),
+                ("ftCreationTime", FILETIME),
+                ("ftLastAccessTime", FILETIME),
+                ("ftLastWriteTime", FILETIME),
+                ("dwVolumeSerialNumber", DWORD),
+                ("nFileSizeHigh", DWORD),
+                ("nFileSizeLow", DWORD),
+                ("nNumberOfLinks", DWORD),
+                ("nFileIndexHigh", DWORD),
+                ("nFileIndexLow", DWORD),
+            ]
 
         flags = FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT
 
         func = ctypes.windll.kernel32.CreateFileW
-        func.argtypes = [c_wchar_p,
-                         DWORD,
-                         DWORD,
-                         c_void_p,
-                         DWORD,
-                         DWORD,
-                         HANDLE]
+        func.argtypes = [c_wchar_p, DWORD, DWORD, c_void_p, DWORD, DWORD, HANDLE]
         func.restype = HANDLE
 
-        hfile = func(path,
-                     0,
-                     FILE_SHARE_READ,
-                     None,
-                     OPEN_EXISTING,
-                     flags,
-                     None)
+        hfile = func(path, 0, FILE_SHARE_READ, None, OPEN_EXISTING, flags, None)
         if hfile is None:
             raise WinError()
 
@@ -200,6 +187,7 @@ class System(object):
     def inode(path):
         if System.is_unix():
             import ctypes
+
             inode = os.lstat(path).st_ino
             # NOTE: See https://bugs.python.org/issue29619 and
             # https://stackoverflow.com/questions/34643289/
@@ -208,11 +196,13 @@ class System(object):
         else:
             # getdirinfo from ntfsutils works on both files and dirs
             info = System.getdirinfo(path)
-            inode = abs(hash((info.dwVolumeSerialNumber,
-                              info.nFileIndexHigh,
-                              info.nFileIndexLow)))
+            inode = abs(
+                hash(
+                    (info.dwVolumeSerialNumber, info.nFileIndexHigh, info.nFileIndexLow)
+                )
+            )
         assert inode >= 0
-        assert inode < 2**64
+        assert inode < 2 ** 64
         return inode
 
     @staticmethod
@@ -238,6 +228,7 @@ class System(object):
     def _wait_for_input_posix(timeout):
         import sys
         import select
+
         try:
             select.select([sys.stdin], [], [], timeout)
         except select.error:

dvc/updater.py

@@ -13,20 +13,20 @@ from dvc.utils import is_binary
 
 
 class Updater(object):  # pragma: no cover
-    URL = 'https://updater.dvc.org'
-    UPDATER_FILE = 'updater'
+    URL = "https://updater.dvc.org"
+    UPDATER_FILE = "updater"
     TIMEOUT = 24 * 60 * 60  # every day
     TIMEOUT_GET = 10
 
     def __init__(self, dvc_dir):
         self.dvc_dir = dvc_dir
         self.updater_file = os.path.join(dvc_dir, self.UPDATER_FILE)
-        self.lock = Lock(dvc_dir, self.updater_file + '.lock')
+        self.lock = Lock(dvc_dir, self.updater_file + ".lock")
         self.current = VERSION_BASE
 
     def _is_outdated_file(self):
         ctime = os.path.getmtime(self.updater_file)
-        outdated = (time.time() - ctime >= self.TIMEOUT)
+        outdated = time.time() - ctime >= self.TIMEOUT
         if outdated:
             logger.debug("'{}' is outdated(".format(self.updater_file))
         return outdated
@@ -40,22 +40,22 @@ class Updater(object):  # pragma: no cover
             logger.debug(msg.format(self.lock.lock_file, action))
 
     def check(self):
-        if os.getenv('CI') or os.getenv('DVC_TEST'):
+        if os.getenv("CI") or os.getenv("DVC_TEST"):
             return
 
-        self._with_lock(self._check, 'checking')
+        self._with_lock(self._check, "checking")
 
     def _check(self):
         if not os.path.exists(self.updater_file) or self._is_outdated_file():
             self.fetch()
             return
 
-        with open(self.updater_file, 'r') as fobj:
+        with open(self.updater_file, "r") as fobj:
             import json
 
             try:
                 info = json.load(fobj)
-                self.latest = info['version']
+                self.latest = info["version"]
             except Exception as exc:
                 msg = "'{}' is not a valid json: {}"
                 logger.debug(msg.format(self.updater_file, exc))
@@ -69,10 +69,10 @@ class Updater(object):  # pragma: no cover
         from dvc.daemon import daemon
 
         if detach:
-            daemon(['updater'])
+            daemon(["updater"])
             return
 
-        self._with_lock(self._get_latest_version, 'fetching')
+        self._with_lock(self._get_latest_version, "fetching")
 
     def _get_latest_version(self):
         import json
@@ -85,12 +85,12 @@ class Updater(object):  # pragma: no cover
             logger.debug(msg.format(exc))
             return
 
-        with open(self.updater_file, 'w+') as fobj:
+        with open(self.updater_file, "w+") as fobj:
             json.dump(info, fobj)
 
     def _is_outdated(self):
-        l_major, l_minor, l_patch = [int(x) for x in self.latest.split('.')]
-        c_major, c_minor, c_patch = [int(x) for x in self.current.split('.')]
+        l_major, l_minor, l_patch = [int(x) for x in self.latest.split(".")]
+        c_major, c_minor, c_patch = [int(x) for x in self.current.split(".")]
 
         if l_major != c_major:
             return l_major > c_major
@@ -105,35 +105,42 @@ class Updater(object):  # pragma: no cover
             return
 
         message = (
-            'Update available {red}{current}{reset} -> {green}{latest}{reset}'
-            + '\n'
+            "Update available {red}{current}{reset} -> {green}{latest}{reset}"
+            + "\n"
             + self._get_update_instructions()
-        ).format(red=colorama.Fore.RED,
-                 reset=colorama.Fore.RESET,
-                 green=colorama.Fore.GREEN,
-                 yellow=colorama.Fore.YELLOW,
-                 blue=colorama.Fore.BLUE,
-                 current=self.current,
-                 latest=self.latest)
+        ).format(
+            red=colorama.Fore.RED,
+            reset=colorama.Fore.RESET,
+            green=colorama.Fore.GREEN,
+            yellow=colorama.Fore.YELLOW,
+            blue=colorama.Fore.BLUE,
+            current=self.current,
+            latest=self.latest,
+        )
 
-        logger.box(message, border_color='yellow')
+        logger.box(message, border_color="yellow")
 
     def _get_update_instructions(self):
         instructions = {
-            'pip': 'Run {yellow}pip{reset} install dvc {blue}--upgrade{reset}',
-            'yum': 'Run {yellow}yum{reset} update dvc',
-            'yay': 'Run {yellow}yay{reset} {blue}-S{reset} dvc',
-            'formula': 'Run {yellow}brew{reset} upgrade dvc',
-            'cask': 'Run {yellow}brew cask{reset} upgrade dvc',
-            'apt': ('Run {yellow}apt-get{reset} install'
-                    ' {blue}--only-upgrade{reset} dvc'),
-            'binary': ('To upgrade follow this steps:\n'
-                       '1. Uninstall dvc binary\n'
-                       '2. Go to {blue}https://dvc.org{reset}\n'
-                       '3. Download and install new binary'),
-            None: ('Find the latest release at\n{blue}'
-                   'https://github.com/iterative/dvc/releases/latest'
-                   '{reset}'),
+            "pip": "Run {yellow}pip{reset} install dvc {blue}--upgrade{reset}",
+            "yum": "Run {yellow}yum{reset} update dvc",
+            "yay": "Run {yellow}yay{reset} {blue}-S{reset} dvc",
+            "formula": "Run {yellow}brew{reset} upgrade dvc",
+            "cask": "Run {yellow}brew cask{reset} upgrade dvc",
+            "apt": (
+                "Run {yellow}apt-get{reset} install" " {blue}--only-upgrade{reset} dvc"
+            ),
+            "binary": (
+                "To upgrade follow this steps:\n"
+                "1. Uninstall dvc binary\n"
+                "2. Go to {blue}https://dvc.org{reset}\n"
+                "3. Download and install new binary"
+            ),
+            None: (
+                "Find the latest release at\n{blue}"
+                "https://github.com/iterative/dvc/releases/latest"
+                "{reset}"
+            ),
         }
 
         package_manager = self._get_package_manager()
@@ -144,47 +151,47 @@ class Updater(object):  # pragma: no cover
         import distro
 
         if not is_binary():
-            return 'pip'
+            return "pip"
 
         package_managers = {
-            'rhel':     'yum',
-            'centos':   'yum',
-            'fedora':   'yum',
-            'amazon':   'yum',
-            'opensuse': 'yum',
-            'ubuntu':   'apt',
-            'debian':   'apt',
+            "rhel": "yum",
+            "centos": "yum",
+            "fedora": "yum",
+            "amazon": "yum",
+            "opensuse": "yum",
+            "ubuntu": "apt",
+            "debian": "apt",
         }
 
         return package_managers.get(distro.id())
 
     def _get_darwin(self):
         if not is_binary():
-            if __file__.startswith('/usr/local/Cellar'):
-                return 'formula'
+            if __file__.startswith("/usr/local/Cellar"):
+                return "formula"
             else:
-                return 'pip'
+                return "pip"
 
         # NOTE: both pkg and cask put dvc binary into /usr/local/bin,
         # so in order to know which method of installation was used,
         # we need to actually call `brew cask`
-        ret = os.system('brew cask ls dvc')
+        ret = os.system("brew cask ls dvc")
         if ret == 0:
-            return 'cask'
+            return "cask"
 
         return None
 
     def _get_windows(self):
-        return None if is_binary() else 'pip'
+        return None if is_binary() else "pip"
 
     def _get_package_manager(self):
         import platform
         from dvc.exceptions import DvcException
 
         m = {
-            'Windows': self._get_windows,
-            'Darwin': self._get_darwin,
-            'Linux': self._get_linux,
+            "Windows": self._get_windows,
+            "Darwin": self._get_darwin,
+            "Linux": self._get_linux,
         }
 
         system = platform.system()

dvc/utils/__init__.py

@@ -11,13 +11,13 @@ import json
 import shutil
 import hashlib
 
-LOCAL_CHUNK_SIZE = 1024*1024
-LARGE_FILE_SIZE = 1024*1024*1024
+LOCAL_CHUNK_SIZE = 1024 * 1024
+LARGE_FILE_SIZE = 1024 * 1024 * 1024
 LARGE_DIR_SIZE = 100
 
 
 def dos2unix(data):
-    return data.replace(b'\r\n', b'\n')
+    return data.replace(b"\r\n", b"\n")
 
 
 def file_md5(fname):
@@ -38,7 +38,7 @@ def file_md5(fname):
             name = os.path.relpath(fname)
             total = 0
 
-        with open(fname, 'rb') as fobj:
+        with open(fname, "rb") as fobj:
             while True:
                 data = fobj.read(LOCAL_CHUNK_SIZE)
                 if not data:
@@ -96,7 +96,7 @@ def dict_filter(d, exclude=[]):
 
 def dict_md5(d, exclude=[]):
     filtered = dict_filter(d, exclude)
-    byts = json.dumps(filtered, sort_keys=True).encode('utf-8')
+    byts = json.dumps(filtered, sort_keys=True).encode("utf-8")
     return bytes_md5(byts)
 
 
@@ -108,12 +108,12 @@ def copyfile(src, dest, no_progress_bar=False, name=None):
     name = name if name else os.path.basename(dest)
     total = os.stat(src).st_size
 
-    fsrc = open(src, 'rb')
+    fsrc = open(src, "rb")
 
     if os.path.isdir(dest):
-        fdest = open(os.path.join(dest, os.path.basename(src)), 'wb+')
+        fdest = open(os.path.join(dest, os.path.basename(src)), "wb+")
     else:
-        fdest = open(dest, 'wb+')
+        fdest = open(dest, "wb+")
 
     while True:
         buf = fsrc.read(LOCAL_CHUNK_SIZE)
@@ -174,13 +174,13 @@ def to_chunks(l, jobs):
     if n == 0:
         n = 1
 
-    return [l[x:x+n] for x in range(0, len(l), n)]
+    return [l[x : x + n] for x in range(0, len(l), n)]
 
 
 # NOTE: Check if we are in a bundle
 # https://pythonhosted.org/PyInstaller/runtime-information.html
 def is_binary():
-    return getattr(sys, 'frozen', False)
+    return getattr(sys, "frozen", False)
 
 
 # NOTE: Fix env variables modified by PyInstaller
@@ -192,8 +192,8 @@ def fix_env(env=None):
         env = env.copy()
 
     if is_binary():
-        lp_key = 'LD_LIBRARY_PATH'
-        lp_orig = env.get(lp_key + '_ORIG', None)
+        lp_key = "LD_LIBRARY_PATH"
+        lp_orig = env.get(lp_key + "_ORIG", None)
         if lp_orig is not None:
             env[lp_key] = lp_orig
         else:

dvc/utils/compat.py

@@ -7,39 +7,39 @@ import sys
 _ver = sys.version_info
 
 #: Python 2.x?
-is_py2 = (_ver[0] == 2)
+is_py2 = _ver[0] == 2
 
 #: Python 3.x?
-is_py3 = (_ver[0] == 3)
+is_py3 = _ver[0] == 3
 
 if is_py2:
-    from urlparse import urlparse, urljoin                 # noqa: F401
-    from StringIO import StringIO                          # noqa: F401
-    from BaseHTTPServer import HTTPServer                  # noqa: F401
+    from urlparse import urlparse, urljoin  # noqa: F401
+    from StringIO import StringIO  # noqa: F401
+    from BaseHTTPServer import HTTPServer  # noqa: F401
     from SimpleHTTPServer import SimpleHTTPRequestHandler  # noqa: F401
-    import ConfigParser                                    # noqa: F401
+    import ConfigParser  # noqa: F401
 
-    builtin_str = str                   # noqa: F821
-    bytes = str                         # noqa: F821
-    str = unicode                       # noqa: F821
-    basestring = basestring             # noqa: F821
+    builtin_str = str  # noqa: F821
+    bytes = str  # noqa: F821
+    str = unicode  # noqa: F821
+    basestring = basestring  # noqa: F821
     numeric_types = (int, long, float)  # noqa: F821
-    integer_types = (int, long)         # noqa: F821
-    input = raw_input                   # noqa: F821
+    integer_types = (int, long)  # noqa: F821
+    input = raw_input  # noqa: F821
 
 elif is_py3:
     from urllib.parse import urlparse, urljoin  # noqa: F401
-    from io import StringIO                     # noqa: F401
-    from http.server import (                   # noqa: F401
-        HTTPServer,                             # noqa: F401
-        SimpleHTTPRequestHandler                # noqa: F401
-    )                                           # noqa: F401
-    import configparser as ConfigParser         # noqa: F401
+    from io import StringIO  # noqa: F401
+    from http.server import (  # noqa: F401
+        HTTPServer,  # noqa: F401
+        SimpleHTTPRequestHandler,  # noqa: F401
+    )  # noqa: F401
+    import configparser as ConfigParser  # noqa: F401
 
-    builtin_str = str             # noqa: F821
-    str = str                     # noqa: F821
-    bytes = bytes                 # noqa: F821
-    basestring = (str, bytes)     # noqa: F821
+    builtin_str = str  # noqa: F821
+    str = str  # noqa: F821
+    bytes = bytes  # noqa: F821
+    basestring = (str, bytes)  # noqa: F821
     numeric_types = (int, float)  # noqa: F821
-    integer_types = (int,)        # noqa: F821
-    input = input                 # noqa: F821
+    integer_types = (int,)  # noqa: F821
+    input = input  # noqa: F821

scripts/ci/requirements.txt

@@ -0,0 +1,2 @@
+-r ../../test-style-requirements.txt
+gitpython  # used by dvc/__init__.py to dynamically create dvc version

test-requirements.txt

@@ -7,5 +7,3 @@ xmltodict>=0.11.0
 awscli>=1.14.18
 google-compute-engine
 pywin32; sys_platform == 'win32'
-Pygments
-collective.checkdocs

test-style-requirements.txt

@@ -0,0 +1,3 @@
+Pygments  # required by collective.checkdocs
+collective.checkdocs
+black

tests/__init__.py

@@ -1,8 +1,11 @@
 import os
+
 # Increasing fd ulimit for tests
-if os.name == 'nt':
+if os.name == "nt":
     import win32file
+
     win32file._setmaxstdio(2048)
 else:
     import resource
+
     resource.setrlimit(resource.RLIMIT_NOFILE, (2048, 2048))

tests/__main__.py

@@ -4,11 +4,12 @@ from subprocess import check_call
 REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 os.chdir(REPO_ROOT)
 
-os.putenv('PATH', '{}:{}'.format(os.path.join(REPO_ROOT, 'bin'),
-                                 os.getenv('PATH')))
-os.putenv('DVC_HOME', REPO_ROOT)
-os.putenv('DVC_TEST', 'true')
+os.putenv("PATH", "{}:{}".format(os.path.join(REPO_ROOT, "bin"), os.getenv("PATH")))
+os.putenv("DVC_HOME", REPO_ROOT)
+os.putenv("DVC_TEST", "true")
 
-cmd = 'nosetests -v --processes=-1 --process-timeout=500 --cover-inclusive ' \
-      '--cover-erase --cover-package=dvc --with-coverage --all-modules'
+cmd = (
+    "nosetests -v --processes=-1 --process-timeout=500 --cover-inclusive "
+    "--cover-erase --cover-package=dvc --with-coverage --all-modules"
+)
 check_call(cmd, shell=True)

tests/basic_env.py

@@ -9,18 +9,16 @@ from dvc.project import Project
 
 
 class TestDir(TestCase):
-    GCP_CREDS_FILE = os.path.abspath(os.path.join('scripts',
-                                                  'ci',
-                                                  'gcp-creds.json'))
-    DATA_DIR = 'data_dir'
-    DATA_SUB_DIR = os.path.join(DATA_DIR, 'data_sub_dir')
-    DATA = os.path.join(DATA_DIR, 'data')
-    DATA_SUB = os.path.join(DATA_SUB_DIR, 'data_sub')
+    GCP_CREDS_FILE = os.path.abspath(os.path.join("scripts", "ci", "gcp-creds.json"))
+    DATA_DIR = "data_dir"
+    DATA_SUB_DIR = os.path.join(DATA_DIR, "data_sub_dir")
+    DATA = os.path.join(DATA_DIR, "data")
+    DATA_SUB = os.path.join(DATA_SUB_DIR, "data_sub")
     DATA_CONTENTS = DATA
     DATA_SUB_CONTENTS = DATA_SUB
-    FOO = 'foo'
+    FOO = "foo"
     FOO_CONTENTS = FOO
-    BAR = 'bar'
+    BAR = "bar"
     # NOTE: len(FOO_CONTENTS) must be != len(BAR_CONTENTS)
     #
     # Our state database relies on file mtime and file size to determine
@@ -30,10 +28,11 @@ class TestDir(TestCase):
     # being able to detect that, thus causing tests to fail. Usually,
     # in tests, we replace foo with bar, so we need to make sure that when we
     # modify a file in our tests, its content length changes.
-    BAR_CONTENTS = BAR + 'r'
-    CODE = 'code.py'
-    CODE_CONTENTS = ('import sys\nimport shutil\n'
-                     'shutil.copyfile(sys.argv[1], sys.argv[2])')
+    BAR_CONTENTS = BAR + "r"
+    CODE = "code.py"
+    CODE_CONTENTS = (
+        "import sys\nimport shutil\n" "shutil.copyfile(sys.argv[1], sys.argv[2])"
+    )
 
     def _pushd(self, d):
         self._saved_dir = os.path.realpath(os.curdir)
@@ -48,13 +47,13 @@ class TestDir(TestCase):
         if len(dname) > 0 and not os.path.isdir(dname):
             os.makedirs(dname)
 
-        with open(name, 'a') as f:
+        with open(name, "a") as f:
             f.write(contents)
 
     @staticmethod
     def mkdtemp():
-        prefix = 'dvc-test.{}.'.format(os.getpid())
-        suffix = '.{}'.format(uuid.uuid4())
+        prefix = "dvc-test.{}.".format(os.getpid())
+        suffix = ".{}".format(uuid.uuid4())
         return tempfile.mkdtemp(prefix=prefix, suffix=suffix)
 
     def setUp(self):
@@ -78,14 +77,14 @@ class TestGit(TestDir):
         super(TestGit, self).setUp()
         self.git = Repo.init()
         self.git.index.add([self.CODE])
-        self.git.index.commit('add code')
+        self.git.index.commit("add code")
 
 
 class TestGitSubmodule(TestGit):
     def setUp(self):
         super(TestGitSubmodule, self).setUp()
         subrepo = Repo.init()
-        subrepo_path = 'subrepo'
+        subrepo_path = "subrepo"
         self.git.create_submodule(subrepo_path, subrepo_path, subrepo.git_dir)
         self._pushd(subrepo_path)
 

tests/test_add.py

@@ -30,20 +30,20 @@ class TestAdd(TestDvc):
         self.assertEqual(len(stage.outs), 1)
         self.assertEqual(len(stage.deps), 0)
         self.assertEqual(stage.cmd, None)
-        self.assertEqual(stage.outs[0].info['md5'], md5)
+        self.assertEqual(stage.outs[0].info["md5"], md5)
 
 
 class TestAddUnupportedFile(TestDvc):
     def test(self):
         with self.assertRaises(DvcException):
-            self.dvc.add('unsupported://unsupported')
+            self.dvc.add("unsupported://unsupported")
 
 
 class TestAddDirectory(TestDvc):
     def test(self):
-        dname = 'directory'
+        dname = "directory"
         os.mkdir(dname)
-        self.create(os.path.join(dname, 'file'), 'file')
+        self.create(os.path.join(dname, "file"), "file")
         stages = self.dvc.add(dname)
         self.assertEqual(len(stages), 1)
         stage = stages[0]
@@ -51,11 +51,11 @@ class TestAddDirectory(TestDvc):
         self.assertEqual(len(stage.deps), 0)
         self.assertEqual(len(stage.outs), 1)
 
-        md5 = stage.outs[0].info['md5']
+        md5 = stage.outs[0].info["md5"]
 
         dir_info = self.dvc.cache.local.load_dir_cache(md5)
         for info in dir_info:
-            self.assertTrue('\\' not in info['relpath'])
+            self.assertTrue("\\" not in info["relpath"])
 
 
 class TestAddDirectoryRecursive(TestDvc):
@@ -66,30 +66,28 @@ class TestAddDirectoryRecursive(TestDvc):
 
 class TestAddCmdDirectoryRecursive(TestDvc):
     def test(self):
-        ret = main(['add',
-                    '--recursive',
-                    self.DATA_DIR])
+        ret = main(["add", "--recursive", self.DATA_DIR])
         self.assertEqual(ret, 0)
 
 
 class TestAddDirectoryWithForwardSlash(TestDvc):
     def test(self):
-        dname = 'directory/'
+        dname = "directory/"
         os.mkdir(dname)
-        self.create(os.path.join(dname, 'file'), 'file')
+        self.create(os.path.join(dname, "file"), "file")
         stages = self.dvc.add(dname)
         self.assertEqual(len(stages), 1)
         stage = stages[0]
         self.assertTrue(stage is not None)
-        self.assertEqual(os.path.abspath('directory.dvc'), stage.path)
+        self.assertEqual(os.path.abspath("directory.dvc"), stage.path)
 
 
 class TestAddTrackedFile(TestDvc):
     def test(self):
-        fname = 'tracked_file'
-        self.create(fname, 'tracked file contents')
+        fname = "tracked_file"
+        self.create(fname, "tracked file contents")
         self.dvc.scm.add([fname])
-        self.dvc.scm.commit('add {}'.format(fname))
+        self.dvc.scm.commit("add {}".format(fname))
 
         with self.assertRaises(OutputAlreadyTrackedError):
             self.dvc.add(fname)
@@ -97,8 +95,8 @@ class TestAddTrackedFile(TestDvc):
 
 class TestAddDirWithExistingCache(TestDvc):
     def test(self):
-        dname = 'a'
-        fname = os.path.join(dname, 'b')
+        dname = "a"
+        fname = os.path.join(dname, "b")
         os.mkdir(dname)
         shutil.copyfile(self.FOO, fname)
 
@@ -132,13 +130,13 @@ class TestAddFileInDir(TestDvc):
         self.assertNotEqual(stage, None)
         self.assertEqual(len(stage.deps), 0)
         self.assertEqual(len(stage.outs), 1)
-        self.assertEqual(stage.relpath, self.DATA_SUB + '.dvc')
+        self.assertEqual(stage.relpath, self.DATA_SUB + ".dvc")
 
 
 class TestAddExternalLocalFile(TestDvc):
     def test(self):
         dname = TestDvc.mkdtemp()
-        fname = os.path.join(dname, 'foo')
+        fname = os.path.join(dname, "foo")
         shutil.copyfile(self.FOO, fname)
 
         stages = self.dvc.add(fname)
@@ -147,10 +145,10 @@ class TestAddExternalLocalFile(TestDvc):
         self.assertNotEqual(stage, None)
         self.assertEqual(len(stage.deps), 0)
         self.assertEqual(len(stage.outs), 1)
-        self.assertEqual(stage.relpath, 'foo.dvc')
+        self.assertEqual(stage.relpath, "foo.dvc")
         self.assertEqual(len(os.listdir(dname)), 1)
         self.assertTrue(os.path.isfile(fname))
-        self.assertTrue(filecmp.cmp(fname, 'foo', shallow=False))
+        self.assertTrue(filecmp.cmp(fname, "foo", shallow=False))
 
 
 class TestAddLocalRemoteFile(TestDvc):
@@ -159,36 +157,34 @@ class TestAddLocalRemoteFile(TestDvc):
         Making sure that 'remote' syntax is handled properly for local outs.
         """
         cwd = os.getcwd()
-        remote = 'myremote'
+        remote = "myremote"
 
-        ret = main(['remote', 'add', remote, cwd])
+        ret = main(["remote", "add", remote, cwd])
         self.assertEqual(ret, 0)
 
         self.dvc = Project()
 
-        foo = 'remote://{}/{}'.format(remote, self.FOO)
-        ret = main(['add', foo])
+        foo = "remote://{}/{}".format(remote, self.FOO)
+        ret = main(["add", foo])
         self.assertEqual(ret, 0)
 
-        with open('foo.dvc', 'r') as fobj:
+        with open("foo.dvc", "r") as fobj:
             d = yaml.safe_load(fobj)
-            self.assertEqual(d['outs'][0]['path'], foo)
+            self.assertEqual(d["outs"][0]["path"], foo)
 
         bar = os.path.join(cwd, self.BAR)
-        ret = main(['add', bar])
+        ret = main(["add", bar])
         self.assertEqual(ret, 0)
 
-        with open('bar.dvc', 'r') as fobj:
+        with open("bar.dvc", "r") as fobj:
             d = yaml.safe_load(fobj)
-            self.assertEqual(d['outs'][0]['path'], bar)
+            self.assertEqual(d["outs"][0]["path"], bar)
 
 
 class TestCmdAdd(TestDvc):
     def test(self):
-        ret = main(['add',
-                    self.FOO])
+        ret = main(["add", self.FOO])
         self.assertEqual(ret, 0)
 
-        ret = main(['add',
-                    'non-existing-file'])
+        ret = main(["add", "non-existing-file"])
         self.assertNotEqual(ret, 0)

tests/test_analytics.py

@@ -11,7 +11,7 @@ def _clean_getenv(key, default=None):
     """
     Remove env vars that affect dvc behavior in tests
     """
-    if key in ['DVC_TEST', 'CI']:
+    if key in ["DVC_TEST", "CI"]:
         return None
     return os.environ.get(key, default)
 
@@ -26,19 +26,20 @@ class TestAnalytics(TestDir):
         self.assertTrue(a.PARAM_SYSTEM_INFO in a.info.keys())
         self.assertNotEqual(a.info[a.PARAM_SYSTEM_INFO], {})
 
-    @mock.patch.object(os, 'getenv', new=_clean_getenv)
-    @mock.patch('requests.post')
+    @mock.patch.object(os, "getenv", new=_clean_getenv)
+    @mock.patch("requests.post")
     def test_send(self, mockpost):
-        ret = main(['daemon', 'analytics', Analytics().dump(), '-v'])
+        ret = main(["daemon", "analytics", Analytics().dump(), "-v"])
         self.assertEqual(ret, 0)
 
         self.assertTrue(mockpost.called)
 
-    @mock.patch.object(os, 'getenv', new=_clean_getenv)
-    @mock.patch.object(requests, 'post',
-                       side_effect=requests.exceptions.RequestException())
+    @mock.patch.object(os, "getenv", new=_clean_getenv)
+    @mock.patch.object(
+        requests, "post", side_effect=requests.exceptions.RequestException()
+    )
     def test_send_failed(self, mockpost):
-        ret = main(['daemon', 'analytics', Analytics().dump(), '-v'])
+        ret = main(["daemon", "analytics", Analytics().dump(), "-v"])
         self.assertEqual(ret, 0)
 
         self.assertTrue(mockpost.called)
@@ -49,13 +50,13 @@ class TestAnalyticsGit(TestAnalytics, TestGit):
 
 
 class TestAnalyticsDvc(TestAnalytics, TestDvc):
-    @mock.patch('requests.post')
+    @mock.patch("requests.post")
     def test_send_disabled(self, mockpost):
-        ret = main(['config', 'core.analytics', 'false'])
+        ret = main(["config", "core.analytics", "false"])
         self.assertEqual(ret, 0)
 
-        with mock.patch.object(os, 'getenv', new=_clean_getenv):
-            ret = main(['daemon', 'analytics', Analytics().dump(), '-v'])
+        with mock.patch.object(os, "getenv", new=_clean_getenv):
+            ret = main(["daemon", "analytics", Analytics().dump(), "-v"])
         self.assertEqual(ret, 0)
 
         self.assertFalse(mockpost.called)

tests/test_cache.py

@@ -11,16 +11,16 @@ from tests.basic_env import TestDvc, TestDir
 class TestCache(TestDvc):
     def setUp(self):
         super(TestCache, self).setUp()
-        self.cache1_md5 = '123'
-        self.cache2_md5 = '234'
-        self.cache1 = os.path.join(self.dvc.cache.local.cache_dir,
-                                   self.cache1_md5[0:2],
-                                   self.cache1_md5[2:])
-        self.cache2 = os.path.join(self.dvc.cache.local.cache_dir,
-                                   self.cache2_md5[0:2],
-                                   self.cache2_md5[2:])
-        self.create(self.cache1, '1')
-        self.create(self.cache2, '2')
+        self.cache1_md5 = "123"
+        self.cache2_md5 = "234"
+        self.cache1 = os.path.join(
+            self.dvc.cache.local.cache_dir, self.cache1_md5[0:2], self.cache1_md5[2:]
+        )
+        self.cache2 = os.path.join(
+            self.dvc.cache.local.cache_dir, self.cache2_md5[0:2], self.cache2_md5[2:]
+        )
+        self.create(self.cache1, "1")
+        self.create(self.cache2, "2")
 
     def test_all(self):
         md5_list = list(Cache(self.dvc).local.all())
@@ -39,11 +39,11 @@ class TestCacheLoadBadDirCache(TestDvc):
         self.assertEqual(len(ret), 0)
 
     def test(self):
-        fname = self.dvc.cache.local.get('123.dir')
-        self.create(fname, '<clearly>not,json')
+        fname = self.dvc.cache.local.get("123.dir")
+        self.create(fname, "<clearly>not,json")
         self._do_test(self.dvc.cache.local.load_dir_cache(fname))
 
-        fname = self.dvc.cache.local.get('234.dir')
+        fname = self.dvc.cache.local.get("234.dir")
         self.create(fname, '{"a": "b"}')
         self._do_test(self.dvc.cache.local.load_dir_cache(fname))
 
@@ -52,93 +52,93 @@ class TestExternalCacheDir(TestDvc):
     def test(self):
         cache_dir = TestDvc.mkdtemp()
 
-        ret = main(['config', 'cache.dir', cache_dir])
+        ret = main(["config", "cache.dir", cache_dir])
         self.assertEqual(ret, 0)
 
-        shutil.rmtree('.dvc/cache')
+        shutil.rmtree(".dvc/cache")
 
-        ret = main(['add', self.FOO])
+        ret = main(["add", self.FOO])
         self.assertEqual(ret, 0)
 
-        ret = main(['add', self.DATA_DIR])
+        ret = main(["add", self.DATA_DIR])
         self.assertEqual(ret, 0)
 
-        self.assertFalse(os.path.exists('.dvc/cache'))
+        self.assertFalse(os.path.exists(".dvc/cache"))
         self.assertNotEqual(len(os.listdir(cache_dir)), 0)
 
 
 class TestSharedCacheDir(TestDir):
     def test(self):
-        cache_dir = os.path.abspath(os.path.join(os.curdir, 'cache'))
-        for d in ['dir1', 'dir2']:
+        cache_dir = os.path.abspath(os.path.join(os.curdir, "cache"))
+        for d in ["dir1", "dir2"]:
             os.mkdir(d)
             os.chdir(d)
 
-            ret = main(['init', '--no-scm'])
+            ret = main(["init", "--no-scm"])
             self.assertEqual(ret, 0)
 
-            ret = main(['config', 'cache.dir', cache_dir])
+            ret = main(["config", "cache.dir", cache_dir])
             self.assertEqual(ret, 0)
 
-            shutil.rmtree(os.path.join('.dvc', 'cache'))
+            shutil.rmtree(os.path.join(".dvc", "cache"))
 
-            with open('common', 'w+') as fd:
-                fd.write('common')
+            with open("common", "w+") as fd:
+                fd.write("common")
 
-            with open('unique', 'w+') as fd:
+            with open("unique", "w+") as fd:
                 fd.write(d)
 
-            ret = main(['add', 'common', 'unique'])
+            ret = main(["add", "common", "unique"])
             self.assertEqual(ret, 0)
 
-            os.chdir('..')
+            os.chdir("..")
 
-        self.assertFalse(os.path.exists(os.path.join('dir1', '.dvc', 'cache')))
-        self.assertFalse(os.path.exists(os.path.join('dir2', '.dvc', 'cache')))
+        self.assertFalse(os.path.exists(os.path.join("dir1", ".dvc", "cache")))
+        self.assertFalse(os.path.exists(os.path.join("dir2", ".dvc", "cache")))
 
-        subdirs = list(filter(
-                        lambda x: os.path.isdir(os.path.join(cache_dir, x)),
-                        os.listdir(cache_dir)))
+        subdirs = list(
+            filter(
+                lambda x: os.path.isdir(os.path.join(cache_dir, x)),
+                os.listdir(cache_dir),
+            )
+        )
         self.assertEqual(len(subdirs), 3)
-        self.assertEqual(len(os.listdir(os.path.join(cache_dir,
-                                                     subdirs[0]))), 1)
-        self.assertEqual(len(os.listdir(os.path.join(cache_dir,
-                                                     subdirs[1]))), 1)
-        self.assertEqual(len(os.listdir(os.path.join(cache_dir,
-                                                     subdirs[2]))), 1)
+        self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[0]))), 1)
+        self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[1]))), 1)
+        self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[2]))), 1)
 
 
 class TestCacheLinkType(TestDvc):
     def test(self):
-        ret = main(['config', 'cache.type', 'reflink,copy'])
+        ret = main(["config", "cache.type", "reflink,copy"])
         self.assertEqual(ret, 0)
 
-        ret = main(['add', self.FOO])
+        ret = main(["add", self.FOO])
         self.assertEqual(ret, 0)
 
 
 class TestCmdCacheDir(TestDvc):
     def test_abs_path(self):
-        dname = os.path.join(os.path.dirname(self._root_dir), 'dir')
-        ret = main(['cache', 'dir', dname])
+        dname = os.path.join(os.path.dirname(self._root_dir), "dir")
+        ret = main(["cache", "dir", dname])
         self.assertEqual(ret, 0)
 
         config = configobj.ConfigObj(self.dvc.config.config_file)
-        self.assertEqual(config['cache']['dir'], dname)
+        self.assertEqual(config["cache"]["dir"], dname)
 
     def test_relative_path(self):
         tmpdir = self.mkdtemp()
         dname = os.path.relpath(tmpdir)
-        ret = main(['cache', 'dir', dname])
+        ret = main(["cache", "dir", dname])
         self.assertEqual(ret, 0)
 
         # NOTE: we are in the project's root and config is in .dvc/, so
         # dir path written to config should be just one level above.
-        rel = os.path.join('..', dname)
+        rel = os.path.join("..", dname)
         config = configobj.ConfigObj(self.dvc.config.config_file)
-        self.assertEqual(config['cache']['dir'], rel)
+        self.assertEqual(config["cache"]["dir"], rel)
 
-        ret = main(['add', self.FOO])
+        ret = main(["add", self.FOO])
         self.assertEqual(ret, 0)
 
         subdirs = os.listdir(tmpdir)

tests/test_checkout.py

@@ -25,11 +25,11 @@ class TestCheckout(TestRepro):
         self.data_dir_stage = stages[0]
         self.assertTrue(self.data_dir_stage is not None)
 
-        self.orig = 'orig'
+        self.orig = "orig"
         shutil.copy(self.FOO, self.orig)
         os.unlink(self.FOO)
 
-        self.orig_dir = 'orig_dir'
+        self.orig_dir = "orig_dir"
         shutil.copytree(self.DATA_DIR, self.orig_dir)
         shutil.rmtree(self.DATA_DIR)
 
@@ -44,10 +44,10 @@ class TestCheckout(TestRepro):
 
 class TestCheckoutSingleStage(TestCheckout):
     def test(self):
-        ret = main(['checkout', '--force', self.foo_stage.path])
+        ret = main(["checkout", "--force", self.foo_stage.path])
         self.assertEqual(ret, 0)
 
-        ret = main(['checkout', '--force', self.data_dir_stage.path])
+        ret = main(["checkout", "--force", self.data_dir_stage.path])
         self.assertEqual(ret, 0)
 
         self._test_checkout()
@@ -57,8 +57,8 @@ class TestCheckoutCorruptedCacheFile(TestRepro):
     def test(self):
         cache = self.foo_stage.outs[0].cache
 
-        with open(cache, 'a') as fd:
-            fd.write('1')
+        with open(cache, "a") as fd:
+            fd.write("1")
 
         self.dvc.checkout(force=True)
 
@@ -69,10 +69,10 @@ class TestCheckoutCorruptedCacheFile(TestRepro):
 class TestCheckoutCorruptedCacheDir(TestDvc):
     def test(self):
         # NOTE: using 'copy' so that cache and link don't have same inode
-        ret = main(['config', 'cache.type', 'copy'])
+        ret = main(["config", "cache.type", "copy"])
         self.assertEqual(ret, 0)
 
-        self.dvc = Project('.')
+        self.dvc = Project(".")
         stages = self.dvc.add(self.DATA_DIR)
         self.assertEqual(len(stages), 1)
         self.assertEqual(len(stages[0].outs), 1)
@@ -84,8 +84,8 @@ class TestCheckoutCorruptedCacheDir(TestDvc):
         checksum = entry[self.dvc.cache.local.PARAM_CHECKSUM]
         cache = self.dvc.cache.local.get(checksum)
 
-        with open(cache, 'w+') as fobj:
-            fobj.write('1')
+        with open(cache, "w+") as fobj:
+            fobj.write("1")
 
         self.dvc.checkout(force=True)
 
@@ -94,29 +94,28 @@ class TestCheckoutCorruptedCacheDir(TestDvc):
 
 class TestCmdCheckout(TestCheckout):
     def test(self):
-        ret = main(['checkout', '--force'])
+        ret = main(["checkout", "--force"])
         self.assertEqual(ret, 0)
         self._test_checkout()
 
 
 class CheckoutBase(TestDvc):
-    GIT_IGNORE = '.gitignore'
+    GIT_IGNORE = ".gitignore"
 
-    def commit_data_file(self, fname, content='random text'):
-        with open(fname, 'w') as fd:
+    def commit_data_file(self, fname, content="random text"):
+        with open(fname, "w") as fd:
             fd.write(content)
         stages = self.dvc.add(fname)
         self.assertEqual(len(stages), 1)
         self.assertTrue(stages[0] is not None)
-        self.dvc.scm.add([fname + '.dvc', '.gitignore'])
-        self.dvc.scm.commit('adding ' + fname)
+        self.dvc.scm.add([fname + ".dvc", ".gitignore"])
+        self.dvc.scm.commit("adding " + fname)
 
     def read_ignored(self):
-        return list(map(lambda s: s.strip('\n'),
-                        open(self.GIT_IGNORE).readlines()))
+        return list(map(lambda s: s.strip("\n"), open(self.GIT_IGNORE).readlines()))
 
     def outs_info(self, stage):
-        FileInfo = collections.namedtuple('FileInfo', 'path inode')
+        FileInfo = collections.namedtuple("FileInfo", "path inode")
 
         paths = [
             os.path.join(root, file)
@@ -125,24 +124,21 @@ class CheckoutBase(TestDvc):
             for file in files
         ]
 
-        return [
-            FileInfo(path=path, inode=System.inode(path))
-            for path in paths
-        ]
+        return [FileInfo(path=path, inode=System.inode(path)) for path in paths]
 
 
 class TestRemoveFilesWhenCheckout(CheckoutBase):
     def test(self):
-        fname = 'file_in_a_branch'
-        branch_master = 'master'
-        branch_1 = 'b1'
+        fname = "file_in_a_branch"
+        branch_master = "master"
+        branch_1 = "b1"
 
         self.dvc.scm.add(self.dvc.scm.untracked_files())
-        self.dvc.scm.commit('add all files')
+        self.dvc.scm.commit("add all files")
 
         # add the file into a separate branch
         self.dvc.scm.checkout(branch_1, True)
-        ret = main(['checkout', '--force'])
+        ret = main(["checkout", "--force"])
         self.assertEqual(ret, 0)
         self.commit_data_file(fname)
 
@@ -152,28 +148,28 @@ class TestRemoveFilesWhenCheckout(CheckoutBase):
 
         # Make sure `dvc checkout` removes the file
         # self.dvc.checkout()
-        ret = main(['checkout', '--force'])
+        ret = main(["checkout", "--force"])
         self.assertEqual(ret, 0)
         self.assertFalse(os.path.exists(fname))
 
 
 class TestCheckoutCleanWorkingDir(CheckoutBase):
-    @patch('dvc.prompt.confirm', return_value=True)
+    @patch("dvc.prompt.confirm", return_value=True)
     def test(self, mock_prompt):
         mock_prompt.return_value = True
 
         stages = self.dvc.add(self.DATA_DIR)
         stage = stages[0]
 
-        working_dir_change = os.path.join(self.DATA_DIR, 'not_cached.txt')
-        with open(working_dir_change, 'w') as f:
-            f.write('not_cached')
+        working_dir_change = os.path.join(self.DATA_DIR, "not_cached.txt")
+        with open(working_dir_change, "w") as f:
+            f.write("not_cached")
 
-        ret = main(['checkout', stage.relpath])
+        ret = main(["checkout", stage.relpath])
         self.assertEqual(ret, 0)
         self.assertFalse(os.path.exists(working_dir_change))
 
-    @patch('dvc.prompt.confirm', return_value=False)
+    @patch("dvc.prompt.confirm", return_value=False)
     def test_force(self, mock_prompt):
         mock_prompt.return_value = False
 
@@ -181,11 +177,11 @@ class TestCheckoutCleanWorkingDir(CheckoutBase):
         self.assertEqual(len(stages), 1)
         stage = stages[0]
 
-        working_dir_change = os.path.join(self.DATA_DIR, 'not_cached.txt')
-        with open(working_dir_change, 'w') as f:
-            f.write('not_cached')
+        working_dir_change = os.path.join(self.DATA_DIR, "not_cached.txt")
+        with open(working_dir_change, "w") as f:
+            f.write("not_cached")
 
-        ret = main(['checkout', stage.relpath])
+        ret = main(["checkout", stage.relpath])
         self.assertNotEqual(ret, 0)
 
         mock_prompt.assert_called()
@@ -196,7 +192,7 @@ class TestCheckoutCleanWorkingDir(CheckoutBase):
 class TestCheckoutSelectiveRemove(CheckoutBase):
     def test(self):
         # Use copy to test for changes in the inodes
-        ret = main(['config', 'cache.type', 'copy'])
+        ret = main(["config", "cache.type", "copy"])
         self.assertEqual(ret, 0)
 
         stages = self.dvc.add(self.DATA_DIR)
@@ -206,7 +202,7 @@ class TestCheckoutSelectiveRemove(CheckoutBase):
 
         os.remove(staged_files[0].path)
 
-        ret = main(['checkout', '--force', stage.relpath])
+        ret = main(["checkout", "--force", stage.relpath])
         self.assertEqual(ret, 0)
 
         checkedout_files = self.outs_info(stage)
@@ -219,20 +215,22 @@ class TestCheckoutSelectiveRemove(CheckoutBase):
 
 class TestGitIgnoreBasic(CheckoutBase):
     def test(self):
-        fname1 = 'file_1'
-        fname2 = 'file_2'
-        fname3 = 'file_3'
+        fname1 = "file_1"
+        fname2 = "file_2"
+        fname3 = "file_3"
 
         self.dvc.scm.add(self.dvc.scm.untracked_files())
-        self.dvc.scm.commit('add all files')
+        self.dvc.scm.commit("add all files")
 
         self.assertFalse(os.path.exists(self.GIT_IGNORE))
 
         self.commit_data_file(fname1)
         self.commit_data_file(fname2)
-        self.dvc.run(cmd='python {} {} {}'.format(self.CODE, self.FOO, fname3),
-                     deps=[self.CODE, self.FOO],
-                     outs_no_cache=[fname3])
+        self.dvc.run(
+            cmd="python {} {} {}".format(self.CODE, self.FOO, fname3),
+            deps=[self.CODE, self.FOO],
+            outs_no_cache=[fname3],
+        )
 
         self.assertTrue(os.path.exists(self.GIT_IGNORE))
 
@@ -240,51 +238,51 @@ class TestGitIgnoreBasic(CheckoutBase):
 
         self.assertEqual(len(ignored), 2)
 
-        self.assertIn('/' + fname1, ignored)
-        self.assertIn('/' + fname2, ignored)
+        self.assertIn("/" + fname1, ignored)
+        self.assertIn("/" + fname2, ignored)
 
 
 class TestGitIgnoreWhenCheckout(CheckoutBase):
     def test(self):
-        fname_master = 'file_in_a_master'
-        branch_master = 'master'
-        fname_branch = 'file_in_a_branch'
-        branch_1 = 'b1'
+        fname_master = "file_in_a_master"
+        branch_master = "master"
+        fname_branch = "file_in_a_branch"
+        branch_1 = "b1"
 
         self.dvc.scm.add(self.dvc.scm.untracked_files())
-        self.dvc.scm.commit('add all files')
+        self.dvc.scm.commit("add all files")
         self.commit_data_file(fname_master)
 
         self.dvc.scm.checkout(branch_1, True)
-        ret = main(['checkout', '--force'])
+        ret = main(["checkout", "--force"])
         self.assertEqual(ret, 0)
         self.commit_data_file(fname_branch)
 
         self.dvc.scm.checkout(branch_master)
-        ret = main(['checkout', '--force'])
+        ret = main(["checkout", "--force"])
         self.assertEqual(ret, 0)
 
         ignored = self.read_ignored()
 
         self.assertEqual(len(ignored), 1)
-        self.assertIn('/' + fname_master, ignored)
+        self.assertIn("/" + fname_master, ignored)
 
         self.dvc.scm.checkout(branch_1)
-        ret = main(['checkout', '--force'])
+        ret = main(["checkout", "--force"])
         self.assertEqual(ret, 0)
         ignored = self.read_ignored()
-        self.assertIn('/' + fname_branch, ignored)
+        self.assertIn("/" + fname_branch, ignored)
 
 
 class TestCheckoutMissingMd5InStageFile(TestRepro):
     def test(self):
-        with open(self.file1_stage, 'r') as fd:
+        with open(self.file1_stage, "r") as fd:
             d = yaml.load(fd)
 
-        del(d[Stage.PARAM_OUTS][0][RemoteLOCAL.PARAM_CHECKSUM])
-        del(d[Stage.PARAM_DEPS][0][RemoteLOCAL.PARAM_CHECKSUM])
+        del (d[Stage.PARAM_OUTS][0][RemoteLOCAL.PARAM_CHECKSUM])
+        del (d[Stage.PARAM_DEPS][0][RemoteLOCAL.PARAM_CHECKSUM])
 
-        with open(self.file1_stage, 'w') as fd:
+        with open(self.file1_stage, "w") as fd:
             yaml.dump(d, fd)
 
         self.dvc.checkout(force=True)
@@ -292,7 +290,7 @@ class TestCheckoutMissingMd5InStageFile(TestRepro):
 
 class TestCheckoutEmptyDir(TestDvc):
     def test(self):
-        dname = 'empty_dir'
+        dname = "empty_dir"
         os.mkdir(dname)
 
         stages = self.dvc.add(dname)
@@ -312,12 +310,10 @@ class TestCheckoutEmptyDir(TestDvc):
 
 class TestCheckoutNotCachedFile(TestDvc):
     def test(self):
-        cmd = 'python {} {} {}'.format(self.CODE, self.FOO, 'out')
+        cmd = "python {} {} {}".format(self.CODE, self.FOO, "out")
 
         self.dvc.add(self.FOO)
-        stage = self.dvc.run(cmd=cmd,
-                             deps=[self.FOO, self.CODE],
-                             outs_no_cache=['out'])
+        stage = self.dvc.run(cmd=cmd, deps=[self.FOO, self.CODE], outs_no_cache=["out"])
         self.assertTrue(stage is not None)
 
         self.dvc.checkout(force=True)
@@ -331,7 +327,7 @@ class TestCheckoutWithDeps(TestRepro):
         self.assertFalse(os.path.exists(self.FOO))
         self.assertFalse(os.path.exists(self.file1))
 
-        ret = main(['checkout', '--force', self.file1_stage, '--with-deps'])
+        ret = main(["checkout", "--force", self.file1_stage, "--with-deps"])
         self.assertEqual(ret, 0)
 
         self.assertTrue(os.path.exists(self.FOO))
@@ -345,7 +341,7 @@ class TestCheckoutDirectory(TestRepro):
         shutil.rmtree(self.DATA_DIR)
         self.assertFalse(os.path.exists(self.DATA_DIR))
 
-        ret = main(['checkout', stage.path])
+        ret = main(["checkout", stage.path])
         self.assertEqual(ret, 0)
 
         self.assertTrue(os.path.exists(self.DATA_DIR))

tests/test_cli.py

@@ -19,38 +19,53 @@ from tests.basic_env import TestDvc
 
 class TestArgParse(TestDvc):
     def test(self):
-        args = parse_args(['init'])
+        args = parse_args(["init"])
         self.assertIsInstance(args.func(args), CmdInit)
 
 
 class TestRun(TestDvc):
     def test(self):
-        dep1 = 'dep1'
-        dep2 = 'dep2'
+        dep1 = "dep1"
+        dep2 = "dep2"
 
-        out1 = 'out1'
-        out2 = 'out2'
+        out1 = "out1"
+        out2 = "out2"
 
-        out_no_cache1 = 'out_no_cache1'
-        out_no_cache2 = 'out_no_cache2'
+        out_no_cache1 = "out_no_cache1"
+        out_no_cache2 = "out_no_cache2"
 
-        fname = 'dvc.dvc'
+        fname = "dvc.dvc"
         cwd = os.curdir
-        cmd = 'cmd'
-        arg1 = 'arg1'
-        arg2 = 'arg2'
+        cmd = "cmd"
+        arg1 = "arg1"
+        arg2 = "arg2"
 
-        args = parse_args(['run',
-                           '-d', dep1,
-                           '--deps', dep2,
-                           '-o', out1,
-                           '--outs', out2,
-                           '-O', out_no_cache1,
-                           '--outs-no-cache', out_no_cache2,
-                           '-f', fname,
-                           '--file', fname,
-                           '-c', cwd,
-                           cmd, arg1, arg2])
+        args = parse_args(
+            [
+                "run",
+                "-d",
+                dep1,
+                "--deps",
+                dep2,
+                "-o",
+                out1,
+                "--outs",
+                out2,
+                "-O",
+                out_no_cache1,
+                "--outs-no-cache",
+                out_no_cache2,
+                "-f",
+                fname,
+                "--file",
+                fname,
+                "-c",
+                cwd,
+                cmd,
+                arg1,
+                arg2,
+            ]
+        )
 
         self.assertIsInstance(args.func(args), CmdRun)
         self.assertEqual(args.deps, [dep1, dep2])
@@ -63,32 +78,30 @@ class TestRun(TestDvc):
 
 class TestPull(TestDvc):
     def test(self):
-        args = parse_args(['pull'])
+        args = parse_args(["pull"])
         self.assertIsInstance(args.func(args), CmdDataPull)
 
 
 class TestPush(TestDvc):
     def test(self):
-        args = parse_args(['push'])
+        args = parse_args(["push"])
         self.assertIsInstance(args.func(args), CmdDataPush)
 
 
 class TestStatus(TestDvc):
     def test(self):
-        args = parse_args(['status'])
+        args = parse_args(["status"])
         self.assertIsInstance(args.func(args), CmdDataStatus)
 
 
 class TestRepro(TestDvc):
     def test(self):
-        target1 = '1'
-        target2 = '2'
+        target1 = "1"
+        target2 = "2"
 
-        args = parse_args(['repro',
-                           target1,
-                           target2,
-                           '-f', '--force',
-                           '-s', '--single-item'])
+        args = parse_args(
+            ["repro", target1, target2, "-f", "--force", "-s", "--single-item"]
+        )
 
         self.assertIsInstance(args.func(args), CmdRepro)
         self.assertEqual(args.targets, [target1, target2])
@@ -98,12 +111,10 @@ class TestRepro(TestDvc):
 
 class TestRemove(TestDvc):
     def test(self):
-        target1 = '1'
-        target2 = '2'
+        target1 = "1"
+        target2 = "2"
 
-        args = parse_args(['remove',
-                           target1,
-                           target2])
+        args = parse_args(["remove", target1, target2])
 
         self.assertIsInstance(args.func(args), CmdRemove)
         self.assertEqual(args.targets, [target1, target2])
@@ -111,12 +122,10 @@ class TestRemove(TestDvc):
 
 class TestAdd(TestDvc):
     def test(self):
-        target1 = '1'
-        target2 = '2'
+        target1 = "1"
+        target2 = "2"
 
-        args = parse_args(['add',
-                           target1,
-                           target2])
+        args = parse_args(["add", target1, target2])
 
         self.assertIsInstance(args.func(args), CmdAdd)
         self.assertEqual(args.targets, [target1, target2])
@@ -124,33 +133,25 @@ class TestAdd(TestDvc):
 
 class TestGC(TestDvc):
     def test(self):
-        args = parse_args(['gc'])
+        args = parse_args(["gc"])
         self.assertIsInstance(args.func(args), CmdGC)
 
 
 class TestGCMultipleProjects(TestDvc):
     def test(self):
-        args = parse_args([
-            'gc',
-            '-p',
-            '/tmp/asdf',
-            '/tmp/xyz'
-        ])
+        args = parse_args(["gc", "-p", "/tmp/asdf", "/tmp/xyz"])
 
         self.assertIsInstance(args.func(args), CmdGC)
 
-        self.assertEqual(args.projects, ['/tmp/asdf', '/tmp/xyz'])
+        self.assertEqual(args.projects, ["/tmp/asdf", "/tmp/xyz"])
 
 
 class TestConfig(TestDvc):
     def test(self):
-        name = 'param'
-        value = '1'
+        name = "param"
+        value = "1"
 
-        args = parse_args(['config',
-                           '-u', '--unset',
-                           name,
-                           value])
+        args = parse_args(["config", "-u", "--unset", name, value])
 
         self.assertIsInstance(args.func(args), CmdConfig)
         self.assertEqual(args.unset, True)
@@ -160,7 +161,7 @@ class TestConfig(TestDvc):
 
 class TestCheckout(TestDvc):
     def test(self):
-        args = parse_args(['checkout'])
+        args = parse_args(["checkout"])
         self.assertIsInstance(args.func(args), CmdCheckout)
 
 

tests/test_config.py

@@ -25,24 +25,24 @@ class TestConfigCLI(TestDvc):
         return True
 
     def test_root(self):
-        ret = main(['root'])
+        ret = main(["root"])
         self.assertEqual(ret, 0)
 
         # NOTE: check that `dvc root` is not blocked with dvc lock
         with self.dvc.lock:
-            ret = main(['root'])
+            ret = main(["root"])
         self.assertEqual(ret, 0)
 
     def _do_test(self, local=False):
-        section = 'setsection'
-        field = 'setfield'
-        section_field = '{}.{}'.format(section, field)
-        value = 'setvalue'
-        newvalue = 'setnewvalue'
+        section = "setsection"
+        field = "setfield"
+        section_field = "{}.{}".format(section, field)
+        value = "setvalue"
+        newvalue = "setnewvalue"
 
-        base = ['config']
+        base = ["config"]
         if local:
-            base.append('--local')
+            base.append("--local")
 
         ret = main(base + [section_field, value])
         self.assertEqual(ret, 0)
@@ -56,7 +56,7 @@ class TestConfigCLI(TestDvc):
         self.assertTrue(self._contains(section, field, newvalue, local))
         self.assertFalse(self._contains(section, field, value, local))
 
-        ret = main(base + [section_field, '--unset'])
+        ret = main(base + [section_field, "--unset"])
         self.assertEqual(ret, 0)
         self.assertFalse(self._contains(section, field, value, local))
 
@@ -69,22 +69,22 @@ class TestConfigCLI(TestDvc):
     def test_non_existing(self):
         # FIXME check stdout/err
 
-        ret = main(['config', 'non_existing_section.field'])
+        ret = main(["config", "non_existing_section.field"])
         self.assertEqual(ret, 1)
 
-        ret = main(['config', 'global.non_existing_field'])
+        ret = main(["config", "global.non_existing_field"])
         self.assertEqual(ret, 1)
 
-        ret = main(['config', 'non_existing_section.field', '-u'])
+        ret = main(["config", "non_existing_section.field", "-u"])
         self.assertEqual(ret, 1)
 
-        ret = main(['config', 'global.non_existing_field', '-u'])
+        ret = main(["config", "global.non_existing_field", "-u"])
         self.assertEqual(ret, 1)
 
-        ret = main(['config', 'core.remote', 'myremote'])
+        ret = main(["config", "core.remote", "myremote"])
         self.assertEqual(ret, 0)
 
-        ret = main(['config', 'core.non_existing_field', '-u'])
+        ret = main(["config", "core.non_existing_field", "-u"])
         self.assertEqual(ret, 1)
 
     def test_failed_write(self):
@@ -92,8 +92,8 @@ class TestConfigCLI(TestDvc):
             system = False
             glob = False
             local = False
-            name = 'core.remote'
-            value = 'myremote'
+            name = "core.remote"
+            value = "myremote"
             unset = False
 
         args = A()

tests/test_data_cloud.py

@@ -14,8 +14,16 @@ import dvc.logger as logger
 from dvc.utils.compat import str
 from dvc.main import main
 from dvc.config import Config
-from dvc.data_cloud import (DataCloud, RemoteS3, RemoteGS, RemoteAzure,
-                            RemoteLOCAL, RemoteSSH, RemoteHDFS, RemoteHTTP)
+from dvc.data_cloud import (
+    DataCloud,
+    RemoteS3,
+    RemoteGS,
+    RemoteAzure,
+    RemoteLOCAL,
+    RemoteSSH,
+    RemoteHDFS,
+    RemoteHTTP,
+)
 from dvc.remote.base import STATUS_OK, STATUS_NEW, STATUS_DELETED
 from dvc.utils import file_md5
 
@@ -24,13 +32,15 @@ from tests.utils import spy
 
 from dvc.utils.compat import StringIO
 
-TEST_REMOTE = 'upstream'
+TEST_REMOTE = "upstream"
 TEST_SECTION = 'remote "{}"'.format(TEST_REMOTE)
-TEST_CONFIG = {Config.SECTION_CORE: {Config.SECTION_CORE_REMOTE: TEST_REMOTE},
-               TEST_SECTION: {Config.SECTION_REMOTE_URL: ''}}
+TEST_CONFIG = {
+    Config.SECTION_CORE: {Config.SECTION_CORE_REMOTE: TEST_REMOTE},
+    TEST_SECTION: {Config.SECTION_REMOTE_URL: ""},
+}
 
-TEST_AWS_REPO_BUCKET = 'dvc-test'
-TEST_GCP_REPO_BUCKET = 'dvc-test'
+TEST_AWS_REPO_BUCKET = "dvc-test"
+TEST_GCP_REPO_BUCKET = "dvc-test"
 
 
 def _should_test_aws():
@@ -59,8 +69,9 @@ def _should_test_gcp():
             os.unlink(creds)
         shutil.copyfile(TestDvc.GCP_CREDS_FILE, creds)
         try:
-            check_output(['gcloud', 'auth', 'activate-service-account',
-                          '--key-file', creds])
+            check_output(
+                ["gcloud", "auth", "activate-service-account", "--key-file", creds]
+            )
         except (CalledProcessError, OSError):
             return False
         return True
@@ -74,8 +85,9 @@ def _should_test_azure():
     elif os.getenv("DVC_TEST_AZURE") == "false":
         return False
 
-    return (os.getenv("AZURE_STORAGE_CONTAINER_NAME")
-            and os.getenv("AZURE_STORAGE_CONNECTION_STRING"))
+    return os.getenv("AZURE_STORAGE_CONTAINER_NAME") and os.getenv(
+        "AZURE_STORAGE_CONNECTION_STRING"
+    )
 
 
 def _should_test_ssh():
@@ -83,11 +95,11 @@ def _should_test_ssh():
         return True
 
     # FIXME: enable on windows
-    if os.name == 'nt':
+    if os.name == "nt":
         return False
 
     try:
-        check_output(['ssh', '-o', 'BatchMode=yes', '127.0.0.1', 'ls'])
+        check_output(["ssh", "-o", "BatchMode=yes", "127.0.0.1", "ls"])
     except (CalledProcessError, IOError):
         return False
 
@@ -95,19 +107,17 @@ def _should_test_ssh():
 
 
 def _should_test_hdfs():
-    if platform.system() != 'Linux':
+    if platform.system() != "Linux":
         return False
 
     try:
-        check_output(['hadoop', 'version'],
-                     shell=True,
-                     executable=os.getenv('SHELL'))
+        check_output(["hadoop", "version"], shell=True, executable=os.getenv("SHELL"))
     except (CalledProcessError, IOError):
         return False
 
-    p = Popen('hadoop fs -ls hdfs://127.0.0.1/',
-              shell=True,
-              executable=os.getenv('SHELL'))
+    p = Popen(
+        "hadoop fs -ls hdfs://127.0.0.1/", shell=True, executable=os.getenv("SHELL")
+    )
     p.communicate()
     if p.returncode != 0:
         return False
@@ -124,45 +134,40 @@ def get_local_url():
 
 
 def get_ssh_url():
-    return 'ssh://{}@127.0.0.1:{}'.format(
-        getpass.getuser(),
-        get_local_storagepath())
+    return "ssh://{}@127.0.0.1:{}".format(getpass.getuser(), get_local_storagepath())
 
 
 def get_hdfs_url():
-    return 'hdfs://{}@127.0.0.1{}'.format(
-        getpass.getuser(),
-        get_local_storagepath())
+    return "hdfs://{}@127.0.0.1{}".format(getpass.getuser(), get_local_storagepath())
 
 
 def get_aws_storagepath():
-    return TEST_AWS_REPO_BUCKET + '/' + str(uuid.uuid4())
+    return TEST_AWS_REPO_BUCKET + "/" + str(uuid.uuid4())
 
 
 def get_aws_url():
-    return 's3://' + get_aws_storagepath()
+    return "s3://" + get_aws_storagepath()
 
 
 def get_gcp_storagepath():
-    return TEST_GCP_REPO_BUCKET + '/' + str(uuid.uuid4())
+    return TEST_GCP_REPO_BUCKET + "/" + str(uuid.uuid4())
 
 
 def get_gcp_url():
-    return 'gs://' + get_gcp_storagepath()
+    return "gs://" + get_gcp_storagepath()
 
 
 def get_azure_url_compat():
     container_name = os.getenv("AZURE_STORAGE_CONTAINER_NAME")
     connection_string = os.getenv("AZURE_STORAGE_CONNECTION_STRING")
     assert container_name is not None
-    return 'azure://ContainerName={};{}'.format(container_name,
-                                                connection_string)
+    return "azure://ContainerName={};{}".format(container_name, connection_string)
 
 
 def get_azure_url():
     container_name = os.getenv("AZURE_STORAGE_CONTAINER_NAME")
     assert container_name is not None
-    return 'azure://{}/{}'.format(container_name, str(uuid.uuid4()))
+    return "azure://{}/{}".format(container_name, str(uuid.uuid4()))
 
 
 class TestDataCloud(TestDvc):
@@ -173,12 +178,14 @@ class TestDataCloud(TestDvc):
     def test(self):
         config = TEST_CONFIG
 
-        clist = [('s3://mybucket/', RemoteS3),
-                 ('gs://mybucket/', RemoteGS),
-                 ('ssh://user@localhost:/', RemoteSSH),
-                 ('http://localhost:8000/', RemoteHTTP),
-                 ('azure://ContainerName=mybucket;conn_string;', RemoteAzure),
-                 (TestDvc.mkdtemp(), RemoteLOCAL)]
+        clist = [
+            ("s3://mybucket/", RemoteS3),
+            ("gs://mybucket/", RemoteGS),
+            ("ssh://user@localhost:/", RemoteSSH),
+            ("http://localhost:8000/", RemoteHTTP),
+            ("azure://ContainerName=mybucket;conn_string;", RemoteAzure),
+            (TestDvc.mkdtemp(), RemoteLOCAL),
+        ]
 
         for scheme, cl in clist:
             remote_url = scheme + str(uuid.uuid4())
@@ -194,12 +201,11 @@ class TestDataCloudBase(TestDvc):
         return False
 
     def _get_url(self):
-        return ''
+        return ""
 
     def _ensure_should_run(self):
         if not self._should_test():
-            raise SkipTest('Test {} is disabled'
-                           .format(self.__class__.__name__))
+            raise SkipTest("Test {} is disabled".format(self.__class__.__name__))
 
     def _setup_cloud(self):
         self._ensure_should_run()
@@ -221,7 +227,7 @@ class TestDataCloudBase(TestDvc):
         self.assertTrue(stage is not None)
         cache = stage.outs[0].cache
         info = stage.outs[0].dumpd()
-        md5 = info['md5']
+        md5 = info["md5"]
 
         stages = self.dvc.add(self.DATA_DIR)
         self.assertEqual(len(stages), 1)
@@ -230,16 +236,16 @@ class TestDataCloudBase(TestDvc):
 
         cache_dir = stage_dir.outs[0].cache
         info_dir = stage_dir.outs[0].dumpd()
-        md5_dir = info_dir['md5']
+        md5_dir = info_dir["md5"]
 
         with self.cloud.project.state:
             # Check status
             status = self.cloud.status([info], show_checksums=True)
-            expected = {md5: {'name': md5, 'status': STATUS_NEW}}
+            expected = {md5: {"name": md5, "status": STATUS_NEW}}
             self.assertEqual(status, expected)
 
             status_dir = self.cloud.status([info_dir], show_checksums=True)
-            expected = {md5_dir: {'name': md5_dir, 'status': STATUS_NEW}}
+            expected = {md5_dir: {"name": md5_dir, "status": STATUS_NEW}}
             self.assertEqual(status_dir, expected)
 
             # Push and check status
@@ -251,40 +257,40 @@ class TestDataCloudBase(TestDvc):
             self.assertTrue(os.path.isfile(cache_dir))
 
             status = self.cloud.status([info], show_checksums=True)
-            expected = {md5: {'name': md5, 'status': STATUS_OK}}
+            expected = {md5: {"name": md5, "status": STATUS_OK}}
             self.assertEqual(status, expected)
 
             status_dir = self.cloud.status([info_dir], show_checksums=True)
-            expected = {md5_dir: {'name': md5_dir, 'status': STATUS_OK}}
+            expected = {md5_dir: {"name": md5_dir, "status": STATUS_OK}}
             self.assertEqual(status_dir, expected)
 
             # Remove and check status
             shutil.rmtree(self.dvc.cache.local.cache_dir)
 
             status = self.cloud.status([info], show_checksums=True)
-            expected = {md5: {'name': md5, 'status': STATUS_DELETED}}
+            expected = {md5: {"name": md5, "status": STATUS_DELETED}}
             self.assertEqual(status, expected)
 
             status_dir = self.cloud.status([info_dir], show_checksums=True)
-            expected = {md5_dir: {'name': md5_dir, 'status': STATUS_DELETED}}
+            expected = {md5_dir: {"name": md5_dir, "status": STATUS_DELETED}}
             self.assertEqual(status_dir, expected)
 
             # Pull and check status
             self.cloud.pull([info])
             self.assertTrue(os.path.exists(cache))
             self.assertTrue(os.path.isfile(cache))
-            with open(cache, 'r') as fd:
+            with open(cache, "r") as fd:
                 self.assertEqual(fd.read(), self.FOO_CONTENTS)
 
             self.cloud.pull([info_dir])
             self.assertTrue(os.path.isfile(cache_dir))
 
             status = self.cloud.status([info], show_checksums=True)
-            expected = {md5: {'name': md5, 'status': STATUS_OK}}
+            expected = {md5: {"name": md5, "status": STATUS_OK}}
             self.assertEqual(status, expected)
 
             status_dir = self.cloud.status([info_dir], show_checksums=True)
-            expected = {md5_dir: {'name': md5_dir, 'status': STATUS_OK}}
+            expected = {md5_dir: {"name": md5_dir, "status": STATUS_OK}}
             self.assertTrue(status_dir, expected)
 
     def test(self):
@@ -374,9 +380,9 @@ class TestDataCloudCLIBase(TestDvc):
         self.assertEqual(ret, 0)
 
     def _test_cloud(self, remote=None):
-        args = ['-v', '-j', '2']
+        args = ["-v", "-j", "2"]
         if remote:
-            args += ['-r', remote]
+            args += ["-r", remote]
         else:
             args += []
 
@@ -393,45 +399,46 @@ class TestDataCloudCLIBase(TestDvc):
         cache_dir = stage_dir.outs[0].cache
 
         # FIXME check status output
-        self.main(['status', '-c', '--show-checksums'] + args)
+        self.main(["status", "-c", "--show-checksums"] + args)
 
-        self.main(['push'] + args)
+        self.main(["push"] + args)
         self.assertTrue(os.path.exists(cache))
         self.assertTrue(os.path.isfile(cache))
         self.assertTrue(os.path.isfile(cache_dir))
 
-        self.main(['status', '-c', '--show-checksums'] + args)
+        self.main(["status", "-c", "--show-checksums"] + args)
 
         shutil.rmtree(self.dvc.cache.local.cache_dir)
 
-        self.main(['status', '-c', '--show-checksums'] + args)
+        self.main(["status", "-c", "--show-checksums"] + args)
 
-        self.main(['fetch'] + args)
+        self.main(["fetch"] + args)
         self.assertTrue(os.path.exists(cache))
         self.assertTrue(os.path.isfile(cache))
         self.assertTrue(os.path.isfile(cache_dir))
 
-        self.main(['pull'] + args)
+        self.main(["pull"] + args)
         self.assertTrue(os.path.exists(cache))
         self.assertTrue(os.path.isfile(cache))
         self.assertTrue(os.path.isfile(cache_dir))
         self.assertTrue(os.path.isfile(self.FOO))
         self.assertTrue(os.path.isdir(self.DATA_DIR))
 
-        with open(cache, 'r') as fd:
+        with open(cache, "r") as fd:
             self.assertEqual(fd.read(), self.FOO_CONTENTS)
         self.assertTrue(os.path.isfile(cache_dir))
 
-        self.main(['status', '-c', '--show-checksums'] + args)
+        self.main(["status", "-c", "--show-checksums"] + args)
 
         # NOTE: check if remote gc works correctly on directories
-        self.main(['gc', '-c', '-f'] + args)
-        shutil.move(self.dvc.cache.local.cache_dir,
-                    self.dvc.cache.local.cache_dir + '.back')
+        self.main(["gc", "-c", "-f"] + args)
+        shutil.move(
+            self.dvc.cache.local.cache_dir, self.dvc.cache.local.cache_dir + ".back"
+        )
 
-        self.main(['fetch'] + args)
+        self.main(["fetch"] + args)
 
-        self.main(['pull', '-f'] + args)
+        self.main(["pull", "-f"] + args)
         self.assertTrue(os.path.exists(cache))
         self.assertTrue(os.path.isfile(cache))
         self.assertTrue(os.path.isfile(cache_dir))
@@ -452,8 +459,8 @@ class TestDataCloudCLIBase(TestDvc):
 class TestCompatRemoteLOCALCLI(TestDataCloudCLIBase):
     def _test(self):
         storagepath = get_local_storagepath()
-        self.main(['config', 'core.cloud', 'local'])
-        self.main(['config', 'local.storagepath', storagepath])
+        self.main(["config", "core.cloud", "local"])
+        self.main(["config", "local.storagepath", storagepath])
 
         self._test_cloud()
 
@@ -462,7 +469,7 @@ class TestRemoteLOCALCLI(TestDataCloudCLIBase):
     def _test(self):
         url = get_local_url()
 
-        self.main(['remote', 'add', TEST_REMOTE, url])
+        self.main(["remote", "add", TEST_REMOTE, url])
 
         self._test_cloud(TEST_REMOTE)
 
@@ -474,7 +481,7 @@ class TestRemoteSSHCLI(TestDataCloudCLIBase):
     def _test(self):
         url = get_ssh_url()
 
-        self.main(['remote', 'add', TEST_REMOTE, url])
+        self.main(["remote", "add", TEST_REMOTE, url])
 
         self._test_cloud(TEST_REMOTE)
 
@@ -486,7 +493,7 @@ class TestRemoteHDFSCLI(TestDataCloudCLIBase):
     def _test(self):
         url = get_hdfs_url()
 
-        self.main(['remote', 'add', TEST_REMOTE, url])
+        self.main(["remote", "add", TEST_REMOTE, url])
 
         self._test_cloud(TEST_REMOTE)
 
@@ -497,8 +504,8 @@ class TestCompatRemoteS3CLI(TestDataCloudCLIBase):
 
     def _test(self):
         storagepath = get_aws_storagepath()
-        self.main(['config', 'core.cloud', 'aws'])
-        self.main(['config', 'aws.storagepath', storagepath])
+        self.main(["config", "core.cloud", "aws"])
+        self.main(["config", "aws.storagepath", storagepath])
 
         self._test_cloud()
 
@@ -510,7 +517,7 @@ class TestRemoteS3CLI(TestDataCloudCLIBase):
     def _test(self):
         url = get_aws_url()
 
-        self.main(['remote', 'add', TEST_REMOTE, url])
+        self.main(["remote", "add", TEST_REMOTE, url])
 
         self._test_cloud(TEST_REMOTE)
 
@@ -521,8 +528,8 @@ class TestCompatRemoteGSCLI(TestDataCloudCLIBase):
 
     def _test(self):
         storagepath = get_gcp_storagepath()
-        self.main(['config', 'core.cloud', 'gcp'])
-        self.main(['config', 'gcp.storagepath', storagepath])
+        self.main(["config", "core.cloud", "gcp"])
+        self.main(["config", "gcp.storagepath", storagepath])
 
         self._test_cloud()
 
@@ -534,7 +541,7 @@ class TestRemoteGSCLI(TestDataCloudCLIBase):
     def _test(self):
         url = get_gcp_url()
 
-        self.main(['remote', 'add', TEST_REMOTE, url])
+        self.main(["remote", "add", TEST_REMOTE, url])
 
         self._test_cloud(TEST_REMOTE)
 
@@ -546,7 +553,7 @@ class TestRemoteAzureCLI(TestDataCloudCLIBase):
     def _test(self):
         url = get_azure_url()
 
-        self.main(['remote', 'add', TEST_REMOTE, url])
+        self.main(["remote", "add", TEST_REMOTE, url])
 
         self._test_cloud(TEST_REMOTE)
 
@@ -557,15 +564,15 @@ class TestDataCloudErrorCLI(TestDvc):
         self.assertNotEqual(ret, 0)
 
     def test_error(self):
-        f = 'non-existing-file'
-        self.main_fail(['status', '-c', f])
-        self.main_fail(['push', f])
-        self.main_fail(['pull', f])
-        self.main_fail(['fetch', f])
+        f = "non-existing-file"
+        self.main_fail(["status", "-c", f])
+        self.main_fail(["push", f])
+        self.main_fail(["pull", f])
+        self.main_fail(["fetch", f])
 
 
 class TestWarnOnOutdatedStage(TestDvc):
-    color_patch = patch.object(logger, 'colorize', new=lambda x, color='': x)
+    color_patch = patch.object(logger, "colorize", new=lambda x, color="": x)
 
     def main(self, args):
         ret = main(args)
@@ -573,26 +580,28 @@ class TestWarnOnOutdatedStage(TestDvc):
 
     def _test(self):
         url = get_local_url()
-        self.main(['remote', 'add', '-d', TEST_REMOTE, url])
+        self.main(["remote", "add", "-d", TEST_REMOTE, url])
 
         self.dvc.add(self.FOO)
-        stage = self.dvc.run(deps=['foo'], outs=['bar'], cmd='echo bar > bar')
-        self.main(['push'])
+        stage = self.dvc.run(deps=["foo"], outs=["bar"], cmd="echo bar > bar")
+        self.main(["push"])
 
         stage_file_path = stage.relpath
-        with open(stage_file_path, 'r') as stage_file:
+        with open(stage_file_path, "r") as stage_file:
             content = yaml.safe_load(stage_file)
-        del (content['outs'][0]['md5'])
-        with open(stage_file_path, 'w') as stage_file:
+        del (content["outs"][0]["md5"])
+        with open(stage_file_path, "w") as stage_file:
             yaml.dump(content, stage_file)
 
         logger.logger.handlers[0].stream = StringIO()
-        self.main(['status', '-c'])
-        self.assertIn('Warning: Output \'bar\'(Stage: \'bar.dvc\') is '
-                      'missing version info. Cache for it will not be '
-                      'collected. Use dvc repro to get your pipeline up to '
-                      'date.',
-                      logger.logger.handlers[0].stream.getvalue())
+        self.main(["status", "-c"])
+        self.assertIn(
+            "Warning: Output 'bar'(Stage: 'bar.dvc') is "
+            "missing version info. Cache for it will not be "
+            "collected. Use dvc repro to get your pipeline up to "
+            "date.",
+            logger.logger.handlers[0].stream.getvalue(),
+        )
 
     def test(self):
         self.color_patch.start()
@@ -616,11 +625,7 @@ class TestRecursiveSyncOperations(TestDataCloudBase):
         return RemoteLOCAL
 
     def _prepare_repo(self):
-        self.main(['remote',
-                   'add',
-                   '-d',
-                   TEST_REMOTE,
-                   self.cloud._cloud.cache_dir])
+        self.main(["remote", "add", "-d", TEST_REMOTE, self.cloud._cloud.cache_dir])
         self.create(self.DATA, self.DATA_CONTENTS)
         self.create(self.DATA_SUB, self.DATA_SUB_CONTENTS)
 
@@ -638,7 +643,7 @@ class TestRecursiveSyncOperations(TestDataCloudBase):
         self.assertFalse(os.path.exists(self.DATA))
         self.assertFalse(os.path.exists(self.DATA_SUB))
 
-        self.main(['pull', '-R', self.DATA_DIR])
+        self.main(["pull", "-R", self.DATA_DIR])
 
         self.assertTrue(os.path.exists(self.DATA))
         self.assertTrue(os.path.exists(self.DATA_SUB))
@@ -655,7 +660,7 @@ class TestRecursiveSyncOperations(TestDataCloudBase):
         self.assertFalse(os.path.exists(local_cache_data_path))
         self.assertFalse(os.path.exists(local_cache_data_sub_path))
 
-        self.main(['fetch', '-R', self.DATA_DIR])
+        self.main(["fetch", "-R", self.DATA_DIR])
 
         self.assertTrue(os.path.exists(local_cache_data_path))
         self.assertTrue(os.path.exists(local_cache_data_sub_path))
@@ -667,7 +672,7 @@ class TestRecursiveSyncOperations(TestDataCloudBase):
         self.assertFalse(os.path.exists(cloud_data_path))
         self.assertFalse(os.path.exists(cloud_data_sub_path))
 
-        self.main(['push', '-R', self.DATA_DIR])
+        self.main(["push", "-R", self.DATA_DIR])
 
         self.assertTrue(os.path.exists(cloud_data_path))
         self.assertTrue(os.path.exists(cloud_data_sub_path))
@@ -687,13 +692,12 @@ class TestRecursiveSyncOperations(TestDataCloudBase):
 
 
 class TestCheckSumRecalculation(TestDvc):
-
     def test(self):
         test_collect = spy(State._collect)
-        with patch.object(State, '_collect', test_collect):
+        with patch.object(State, "_collect", test_collect):
             url = get_local_url()
-            main(['remote', 'add', '-d', TEST_REMOTE, url])
-            main(['add', self.FOO])
-            main(['push'])
-            main(['run', '-d', self.FOO, 'echo foo'])
+            main(["remote", "add", "-d", TEST_REMOTE, url])
+            main(["add", self.FOO])
+            main(["push"])
+            main(["run", "-d", self.FOO, "echo foo"])
         self.assertEqual(test_collect.mock.call_count, 1)

tests/test_dependency.py

@@ -8,8 +8,8 @@ class TestDepScheme(TestOutScheme):
     MAP = DEP_MAP
 
     TESTS = TestOutScheme.TESTS.copy()
-    TESTS['http://example.com/path/to/file'] = 'http'
-    TESTS['https://example.com/path/to/file'] = 'https'
+    TESTS["http://example.com/path/to/file"] = "http"
+    TESTS["https://example.com/path/to/file"] = "https"
 
     def _get(self, path):
         return _get(Stage(self.dvc), path, None)

tests/test_destroy.py

@@ -7,13 +7,13 @@ from tests.test_repro import TestRepro
 
 class TestDestroyNoConfirmation(TestRepro):
     def test(self):
-        ret = main(['destroy'])
+        ret = main(["destroy"])
         self.assertNotEqual(ret, 0)
 
 
 class TestDestroyForce(TestRepro):
     def test(self):
-        ret = main(['destroy', '-f'])
+        ret = main(["destroy", "-f"])
         self.assertEqual(ret, 0)
 
         self.assertFalse(os.path.exists(self.dvc.dvc_dir))

tests/test_gc.py

@@ -16,11 +16,12 @@ class TestGC(TestDvc):
 
         self.dvc.add(self.FOO)
         self.dvc.add(self.DATA_DIR)
-        self.good_cache = [self.dvc.cache.local.get(
-            md5) for md5 in self.dvc.cache.local.all()]
+        self.good_cache = [
+            self.dvc.cache.local.get(md5) for md5 in self.dvc.cache.local.all()
+        ]
 
         self.bad_cache = []
-        for i in ['123', '234', '345']:
+        for i in ["123", "234", "345"]:
             path = os.path.join(self.dvc.cache.local.cache_dir, i[0:2], i[2:])
             self.create(path, i)
             self.bad_cache.append(path)
@@ -30,7 +31,7 @@ class TestGC(TestDvc):
         self._test_gc()
 
     def test_cli(self):
-        ret = main(['gc', '-f'])
+        ret = main(["gc", "-f"])
         self.assertEqual(ret, 0)
         self._test_gc()
 
@@ -46,47 +47,47 @@ class TestGC(TestDvc):
 class TestGCBranchesTags(TestDvc):
     def _check_cache(self, num):
         total = 0
-        for root, dirs, files in os.walk(os.path.join('.dvc', 'cache')):
+        for root, dirs, files in os.walk(os.path.join(".dvc", "cache")):
             total += len(files)
         self.assertEqual(total, num)
 
     def test(self):
-        fname = 'file'
+        fname = "file"
 
-        with open(fname, 'w+') as fobj:
-            fobj.write('v1.0')
+        with open(fname, "w+") as fobj:
+            fobj.write("v1.0")
 
         stages = self.dvc.add(fname)
         self.assertEqual(len(stages), 1)
-        self.dvc.scm.add(['.gitignore', stages[0].relpath])
-        self.dvc.scm.commit('v1.0')
-        self.dvc.scm.tag('v1.0')
+        self.dvc.scm.add([".gitignore", stages[0].relpath])
+        self.dvc.scm.commit("v1.0")
+        self.dvc.scm.tag("v1.0")
 
-        self.dvc.scm.checkout('test', create_new=True)
+        self.dvc.scm.checkout("test", create_new=True)
         self.dvc.remove(stages[0].relpath, outs_only=True)
-        with open(fname, 'w+') as fobj:
-            fobj.write('test')
+        with open(fname, "w+") as fobj:
+            fobj.write("test")
         stages = self.dvc.add(fname)
         self.assertEqual(len(stages), 1)
-        self.dvc.scm.add(['.gitignore', stages[0].relpath])
-        self.dvc.scm.commit('test')
+        self.dvc.scm.add([".gitignore", stages[0].relpath])
+        self.dvc.scm.commit("test")
 
-        self.dvc.scm.checkout('master')
+        self.dvc.scm.checkout("master")
         self.dvc.remove(stages[0].relpath, outs_only=True)
-        with open(fname, 'w+') as fobj:
-            fobj.write('trash')
+        with open(fname, "w+") as fobj:
+            fobj.write("trash")
         stages = self.dvc.add(fname)
         self.assertEqual(len(stages), 1)
-        self.dvc.scm.add(['.gitignore', stages[0].relpath])
-        self.dvc.scm.commit('trash')
+        self.dvc.scm.add([".gitignore", stages[0].relpath])
+        self.dvc.scm.commit("trash")
 
         self.dvc.remove(stages[0].relpath, outs_only=True)
-        with open(fname, 'w+') as fobj:
-            fobj.write('master')
+        with open(fname, "w+") as fobj:
+            fobj.write("master")
         stages = self.dvc.add(fname)
         self.assertEqual(len(stages), 1)
-        self.dvc.scm.add(['.gitignore', stages[0].relpath])
-        self.dvc.scm.commit('master')
+        self.dvc.scm.add([".gitignore", stages[0].relpath])
+        self.dvc.scm.commit("master")
 
         self._check_cache(4)
 
@@ -106,7 +107,7 @@ class TestGCBranchesTags(TestDvc):
 class TestGCMultipleProjects(TestDvc):
     def _check_cache(self, num):
         total = 0
-        for root, dirs, files in os.walk(os.path.join('.dvc', 'cache')):
+        for root, dirs, files in os.walk(os.path.join(".dvc", "cache")):
             total += len(files)
         self.assertEqual(total, num)
 
@@ -116,14 +117,11 @@ class TestGCMultipleProjects(TestDvc):
         self.additional_git = Repo.init(self.additional_path)
         self.additional_dvc = Project.init(self.additional_path)
 
-        cache_path = os.path.join(self._root_dir, '.dvc', 'cache')
-        config_path = os.path.join(self.additional_path, '.dvc',
-                                   'config.local')
+        cache_path = os.path.join(self._root_dir, ".dvc", "cache")
+        config_path = os.path.join(self.additional_path, ".dvc", "config.local")
         cfg = configobj.ConfigObj()
         cfg.filename = config_path
-        cfg['cache'] = {
-            'dir': cache_path
-        }
+        cfg["cache"] = {"dir": cache_path}
         cfg.write()
 
         self.additional_dvc = Project(self.additional_path)
@@ -131,17 +129,17 @@ class TestGCMultipleProjects(TestDvc):
     def test(self):
 
         # ADD FILE ONLY IN MAIN PROJECT
-        fname = 'only_in_first'
-        with open(fname, 'w+') as fobj:
-            fobj.write('only in main project')
+        fname = "only_in_first"
+        with open(fname, "w+") as fobj:
+            fobj.write("only in main project")
 
         stages = self.dvc.add(fname)
         self.assertEqual(len(stages), 1)
 
         # ADD FILE IN MAIN PROJECT THAT IS ALSO IN SECOND PROJECT
-        fname = 'in_both'
-        with open(fname, 'w+') as fobj:
-            fobj.write('in both projects')
+        fname = "in_both"
+        with open(fname, "w+") as fobj:
+            fobj.write("in both projects")
 
         stages = self.dvc.add(fname)
         self.assertEqual(len(stages), 1)
@@ -149,17 +147,17 @@ class TestGCMultipleProjects(TestDvc):
         cwd = os.getcwd()
         os.chdir(self.additional_path)
         # ADD FILE ONLY IN SECOND PROJECT
-        fname = os.path.join(self.additional_path, 'only_in_second')
-        with open(fname, 'w+') as fobj:
-            fobj.write('only in additional project')
+        fname = os.path.join(self.additional_path, "only_in_second")
+        with open(fname, "w+") as fobj:
+            fobj.write("only in additional project")
 
         stages = self.additional_dvc.add(fname)
         self.assertEqual(len(stages), 1)
 
         # ADD FILE IN SECOND PROJECT THAT IS ALSO IN MAIN PROJECT
-        fname = os.path.join(self.additional_path, 'in_both')
-        with open(fname, 'w+') as fobj:
-            fobj.write('in both projects')
+        fname = os.path.join(self.additional_path, "in_both")
+        with open(fname, "w+") as fobj:
+            fobj.write("in both projects")
 
         stages = self.additional_dvc.add(fname)
         self.assertEqual(len(stages), 1)

tests/test_import.py

@@ -14,15 +14,15 @@ from dvc.utils.compat import StringIO
 
 class TestCmdImport(TestDvc):
     def test(self):
-        ret = main(['import', self.FOO, 'import'])
+        ret = main(["import", self.FOO, "import"])
         self.assertEqual(ret, 0)
-        self.assertTrue(os.path.exists('import.dvc'))
+        self.assertTrue(os.path.exists("import.dvc"))
 
-        ret = main(['import', 'non-existing-file', 'import'])
+        ret = main(["import", "non-existing-file", "import"])
         self.assertNotEqual(ret, 0)
 
     def test_unsupported(self):
-        ret = main(['import', 'unsupported://path', 'import_unsupported'])
+        ret = main(["import", "unsupported://path", "import_unsupported"])
         self.assertNotEqual(ret, 0)
 
 
@@ -32,35 +32,37 @@ class TestDefaultOutput(TestDvc):
         filename = str(uuid4())
         tmpfile = os.path.join(tmpdir, filename)
 
-        with open(tmpfile, 'w') as fd:
-            fd.write('content')
+        with open(tmpfile, "w") as fd:
+            fd.write("content")
 
-        ret = main(['import', tmpfile])
+        ret = main(["import", tmpfile])
         self.assertEqual(ret, 0)
         self.assertTrue(os.path.exists(filename))
-        self.assertEqual(open(filename).read(), 'content')
+        self.assertEqual(open(filename).read(), "content")
 
 
 class TestFailedImportMessage(TestDvc):
-    color_patch = patch.object(logger, 'colorize', new=lambda x, color='': x)
+    color_patch = patch.object(logger, "colorize", new=lambda x, color="": x)
 
     def test(self):
         self.color_patch.start()
         self._test()
         self.color_patch.stop()
 
-    @patch('dvc.command.imp.urlparse')
+    @patch("dvc.command.imp.urlparse")
     def _test(self, imp_urlparse_patch):
         logger.logger.handlers[1].stream = StringIO()
-        page_address = 'http://somesite.com/file_name'
+        page_address = "http://somesite.com/file_name"
 
         def dvc_exception(*args, **kwargs):
-            raise DvcException('message')
+            raise DvcException("message")
 
         imp_urlparse_patch.side_effect = dvc_exception
-        main(['import', page_address])
-        self.assertIn('Error: failed to import '
-                      'http://somesite.com/file_name. You could also try '
-                      'downloading it manually and adding it with `dvc add` '
-                      'command.',
-                      logger.logger.handlers[1].stream.getvalue())
+        main(["import", page_address])
+        self.assertIn(
+            "Error: failed to import "
+            "http://somesite.com/file_name. You could also try "
+            "downloading it manually and adding it with `dvc add` "
+            "command.",
+            logger.logger.handlers[1].stream.getvalue(),
+        )

tests/test_init.py

@@ -17,7 +17,7 @@ class TestInit(TestGit):
         self._test_init()
 
     def test_cli(self):
-        ret = main(['init'])
+        ret = main(["init"])
         self.assertEqual(ret, 0)
 
         self._test_init()
@@ -25,15 +25,15 @@ class TestInit(TestGit):
 
 class TestDoubleInit(TestInit):
     def test(self):
-        ret = main(['init'])
+        ret = main(["init"])
         self.assertEqual(ret, 0)
         self._test_init()
 
-        ret = main(['init'])
+        ret = main(["init"])
         self.assertNotEqual(ret, 0)
         self._test_init()
 
-        ret = main(['init', '--force'])
+        ret = main(["init", "--force"])
         self.assertEqual(ret, 0)
         self._test_init()
 
@@ -44,7 +44,7 @@ class TestInitNoSCMFail(TestDir):
             Project.init()
 
     def test_cli(self):
-        ret = main(['init'])
+        ret = main(["init"])
         self.assertNotEqual(ret, 0)
 
 
@@ -59,7 +59,7 @@ class TestInitNoSCM(TestDir):
         self._test_init()
 
     def test_cli(self):
-        ret = main(['init', '--no-scm'])
+        ret = main(["init", "--no-scm"])
         self.assertEqual(ret, 0)
 
         self._test_init()

tests/test_install.py

@@ -7,20 +7,20 @@ from tests.basic_env import TestDvc
 
 class TestInstall(TestDvc):
     def test(self):
-        ret = main(['install'])
+        ret = main(["install"])
         self.assertEqual(ret, 0)
 
-        ret = main(['install'])
+        ret = main(["install"])
         self.assertNotEqual(ret, 0)
 
         def hook(name):
-            return os.path.join('.git', 'hooks', name)
+            return os.path.join(".git", "hooks", name)
 
-        self.assertTrue(os.path.isfile(hook('post-checkout')))
-        self.assertTrue(os.path.isfile(hook('pre-commit')))
+        self.assertTrue(os.path.isfile(hook("post-checkout")))
+        self.assertTrue(os.path.isfile(hook("pre-commit")))
 
         self.dvc.add(self.FOO)
         os.unlink(self.FOO)
 
-        self.dvc.scm.checkout('branch', create_new=True)
+        self.dvc.scm.checkout("branch", create_new=True)
         self.assertTrue(os.path.isfile(self.FOO))

tests/test_lock.py

@@ -17,5 +17,5 @@ class TestLock(TestDvc):
     def test_cli(self):
         lock = Lock(self.dvc.dvc_dir)
         with lock:
-            ret = main(['add', self.FOO])
+            ret = main(["add", self.FOO])
             self.assertEqual(ret, 1)

tests/test_logger.py

@@ -11,7 +11,7 @@ from dvc.utils.compat import StringIO
 
 class TestLogger(TestCase):
     handlers = logger.logger.handlers
-    color_patch = patch.object(logger, 'colorize', new=lambda x, color='': x)
+    color_patch = patch.object(logger, "colorize", new=lambda x, color="": x)
 
     def setUp(self):
         logger.logger.handlers[0].stream = StringIO()
@@ -33,122 +33,124 @@ class TestLogger(TestCase):
         return logger.logger.handlers[1].stream.getvalue()
 
     def test_info(self):
-        logger.info('message')
+        logger.info("message")
 
-        self.assertEqual(self.stdout, 'message\n')
+        self.assertEqual(self.stdout, "message\n")
 
     def test_debug(self):
         with logger.verbose():
-            logger.debug('message')
+            logger.debug("message")
 
-        self.assertEqual(self.stdout, 'Debug: message\n')
+        self.assertEqual(self.stdout, "Debug: message\n")
 
     def test_warning(self):
-        logger.warning('message')
+        logger.warning("message")
 
-        self.assertEqual(self.stdout, 'Warning: message\n')
+        self.assertEqual(self.stdout, "Warning: message\n")
 
     def test_error(self):
-        logger.error('message')
+        logger.error("message")
 
         output = (
-            'Error: message\n'
-            '\n'
-            'Having any troubles? Hit us up at https://dvc.org/support,'
-            ' we are always happy to help!\n'
+            "Error: message\n"
+            "\n"
+            "Having any troubles? Hit us up at https://dvc.org/support,"
+            " we are always happy to help!\n"
         )
 
         self.assertEqual(self.stderr, output)
 
     def test_error_with_exception(self):
         try:
-            raise Exception('exception description')
+            raise Exception("exception description")
         except Exception:
-            logger.error('')
+            logger.error("")
 
         output = (
-            'Error: exception description\n'
-            '\n'
-            'Having any troubles? Hit us up at https://dvc.org/support,'
-            ' we are always happy to help!\n'
+            "Error: exception description\n"
+            "\n"
+            "Having any troubles? Hit us up at https://dvc.org/support,"
+            " we are always happy to help!\n"
         )
 
         self.assertEqual(self.stderr, output)
 
     def test_error_with_exception_and_message(self):
         try:
-            raise Exception('exception description')
+            raise Exception("exception description")
         except Exception:
-            logger.error('message')
+            logger.error("message")
 
         output = (
-            'Error: message - exception description\n'
-            '\n'
-            'Having any troubles? Hit us up at https://dvc.org/support,'
-            ' we are always happy to help!\n'
+            "Error: message - exception description\n"
+            "\n"
+            "Having any troubles? Hit us up at https://dvc.org/support,"
+            " we are always happy to help!\n"
         )
 
         self.assertEqual(self.stderr, output)
 
     def test_error_with_chained_exception_and_message(self):
         try:
-            raise DvcException('exception', cause=Exception('cause'))
+            raise DvcException("exception", cause=Exception("cause"))
         except Exception:
-            logger.error('message')
+            logger.error("message")
 
         output = (
-            'Error: message - exception: cause\n'
-            '\n'
-            'Having any troubles? Hit us up at https://dvc.org/support,'
-            ' we are always happy to help!\n'
+            "Error: message - exception: cause\n"
+            "\n"
+            "Having any troubles? Hit us up at https://dvc.org/support,"
+            " we are always happy to help!\n"
         )
 
         self.assertEqual(self.stderr, output)
 
     def test_traceback(self):
-        stack_trace1 = 'stack_trace1\n'
-        stack_trace2 = 'stack_trace2\n'
+        stack_trace1 = "stack_trace1\n"
+        stack_trace2 = "stack_trace2\n"
         try:
-            exc1 = Exception('exception1')
-            exc2 = DvcException('exception2', cause=exc1)
+            exc1 = Exception("exception1")
+            exc2 = DvcException("exception2", cause=exc1)
             exc2.cause_tb = stack_trace1
-            exc3 = DvcException('exception3', cause=exc2)
+            exc3 = DvcException("exception3", cause=exc2)
             exc3.cause_tb = stack_trace2
             raise exc3
         except Exception:
             stack_trace3 = traceback.format_exc()
             with logger.verbose():
-                logger.error('message')
+                logger.error("message")
 
         output = (
-            'Error: message - exception3: exception2: exception1\n'
-            '{line}\n'
-            '{stack_trace1}'
-            '\n'
-            '{stack_trace2}'
-            '\n'
-            '{stack_trace3}'
-            '{line}\n'
-            '\n'
-            'Having any troubles? Hit us up at https://dvc.org/support,'
-            ' we are always happy to help!\n'
-        ).format(line='-' * 60,
-                 stack_trace1=stack_trace1,
-                 stack_trace2=stack_trace2,
-                 stack_trace3=stack_trace3)
+            "Error: message - exception3: exception2: exception1\n"
+            "{line}\n"
+            "{stack_trace1}"
+            "\n"
+            "{stack_trace2}"
+            "\n"
+            "{stack_trace3}"
+            "{line}\n"
+            "\n"
+            "Having any troubles? Hit us up at https://dvc.org/support,"
+            " we are always happy to help!\n"
+        ).format(
+            line="-" * 60,
+            stack_trace1=stack_trace1,
+            stack_trace2=stack_trace2,
+            stack_trace3=stack_trace3,
+        )
 
         self.assertEqual(self.stderr, output)
 
     def test_box(self):
-        logger.box('message')
+        logger.box("message")
 
         output = (
-            '+-----------------+\n'
-            '|                 |\n'
-            '|     message     |\n'
-            '|                 |\n'
-            '+-----------------+\n'
-            '\n'
+            "+-----------------+\n"
+            "|                 |\n"
+            "|     message     |\n"
+            "|                 |\n"
+            "+-----------------+\n"
+            "\n"
         )
 
         self.assertEqual(self.stdout, output)

tests/test_metrics.py

@@ -10,215 +10,241 @@ from tests.basic_env import TestDvc
 class TestMetrics(TestDvc):
     def setUp(self):
         super(TestMetrics, self).setUp()
-        self.dvc.scm.commit('init')
+        self.dvc.scm.commit("init")
 
-        for branch in ['foo', 'bar', 'baz']:
+        for branch in ["foo", "bar", "baz"]:
             self.dvc.scm.checkout(branch, create_new=True)
 
-            with open('metric', 'w+') as fd:
+            with open("metric", "w+") as fd:
                 fd.write(branch)
 
-            with open('metric_json', 'w+') as fd:
-                json.dump({'branch': branch}, fd)
+            with open("metric_json", "w+") as fd:
+                json.dump({"branch": branch}, fd)
 
-            with open('metric_csv', 'w+') as fd:
+            with open("metric_csv", "w+") as fd:
                 fd.write(branch)
 
-            with open('metric_hcsv', 'w+') as fd:
-                fd.write('branch\n')
+            with open("metric_hcsv", "w+") as fd:
+                fd.write("branch\n")
                 fd.write(branch)
 
-            with open('metric_tsv', 'w+') as fd:
+            with open("metric_tsv", "w+") as fd:
                 fd.write(branch)
 
-            with open('metric_htsv', 'w+') as fd:
-                fd.write('branch\n')
+            with open("metric_htsv", "w+") as fd:
+                fd.write("branch\n")
                 fd.write(branch)
 
-            self.dvc.scm.add(['metric',
-                              'metric_json',
-                              'metric_tsv',
-                              'metric_htsv',
-                              'metric_csv',
-                              'metric_hcsv'])
-            self.dvc.scm.commit('metric')
+            self.dvc.scm.add(
+                [
+                    "metric",
+                    "metric_json",
+                    "metric_tsv",
+                    "metric_htsv",
+                    "metric_csv",
+                    "metric_hcsv",
+                ]
+            )
+            self.dvc.scm.commit("metric")
 
-        self.dvc.scm.checkout('master')
+        self.dvc.scm.checkout("master")
 
     def test(self):
-        ret = self.dvc.metrics_show('metric', all_branches=True)
+        ret = self.dvc.metrics_show("metric", all_branches=True)
         self.assertEqual(len(ret), 3)
-        self.assertTrue(ret['foo']['metric'] == 'foo')
-        self.assertTrue(ret['bar']['metric'] == 'bar')
-        self.assertTrue(ret['baz']['metric'] == 'baz')
+        self.assertTrue(ret["foo"]["metric"] == "foo")
+        self.assertTrue(ret["bar"]["metric"] == "bar")
+        self.assertTrue(ret["baz"]["metric"] == "baz")
 
-        ret = self.dvc.metrics_show('metric_json',
-                                    typ='json',
-                                    xpath='branch',
-                                    all_branches=True)
+        ret = self.dvc.metrics_show(
+            "metric_json", typ="json", xpath="branch", all_branches=True
+        )
         self.assertEqual(len(ret), 3)
-        self.assertTrue(ret['foo']['metric_json'] == ['foo'])
-        self.assertTrue(ret['bar']['metric_json'] == ['bar'])
-        self.assertTrue(ret['baz']['metric_json'] == ['baz'])
+        self.assertTrue(ret["foo"]["metric_json"] == ["foo"])
+        self.assertTrue(ret["bar"]["metric_json"] == ["bar"])
+        self.assertTrue(ret["baz"]["metric_json"] == ["baz"])
 
-        ret = self.dvc.metrics_show('metric_tsv',
-                                    typ='tsv',
-                                    xpath='0,0',
-                                    all_branches=True)
+        ret = self.dvc.metrics_show(
+            "metric_tsv", typ="tsv", xpath="0,0", all_branches=True
+        )
         self.assertEqual(len(ret), 3)
-        self.assertTrue(ret['foo']['metric_tsv'] == ['foo'])
-        self.assertTrue(ret['bar']['metric_tsv'] == ['bar'])
-        self.assertTrue(ret['baz']['metric_tsv'] == ['baz'])
+        self.assertTrue(ret["foo"]["metric_tsv"] == ["foo"])
+        self.assertTrue(ret["bar"]["metric_tsv"] == ["bar"])
+        self.assertTrue(ret["baz"]["metric_tsv"] == ["baz"])
 
-        ret = self.dvc.metrics_show('metric_htsv',
-                                    typ='htsv',
-                                    xpath='branch,0',
-                                    all_branches=True)
+        ret = self.dvc.metrics_show(
+            "metric_htsv", typ="htsv", xpath="branch,0", all_branches=True
+        )
         self.assertEqual(len(ret), 3)
-        self.assertTrue(ret['foo']['metric_htsv'] == ['foo'])
-        self.assertTrue(ret['bar']['metric_htsv'] == ['bar'])
-        self.assertTrue(ret['baz']['metric_htsv'] == ['baz'])
+        self.assertTrue(ret["foo"]["metric_htsv"] == ["foo"])
+        self.assertTrue(ret["bar"]["metric_htsv"] == ["bar"])
+        self.assertTrue(ret["baz"]["metric_htsv"] == ["baz"])
 
-        ret = self.dvc.metrics_show('metric_csv',
-                                    typ='csv',
-                                    xpath='0,0',
-                                    all_branches=True)
+        ret = self.dvc.metrics_show(
+            "metric_csv", typ="csv", xpath="0,0", all_branches=True
+        )
         self.assertEqual(len(ret), 3)
-        self.assertTrue(ret['foo']['metric_csv'] == ['foo'])
-        self.assertTrue(ret['bar']['metric_csv'] == ['bar'])
-        self.assertTrue(ret['baz']['metric_csv'] == ['baz'])
+        self.assertTrue(ret["foo"]["metric_csv"] == ["foo"])
+        self.assertTrue(ret["bar"]["metric_csv"] == ["bar"])
+        self.assertTrue(ret["baz"]["metric_csv"] == ["baz"])
 
-        ret = self.dvc.metrics_show('metric_hcsv',
-                                    typ='hcsv',
-                                    xpath='branch,0',
-                                    all_branches=True)
+        ret = self.dvc.metrics_show(
+            "metric_hcsv", typ="hcsv", xpath="branch,0", all_branches=True
+        )
         self.assertEqual(len(ret), 3)
-        self.assertTrue(ret['foo']['metric_hcsv'] == ['foo'])
-        self.assertTrue(ret['bar']['metric_hcsv'] == ['bar'])
-        self.assertTrue(ret['baz']['metric_hcsv'] == ['baz'])
+        self.assertTrue(ret["foo"]["metric_hcsv"] == ["foo"])
+        self.assertTrue(ret["bar"]["metric_hcsv"] == ["bar"])
+        self.assertTrue(ret["baz"]["metric_hcsv"] == ["baz"])
 
 
 class TestMetricsReproCLI(TestDvc):
     def test(self):
-        stage = self.dvc.run(metrics_no_cache=['metrics'],
-                             cmd='python {} {} {}'.format(self.CODE,
-                                                          self.FOO,
-                                                          'metrics'))
+        stage = self.dvc.run(
+            metrics_no_cache=["metrics"],
+            cmd="python {} {} {}".format(self.CODE, self.FOO, "metrics"),
+        )
 
-        ret = main(['repro', '-m', stage.path])
+        ret = main(["repro", "-m", stage.path])
         self.assertEqual(ret, 0)
 
-        ret = main(['metrics', 'remove', 'metrics'])
+        ret = main(["metrics", "remove", "metrics"])
         self.assertEqual(ret, 0)
 
-        ret = main(['repro', '-f', '-m', stage.path])
+        ret = main(["repro", "-f", "-m", stage.path])
         self.assertNotEqual(ret, 0)
 
-        ret = main(['metrics', 'add', 'metrics'])
+        ret = main(["metrics", "add", "metrics"])
         self.assertEqual(ret, 0)
 
-        ret = main(['metrics', 'modify', '-t', 'csv', '-x', '0,0', 'metrics'])
+        ret = main(["metrics", "modify", "-t", "csv", "-x", "0,0", "metrics"])
         self.assertEqual(ret, 0)
 
-        ret = main(['repro', '-f', '-m', stage.path])
+        ret = main(["repro", "-f", "-m", stage.path])
         self.assertEqual(ret, 0)
 
     def test_dir(self):
-        os.mkdir('metrics_dir')
+        os.mkdir("metrics_dir")
 
         with self.assertRaises(DvcException):
-            self.dvc.run(metrics_no_cache=['metrics_dir'])
+            self.dvc.run(metrics_no_cache=["metrics_dir"])
 
     def test_binary(self):
-        with open('metrics_bin', 'w+') as fd:
-            fd.write('\0\0\0\0\0\0\0\0')
+        with open("metrics_bin", "w+") as fd:
+            fd.write("\0\0\0\0\0\0\0\0")
 
         with self.assertRaises(DvcException):
-            self.dvc.run(metrics_no_cache=['metrics_bin'])
+            self.dvc.run(metrics_no_cache=["metrics_bin"])
 
 
 class TestMetricsCLICompat(TestMetrics):
     def test(self):
         # FIXME check output
-        ret = main(['metrics', 'show', '--all-branches', 'metric', '-v'])
+        ret = main(["metrics", "show", "--all-branches", "metric", "-v"])
         self.assertEqual(ret, 0)
 
-        ret = main(['metrics', 'show', '--all-branches', 'metric_json',
-                    '--json-path', 'branch'])
+        ret = main(
+            [
+                "metrics",
+                "show",
+                "--all-branches",
+                "metric_json",
+                "--json-path",
+                "branch",
+            ]
+        )
         self.assertEqual(ret, 0)
 
-        ret = main(['metrics', 'show', '--all-branches', 'metric_tsv',
-                    '--tsv-path', '0,0'])
+        ret = main(
+            ["metrics", "show", "--all-branches", "metric_tsv", "--tsv-path", "0,0"]
+        )
         self.assertEqual(ret, 0)
 
-        ret = main(['metrics', 'show', '--all-branches', 'metric_htsv',
-                    '--htsv-path', 'branch,0'])
+        ret = main(
+            [
+                "metrics",
+                "show",
+                "--all-branches",
+                "metric_htsv",
+                "--htsv-path",
+                "branch,0",
+            ]
+        )
         self.assertEqual(ret, 0)
 
-        ret = main(['metrics', 'show', '--all-branches', 'metric_csv',
-                    '--csv-path', '0,0'])
+        ret = main(
+            ["metrics", "show", "--all-branches", "metric_csv", "--csv-path", "0,0"]
+        )
         self.assertEqual(ret, 0)
 
-        ret = main(['metrics', 'show', '--all-branches', 'metric_hcsv',
-                    '--hcsv-path', 'branch,0'])
+        ret = main(
+            [
+                "metrics",
+                "show",
+                "--all-branches",
+                "metric_hcsv",
+                "--hcsv-path",
+                "branch,0",
+            ]
+        )
         self.assertEqual(ret, 0)
 
 
 class TestMetricsCLI(TestMetrics):
     def test(self):
         # FIXME check output
-        ret = main(['metrics', 'show', '-a', 'metric', '-v'])
+        ret = main(["metrics", "show", "-a", "metric", "-v"])
         self.assertEqual(ret, 0)
 
-        ret = main(['metrics', 'show', '-a', 'metric_json',
-                    '-t', 'json', '-x', 'branch'])
+        ret = main(
+            ["metrics", "show", "-a", "metric_json", "-t", "json", "-x", "branch"]
+        )
         self.assertEqual(ret, 0)
 
-        ret = main(['metrics', 'show', '-a', 'metric_tsv',
-                    '-t', 'tsv', '-x', '0,0'])
+        ret = main(["metrics", "show", "-a", "metric_tsv", "-t", "tsv", "-x", "0,0"])
         self.assertEqual(ret, 0)
 
-        ret = main(['metrics', 'show', '-a', 'metric_htsv',
-                    '-t', 'htsv', '-x', 'branch,0'])
+        ret = main(
+            ["metrics", "show", "-a", "metric_htsv", "-t", "htsv", "-x", "branch,0"]
+        )
         self.assertEqual(ret, 0)
 
-        ret = main(['metrics', 'show', '-a', 'metric_csv',
-                    '-t', 'csv', '-x', '0,0'])
+        ret = main(["metrics", "show", "-a", "metric_csv", "-t", "csv", "-x", "0,0"])
         self.assertEqual(ret, 0)
 
-        ret = main(['metrics', 'show', '-a', 'metric_hcsv',
-                    '-t', 'hcsv', '-x', 'branch,0'])
+        ret = main(
+            ["metrics", "show", "-a", "metric_hcsv", "-t", "hcsv", "-x", "branch,0"]
+        )
         self.assertEqual(ret, 0)
 
     def test_dir(self):
-        os.mkdir('metrics_dir')
+        os.mkdir("metrics_dir")
 
         with self.assertRaises(DvcException):
-            self.dvc.run(outs_no_cache=['metrics_dir'])
-            self.dvc.metrics_add('metrics_dir')
+            self.dvc.run(outs_no_cache=["metrics_dir"])
+            self.dvc.metrics_add("metrics_dir")
 
     def test_binary(self):
-        with open('metrics_bin', 'w+') as fd:
-            fd.write('\0\0\0\0\0\0\0\0')
+        with open("metrics_bin", "w+") as fd:
+            fd.write("\0\0\0\0\0\0\0\0")
 
         with self.assertRaises(DvcException):
-            self.dvc.run(outs_no_cache=['metrics_bin'])
-            self.dvc.metrics_add('metrics_bin')
+            self.dvc.run(outs_no_cache=["metrics_bin"])
+            self.dvc.metrics_add("metrics_bin")
 
     def test_non_existing(self):
-        ret = main(['metrics', 'add', 'non-existing'])
+        ret = main(["metrics", "add", "non-existing"])
         self.assertNotEqual(ret, 0)
 
-        ret = main(['metrics', 'modify', 'non-existing'])
+        ret = main(["metrics", "modify", "non-existing"])
         self.assertNotEqual(ret, 0)
 
-        ret = main(['metrics', 'remove', 'non-existing'])
+        ret = main(["metrics", "remove", "non-existing"])
         self.assertNotEqual(ret, 0)
 
 
 class TestNoMetrics(TestDvc):
     def test(self):
-        ret = main(['metrics', 'show'])
+        ret = main(["metrics", "show"])
         self.assertNotEqual(ret, 0)
 
 
@@ -227,34 +253,38 @@ class TestCachedMetrics(TestDvc):
         self.dvc.scm.checkout(branch)
         self.dvc.checkout(force=True)
 
-        with open('metrics.json', 'w+') as fd:
-            json.dump({'metrics': branch}, fd)
+        with open("metrics.json", "w+") as fd:
+            json.dump({"metrics": branch}, fd)
 
-        stages = self.dvc.add('metrics.json')
+        stages = self.dvc.add("metrics.json")
         self.assertEqual(len(stages), 1)
         stage = stages[0]
         self.assertNotEqual(stage, None)
 
-        self.dvc.scm.add(['.gitignore', 'metrics.json.dvc'])
+        self.dvc.scm.add([".gitignore", "metrics.json.dvc"])
         self.dvc.scm.commit(branch)
 
     def test(self):
-        self.dvc.scm.commit('init')
+        self.dvc.scm.commit("init")
 
-        self.dvc.scm.branch('one')
-        self.dvc.scm.branch('two')
+        self.dvc.scm.branch("one")
+        self.dvc.scm.branch("two")
 
-        self._do_write('master')
-        self._do_write('one')
-        self._do_write('two')
+        self._do_write("master")
+        self._do_write("one")
+        self._do_write("two")
 
-        self.dvc = Project('.')
+        self.dvc = Project(".")
 
-        res = self.dvc.metrics_show('metrics.json',
-                                    all_branches=True,
-                                    typ='json',
-                                    xpath='metrics')
+        res = self.dvc.metrics_show(
+            "metrics.json", all_branches=True, typ="json", xpath="metrics"
+        )
 
-        self.assertEqual(res, {"master": {"metrics.json": ["master"]},
-                               "one": {"metrics.json": ["one"]},
-                               "two": {"metrics.json": ["two"]}})
+        self.assertEqual(
+            res,
+            {
+                "master": {"metrics.json": ["master"]},
+                "one": {"metrics.json": ["one"]},
+                "two": {"metrics.json": ["two"]},
+            },
+        )

tests/test_move.py

@@ -11,7 +11,7 @@ from tests.utils import load_stage_file
 
 class TestMove(TestDvc):
     def test(self):
-        dst = self.FOO + '1'
+        dst = self.FOO + "1"
         self.dvc.add(self.FOO)
         self.dvc.move(self.FOO, dst)
 
@@ -22,12 +22,12 @@ class TestMove(TestDvc):
 class TestMoveNonExistentFile(TestDvc):
     def test(self):
         with self.assertRaises(DvcException):
-            self.dvc.move('non_existent_file', 'dst')
+            self.dvc.move("non_existent_file", "dst")
 
 
 class TestMoveDirectory(TestDvc):
     def test(self):
-        dst = 'dst'
+        dst = "dst"
         stages = self.dvc.add(self.DATA_DIR)
         self.assertEqual(len(stages), 1)
         self.assertTrue(stages[0] is not None)
@@ -42,10 +42,10 @@ class TestCmdMove(TestDvc):
         self.assertEqual(len(stages), 1)
         self.assertTrue(stages[0] is not None)
 
-        ret = main(['move', self.FOO, self.FOO + '1'])
+        ret = main(["move", self.FOO, self.FOO + "1"])
         self.assertEqual(ret, 0)
 
-        ret = main(['move', 'non-existing-file', 'dst'])
+        ret = main(["move", "non-existing-file", "dst"])
         self.assertNotEqual(ret, 0)
 
 
@@ -55,29 +55,29 @@ class TestMoveNotDataSource(TestRepro):
 
         self.dvc = Project(self._root_dir)
         with self.assertRaises(MoveNotDataSourceError):
-            self.dvc.move(self.file1, 'dst')
+            self.dvc.move(self.file1, "dst")
 
-        ret = main(['move', self.file1, 'dst'])
+        ret = main(["move", self.file1, "dst"])
         self.assertNotEqual(ret, 0)
 
 
 class TestMoveFileWithExtension(TestDvc):
     def test(self):
-        with open(os.path.join(self.dvc.root_dir, 'file.csv'), 'w') as fd:
-            fd.write('1,2,3\n')
+        with open(os.path.join(self.dvc.root_dir, "file.csv"), "w") as fd:
+            fd.write("1,2,3\n")
 
-        self.dvc.add('file.csv')
+        self.dvc.add("file.csv")
 
-        self.assertTrue(os.path.exists('file.csv'))
-        self.assertTrue(os.path.exists('file.csv.dvc'))
+        self.assertTrue(os.path.exists("file.csv"))
+        self.assertTrue(os.path.exists("file.csv.dvc"))
 
-        ret = main(['move', 'file.csv', 'other_name.csv'])
+        ret = main(["move", "file.csv", "other_name.csv"])
         self.assertEqual(ret, 0)
 
-        self.assertFalse(os.path.exists('file.csv'))
-        self.assertFalse(os.path.exists('file.csv.dvc'))
-        self.assertTrue(os.path.exists('other_name.csv'))
-        self.assertTrue(os.path.exists('other_name.csv.dvc'))
+        self.assertFalse(os.path.exists("file.csv"))
+        self.assertFalse(os.path.exists("file.csv.dvc"))
+        self.assertTrue(os.path.exists("other_name.csv"))
+        self.assertTrue(os.path.exists("other_name.csv.dvc"))
 
 
 class TestMoveFileToDirectory(TestDvc):
@@ -101,14 +101,14 @@ class TestMoveFileToDirectory(TestDvc):
 class TestMoveFileToDirectoryWithoutSpecifiedTargetName(TestDvc):
     def test(self):
         foo_stage_file_path = self.FOO + Stage.STAGE_FILE_SUFFIX
-        ret = main(['add', self.FOO])
+        ret = main(["add", self.FOO])
         self.assertEqual(ret, 0)
         self.assertTrue(os.path.exists(foo_stage_file_path))
 
         target_foo_path = os.path.join(self.DATA_DIR, self.FOO)
         target_foo_stage_file_path = target_foo_path + Stage.STAGE_FILE_SUFFIX
 
-        ret = main(['move', self.FOO, self.DATA_DIR])
+        ret = main(["move", self.FOO, self.DATA_DIR])
         self.assertEqual(ret, 0)
 
         self.assertFalse(os.path.exists(self.FOO))
@@ -118,41 +118,39 @@ class TestMoveFileToDirectoryWithoutSpecifiedTargetName(TestDvc):
         self.assertTrue(os.path.exists(target_foo_stage_file_path))
 
         new_stage = load_stage_file(target_foo_stage_file_path)
-        self.assertEqual(self.FOO, new_stage['outs'][0]['path'])
+        self.assertEqual(self.FOO, new_stage["outs"][0]["path"])
 
 
 class TestMoveDirectoryShouldNotOverwriteExisting(TestDvc):
     def test(self):
-        new_dir_name = 'data_dir2'
+        new_dir_name = "data_dir2"
         os.mkdir(new_dir_name)
-        ret = main(['add', self.DATA_DIR])
+        ret = main(["add", self.DATA_DIR])
         self.assertEqual(ret, 0)
 
-        ret = main(['move', self.DATA_DIR, new_dir_name])
+        ret = main(["move", self.DATA_DIR, new_dir_name])
         self.assertEqual(ret, 0)
 
         self.assertFalse(os.path.exists(self.DATA_DIR))
-        self.assertFalse(
-            os.path.exists(self.DATA_DIR + Stage.STAGE_FILE_SUFFIX))
+        self.assertFalse(os.path.exists(self.DATA_DIR + Stage.STAGE_FILE_SUFFIX))
 
         self.assertTrue(os.path.exists(new_dir_name))
         self.assertTrue(os.path.isfile(new_dir_name + Stage.STAGE_FILE_SUFFIX))
 
-        self.assertEqual(os.listdir(new_dir_name),
-                         [self.DATA_DIR])
+        self.assertEqual(os.listdir(new_dir_name), [self.DATA_DIR])
 
 
 class TestMoveFileBetweenDirectories(TestDvc):
     def test(self):
         data_stage_file = self.DATA + Stage.STAGE_FILE_SUFFIX
-        ret = main(['add', self.DATA])
+        ret = main(["add", self.DATA])
         self.assertEqual(ret, 0)
         self.assertTrue(os.path.exists(data_stage_file))
 
-        new_data_dir = 'data_dir2'
+        new_data_dir = "data_dir2"
         os.makedirs(new_data_dir)
 
-        ret = main(['move', self.DATA, new_data_dir])
+        ret = main(["move", self.DATA, new_data_dir])
         self.assertEqual(ret, 0)
 
         new_data_path = os.path.join(new_data_dir, os.path.basename(self.DATA))
@@ -165,5 +163,4 @@ class TestMoveFileBetweenDirectories(TestDvc):
         self.assertTrue(os.path.exists(new_data_stage_file))
 
         new_stage_file = load_stage_file(new_data_stage_file)
-        self.assertEqual(os.path.basename(self.DATA),
-                         new_stage_file['outs'][0]['path'])
+        self.assertEqual(os.path.basename(self.DATA), new_stage_file["outs"][0]["path"])

tests/test_output.py

@@ -8,18 +8,18 @@ class TestOutScheme(TestDvc):
     MAP = OUTS_MAP
 
     TESTS = {
-        's3://bucket/path': 's3',
-        'gs://bucket/path': 'gs',
-        'ssh://example.com:/dir/path': 'ssh',
-        'hdfs://example.com/dir/path': 'hdfs',
-        'path/to/file': 'local',
-        'path\\to\\file': 'local',
-        'file': 'local',
-        './file': 'local',
-        '.\\file': 'local',
-        '../file': 'local',
-        '..\\file': 'local',
-        'unknown://path': 'local',
+        "s3://bucket/path": "s3",
+        "gs://bucket/path": "gs",
+        "ssh://example.com:/dir/path": "ssh",
+        "hdfs://example.com/dir/path": "hdfs",
+        "path/to/file": "local",
+        "path\\to\\file": "local",
+        "file": "local",
+        "./file": "local",
+        ".\\file": "local",
+        "../file": "local",
+        "..\\file": "local",
+        "unknown://path": "local",
     }
 
     def _get(self, path):

tests/test_pipeline.py

@@ -8,184 +8,184 @@ import os
 class TestPipelineShowSingle(TestDvc):
     def setUp(self):
         super(TestPipelineShowSingle, self).setUp()
-        self.stage = 'foo.dvc'
-        self.dotFile = 'graph.dot'
-        ret = main(['add', self.FOO])
+        self.stage = "foo.dvc"
+        self.dotFile = "graph.dot"
+        ret = main(["add", self.FOO])
         self.assertEqual(ret, 0)
 
     def test(self):
-        ret = main(['pipeline', 'show', self.stage])
+        ret = main(["pipeline", "show", self.stage])
         self.assertEqual(ret, 0)
 
     def test_commands(self):
-        ret = main(['pipeline', 'show', self.stage, '--commands'])
+        ret = main(["pipeline", "show", self.stage, "--commands"])
         self.assertEqual(ret, 0)
 
     def test_outs(self):
-        ret = main(['pipeline', 'show', self.stage, '--outs'])
+        ret = main(["pipeline", "show", self.stage, "--outs"])
         self.assertEqual(ret, 0)
 
     def test_ascii(self):
-        ret = main(['pipeline', 'show', '--ascii', self.stage])
+        ret = main(["pipeline", "show", "--ascii", self.stage])
         self.assertEqual(ret, 0)
 
     def test_dot(self):
-        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.stage])
+        ret = main(["pipeline", "show", "--dot", self.dotFile, self.stage])
         self.assertEqual(ret, 0)
         self.assertTrue(os.path.isfile(self.dotFile))
 
     def test_ascii_commands(self):
-        ret = main(['pipeline', 'show', '--ascii', self.stage, '--commands'])
+        ret = main(["pipeline", "show", "--ascii", self.stage, "--commands"])
         self.assertEqual(ret, 0)
 
     def test_ascii_outs(self):
-        ret = main(['pipeline', 'show', '--ascii', self.stage, '--outs'])
+        ret = main(["pipeline", "show", "--ascii", self.stage, "--outs"])
         self.assertEqual(ret, 0)
 
     def test_dot_commands(self):
-        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.stage,
-                    '--commands'])
+        ret = main(
+            ["pipeline", "show", "--dot", self.dotFile, self.stage, "--commands"]
+        )
         self.assertEqual(ret, 0)
         self.assertTrue(os.path.isfile(self.dotFile))
 
     def test_dot_outs(self):
-        ret = main(['pipeline', 'show', '--dot', self.dotFile, self.stage,
-                    '--outs'])
+        ret = main(["pipeline", "show", "--dot", self.dotFile, self.stage, "--outs"])
         self.assertEqual(ret, 0)
         self.assertTrue(os.path.isfile(self.dotFile))
 
     def test_not_dvc_file(self):
-        ret = main(['pipeline', 'show', self.FOO])
+        ret = main(["pipeline", "show", self.FOO])
         self.assertNotEqual(ret, 0)
 
     def test_non_existing(self):
-        ret = main(['pipeline', 'show', 'non-existing'])
+        ret = main(["pipeline", "show", "non-existing"])
         self.assertNotEqual(ret, 0)
 
 
 class TestPipelineShow(TestRepro):
     def setUp(self):
         super(TestPipelineShow, self).setUp()
-        self.dotFile = 'graph.dot'
+        self.dotFile = "graph.dot"
 
     def test(self):
-        ret = main(['pipeline', 'show', self.file1_stage])
+        ret = main(["pipeline", "show", self.file1_stage])
         self.assertEqual(ret, 0)
 
     def test_commands(self):
-        ret = main(['pipeline', 'show', self.file1_stage, '--commands'])
+        ret = main(["pipeline", "show", self.file1_stage, "--commands"])
         self.assertEqual(ret, 0)
 
     def test_outs(self):
-        ret = main(['pipeline', 'show', self.file1_stage, '--outs'])
+        ret = main(["pipeline", "show", self.file1_stage, "--outs"])
         self.assertEqual(ret, 0)
 
     def test_ascii(self):
-        ret = main(['pipeline', 'show', '--ascii', self.file1_stage])
+        ret = main(["pipeline", "show", "--ascii", self.file1_stage])
         self.assertEqual(ret, 0)
 
     def test_dot(self):
-        ret = main(['pipeline', 'show', '--dot', self.dotFile,
-                    self.file1_stage])
+        ret = main(["pipeline", "show", "--dot", self.dotFile, self.file1_stage])
         self.assertEqual(ret, 0)
         self.assertTrue(os.path.isfile(self.dotFile))
 
     def test_ascii_commands(self):
-        ret = main(['pipeline', 'show', '--ascii', self.file1_stage,
-                    '--commands'])
+        ret = main(["pipeline", "show", "--ascii", self.file1_stage, "--commands"])
         self.assertEqual(ret, 0)
 
     def test_ascii_outs(self):
-        ret = main(['pipeline', 'show', '--ascii', self.file1_stage, '--outs'])
+        ret = main(["pipeline", "show", "--ascii", self.file1_stage, "--outs"])
         self.assertEqual(ret, 0)
 
     def test_dot_commands(self):
-        ret = main(['pipeline', 'show', '--dot', self.dotFile,
-                    self.file1_stage, '--commands'])
+        ret = main(
+            ["pipeline", "show", "--dot", self.dotFile, self.file1_stage, "--commands"]
+        )
         self.assertEqual(ret, 0)
         self.assertTrue(os.path.isfile(self.dotFile))
 
     def test_dot_outs(self):
-        ret = main(['pipeline', 'show', '--dot', self.dotFile,
-                    self.file1_stage, '--outs'])
+        ret = main(
+            ["pipeline", "show", "--dot", self.dotFile, self.file1_stage, "--outs"]
+        )
         self.assertEqual(ret, 0)
         self.assertTrue(os.path.isfile(self.dotFile))
 
     def test_not_dvc_file(self):
-        ret = main(['pipeline', 'show', self.file1])
+        ret = main(["pipeline", "show", self.file1])
         self.assertNotEqual(ret, 0)
 
     def test_non_existing(self):
-        ret = main(['pipeline', 'show', 'non-existing'])
+        ret = main(["pipeline", "show", "non-existing"])
         self.assertNotEqual(ret, 0)
 
 
 class TestPipelineShowDeep(TestReproChangedDeepData):
     def setUp(self):
         super(TestPipelineShowDeep, self).setUp()
-        self.dotFile = 'graph.dot'
+        self.dotFile = "graph.dot"
 
     def test(self):
-        ret = main(['pipeline', 'show', self.file1_stage])
+        ret = main(["pipeline", "show", self.file1_stage])
         self.assertEqual(ret, 0)
 
     def test_commands(self):
-        ret = main(['pipeline', 'show', self.file1_stage, '--commands'])
+        ret = main(["pipeline", "show", self.file1_stage, "--commands"])
         self.assertEqual(ret, 0)
 
     def test_outs(self):
-        ret = main(['pipeline', 'show', self.file1_stage, '--outs'])
+        ret = main(["pipeline", "show", self.file1_stage, "--outs"])
         self.assertEqual(ret, 0)
 
     def test_ascii(self):
-        ret = main(['pipeline', 'show', '--ascii', self.file1_stage])
+        ret = main(["pipeline", "show", "--ascii", self.file1_stage])
         self.assertEqual(ret, 0)
 
     def test_dot(self):
-        ret = main(['pipeline', 'show', '--dot', self.dotFile,
-                    self.file1_stage])
+        ret = main(["pipeline", "show", "--dot", self.dotFile, self.file1_stage])
         self.assertEqual(ret, 0)
         self.assertTrue(os.path.isfile(self.dotFile))
 
     def test_ascii_commands(self):
-        ret = main(['pipeline', 'show', '--ascii', self.file1_stage,
-                    '--commands'])
+        ret = main(["pipeline", "show", "--ascii", self.file1_stage, "--commands"])
         self.assertEqual(ret, 0)
 
     def test_ascii_outs(self):
-        ret = main(['pipeline', 'show', '--ascii', self.file1_stage, '--outs'])
+        ret = main(["pipeline", "show", "--ascii", self.file1_stage, "--outs"])
         self.assertEqual(ret, 0)
 
     def test_dot_commands(self):
-        ret = main(['pipeline', 'show', '--dot', self.dotFile,
-                    self.file1_stage, '--commands'])
+        ret = main(
+            ["pipeline", "show", "--dot", self.dotFile, self.file1_stage, "--commands"]
+        )
         self.assertEqual(ret, 0)
         self.assertTrue(os.path.isfile(self.dotFile))
 
     def test_dot_outs(self):
-        ret = main(['pipeline', 'show', '--dot', self.dotFile,
-                    self.file1_stage, '--outs'])
+        ret = main(
+            ["pipeline", "show", "--dot", self.dotFile, self.file1_stage, "--outs"]
+        )
         self.assertEqual(ret, 0)
         self.assertTrue(os.path.isfile(self.dotFile))
 
     def test_not_dvc_file(self):
-        ret = main(['pipeline', 'show', self.file1])
+        ret = main(["pipeline", "show", self.file1])
         self.assertNotEqual(ret, 0)
 
     def test_non_existing(self):
-        ret = main(['pipeline', 'show', 'non-existing'])
+        ret = main(["pipeline", "show", "non-existing"])
         self.assertNotEqual(ret, 0)
 
 
 class TestPipelineListEmpty(TestDvc):
     def test(self):
-        ret = main(['pipeline', 'list'])
+        ret = main(["pipeline", "list"])
         self.assertEqual(ret, 0)
 
 
 class TestPipelineListSingle(TestPipelineShowDeep):
     def test(self):
-        ret = main(['pipeline', 'list'])
+        ret = main(["pipeline", "list"])
         self.assertEqual(ret, 0)
 
 
@@ -195,9 +195,9 @@ class TestProjectPipeline(TestDvc):
         self.assertEqual(len(pipelines), 0)
 
     def one_pipeline(self):
-        self.dvc.add('foo')
-        self.dvc.run(deps=['foo'], outs=['bar'], cmd='')
-        self.dvc.run(deps=['bar'], outs=['baz'], cmd='echo baz > baz')
+        self.dvc.add("foo")
+        self.dvc.run(deps=["foo"], outs=["bar"], cmd="")
+        self.dvc.run(deps=["bar"], outs=["baz"], cmd="echo baz > baz")
         pipelines = self.dvc.pipelines()
 
         self.assertEqual(len(pipelines), 1)
@@ -205,11 +205,11 @@ class TestProjectPipeline(TestDvc):
         self.assertEqual(pipelines[0].edges, 2)
 
     def two_pipelines(self):
-        self.dvc.add('foo')
-        self.dvc.run(deps=['foo'], outs=['bar'], cmd='')
-        self.dvc.run(deps=['bar'], outs=['baz'], cmd='echo baz > baz')
+        self.dvc.add("foo")
+        self.dvc.run(deps=["foo"], outs=["bar"], cmd="")
+        self.dvc.run(deps=["bar"], outs=["baz"], cmd="echo baz > baz")
 
-        self.dvc.add('code.py')
+        self.dvc.add("code.py")
 
         pipelines = self.dvc.pipelines()
 
@@ -220,8 +220,8 @@ class TestProjectPipeline(TestDvc):
         self.assertEqual(pipelines[1].edges, 0)
 
     def locked_stage(self):
-        self.dvc.add('foo')
-        self.dvc.lock_stage('foo.dvc')
+        self.dvc.add("foo")
+        self.dvc.lock_stage("foo.dvc")
 
         pipelines = self.dvc.pipelines()
         self.assertEqual(len(pipelines), 0)

tests/test_remote.py

@@ -10,30 +10,30 @@ from tests.basic_env import TestDvc
 
 class TestRemote(TestDvc):
     def test(self):
-        remotes = ['a', 'b', 'c']
+        remotes = ["a", "b", "c"]
 
-        self.assertEqual(main(['remote', 'list']), 0)
-        self.assertNotEqual(main(['remote', 'remove', remotes[0]]), 0)
+        self.assertEqual(main(["remote", "list"]), 0)
+        self.assertNotEqual(main(["remote", "remove", remotes[0]]), 0)
 
         for r in remotes:
-            self.assertEqual(main(['remote', 'add', '--default', r,
-                                   's3://bucket/name']), 0)
+            self.assertEqual(
+                main(["remote", "add", "--default", r, "s3://bucket/name"]), 0
+            )
 
-        self.assertEqual(main(['remote', 'list']), 0)
+        self.assertEqual(main(["remote", "list"]), 0)
 
-        self.assertEqual(main(['remote', 'remove', remotes[0]]), 0)
-        self.assertEqual(main(['remote', 'modify', remotes[0],
-                               'option', 'value']), 0)
+        self.assertEqual(main(["remote", "remove", remotes[0]]), 0)
+        self.assertEqual(main(["remote", "modify", remotes[0], "option", "value"]), 0)
 
-        self.assertEqual(main(['remote', 'list']), 0)
+        self.assertEqual(main(["remote", "list"]), 0)
 
     def test_failed_write(self):
         class A(object):
             system = False
             glob = False
             local = False
-            name = 'myremote'
-            url = 's3://remote'
+            name = "myremote"
+            url = "s3://remote"
             unset = False
 
         args = A()
@@ -44,40 +44,42 @@ class TestRemote(TestDvc):
         self.assertNotEqual(ret, 0)
 
     def test_relative_path(self):
-        dname = os.path.join('..', 'path', 'to', 'dir')
-        ret = main(['remote', 'add', 'mylocal', dname])
+        dname = os.path.join("..", "path", "to", "dir")
+        ret = main(["remote", "add", "mylocal", dname])
         self.assertEqual(ret, 0)
 
         # NOTE: we are in the project's root and config is in .dvc/, so
         # dir path written to config should be just one level above.
-        rel = os.path.join('..', dname)
+        rel = os.path.join("..", dname)
         config = configobj.ConfigObj(self.dvc.config.config_file)
-        self.assertEqual(config['remote "mylocal"']['url'], rel)
+        self.assertEqual(config['remote "mylocal"']["url"], rel)
 
 
 class TestRemoteRemoveDefault(TestDvc):
     def test(self):
-        remote = 'mys3'
-        self.assertEqual(main(['remote', 'add', '--default', remote,
-                               's3://bucket/name']), 0)
-        self.assertEqual(main(['remote', 'modify', remote, 'profile',
-                               'default']), 0)
-        self.assertEqual(main(['config', '--local', 'core.remote', remote]), 0)
+        remote = "mys3"
+        self.assertEqual(
+            main(["remote", "add", "--default", remote, "s3://bucket/name"]), 0
+        )
+        self.assertEqual(main(["remote", "modify", remote, "profile", "default"]), 0)
+        self.assertEqual(main(["config", "--local", "core.remote", remote]), 0)
 
-        config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir,
-                                                  Config.CONFIG))
-        local_config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir,
-                                                        Config.CONFIG_LOCAL))
-        self.assertEqual(config[Config.SECTION_CORE]
-                               [Config.SECTION_CORE_REMOTE], remote)
-        self.assertEqual(local_config[Config.SECTION_CORE]
-                                     [Config.SECTION_CORE_REMOTE], remote)
+        config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, Config.CONFIG))
+        local_config = configobj.ConfigObj(
+            os.path.join(self.dvc.dvc_dir, Config.CONFIG_LOCAL)
+        )
+        self.assertEqual(
+            config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE], remote
+        )
+        self.assertEqual(
+            local_config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE], remote
+        )
 
-        self.assertEqual(main(['remote', 'remove', remote]), 0)
-        config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir,
-                                                  Config.CONFIG))
-        local_config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir,
-                                                        Config.CONFIG_LOCAL))
+        self.assertEqual(main(["remote", "remove", remote]), 0)
+        config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, Config.CONFIG))
+        local_config = configobj.ConfigObj(
+            os.path.join(self.dvc.dvc_dir, Config.CONFIG_LOCAL)
+        )
         section = Config.SECTION_REMOTE_FMT.format(remote)
         self.assertTrue(section not in config.keys())
 
@@ -92,31 +94,31 @@ class TestRemoteRemoveDefault(TestDvc):
 
 class TestRemoteRemove(TestDvc):
     def test(self):
-        ret = main(['config', 'core.jobs', '1'])
+        ret = main(["config", "core.jobs", "1"])
         self.assertEqual(ret, 0)
 
-        remote = 'mys3'
-        ret = main(['remote', 'add', remote, 's3://bucket/name'])
+        remote = "mys3"
+        ret = main(["remote", "add", remote, "s3://bucket/name"])
         self.assertEqual(ret, 0)
 
-        ret = main(['remote', 'remove', remote])
+        ret = main(["remote", "remove", remote])
         self.assertEqual(ret, 0)
 
 
 class TestRemoteDefault(TestDvc):
     def test(self):
-        remote = 'mys3'
-        ret = main(['remote', 'add', 'mys3', 's3://bucket/path'])
+        remote = "mys3"
+        ret = main(["remote", "add", "mys3", "s3://bucket/path"])
         self.assertEqual(ret, 0)
 
-        ret = main(['remote', 'default', 'mys3'])
+        ret = main(["remote", "default", "mys3"])
         self.assertEqual(ret, 0)
         config_file = os.path.join(self.dvc.dvc_dir, Config.CONFIG)
         config = configobj.ConfigObj(config_file)
         default = config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE]
         self.assertEqual(default, remote)
 
-        ret = main(['remote', 'default', '--unset'])
+        ret = main(["remote", "default", "--unset"])
         self.assertEqual(ret, 0)
         config = configobj.ConfigObj(config_file)
         core = config.get(Config.SECTION_CORE)

tests/test_remove.py

@@ -30,7 +30,7 @@ class TestRemove(TestDvc):
 class TestRemoveNonExistentFile(TestDvc):
     def test(self):
         with self.assertRaises(StageFileDoesNotExistError):
-            self.dvc.remove('non_existent_dvc_file')
+            self.dvc.remove("non_existent_dvc_file")
 
 
 class TestRemoveDirectory(TestDvc):
@@ -51,28 +51,26 @@ class TestCmdRemove(TestDvc):
         self.assertEqual(len(stages), 1)
         stage = stages[0]
         self.assertTrue(stage is not None)
-        ret = main(['remove',
-                    stage.path])
+        ret = main(["remove", stage.path])
         self.assertEqual(ret, 0)
 
-        ret = main(['remove',
-                    'non-existing-dvc-file'])
+        ret = main(["remove", "non-existing-dvc-file"])
         self.assertNotEqual(ret, 0)
 
 
 class TestRemovePurge(TestDvc):
     def test(self):
         dvcfile = self.dvc.add(self.FOO)[0].path
-        ret = main(['remove', '--purge', '--force', dvcfile])
+        ret = main(["remove", "--purge", "--force", dvcfile])
 
         self.assertEqual(ret, 0)
         self.assertFalse(os.path.exists(self.FOO))
         self.assertFalse(os.path.exists(dvcfile))
 
-    @patch('dvc.prompt.confirm', return_value=False)
+    @patch("dvc.prompt.confirm", return_value=False)
     def test_force(self, mock_prompt):
         dvcfile = self.dvc.add(self.FOO)[0].path
-        ret = main(['remove', '--purge', dvcfile])
+        ret = main(["remove", "--purge", dvcfile])
 
         mock_prompt.assert_called()
         self.assertEqual(ret, 1)

tests/test_run.py

@@ -10,33 +10,37 @@ from dvc.system import System
 from dvc.stage import Stage
 from dvc.stage import StageFileBadNameError, MissingDep
 from dvc.stage import StageBadCwdError, StageFileAlreadyExistsError
-from dvc.exceptions import (OutputDuplicationError,
-                            CircularDependencyError,
-                            CyclicGraphError,
-                            ArgumentDuplicationError,
-                            WorkingDirectoryAsOutputError)
+from dvc.exceptions import (
+    OutputDuplicationError,
+    CircularDependencyError,
+    CyclicGraphError,
+    ArgumentDuplicationError,
+    WorkingDirectoryAsOutputError,
+)
 
 from tests.basic_env import TestDvc
 
 
 class TestRun(TestDvc):
     def test(self):
-        cmd = 'python {} {} {}'.format(self.CODE, self.FOO, 'out')
+        cmd = "python {} {} {}".format(self.CODE, self.FOO, "out")
         deps = [self.FOO, self.CODE]
-        outs = [os.path.join(self.dvc.root_dir, 'out')]
+        outs = [os.path.join(self.dvc.root_dir, "out")]
         outs_no_cache = []
-        fname = 'out.dvc'
+        fname = "out.dvc"
         cwd = os.curdir
 
         self.dvc.add(self.FOO)
-        stage = self.dvc.run(cmd=cmd,
-                             deps=deps,
-                             outs=outs,
-                             outs_no_cache=outs_no_cache,
-                             fname=fname,
-                             cwd=cwd)
+        stage = self.dvc.run(
+            cmd=cmd,
+            deps=deps,
+            outs=outs,
+            outs_no_cache=outs_no_cache,
+            fname=fname,
+            cwd=cwd,
+        )
 
-        self.assertTrue(filecmp.cmp(self.FOO, 'out', shallow=False))
+        self.assertTrue(filecmp.cmp(self.FOO, "out", shallow=False))
         self.assertTrue(os.path.isfile(stage.path))
         self.assertEqual(stage.cmd, cmd)
         self.assertEqual(len(stage.deps), len(deps))
@@ -46,301 +50,338 @@ class TestRun(TestDvc):
         self.assertTrue(stage.path, fname)
 
         with self.assertRaises(OutputDuplicationError):
-            stage = self.dvc.run(cmd=cmd,
-                                 deps=deps,
-                                 outs=outs,
-                                 outs_no_cache=outs_no_cache,
-                                 fname='duplicate' + fname,
-                                 cwd=cwd)
+            stage = self.dvc.run(
+                cmd=cmd,
+                deps=deps,
+                outs=outs,
+                outs_no_cache=outs_no_cache,
+                fname="duplicate" + fname,
+                cwd=cwd,
+            )
 
 
 class TestRunEmpty(TestDvc):
     def test(self):
-        self.dvc.run(cmd='',
-                     deps=[],
-                     outs=[],
-                     outs_no_cache=[],
-                     fname='empty.dvc',
-                     cwd=os.curdir)
+        self.dvc.run(
+            cmd="", deps=[], outs=[], outs_no_cache=[], fname="empty.dvc", cwd=os.curdir
+        )
 
 
 class TestRunMissingDep(TestDvc):
     def test(self):
         with self.assertRaises(MissingDep):
-            self.dvc.run(cmd='',
-                         deps=['non-existing-dep'],
-                         outs=[],
-                         outs_no_cache=[],
-                         fname='empty.dvc',
-                         cwd=os.curdir)
+            self.dvc.run(
+                cmd="",
+                deps=["non-existing-dep"],
+                outs=[],
+                outs_no_cache=[],
+                fname="empty.dvc",
+                cwd=os.curdir,
+            )
 
 
 class TestRunBadStageFilename(TestDvc):
     def test(self):
         with self.assertRaises(StageFileBadNameError):
-            self.dvc.run(cmd='',
-                         deps=[],
-                         outs=[],
-                         outs_no_cache=[],
-                         fname='empty',
-                         cwd=os.curdir)
+            self.dvc.run(
+                cmd="", deps=[], outs=[], outs_no_cache=[], fname="empty", cwd=os.curdir
+            )
 
         with self.assertRaises(StageFileBadNameError):
-            self.dvc.run(cmd='',
-                         deps=[],
-                         outs=[],
-                         outs_no_cache=[],
-                         fname=os.path.join(self.DATA_DIR, 'empty.dvc'),
-                         cwd=os.curdir)
+            self.dvc.run(
+                cmd="",
+                deps=[],
+                outs=[],
+                outs_no_cache=[],
+                fname=os.path.join(self.DATA_DIR, "empty.dvc"),
+                cwd=os.curdir,
+            )
 
 
 class TestRunNoExec(TestDvc):
     def test(self):
-        self.dvc.run(cmd='python {} {} {}'.format(self.CODE, self.FOO, 'out'),
-                     no_exec=True)
-        self.assertFalse(os.path.exists('out'))
+        self.dvc.run(
+            cmd="python {} {} {}".format(self.CODE, self.FOO, "out"), no_exec=True
+        )
+        self.assertFalse(os.path.exists("out"))
 
 
 class TestRunCircularDependency(TestDvc):
     def test(self):
         with self.assertRaises(CircularDependencyError):
-            self.dvc.run(cmd='',
-                         deps=[self.FOO],
-                         outs=[self.FOO],
-                         fname='circular-dependency.dvc')
+            self.dvc.run(
+                cmd="",
+                deps=[self.FOO],
+                outs=[self.FOO],
+                fname="circular-dependency.dvc",
+            )
 
     def test_outs_no_cache(self):
         with self.assertRaises(CircularDependencyError):
-            self.dvc.run(cmd='',
-                         deps=[self.FOO],
-                         outs_no_cache=[self.FOO],
-                         fname='circular-dependency.dvc')
+            self.dvc.run(
+                cmd="",
+                deps=[self.FOO],
+                outs_no_cache=[self.FOO],
+                fname="circular-dependency.dvc",
+            )
 
     def test_non_normalized_paths(self):
         with self.assertRaises(CircularDependencyError):
-            self.dvc.run(cmd='',
-                         deps=['./foo'],
-                         outs=['foo'],
-                         fname='circular-dependency.dvc')
+            self.dvc.run(
+                cmd="", deps=["./foo"], outs=["foo"], fname="circular-dependency.dvc"
+            )
 
     def test_graph(self):
-        self.dvc.run(deps=[self.FOO],
-                     outs=['bar.txt'],
-                     cmd='echo bar > bar.txt')
+        self.dvc.run(deps=[self.FOO], outs=["bar.txt"], cmd="echo bar > bar.txt")
 
-        self.dvc.run(deps=['bar.txt'],
-                     outs=['baz.txt'],
-                     cmd='echo baz > baz.txt')
+        self.dvc.run(deps=["bar.txt"], outs=["baz.txt"], cmd="echo baz > baz.txt")
 
         with self.assertRaises(CyclicGraphError):
-            self.dvc.run(deps=['baz.txt'],
-                         outs=[self.FOO],
-                         cmd='echo baz > foo')
+            self.dvc.run(deps=["baz.txt"], outs=[self.FOO], cmd="echo baz > foo")
 
 
 class TestRunDuplicatedArguments(TestDvc):
     def test(self):
         with self.assertRaises(ArgumentDuplicationError):
-            self.dvc.run(cmd='',
-                         deps=[],
-                         outs=[self.FOO, self.FOO],
-                         fname='circular-dependency.dvc')
+            self.dvc.run(
+                cmd="",
+                deps=[],
+                outs=[self.FOO, self.FOO],
+                fname="circular-dependency.dvc",
+            )
 
     def test_outs_no_cache(self):
         with self.assertRaises(ArgumentDuplicationError):
-            self.dvc.run(cmd='',
-                         outs=[self.FOO],
-                         outs_no_cache=[self.FOO],
-                         fname='circular-dependency.dvc')
+            self.dvc.run(
+                cmd="",
+                outs=[self.FOO],
+                outs_no_cache=[self.FOO],
+                fname="circular-dependency.dvc",
+            )
 
     def test_non_normalized_paths(self):
         with self.assertRaises(ArgumentDuplicationError):
-            self.dvc.run(cmd='',
-                         deps=[],
-                         outs=['foo', './foo'],
-                         fname='circular-dependency.dvc')
+            self.dvc.run(
+                cmd="", deps=[], outs=["foo", "./foo"], fname="circular-dependency.dvc"
+            )
 
 
 class TestRunWorkingDirectoryAsOutput(TestDvc):
     def test(self):
-        self.dvc.run(cmd='',
-                     deps=[],
-                     outs=[self.DATA_DIR])
+        self.dvc.run(cmd="", deps=[], outs=[self.DATA_DIR])
 
         with self.assertRaises(WorkingDirectoryAsOutputError):
-            self.dvc.run(cmd='',
-                         cwd=self.DATA_DIR,
-                         outs=[self.FOO],
-                         fname='inside-cwd.dvc')
+            self.dvc.run(
+                cmd="", cwd=self.DATA_DIR, outs=[self.FOO], fname="inside-cwd.dvc"
+            )
 
 
 class TestRunBadCwd(TestDvc):
     def test(self):
         with self.assertRaises(StageBadCwdError):
-            self.dvc.run(cmd='',
-                         cwd=self.mkdtemp())
+            self.dvc.run(cmd="", cwd=self.mkdtemp())
 
 
 class TestCmdRun(TestDvc):
     def test_run(self):
-        ret = main(['run',
-                    '-d', self.FOO,
-                    '-d', self.CODE,
-                    '-o', 'out',
-                    '-f', 'out.dvc',
-                    'python', self.CODE, self.FOO, 'out'])
+        ret = main(
+            [
+                "run",
+                "-d",
+                self.FOO,
+                "-d",
+                self.CODE,
+                "-o",
+                "out",
+                "-f",
+                "out.dvc",
+                "python",
+                self.CODE,
+                self.FOO,
+                "out",
+            ]
+        )
 
-        stage = Stage.load(self.dvc, fname='out.dvc')
+        stage = Stage.load(self.dvc, fname="out.dvc")
 
         self.assertEqual(ret, 0)
-        self.assertTrue(os.path.isfile('out'))
-        self.assertTrue(os.path.isfile('out.dvc'))
-        self.assertTrue(filecmp.cmp(self.FOO, 'out', shallow=False))
-        self.assertEqual(stage.cmd, 'python code.py foo out')
+        self.assertTrue(os.path.isfile("out"))
+        self.assertTrue(os.path.isfile("out.dvc"))
+        self.assertTrue(filecmp.cmp(self.FOO, "out", shallow=False))
+        self.assertEqual(stage.cmd, "python code.py foo out")
 
     def test_run_args_from_cli(self):
-        ret = main(['run', 'echo', 'foo'])
-        stage = Stage.load(self.dvc, fname='Dvcfile')
+        ret = main(["run", "echo", "foo"])
+        stage = Stage.load(self.dvc, fname="Dvcfile")
         self.assertEqual(ret, 0)
-        self.assertEqual(stage.cmd, 'echo foo')
+        self.assertEqual(stage.cmd, "echo foo")
 
     def test_run_bad_command(self):
-        ret = main(['run', 'non-existing-command'])
+        ret = main(["run", "non-existing-command"])
         self.assertNotEqual(ret, 0)
 
     def test_run_args_with_spaces(self):
-        ret = main(['run', 'echo', 'foo bar'])
-        stage = Stage.load(self.dvc, fname='Dvcfile')
+        ret = main(["run", "echo", "foo bar"])
+        stage = Stage.load(self.dvc, fname="Dvcfile")
         self.assertEqual(ret, 0)
         self.assertEqual(stage.cmd, 'echo "foo bar"')
 
-    @mock.patch.object(subprocess, 'Popen', side_effect=KeyboardInterrupt)
+    @mock.patch.object(subprocess, "Popen", side_effect=KeyboardInterrupt)
     def test_keyboard_interrupt(self, mock_popen):
-        ret = main(['run', 'mycmd'])
+        ret = main(["run", "mycmd"])
         self.assertEqual(ret, 252)
 
 
 class TestRunRemoveOuts(TestDvc):
     def test(self):
-        with open(self.CODE, 'w+') as fobj:
+        with open(self.CODE, "w+") as fobj:
             fobj.write("import sys\n")
             fobj.write("import os\n")
             fobj.write("if os.path.exists(sys.argv[1]):\n")
             fobj.write("    sys.exit(1)\n")
             fobj.write("open(sys.argv[1], 'w+').close()\n")
 
-        ret = main(['run',
-                    '--remove-outs',
-                    '-d', self.CODE,
-                    '-o', self.FOO,
-                    'python', self.CODE, self.FOO])
+        ret = main(
+            [
+                "run",
+                "--remove-outs",
+                "-d",
+                self.CODE,
+                "-o",
+                self.FOO,
+                "python",
+                self.CODE,
+                self.FOO,
+            ]
+        )
         self.assertEqual(ret, 0)
 
 
 class TestRunUnprotectOutsCopy(TestDvc):
     def test(self):
-        with open(self.CODE, 'w+') as fobj:
+        with open(self.CODE, "w+") as fobj:
             fobj.write("import sys\n")
             fobj.write("with open(sys.argv[1], 'a+') as fobj:\n")
             fobj.write("    fobj.write('foo')\n")
 
-        ret = main(['config', 'cache.protected', 'true'])
+        ret = main(["config", "cache.protected", "true"])
         self.assertEqual(ret, 0)
 
-        ret = main(['config', 'cache.type', 'copy'])
+        ret = main(["config", "cache.type", "copy"])
         self.assertEqual(ret, 0)
 
-        ret = main(['run',
-                    '-d', self.CODE,
-                    '-o', self.FOO,
-                    'python', self.CODE, self.FOO])
+        ret = main(
+            ["run", "-d", self.CODE, "-o", self.FOO, "python", self.CODE, self.FOO]
+        )
         self.assertEqual(ret, 0)
         self.assertFalse(os.access(self.FOO, os.W_OK))
-        self.assertEqual(open(self.FOO, 'r').read(), 'foofoo')
+        self.assertEqual(open(self.FOO, "r").read(), "foofoo")
 
-        ret = main(['run',
-                    '--overwrite-dvcfile',
-                    '--ignore-build-cache',
-                    '-d', self.CODE,
-                    '-o', self.FOO,
-                    'python', self.CODE, self.FOO])
+        ret = main(
+            [
+                "run",
+                "--overwrite-dvcfile",
+                "--ignore-build-cache",
+                "-d",
+                self.CODE,
+                "-o",
+                self.FOO,
+                "python",
+                self.CODE,
+                self.FOO,
+            ]
+        )
         self.assertEqual(ret, 0)
         self.assertFalse(os.access(self.FOO, os.W_OK))
-        self.assertEqual(open(self.FOO, 'r').read(), 'foofoofoo')
+        self.assertEqual(open(self.FOO, "r").read(), "foofoofoo")
 
 
 class TestRunUnprotectOutsSymlink(TestDvc):
     def test(self):
-        with open(self.CODE, 'w+') as fobj:
+        with open(self.CODE, "w+") as fobj:
             fobj.write("import sys\n")
             fobj.write("import os\n")
             fobj.write("assert os.path.exists(sys.argv[1])\n")
             fobj.write("with open(sys.argv[1], 'a+') as fobj:\n")
             fobj.write("    fobj.write('foo')\n")
 
-        ret = main(['config', 'cache.protected', 'true'])
+        ret = main(["config", "cache.protected", "true"])
         self.assertEqual(ret, 0)
 
-        ret = main(['config', 'cache.type', 'symlink'])
+        ret = main(["config", "cache.type", "symlink"])
         self.assertEqual(ret, 0)
 
         self.assertEqual(ret, 0)
-        ret = main(['run',
-                    '-d', self.CODE,
-                    '-o', self.FOO,
-                    'python', self.CODE, self.FOO])
+        ret = main(
+            ["run", "-d", self.CODE, "-o", self.FOO, "python", self.CODE, self.FOO]
+        )
         self.assertEqual(ret, 0)
         self.assertFalse(os.access(self.FOO, os.W_OK))
         self.assertTrue(System.is_symlink(self.FOO))
-        self.assertEqual(open(self.FOO, 'r').read(), 'foofoo')
+        self.assertEqual(open(self.FOO, "r").read(), "foofoo")
 
-        ret = main(['run',
-                    '--overwrite-dvcfile',
-                    '--ignore-build-cache',
-                    '-d', self.CODE,
-                    '-o', self.FOO,
-                    'python', self.CODE, self.FOO])
+        ret = main(
+            [
+                "run",
+                "--overwrite-dvcfile",
+                "--ignore-build-cache",
+                "-d",
+                self.CODE,
+                "-o",
+                self.FOO,
+                "python",
+                self.CODE,
+                self.FOO,
+            ]
+        )
         self.assertEqual(ret, 0)
         self.assertFalse(os.access(self.FOO, os.W_OK))
         self.assertTrue(System.is_symlink(self.FOO))
-        self.assertEqual(open(self.FOO, 'r').read(), 'foofoofoo')
+        self.assertEqual(open(self.FOO, "r").read(), "foofoofoo")
 
 
 class TestRunUnprotectOutsHardlink(TestDvc):
     def test(self):
-        with open(self.CODE, 'w+') as fobj:
+        with open(self.CODE, "w+") as fobj:
             fobj.write("import sys\n")
             fobj.write("import os\n")
             fobj.write("assert os.path.exists(sys.argv[1])\n")
             fobj.write("with open(sys.argv[1], 'a+') as fobj:\n")
             fobj.write("    fobj.write('foo')\n")
 
-        ret = main(['config', 'cache.protected', 'true'])
+        ret = main(["config", "cache.protected", "true"])
         self.assertEqual(ret, 0)
 
-        ret = main(['config', 'cache.type', 'hardlink'])
+        ret = main(["config", "cache.type", "hardlink"])
         self.assertEqual(ret, 0)
 
         self.assertEqual(ret, 0)
-        ret = main(['run',
-                    '-d', self.CODE,
-                    '-o', self.FOO,
-                    'python', self.CODE, self.FOO])
+        ret = main(
+            ["run", "-d", self.CODE, "-o", self.FOO, "python", self.CODE, self.FOO]
+        )
         self.assertEqual(ret, 0)
         self.assertFalse(os.access(self.FOO, os.W_OK))
         self.assertTrue(System.is_hardlink(self.FOO))
-        self.assertEqual(open(self.FOO, 'r').read(), 'foofoo')
+        self.assertEqual(open(self.FOO, "r").read(), "foofoo")
 
-        ret = main(['run',
-                    '--overwrite-dvcfile',
-                    '--ignore-build-cache',
-                    '-d', self.CODE,
-                    '-o', self.FOO,
-                    'python', self.CODE, self.FOO])
+        ret = main(
+            [
+                "run",
+                "--overwrite-dvcfile",
+                "--ignore-build-cache",
+                "-d",
+                self.CODE,
+                "-o",
+                self.FOO,
+                "python",
+                self.CODE,
+                self.FOO,
+            ]
+        )
         self.assertEqual(ret, 0)
         self.assertFalse(os.access(self.FOO, os.W_OK))
         self.assertTrue(System.is_hardlink(self.FOO))
-        self.assertEqual(open(self.FOO, 'r').read(), 'foofoofoo')
+        self.assertEqual(open(self.FOO, "r").read(), "foofoofoo")
 
 
 class TestCmdRunOverwrite(TestDvc):
@@ -352,64 +393,94 @@ class TestCmdRunOverwrite(TestDvc):
         # below don't change).
         import time
 
-        ret = main(['run',
-                    '-d', self.FOO,
-                    '-d', self.CODE,
-                    '-o', 'out',
-                    '-f', 'out.dvc',
-                    'python', self.CODE, self.FOO, 'out'])
+        ret = main(
+            [
+                "run",
+                "-d",
+                self.FOO,
+                "-d",
+                self.CODE,
+                "-o",
+                "out",
+                "-f",
+                "out.dvc",
+                "python",
+                self.CODE,
+                self.FOO,
+                "out",
+            ]
+        )
         self.assertEqual(ret, 0)
 
-        stage_mtime = os.path.getmtime('out.dvc')
+        stage_mtime = os.path.getmtime("out.dvc")
 
         time.sleep(1)
 
-        ret = main(['run',
-                    '-d', self.FOO,
-                    '-d', self.CODE,
-                    '-o', 'out',
-                    '-f', 'out.dvc',
-                    'python', self.CODE, self.FOO, 'out'])
+        ret = main(
+            [
+                "run",
+                "-d",
+                self.FOO,
+                "-d",
+                self.CODE,
+                "-o",
+                "out",
+                "-f",
+                "out.dvc",
+                "python",
+                self.CODE,
+                self.FOO,
+                "out",
+            ]
+        )
         self.assertEqual(ret, 0)
 
         # NOTE: check that dvcfile was NOT overwritten
-        self.assertEqual(stage_mtime, os.path.getmtime('out.dvc'))
-        stage_mtime = os.path.getmtime('out.dvc')
+        self.assertEqual(stage_mtime, os.path.getmtime("out.dvc"))
+        stage_mtime = os.path.getmtime("out.dvc")
 
         time.sleep(1)
 
-        ret = main(['run',
-                    '-d', self.FOO,
-                    '-d', self.CODE,
-                    '--overwrite-dvcfile',
-                    '--ignore-build-cache',
-                    '-o', 'out',
-                    '-f', 'out.dvc',
-                    'python', self.CODE, self.FOO, 'out'])
+        ret = main(
+            [
+                "run",
+                "-d",
+                self.FOO,
+                "-d",
+                self.CODE,
+                "--overwrite-dvcfile",
+                "--ignore-build-cache",
+                "-o",
+                "out",
+                "-f",
+                "out.dvc",
+                "python",
+                self.CODE,
+                self.FOO,
+                "out",
+            ]
+        )
         self.assertEqual(ret, 0)
 
         # NOTE: check that dvcfile was overwritten
-        self.assertNotEqual(stage_mtime, os.path.getmtime('out.dvc'))
-        stage_mtime = os.path.getmtime('out.dvc')
+        self.assertNotEqual(stage_mtime, os.path.getmtime("out.dvc"))
+        stage_mtime = os.path.getmtime("out.dvc")
 
         time.sleep(1)
 
-        ret = main(['run',
-                    '--overwrite-dvcfile',
-                    '-f', 'out.dvc',
-                    '-d', self.BAR])
+        ret = main(["run", "--overwrite-dvcfile", "-f", "out.dvc", "-d", self.BAR])
         self.assertEqual(ret, 0)
 
         # NOTE: check that dvcfile was overwritten
-        self.assertNotEqual(stage_mtime, os.path.getmtime('out.dvc'))
+        self.assertNotEqual(stage_mtime, os.path.getmtime("out.dvc"))
 
 
 class TestRunDeterministicBase(TestDvc):
     def setUp(self):
         super(TestRunDeterministicBase, self).setUp()
-        self.out_file = 'out'
-        self.stage_file = self.out_file + '.dvc'
-        self.cmd = 'python {} {} {}'.format(self.CODE, self.FOO, self.out_file)
+        self.out_file = "out"
+        self.stage_file = self.out_file + ".dvc"
+        self.cmd = "python {} {} {}".format(self.CODE, self.FOO, self.out_file)
         self.deps = [self.FOO, self.CODE]
         self.outs = [self.out_file]
         self.overwrite = False
@@ -418,12 +489,14 @@ class TestRunDeterministicBase(TestDvc):
         self._run()
 
     def _run(self):
-        self.stage = self.dvc.run(cmd=self.cmd,
-                                  fname=self.stage_file,
-                                  overwrite=self.overwrite,
-                                  ignore_build_cache=self.ignore_build_cache,
-                                  deps=self.deps,
-                                  outs=self.outs)
+        self.stage = self.dvc.run(
+            cmd=self.cmd,
+            fname=self.stage_file,
+            overwrite=self.overwrite,
+            ignore_build_cache=self.ignore_build_cache,
+            deps=self.deps,
+            outs=self.outs,
+        )
 
 
 class TestRunDeterministic(TestRunDeterministicBase):
@@ -485,6 +558,6 @@ class TestRunDeterministicChangedOut(TestRunDeterministicBase):
 
 class TestRunDeterministicChangedCmd(TestRunDeterministicBase):
     def test(self):
-        self.cmd += ' arg'
+        self.cmd += " arg"
         with self.assertRaises(StageFileAlreadyExistsError):
             self._run()

tests/test_scm.py

@@ -22,9 +22,9 @@ class TestSCMGit(TestGit):
 
     def test_commit(self):
         G = Git(self._root_dir)
-        G.add(['foo'])
-        G.commit('add')
-        self.assertTrue('foo' in self.git.git.ls_files())
+        G.add(["foo"])
+        G.commit("add")
+        self.assertTrue("foo" in self.git.git.ls_files())
 
 
 class TestSCMGitSubmodule(TestGitSubmodule):
@@ -36,17 +36,16 @@ class TestSCMGitSubmodule(TestGitSubmodule):
 
     def test_commit_in_submodule(self):
         G = Git(self._root_dir)
-        G.add(['foo'])
-        G.commit('add')
-        self.assertTrue('foo' in self.git.git.ls_files())
+        G.add(["foo"])
+        G.commit("add")
+        self.assertTrue("foo" in self.git.git.ls_files())
 
 
 class TestIgnore(TestGit):
     def _count_gitignore(self):
-        with open(Git.GITIGNORE, 'r') as fd:
+        with open(Git.GITIGNORE, "r") as fd:
             lines = fd.readlines()
-            return len(list(filter(lambda x: x.strip() == '/' + self.FOO,
-                                   lines)))
+            return len(list(filter(lambda x: x.strip() == "/" + self.FOO, lines)))
 
     def test(self):
         git = Git(self._root_dir)

tests/test_stage.py

@@ -25,7 +25,7 @@ class TestSchemaCmd(TestSchema):
         Stage.validate(d)
 
     def test_cmd_str(self):
-        d = {Stage.PARAM_CMD: 'cmd'}
+        d = {Stage.PARAM_CMD: "cmd"}
         Stage.validate(d)
 
 
@@ -52,11 +52,11 @@ class TestSchemaDepsOuts(TestSchema):
         Stage.validate(d)
 
     def test_list(self):
-        lst = [{OutputLOCAL.PARAM_PATH: 'foo',
-                RemoteLOCAL.PARAM_CHECKSUM: '123'},
-               {OutputLOCAL.PARAM_PATH: 'bar',
-                RemoteLOCAL.PARAM_CHECKSUM: None},
-               {OutputLOCAL.PARAM_PATH: 'baz'}]
+        lst = [
+            {OutputLOCAL.PARAM_PATH: "foo", RemoteLOCAL.PARAM_CHECKSUM: "123"},
+            {OutputLOCAL.PARAM_PATH: "bar", RemoteLOCAL.PARAM_CHECKSUM: None},
+            {OutputLOCAL.PARAM_PATH: "baz"},
+        ]
         d = {Stage.PARAM_DEPS: lst}
         Stage.validate(d)
 
@@ -69,26 +69,27 @@ class TestSchemaDepsOuts(TestSchema):
 class TestReload(TestDvc):
     def test(self):
         import yaml
+
         stages = self.dvc.add(self.FOO)
         self.assertEqual(len(stages), 1)
         stage = stages[0]
         self.assertTrue(stage is not None)
 
-        with open(stage.relpath, 'r') as fobj:
+        with open(stage.relpath, "r") as fobj:
             d = yaml.safe_load(fobj)
 
         # NOTE: checking that reloaded stage didn't change its checksum
-        md5 = '11111111111111111111111111111111'
+        md5 = "11111111111111111111111111111111"
         d[stage.PARAM_MD5] = md5
 
-        with open(stage.relpath, 'w') as fobj:
+        with open(stage.relpath, "w") as fobj:
             yaml.safe_dump(d, fobj, default_flow_style=False)
 
         stage = Stage.load(self.dvc, stage.relpath)
         self.assertTrue(stage is not None)
         stage.dump()
 
-        with open(stage.relpath, 'r') as fobj:
+        with open(stage.relpath, "r") as fobj:
             d = yaml.safe_load(fobj)
 
         self.assertEqual(d[stage.PARAM_MD5], md5)

tests/test_state.py

@@ -21,8 +21,8 @@ class TestState(TestDvc):
             self.assertEqual(entry_md5, md5)
 
             os.unlink(path)
-            with open(path, 'a') as fd:
-                fd.write('1')
+            with open(path, "a") as fd:
+                fd.write("1")
 
             md5 = file_md5(path)[0]
             entry_md5 = state.update(path)
@@ -33,14 +33,14 @@ class TestStateOverflow(TestDvc):
     def test(self):
         # NOTE: trying to add more entries than state can handle,
         # to see if it will clean up and vacuum successfully
-        ret = main(['config', 'state.row_limit', '10'])
+        ret = main(["config", "state.row_limit", "10"])
         self.assertEqual(ret, 0)
 
-        dname = 'dir'
+        dname = "dir"
         os.mkdir(dname)
         for i in range(20):
-            with open(os.path.join(dname, str(i)), 'w+') as fobj:
+            with open(os.path.join(dname, str(i)), "w+") as fobj:
                 fobj.write(str(i))
 
-        ret = main(['add', 'dir'])
+        ret = main(["add", "dir"])
         self.assertEqual(ret, 0)

tests/test_status.py

@@ -9,11 +9,11 @@ class TestStatus(TestDvc):
     def test_quiet(self):
         self.dvc.add(self.FOO)
 
-        ret = main(['status', '--quiet'])
+        ret = main(["status", "--quiet"])
         self.assertEqual(ret, 0)
 
         os.remove(self.FOO)
         os.rename(self.BAR, self.FOO)
 
-        ret = main(['status', '--quiet'])
+        ret = main(["status", "--quiet"])
         self.assertEqual(ret, 1)

tests/test_system.py

@@ -6,11 +6,11 @@ from tests.basic_env import TestDir
 
 class TestSystem(TestDir):
     def test_getdirinfo(self):
-        if os.name != 'nt':
+        if os.name != "nt":
             return
 
         # simulate someone holding an exclussive access
-        locked = 'locked'
+        locked = "locked"
         fd = os.open(locked, os.O_WRONLY | os.O_CREAT | os.O_EXCL)
 
         # should not raise WinError

tests/test_unprotect.py

@@ -8,23 +8,24 @@ from tests.basic_env import TestDvc
 
 class TestUnprotect(TestDvc):
     def test(self):
-        ret = main(['config', 'cache.protected', 'true'])
+        ret = main(["config", "cache.protected", "true"])
         self.assertEqual(ret, 0)
 
-        ret = main(['add', self.FOO])
+        ret = main(["add", self.FOO])
         self.assertEqual(ret, 0)
 
-        self.assertNotEqual(os.stat(self.FOO).st_mode & (stat.S_IREAD |
-                                                         stat.S_IRGRP |
-                                                         stat.S_IROTH), 0)
-        self.assertEqual(os.stat(self.FOO).st_mode & (stat.S_IWRITE |
-                                                      stat.S_IWGRP |
-                                                      stat.S_IWOTH), 0)
+        self.assertNotEqual(
+            os.stat(self.FOO).st_mode & (stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH), 0
+        )
+        self.assertEqual(
+            os.stat(self.FOO).st_mode & (stat.S_IWRITE | stat.S_IWGRP | stat.S_IWOTH), 0
+        )
 
-        ret = main(['unprotect', self.FOO])
+        ret = main(["unprotect", self.FOO])
         self.assertEqual(ret, 0)
 
-        self.assertNotEqual(os.stat(self.FOO).st_mode & (stat.S_IREAD |
-                                                         stat.S_IRGRP |
-                                                         stat.S_IROTH |
-                                                         stat.S_IWRITE), 0)
+        self.assertNotEqual(
+            os.stat(self.FOO).st_mode
+            & (stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH | stat.S_IWRITE),
+            0,
+        )