diff --git a/.travis.yml b/.travis.yml index f4e755587..ebd157762 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,13 @@ language: python cache: pip -sudo: required -dist: trusty -group: deprecated-2017Q4 -python: - - "2.7" - - "3.4" - - "3.5" - - "3.6" +sudo: true +dist: xenial +python: "3.7" +env: + - | + PYENV_ROOT="$HOME/.pyenv" + PATH="$PYENV_ROOT/bin:$PATH" + PYTHON_CONFIGURE_OPTS="--enable-framework" # Travis doesn't support testing python on Mac yet, so we need # to workaround it by installing it directly with brew. # @@ -16,10 +16,22 @@ python: # version, that causes fpm to crash. matrix: include: - # 3.7 needs Xenial image because of https://github.com/travis-ci/travis-ci/issues/9069 - - python: "3.7" - dist: xenial - sudo: true + - name: "2.7" + env: + - PYTHON_VER=2.7.13 + - name: "3.4" + env: + - PYTHON_VER=3.4.6 + - name: "3.5" + env: + - PYTHON_VER=3.5.3 + - name: "3.6" + env: + - PYTHON_VER=3.6.2 + - name: "3.7" + env: + # NOTE: using default python + - PYTHON_VER= - if: type = cron OR tag is present os: osx osx_image: xcode7.3 @@ -27,8 +39,6 @@ matrix: rvm: 2.4.0 env: - PYTHON_VER=2.7.10 - - PYENV_ROOT="$HOME/.pyenv" - - PATH="$PYENV_ROOT/bin:$PATH" - CPPFLAGS="-I/usr/local/opt/openssl/include $CPPFLAGS" - LDFLAGS="-L/usr/local/opt/openssl/lib $LDFLAGS" - if: type = cron OR tag is present @@ -38,8 +48,6 @@ matrix: rvm: 2.4.0 env: - PYTHON_VER=3.4.3 - - PYENV_ROOT="$HOME/.pyenv" - - PATH="$PYENV_ROOT/bin:$PATH" - if: type = cron OR tag is present os: osx osx_image: xcode7.3 @@ -47,19 +55,23 @@ matrix: rvm: 2.4.0 env: - PYTHON_VER=3.5.0 - - PYENV_ROOT="$HOME/.pyenv" - - PATH="$PYENV_ROOT/bin:$PATH" before_install: + - pip install -r scripts/ci/requirements.txt + # stop the build if there are any readme formatting errors + - python setup.py checkdocs + # stop the build if there are any black errors + - black dvc --check + - black tests --check - bash ./scripts/ci/before_install.sh - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then + - if [[ -n "$PYTHON_VER" ]]; then eval "$(pyenv init -)"; - PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing --keep --verbose "$PYTHON_VER"; + pyenv install --skip-existing --keep --verbose "$PYTHON_VER"; pyenv shell "$PYTHON_VER"; + fi + - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install openssl; brew cask install google-cloud-sdk; fi - - which python - - which pip - python --version - if [[ "$TRAVIS_OS_NAME" != "osx" ]]; then bash ./scripts/ci/install_hadoop.sh; @@ -79,13 +91,6 @@ before_install: fi install: - bash ./scripts/ci/install.sh - - pip install flake8 -before_script: - # stop the build if there are any readme formatting errors - - python setup.py checkdocs - # stop the build if there are any flake8 errors - - flake8 dvc - - flake8 tests script: - bash ./scripts/ci/script.sh deploy: diff --git a/dvc/__init__.py b/dvc/__init__.py index 00afe0a8a..d59a29a09 100644 --- a/dvc/__init__.py +++ b/dvc/__init__.py @@ -10,12 +10,12 @@ import os import warnings -VERSION_BASE = '0.24.3' +VERSION_BASE = "0.24.3" __version__ = VERSION_BASE PACKAGEPATH = os.path.abspath(os.path.dirname(__file__)) HOMEPATH = os.path.dirname(PACKAGEPATH) -VERSIONPATH = os.path.join(PACKAGEPATH, 'version.py') +VERSIONPATH = os.path.join(PACKAGEPATH, "version.py") def _update_version_file(): @@ -30,13 +30,13 @@ def _update_version_file(): sha = repo.head.object.hexsha short_sha = repo.git.rev_parse(sha, short=6) - dirty = '.mod' if repo.is_dirty() else '' - ver = '{}+{}{}'.format(__version__, short_sha, dirty) + dirty = ".mod" if repo.is_dirty() else "" + ver = "{}+{}{}".format(__version__, short_sha, dirty) # Write a helper file, that will be installed with the package # and will provide a true version of the installed dvc - with open(VERSIONPATH, 'w+') as fobj: - fobj.write('# AUTOGENERATED by dvc/__init__.py\n') + with open(VERSIONPATH, "w+") as fobj: + fobj.write("# AUTOGENERATED by dvc/__init__.py\n") fobj.write('version = "{}"\n'.format(ver)) return ver @@ -48,11 +48,13 @@ def _remove_version_file(): os.unlink(VERSIONPATH) -if os.path.exists(os.path.join(HOMEPATH, 'setup.py')): +if os.path.exists(os.path.join(HOMEPATH, "setup.py")): # dvc is run directly from source without installation or # __version__ is called from setup.py - if os.getenv('APPVEYOR_REPO_TAG', '').lower() != 'true' \ - and os.getenv('TRAVIS_TAG', '') == '': + if ( + os.getenv("APPVEYOR_REPO_TAG", "").lower() != "true" + and os.getenv("TRAVIS_TAG", "") == "" + ): __version__ = _update_version_file() else: # pragma: no cover _remove_version_file() @@ -61,6 +63,7 @@ else: # pragma: no cover # auto-generated version.py to help us provide a true version try: from dvc.version import version + __version__ = version except Exception: pass diff --git a/dvc/__main__.py b/dvc/__main__.py index b338cad48..357464370 100644 --- a/dvc/__main__.py +++ b/dvc/__main__.py @@ -6,5 +6,5 @@ import sys from dvc.main import main -if __name__ == '__main__': +if __name__ == "__main__": main(sys.argv[1:]) diff --git a/dvc/analytics.py b/dvc/analytics.py index 1a14a9c88..58ec7f3d4 100644 --- a/dvc/analytics.py +++ b/dvc/analytics.py @@ -18,32 +18,33 @@ class Analytics(object): Args: info (dict): optional existing analytics report. """ - URL = 'https://analytics.dvc.org' + + URL = "https://analytics.dvc.org" TIMEOUT_POST = 5 - USER_ID_FILE = 'user_id' + USER_ID_FILE = "user_id" - PARAM_DVC_VERSION = 'dvc_version' - PARAM_USER_ID = 'user_id' - PARAM_SYSTEM_INFO = 'system_info' + PARAM_DVC_VERSION = "dvc_version" + PARAM_USER_ID = "user_id" + PARAM_SYSTEM_INFO = "system_info" - PARAM_OS = 'os' + PARAM_OS = "os" - PARAM_WINDOWS_VERSION_MAJOR = 'windows_version_major' - PARAM_WINDOWS_VERSION_MINOR = 'windows_version_minor' - PARAM_WINDOWS_VERSION_BUILD = 'windows_version_build' - PARAM_WINDOWS_VERSION_SERVICE_PACK = 'windows_version_service_pack' + PARAM_WINDOWS_VERSION_MAJOR = "windows_version_major" + PARAM_WINDOWS_VERSION_MINOR = "windows_version_minor" + PARAM_WINDOWS_VERSION_BUILD = "windows_version_build" + PARAM_WINDOWS_VERSION_SERVICE_PACK = "windows_version_service_pack" - PARAM_MAC_VERSION = 'mac_version' + PARAM_MAC_VERSION = "mac_version" - PARAM_LINUX_DISTRO = 'linux_distro' - PARAM_LINUX_DISTRO_VERSION = 'linux_distro_version' - PARAM_LINUX_DISTRO_LIKE = 'linux_distro_like' + PARAM_LINUX_DISTRO = "linux_distro" + PARAM_LINUX_DISTRO_VERSION = "linux_distro_version" + PARAM_LINUX_DISTRO_LIKE = "linux_distro_like" - PARAM_SCM_CLASS = 'scm_class' - PARAM_IS_BINARY = 'is_binary' - PARAM_CMD_CLASS = 'cmd_class' - PARAM_CMD_RETURN_CODE = 'cmd_return_code' + PARAM_SCM_CLASS = "scm_class" + PARAM_IS_BINARY = "is_binary" + PARAM_CMD_CLASS = "cmd_class" + PARAM_CMD_RETURN_CODE = "cmd_return_code" def __init__(self, info=None): from dvc.config import Config @@ -62,7 +63,7 @@ class Analytics(object): raise self.user_id_file = os.path.join(cdir, self.USER_ID_FILE) - self.user_id_file_lock = Lock(cdir, self.USER_ID_FILE + '.lock') + self.user_id_file_lock = Lock(cdir, self.USER_ID_FILE + ".lock") @staticmethod def load(path): @@ -71,7 +72,7 @@ class Analytics(object): Args: path (str): path to json file with analytics report. """ - with open(path, 'r') as fobj: + with open(path, "r") as fobj: analytics = Analytics(info=json.load(fobj)) os.unlink(path) return analytics @@ -79,7 +80,7 @@ class Analytics(object): def _write_user_id(self): import uuid - with open(self.user_id_file, 'w+') as fobj: + with open(self.user_id_file, "w+") as fobj: user_id = str(uuid.uuid4()) info = {self.PARAM_USER_ID: user_id} json.dump(info, fobj) @@ -89,7 +90,7 @@ class Analytics(object): if not os.path.exists(self.user_id_file): return None - with open(self.user_id_file, 'r') as fobj: + with open(self.user_id_file, "r") as fobj: try: info = json.load(fobj) return info[self.PARAM_USER_ID] @@ -112,9 +113,10 @@ class Analytics(object): def _collect_windows(self): import sys + version = sys.getwindowsversion() # pylint: disable=no-member info = {} - info[self.PARAM_OS] = 'windows' + info[self.PARAM_OS] = "windows" info[self.PARAM_WINDOWS_VERSION_MAJOR] = version.major info[self.PARAM_WINDOWS_VERSION_MINOR] = version.minor info[self.PARAM_WINDOWS_VERSION_BUILD] = version.build @@ -123,15 +125,17 @@ class Analytics(object): def _collect_darwin(self): import platform + info = {} - info[self.PARAM_OS] = 'mac' + info[self.PARAM_OS] = "mac" info[self.PARAM_MAC_VERSION] = platform.mac_ver()[0] return info def _collect_linux(self): import distro + info = {} - info[self.PARAM_OS] = 'linux' + info[self.PARAM_OS] = "linux" info[self.PARAM_LINUX_DISTRO] = distro.id() info[self.PARAM_LINUX_DISTRO_VERSION] = distro.version() info[self.PARAM_LINUX_DISTRO_LIKE] = distro.like() @@ -139,15 +143,16 @@ class Analytics(object): def _collect_system_info(self): import platform + system = platform.system() - if system == 'Windows': + if system == "Windows": return self._collect_windows() - if system == 'Darwin': + if system == "Darwin": return self._collect_darwin() - if system == 'Linux': + if system == "Linux": return self._collect_linux() raise NotImplementedError @@ -180,7 +185,7 @@ class Analytics(object): if ret is not None: self.info[self.PARAM_CMD_RETURN_CODE] = ret - if args is not None and hasattr(args, 'func'): + if args is not None and hasattr(args, "func"): assert args.func != CmdDaemonAnalytics self.info[self.PARAM_CMD_CLASS] = args.func.__name__ @@ -192,7 +197,7 @@ class Analytics(object): """ import tempfile - with tempfile.NamedTemporaryFile(delete=False, mode='w') as fobj: + with tempfile.NamedTemporaryFile(delete=False, mode="w") as fobj: json.dump(self.info, fobj) return fobj.name @@ -203,13 +208,13 @@ class Analytics(object): from dvc.exceptions import NotDvcProjectError from dvc.command.daemon import CmdDaemonBase - if os.getenv('DVC_TEST'): + if os.getenv("DVC_TEST"): return False if isinstance(cmd, CmdDaemonBase): return False - if cmd is None or not hasattr(cmd, 'config'): + if cmd is None or not hasattr(cmd, "config"): try: dvc_dir = Project.find_dvc_dir() config = Config(dvc_dir) @@ -223,8 +228,7 @@ class Analytics(object): core = config.config.get(Config.SECTION_CORE, {}) enabled = core.get(Config.SECTION_CORE_ANALYTICS, True) - logger.debug("Analytics is {}abled." - .format('en' if enabled else 'dis')) + logger.debug("Analytics is {}abled.".format("en" if enabled else "dis")) return enabled @staticmethod @@ -242,7 +246,7 @@ class Analytics(object): analytics = Analytics() analytics.collect_cmd(args, ret) - daemon(['analytics', analytics.dump()]) + daemon(["analytics", analytics.dump()]) def send(self): """Collect and send analytics.""" diff --git a/dvc/cache.py b/dvc/cache.py index b66f638f2..02ddd4871 100644 --- a/dvc/cache.py +++ b/dvc/cache.py @@ -14,7 +14,8 @@ class Cache(object): project (dvc.project.Project): project instance that this cache belongs to. """ - CACHE_DIR = 'cache' + + CACHE_DIR = "cache" def __init__(self, project): from dvc.remote import Remote @@ -34,11 +35,10 @@ class Cache(object): protected = config.get(Config.SECTION_CACHE_PROTECTED) sect = { - Config.PRIVATE_CWD: config.get(Config.PRIVATE_CWD, - project.dvc_dir), + Config.PRIVATE_CWD: config.get(Config.PRIVATE_CWD, project.dvc_dir), Config.SECTION_REMOTE_URL: cache_dir, Config.SECTION_CACHE_TYPE: cache_type, - Config.SECTION_CACHE_PROTECTED: protected + Config.SECTION_CACHE_PROTECTED: protected, } self._local = Remote(project, sect) diff --git a/dvc/cli.py b/dvc/cli.py index 627aa4a2d..8bf4e6e64 100644 --- a/dvc/cli.py +++ b/dvc/cli.py @@ -61,6 +61,7 @@ COMMANDS = [ class DvcParser(argparse.ArgumentParser): """Custom parser class for dvc CLI.""" + def error(self, message): """Custom error method. Args: @@ -77,6 +78,7 @@ class DvcParser(argparse.ArgumentParser): class VersionAction(argparse.Action): # pragma: no cover # pylint: disable=too-few-public-methods """Shows dvc version and exits.""" + def __call__(self, parser, namespace, values, option_string=None): print(VERSION) sys.exit(0) @@ -96,41 +98,34 @@ def parse_args(argv=None): log_level_group = parent_parser.add_mutually_exclusive_group() log_level_group.add_argument( - '-q', - '--quiet', - action='store_true', - default=False, - help='Be quiet.') + "-q", "--quiet", action="store_true", default=False, help="Be quiet." + ) log_level_group.add_argument( - '-v', - '--verbose', - action='store_true', - default=False, - help='Be verbose.') + "-v", "--verbose", action="store_true", default=False, help="Be verbose." + ) # Main parser - desc = 'Data Version Control' + desc = "Data Version Control" parser = DvcParser( - prog='dvc', + prog="dvc", description=desc, parents=[parent_parser], - formatter_class=argparse.RawTextHelpFormatter) + formatter_class=argparse.RawTextHelpFormatter, + ) # NOTE: On some python versions action='version' prints to stderr # instead of stdout https://bugs.python.org/issue18920 parser.add_argument( - '-V', - '--version', - action=VersionAction, - nargs=0, - help="Show program's version.") + "-V", "--version", action=VersionAction, nargs=0, help="Show program's version." + ) # Sub commands subparsers = parser.add_subparsers( - title='Available Commands', - metavar='COMMAND', - dest='cmd', - help='Use dvc COMMAND --help for command-specific help.') + title="Available Commands", + metavar="COMMAND", + dest="cmd", + help="Use dvc COMMAND --help for command-specific help.", + ) fix_subparsers(subparsers) diff --git a/dvc/command/add.py b/dvc/command/add.py index 7c6a66650..1f8241a8b 100644 --- a/dvc/command/add.py +++ b/dvc/command/add.py @@ -11,26 +11,22 @@ class CmdAdd(CmdBase): try: self.project.add(target, recursive=self.args.recursive) except DvcException: - logger.error('failed to add file') + logger.error("failed to add file") return 1 return 0 def add_parser(subparsers, parent_parser): - ADD_HELP = 'Add files/directories to dvc.' + ADD_HELP = "Add files/directories to dvc." add_parser = subparsers.add_parser( - 'add', - parents=[parent_parser], - description=ADD_HELP, - help=ADD_HELP) + "add", parents=[parent_parser], description=ADD_HELP, help=ADD_HELP + ) add_parser.add_argument( - '-R', - '--recursive', - action='store_true', + "-R", + "--recursive", + action="store_true", default=False, - help='Recursively add each file under the directory.') - add_parser.add_argument( - 'targets', - nargs='+', - help='Input files/directories.') + help="Recursively add each file under the directory.", + ) + add_parser.add_argument("targets", nargs="+", help="Input files/directories.") add_parser.set_defaults(func=CmdAdd) diff --git a/dvc/command/base.py b/dvc/command/base.py index 7ca1e413e..225d030a5 100644 --- a/dvc/command/base.py +++ b/dvc/command/base.py @@ -15,7 +15,7 @@ def fix_subparsers(subparsers): if is_py3: # pragma: no cover subparsers.required = True - subparsers.dest = 'cmd' + subparsers.dest = "cmd" class CmdBase(object): @@ -31,6 +31,7 @@ class CmdBase(object): def default_targets(self): """Default targets for `dvc repro` and `dvc pipeline`.""" from dvc.stage import Stage + msg = "assuming default target '{}'.".format(Stage.STAGE_FILE) logger.warning(msg) return [Stage.STAGE_FILE] @@ -51,7 +52,7 @@ class CmdBase(object): with self.project.lock: return self.run() except LockError: - logger.error('failed to lock before running a command') + logger.error("failed to lock before running a command") return 1 # Abstract methods that have to be implemented by any inheritance class diff --git a/dvc/command/cache.py b/dvc/command/cache.py index 8464e745a..76bc1a547 100644 --- a/dvc/command/cache.py +++ b/dvc/command/cache.py @@ -9,9 +9,10 @@ from dvc.command.config import CmdConfig class CmdCacheDir(CmdConfig): def run(self): - self.args.name = 'cache.dir' - self.args.value = CmdRemoteAdd.resolve_path(self.args.value, - self.configobj.filename) + self.args.name = "cache.dir" + self.args.value = CmdRemoteAdd.resolve_path( + self.args.value, self.configobj.filename + ) return super(CmdCacheDir, self).run() @@ -19,39 +20,36 @@ class CmdCacheDir(CmdConfig): def add_parser(subparsers, parent_parser): from dvc.command.config import parent_config_parser - CACHE_HELP = 'Manage cache settings.' + CACHE_HELP = "Manage cache settings." cache_parser = subparsers.add_parser( - 'cache', - parents=[parent_parser], - description=CACHE_HELP, - help=CACHE_HELP) + "cache", parents=[parent_parser], description=CACHE_HELP, help=CACHE_HELP + ) cache_subparsers = cache_parser.add_subparsers( - dest='cmd', - help='Use dvc cache CMD --help for command-specific help.') + dest="cmd", help="Use dvc cache CMD --help for command-specific help." + ) fix_subparsers(cache_subparsers) parent_cache_config_parser = argparse.ArgumentParser( - add_help=False, - parents=[parent_config_parser]) - CACHE_DIR_HELP = 'Configure cache directory location.' + add_help=False, parents=[parent_config_parser] + ) + CACHE_DIR_HELP = "Configure cache directory location." cache_dir_parser = cache_subparsers.add_parser( - 'dir', + "dir", parents=[parent_cache_config_parser], description=CACHE_DIR_HELP, - help=CACHE_DIR_HELP) + help=CACHE_DIR_HELP, + ) cache_dir_parser.add_argument( - '-u', - '--unset', - default=False, - action='store_true', - help='Unset option.') + "-u", "--unset", default=False, action="store_true", help="Unset option." + ) cache_dir_parser.add_argument( - 'value', - nargs='?', + "value", + nargs="?", default=None, help="Path to cache directory. Relative paths are resolved relative " - "to the current directory and saved to config relative to the " - "config file location.") + "to the current directory and saved to config relative to the " + "config file location.", + ) cache_dir_parser.set_defaults(func=CmdCacheDir) diff --git a/dvc/command/checkout.py b/dvc/command/checkout.py index 61cabd15c..790a5dfe7 100644 --- a/dvc/command/checkout.py +++ b/dvc/command/checkout.py @@ -9,33 +9,33 @@ class CmdCheckout(CmdBase): self.project.checkout(force=self.args.force) else: for target in self.args.targets: - self.project.checkout(target=target, - with_deps=self.args.with_deps, - force=self.args.force) + self.project.checkout( + target=target, with_deps=self.args.with_deps, force=self.args.force + ) return 0 def add_parser(subparsers, parent_parser): - CHECKOUT_HELP = 'Checkout data files from cache.' + CHECKOUT_HELP = "Checkout data files from cache." checkout_parser = subparsers.add_parser( - 'checkout', + "checkout", parents=[parent_parser], description=CHECKOUT_HELP, - help=CHECKOUT_HELP) + help=CHECKOUT_HELP, + ) checkout_parser.add_argument( - '-d', - '--with-deps', - action='store_true', + "-d", + "--with-deps", + action="store_true", default=False, - help='Checkout all dependencies of the specified target.') + help="Checkout all dependencies of the specified target.", + ) checkout_parser.add_argument( - '-f', - '--force', - action='store_true', + "-f", + "--force", + action="store_true", default=False, - help='Do not prompt when removing working directory files.') - checkout_parser.add_argument( - 'targets', - nargs='*', - help='DVC files.') + help="Do not prompt when removing working directory files.", + ) + checkout_parser.add_argument("targets", nargs="*", help="DVC files.") checkout_parser.set_defaults(func=CmdCheckout) diff --git a/dvc/command/config.py b/dvc/command/config.py index 9d4c058e8..cdbede8e1 100644 --- a/dvc/command/config.py +++ b/dvc/command/config.py @@ -64,13 +64,12 @@ class CmdConfig(CmdBase): self.config.set(self.configobj, section, opt, value) self.config.save(self.configobj) except DvcException: - logger.error("failed to set '{}.{}' to '{}'" - .format(section, opt, value)) + logger.error("failed to set '{}.{}' to '{}'".format(section, opt, value)) return 1 return 0 def run(self): - section, opt = self.args.name.lower().strip().split('.', 1) + section, opt = self.args.name.lower().strip().split(".", 1) if self.args.unset: return self._unset(section, opt) @@ -82,42 +81,31 @@ class CmdConfig(CmdBase): parent_config_parser = argparse.ArgumentParser(add_help=False) parent_config_parser.add_argument( - '--global', - dest='glob', - action='store_true', - default=False, - help='Use global config.') + "--global", + dest="glob", + action="store_true", + default=False, + help="Use global config.", +) parent_config_parser.add_argument( - '--system', - action='store_true', - default=False, - help='Use system config.') + "--system", action="store_true", default=False, help="Use system config." +) parent_config_parser.add_argument( - '--local', - action='store_true', - default=False, - help='Use local config.') + "--local", action="store_true", default=False, help="Use local config." +) def add_parser(subparsers, parent_parser): - CONFIG_HELP = 'Get or set config options.' + CONFIG_HELP = "Get or set config options." config_parser = subparsers.add_parser( - 'config', + "config", parents=[parent_config_parser, parent_parser], description=CONFIG_HELP, - help=CONFIG_HELP) + help=CONFIG_HELP, + ) config_parser.add_argument( - '-u', - '--unset', - default=False, - action='store_true', - help='Unset option.') - config_parser.add_argument( - 'name', - help='Option name.') - config_parser.add_argument( - 'value', - nargs='?', - default=None, - help='Option value.') + "-u", "--unset", default=False, action="store_true", help="Unset option." + ) + config_parser.add_argument("name", help="Option name.") + config_parser.add_argument("value", nargs="?", default=None, help="Option value.") config_parser.set_defaults(func=CmdConfig) diff --git a/dvc/command/daemon.py b/dvc/command/daemon.py index a5c864313..70e056977 100644 --- a/dvc/command/daemon.py +++ b/dvc/command/daemon.py @@ -38,34 +38,32 @@ class CmdDaemonAnalytics(CmdDaemonBase): def add_parser(subparsers, parent_parser): - DAEMON_HELP = 'Service daemon.' + DAEMON_HELP = "Service daemon." daemon_parser = subparsers.add_parser( - 'daemon', - parents=[parent_parser], - description=DAEMON_HELP, - help=DAEMON_HELP) + "daemon", parents=[parent_parser], description=DAEMON_HELP, help=DAEMON_HELP + ) daemon_subparsers = daemon_parser.add_subparsers( - dest='cmd', - help='Use dvc daemon CMD --help for command-specific help.') + dest="cmd", help="Use dvc daemon CMD --help for command-specific help." + ) fix_subparsers(daemon_subparsers) - DAEMON_UPDATER_HELP = 'Fetch latest available version.' + DAEMON_UPDATER_HELP = "Fetch latest available version." daemon_updater_parser = daemon_subparsers.add_parser( - 'updater', + "updater", parents=[parent_parser], description=DAEMON_UPDATER_HELP, - help=DAEMON_UPDATER_HELP) + help=DAEMON_UPDATER_HELP, + ) daemon_updater_parser.set_defaults(func=CmdDaemonUpdater) - DAEMON_ANALYTICS_HELP = 'Send dvc usage analytics.' + DAEMON_ANALYTICS_HELP = "Send dvc usage analytics." daemon_analytics_parser = daemon_subparsers.add_parser( - 'analytics', + "analytics", parents=[parent_parser], description=DAEMON_ANALYTICS_HELP, - help=DAEMON_ANALYTICS_HELP) - daemon_analytics_parser.add_argument( - 'target', - help="Analytics file.") + help=DAEMON_ANALYTICS_HELP, + ) + daemon_analytics_parser.add_argument("target", help="Analytics file.") daemon_analytics_parser.set_defaults(func=CmdDaemonAnalytics) diff --git a/dvc/command/data_sync.py b/dvc/command/data_sync.py index e1e3eec61..ac5037895 100644 --- a/dvc/command/data_sync.py +++ b/dvc/command/data_sync.py @@ -24,17 +24,19 @@ class CmdDataBase(CmdBase): class CmdDataPull(CmdDataBase): def do_run(self, target=None): try: - self.project.pull(target=target, - jobs=self.args.jobs, - remote=self.args.remote, - show_checksums=self.args.show_checksums, - all_branches=self.args.all_branches, - all_tags=self.args.all_tags, - with_deps=self.args.with_deps, - force=self.args.force, - recursive=self.args.recursive) + self.project.pull( + target=target, + jobs=self.args.jobs, + remote=self.args.remote, + show_checksums=self.args.show_checksums, + all_branches=self.args.all_branches, + all_tags=self.args.all_tags, + with_deps=self.args.with_deps, + force=self.args.force, + recursive=self.args.recursive, + ) except Exception: - logger.error('failed to pull data from the cloud') + logger.error("failed to pull data from the cloud") return 1 return 0 @@ -42,16 +44,18 @@ class CmdDataPull(CmdDataBase): class CmdDataPush(CmdDataBase): def do_run(self, target=None): try: - self.project.push(target=target, - jobs=self.args.jobs, - remote=self.args.remote, - show_checksums=self.args.show_checksums, - all_branches=self.args.all_branches, - all_tags=self.args.all_tags, - with_deps=self.args.with_deps, - recursive=self.args.recursive) + self.project.push( + target=target, + jobs=self.args.jobs, + remote=self.args.remote, + show_checksums=self.args.show_checksums, + all_branches=self.args.all_branches, + all_tags=self.args.all_tags, + with_deps=self.args.with_deps, + recursive=self.args.recursive, + ) except Exception: - logger.error('failed to push data to the cloud') + logger.error("failed to push data to the cloud") return 1 return 0 @@ -59,16 +63,18 @@ class CmdDataPush(CmdDataBase): class CmdDataFetch(CmdDataBase): def do_run(self, target=None): try: - self.project.fetch(target=target, - jobs=self.args.jobs, - remote=self.args.remote, - show_checksums=self.args.show_checksums, - all_branches=self.args.all_branches, - all_tags=self.args.all_tags, - with_deps=self.args.with_deps, - recursive=self.args.recursive) + self.project.fetch( + target=target, + jobs=self.args.jobs, + remote=self.args.remote, + show_checksums=self.args.show_checksums, + all_branches=self.args.all_branches, + all_tags=self.args.all_tags, + with_deps=self.args.with_deps, + recursive=self.args.recursive, + ) except Exception: - logger.error('failed to fetch data from the cloud') + logger.error("failed to fetch data from the cloud") return 1 return 0 @@ -78,178 +84,179 @@ def add_parser(subparsers, parent_parser): # Parent parser used in pull/push/status parent_cache_parser = argparse.ArgumentParser( - add_help=False, - parents=[parent_parser]) + add_help=False, parents=[parent_parser] + ) parent_cache_parser.add_argument( - '-j', - '--jobs', + "-j", + "--jobs", type=int, default=None, - help='Number of jobs to run simultaneously.') + help="Number of jobs to run simultaneously.", + ) parent_cache_parser.add_argument( - '--show-checksums', - action='store_true', + "--show-checksums", + action="store_true", default=False, - help='Show checksums instead of file names.') + help="Show checksums instead of file names.", + ) parent_cache_parser.add_argument( - 'targets', - nargs='*', - default=None, - help='DVC files.') + "targets", nargs="*", default=None, help="DVC files." + ) # Pull - PULL_HELP = 'Pull data files from the cloud.' + PULL_HELP = "Pull data files from the cloud." pull_parser = subparsers.add_parser( - 'pull', - parents=[parent_cache_parser], - description=PULL_HELP, - help=PULL_HELP) + "pull", parents=[parent_cache_parser], description=PULL_HELP, help=PULL_HELP + ) + pull_parser.add_argument("-r", "--remote", help="Remote repository to pull from.") pull_parser.add_argument( - '-r', - '--remote', - help='Remote repository to pull from.') - pull_parser.add_argument( - '-a', - '--all-branches', - action='store_true', + "-a", + "--all-branches", + action="store_true", default=False, - help='Fetch cache for all branches.') + help="Fetch cache for all branches.", + ) pull_parser.add_argument( - '-T', - '--all-tags', - action='store_true', + "-T", + "--all-tags", + action="store_true", default=False, - help='Fetch cache for all tags.') + help="Fetch cache for all tags.", + ) pull_parser.add_argument( - '-d', - '--with-deps', - action='store_true', + "-d", + "--with-deps", + action="store_true", default=False, - help='Fetch cache for all dependencies of the specified target.') + help="Fetch cache for all dependencies of the specified target.", + ) pull_parser.add_argument( - '-f', - '--force', - action='store_true', + "-f", + "--force", + action="store_true", default=False, - help='Do not prompt when removing working directory files.') + help="Do not prompt when removing working directory files.", + ) pull_parser.add_argument( - '-R', - '--recursive', - action='store_true', + "-R", + "--recursive", + action="store_true", default=False, - help='Pull cache for subdirectories of the specified directory.') + help="Pull cache for subdirectories of the specified directory.", + ) pull_parser.set_defaults(func=CmdDataPull) # Push - PUSH_HELP = 'Push data files to the cloud.' + PUSH_HELP = "Push data files to the cloud." push_parser = subparsers.add_parser( - 'push', - parents=[parent_cache_parser], - description=PUSH_HELP, - help=PUSH_HELP) + "push", parents=[parent_cache_parser], description=PUSH_HELP, help=PUSH_HELP + ) + push_parser.add_argument("-r", "--remote", help="Remote repository to push to.") push_parser.add_argument( - '-r', - '--remote', - help='Remote repository to push to.') - push_parser.add_argument( - '-a', - '--all-branches', - action='store_true', + "-a", + "--all-branches", + action="store_true", default=False, - help='Push cache for all branches.') + help="Push cache for all branches.", + ) push_parser.add_argument( - '-T', - '--all-tags', - action='store_true', + "-T", + "--all-tags", + action="store_true", default=False, - help='Push cache for all tags.') + help="Push cache for all tags.", + ) push_parser.add_argument( - '-d', - '--with-deps', - action='store_true', + "-d", + "--with-deps", + action="store_true", default=False, - help='Push cache for all dependencies of the specified target.') + help="Push cache for all dependencies of the specified target.", + ) push_parser.add_argument( - '-R', - '--recursive', - action='store_true', + "-R", + "--recursive", + action="store_true", default=False, - help='Push cache from subdirectories of specified directory.') + help="Push cache from subdirectories of specified directory.", + ) push_parser.set_defaults(func=CmdDataPush) # Fetch - FETCH_HELP = 'Fetch data files from the cloud.' + FETCH_HELP = "Fetch data files from the cloud." fetch_parser = subparsers.add_parser( - 'fetch', - parents=[parent_cache_parser], - description=FETCH_HELP, - help=FETCH_HELP) + "fetch", parents=[parent_cache_parser], description=FETCH_HELP, help=FETCH_HELP + ) + fetch_parser.add_argument("-r", "--remote", help="Remote repository to fetch from.") fetch_parser.add_argument( - '-r', - '--remote', - help='Remote repository to fetch from.') - fetch_parser.add_argument( - '-a', - '--all-branches', - action='store_true', + "-a", + "--all-branches", + action="store_true", default=False, - help='Fetch cache for all branches.') + help="Fetch cache for all branches.", + ) fetch_parser.add_argument( - '-T', - '--all-tags', - action='store_true', + "-T", + "--all-tags", + action="store_true", default=False, - help='Fetch cache for all tags.') + help="Fetch cache for all tags.", + ) fetch_parser.add_argument( - '-d', - '--with-deps', - action='store_true', + "-d", + "--with-deps", + action="store_true", default=False, - help='Fetch cache for all dependencies of the ' - 'specified target.') + help="Fetch cache for all dependencies of the " "specified target.", + ) fetch_parser.add_argument( - '-R', - '--recursive', - action='store_true', + "-R", + "--recursive", + action="store_true", default=False, - help='Fetch cache for subdirectories of specified directory.') + help="Fetch cache for subdirectories of specified directory.", + ) fetch_parser.set_defaults(func=CmdDataFetch) # Status - STATUS_HELP = 'Show the project status.' + STATUS_HELP = "Show the project status." status_parser = subparsers.add_parser( - 'status', + "status", parents=[parent_cache_parser], description=STATUS_HELP, - help=STATUS_HELP) + help=STATUS_HELP, + ) status_parser.add_argument( - '-c', - '--cloud', - action='store_true', + "-c", + "--cloud", + action="store_true", default=False, - help='Show status of a local cache compared to a remote repository.') + help="Show status of a local cache compared to a remote repository.", + ) status_parser.add_argument( - '-r', - '--remote', - help='Remote repository to compare local cache to.') + "-r", "--remote", help="Remote repository to compare local cache to." + ) status_parser.add_argument( - '-a', - '--all-branches', - action='store_true', + "-a", + "--all-branches", + action="store_true", default=False, - help='Show status of a local cache compared to a remote repository ' - 'for all branches.') + help="Show status of a local cache compared to a remote repository " + "for all branches.", + ) status_parser.add_argument( - '-T', - '--all-tags', - action='store_true', + "-T", + "--all-tags", + action="store_true", default=False, - help='Show status of a local cache compared to a remote repository ' - 'for all tags.') + help="Show status of a local cache compared to a remote repository " + "for all tags.", + ) status_parser.add_argument( - '-d', - '--with-deps', - action='store_true', + "-d", + "--with-deps", + action="store_true", default=False, - help='Show status for all dependencies of the specified target.') + help="Show status for all dependencies of the specified target.", + ) status_parser.set_defaults(func=CmdDataStatus) diff --git a/dvc/command/destroy.py b/dvc/command/destroy.py index 6003decde..e9eea5998 100644 --- a/dvc/command/destroy.py +++ b/dvc/command/destroy.py @@ -10,10 +10,10 @@ class CmdDestroy(CmdBase): def run_cmd(self): try: statement = ( - 'This will destroy all information about your pipelines,' - ' all data files, as well as cache in .dvc/cache.' - '\n' - 'Are you sure you want to continue?' + "This will destroy all information about your pipelines," + " all data files, as well as cache in .dvc/cache." + "\n" + "Are you sure you want to continue?" ) if not self.args.force and not prompt.confirm(statement): @@ -24,23 +24,19 @@ class CmdDestroy(CmdBase): self.project.destroy() except Exception: - logger.error('failed to destroy DVC') + logger.error("failed to destroy DVC") return 1 return 0 def add_parser(subparsers, parent_parser): - DESTROY_HELP = "Destroy dvc. Will remove all project's information, " \ - "data files and cache." + DESTROY_HELP = ( + "Destroy dvc. Will remove all project's information, " "data files and cache." + ) destroy_parser = subparsers.add_parser( - 'destroy', - parents=[parent_parser], - description=DESTROY_HELP, - help=DESTROY_HELP) + "destroy", parents=[parent_parser], description=DESTROY_HELP, help=DESTROY_HELP + ) destroy_parser.add_argument( - '-f', - '--force', - action='store_true', - default=False, - help='Force destruction.') + "-f", "--force", action="store_true", default=False, help="Force destruction." + ) destroy_parser.set_defaults(func=CmdDestroy) diff --git a/dvc/command/gc.py b/dvc/command/gc.py index 3e74f66dd..7112d8f98 100644 --- a/dvc/command/gc.py +++ b/dvc/command/gc.py @@ -9,86 +9,91 @@ from dvc.command.base import CmdBase class CmdGC(CmdBase): def run(self): - msg = 'this will remove all cache except the cache that is used in ' + msg = "this will remove all cache except the cache that is used in " if not self.args.all_branches and not self.args.all_tags: - msg += 'the current git branch' + msg += "the current git branch" elif self.args.all_branches and not self.args.all_tags: - msg += 'all git branches' + msg += "all git branches" elif not self.args.all_branches and self.args.all_tags: - msg += 'all git tags' + msg += "all git tags" else: - msg += 'all git branches and all git tags' + msg += "all git branches and all git tags" if self.args.projects is not None and len(self.args.projects) > 0: - msg += ' of the current and the following projects:' + msg += " of the current and the following projects:" for project_path in self.args.projects: - msg += '\n - %s' % os.path.abspath(project_path) + msg += "\n - %s" % os.path.abspath(project_path) else: - msg += ' of the current project.' + msg += " of the current project." logger.warning(msg) - msg = 'Are you sure you want to proceed?' + msg = "Are you sure you want to proceed?" if not self.args.force and not prompt.confirm(msg): return 1 - self.project.gc(all_branches=self.args.all_branches, - all_tags=self.args.all_tags, - cloud=self.args.cloud, - remote=self.args.remote, - force=self.args.force, - jobs=self.args.jobs, - projects=self.args.projects) + self.project.gc( + all_branches=self.args.all_branches, + all_tags=self.args.all_tags, + cloud=self.args.cloud, + remote=self.args.remote, + force=self.args.force, + jobs=self.args.jobs, + projects=self.args.projects, + ) return 0 def add_parser(subparsers, parent_parser): - GC_HELP = 'Collect garbage.' + GC_HELP = "Collect garbage." gc_parser = subparsers.add_parser( - 'gc', - parents=[parent_parser], - description=GC_HELP, - help=GC_HELP) + "gc", parents=[parent_parser], description=GC_HELP, help=GC_HELP + ) gc_parser.add_argument( - '-a', - '--all-branches', - action='store_true', + "-a", + "--all-branches", + action="store_true", default=False, - help='Collect garbage for all branches.') + help="Collect garbage for all branches.", + ) gc_parser.add_argument( - '-T', - '--all-tags', - action='store_true', + "-T", + "--all-tags", + action="store_true", default=False, - help='Collect garbage for all tags.') + help="Collect garbage for all tags.", + ) gc_parser.add_argument( - '-c', - '--cloud', - action='store_true', + "-c", + "--cloud", + action="store_true", default=False, - help='Collect garbage in remote repository.') + help="Collect garbage in remote repository.", + ) gc_parser.add_argument( - '-r', - '--remote', - help='Remote repository to collect garbage in.') + "-r", "--remote", help="Remote repository to collect garbage in." + ) gc_parser.add_argument( - '-f', - '--force', - action='store_true', + "-f", + "--force", + action="store_true", default=False, - help='Force garbage collection.') + help="Force garbage collection.", + ) gc_parser.add_argument( - '-j', - '--jobs', + "-j", + "--jobs", type=int, default=None, - help='Number of jobs to run simultaneously.') + help="Number of jobs to run simultaneously.", + ) gc_parser.add_argument( - '-p', - '--projects', + "-p", + "--projects", type=str, - nargs='*', + nargs="*", default=None, - help='Collect garbage for all given projects.') + help="Collect garbage for all given projects.", + ) gc_parser.set_defaults(func=CmdGC) diff --git a/dvc/command/imp.py b/dvc/command/imp.py index 35f8b93c5..37611e56f 100644 --- a/dvc/command/imp.py +++ b/dvc/command/imp.py @@ -11,30 +11,29 @@ from dvc.command.base import CmdBase class CmdImport(CmdBase): def run(self): try: - default_out = os.path.basename( - urlparse(self.args.url).path - ) + default_out = os.path.basename(urlparse(self.args.url).path) out = self.args.out or default_out self.project.imp(self.args.url, out) except DvcException: - logger.error('failed to import {}. You could also try downloading ' - 'it manually and adding it with `dvc add` command.' - .format(self.args.url)) + logger.error( + "failed to import {}. You could also try downloading " + "it manually and adding it with `dvc add` command.".format( + self.args.url + ) + ) return 1 return 0 def add_parser(subparsers, parent_parser): - IMPORT_HELP = 'Import files from URL.' + IMPORT_HELP = "Import files from URL." import_parser = subparsers.add_parser( - 'import', - parents=[parent_parser], - description=IMPORT_HELP, - help=IMPORT_HELP) + "import", parents=[parent_parser], description=IMPORT_HELP, help=IMPORT_HELP + ) import_parser.add_argument( - 'url', + "url", help="URL. Supported urls: " "'/path/to/file', " "'C:\\\\path\\to\\file', " @@ -44,9 +43,7 @@ def add_parser(subparsers, parent_parser): "'hdfs://example.com/path/to/file', " "'ssh://example.com:/path/to/file', " "'remote://myremote/path/to/file'(see " - "`dvc remote` commands). ") - import_parser.add_argument( - 'out', - nargs='?', - help='Output.') + "`dvc remote` commands). ", + ) + import_parser.add_argument("out", nargs="?", help="Output.") import_parser.set_defaults(func=CmdImport) diff --git a/dvc/command/init.py b/dvc/command/init.py index b09daf440..2eff8d0d1 100644 --- a/dvc/command/init.py +++ b/dvc/command/init.py @@ -11,35 +11,34 @@ class CmdInit(object): from dvc.project import Project, InitError try: - self.project = Project.init('.', - no_scm=self.args.no_scm, - force=self.args.force) + self.project = Project.init( + ".", no_scm=self.args.no_scm, force=self.args.force + ) self.config = self.project.config except InitError: - logger.error('failed to initiate dvc') + logger.error("failed to initiate dvc") return 1 return 0 def add_parser(subparsers, parent_parser): """Setup parser for `dvc init`.""" - INIT_HELP = 'Initialize dvc over a directory ' \ - '(should already be a git dir).' + INIT_HELP = "Initialize dvc over a directory " "(should already be a git dir)." init_parser = subparsers.add_parser( - 'init', - parents=[parent_parser], - description=INIT_HELP, - help=INIT_HELP) + "init", parents=[parent_parser], description=INIT_HELP, help=INIT_HELP + ) init_parser.add_argument( - '--no-scm', - action='store_true', + "--no-scm", + action="store_true", default=False, help="Initiate dvc in directory that is " - "not tracked by any scm tool (e.g. git).") + "not tracked by any scm tool (e.g. git).", + ) init_parser.add_argument( - '-f', - '--force', - action='store_true', + "-f", + "--force", + action="store_true", default=False, - help="Overwrite '.dvc' if it exists. Will remove all local cache.") + help="Overwrite '.dvc' if it exists. Will remove all local cache.", + ) init_parser.set_defaults(func=CmdInit) diff --git a/dvc/command/install.py b/dvc/command/install.py index 89f2ee7e5..3fc5e554a 100644 --- a/dvc/command/install.py +++ b/dvc/command/install.py @@ -9,16 +9,14 @@ class CmdInstall(CmdBase): try: self.project.install() except Exception: - logger.error('failed to install dvc hooks') + logger.error("failed to install dvc hooks") return 1 return 0 def add_parser(subparsers, parent_parser): - INSTALL_HELP = 'Install dvc hooks into the repository.' + INSTALL_HELP = "Install dvc hooks into the repository." install_parser = subparsers.add_parser( - 'install', - parents=[parent_parser], - description=INSTALL_HELP, - help=INSTALL_HELP) + "install", parents=[parent_parser], description=INSTALL_HELP, help=INSTALL_HELP + ) install_parser.set_defaults(func=CmdInstall) diff --git a/dvc/command/lock.py b/dvc/command/lock.py index 29109f973..969cd9f26 100644 --- a/dvc/command/lock.py +++ b/dvc/command/lock.py @@ -12,8 +12,7 @@ class CmdLockBase(CmdBase): self.project.lock_stage(target, unlock=unlock) except DvcException: logger.error( - "failed to {}lock '{}'" - .format('un' if unlock else '', target) + "failed to {}lock '{}'".format("un" if unlock else "", target) ) return 1 @@ -31,26 +30,16 @@ class CmdUnlock(CmdLockBase): def add_parser(subparsers, parent_parser): - LOCK_HELP = 'Lock DVC file.' + LOCK_HELP = "Lock DVC file." lock_parser = subparsers.add_parser( - 'lock', - parents=[parent_parser], - description=LOCK_HELP, - help=LOCK_HELP) - lock_parser.add_argument( - 'targets', - nargs='+', - help='DVC files.') + "lock", parents=[parent_parser], description=LOCK_HELP, help=LOCK_HELP + ) + lock_parser.add_argument("targets", nargs="+", help="DVC files.") lock_parser.set_defaults(func=CmdLock) - UNLOCK_HELP = 'Unlock DVC file.' + UNLOCK_HELP = "Unlock DVC file." unlock_parser = subparsers.add_parser( - 'unlock', - parents=[parent_parser], - description=UNLOCK_HELP, - help=UNLOCK_HELP) - unlock_parser.add_argument( - 'targets', - nargs='+', - help='DVC files.') + "unlock", parents=[parent_parser], description=UNLOCK_HELP, help=UNLOCK_HELP + ) + unlock_parser.add_argument("targets", nargs="+", help="DVC files.") unlock_parser.set_defaults(func=CmdUnlock) diff --git a/dvc/command/metrics.py b/dvc/command/metrics.py index a748d0b00..285ec3a84 100644 --- a/dvc/command/metrics.py +++ b/dvc/command/metrics.py @@ -10,31 +10,33 @@ class CmdMetricsShow(CmdBase): try: # backward compatibility if self.args.json_path: - typ = 'json' + typ = "json" xpath = self.args.json_path elif self.args.tsv_path: - typ = 'tsv' + typ = "tsv" xpath = self.args.tsv_path elif self.args.htsv_path: - typ = 'htsv' + typ = "htsv" xpath = self.args.htsv_path elif self.args.csv_path: - typ = 'csv' + typ = "csv" xpath = self.args.csv_path elif self.args.hcsv_path: - typ = 'hcsv' + typ = "hcsv" xpath = self.args.hcsv_path else: typ = self.args.type xpath = self.args.xpath - self.project.metrics_show(self.args.path, - typ=typ, - xpath=xpath, - all_branches=self.args.all_branches, - all_tags=self.args.all_tags) + self.project.metrics_show( + self.args.path, + typ=typ, + xpath=xpath, + all_branches=self.args.all_branches, + all_tags=self.args.all_tags, + ) except DvcException: - logger.error('failed to show metrics') + logger.error("failed to show metrics") return 1 return 0 @@ -43,11 +45,11 @@ class CmdMetricsShow(CmdBase): class CmdMetricsModify(CmdBase): def run(self): try: - self.project.metrics_modify(self.args.path, - typ=self.args.type, - xpath=self.args.xpath) + self.project.metrics_modify( + self.args.path, typ=self.args.type, xpath=self.args.xpath + ) except DvcException: - logger.error('failed to modify metrics') + logger.error("failed to modify metrics") return 1 return 0 @@ -56,9 +58,7 @@ class CmdMetricsModify(CmdBase): class CmdMetricsAdd(CmdBase): def run(self): try: - self.project.metrics_add(self.args.path, - self.args.type, - self.args.xpath) + self.project.metrics_add(self.args.path, self.args.type, self.args.xpath) except DvcException: msg = "failed to add metric file '{}'".format(self.args.path) logger.error(msg) @@ -80,112 +80,99 @@ class CmdMetricsRemove(CmdBase): def add_parser(subparsers, parent_parser): - METRICS_HELP = 'Get metrics from all branches.' + METRICS_HELP = "Get metrics from all branches." metrics_parser = subparsers.add_parser( - 'metrics', - parents=[parent_parser], - description=METRICS_HELP, - help=METRICS_HELP) + "metrics", parents=[parent_parser], description=METRICS_HELP, help=METRICS_HELP + ) metrics_subparsers = metrics_parser.add_subparsers( - dest='cmd', - help="Use dvc metrics CMD --help for command-specific help.") + dest="cmd", help="Use dvc metrics CMD --help for command-specific help." + ) fix_subparsers(metrics_subparsers) - METRICS_SHOW_HELP = 'Show metrics.' + METRICS_SHOW_HELP = "Show metrics." metrics_show_parser = metrics_subparsers.add_parser( - 'show', + "show", parents=[parent_parser], description=METRICS_SHOW_HELP, - help=METRICS_SHOW_HELP) + help=METRICS_SHOW_HELP, + ) + metrics_show_parser.add_argument("path", nargs="?", help="Path to metrics file.") metrics_show_parser.add_argument( - 'path', - nargs='?', - help='Path to metrics file.') + "-t", "--type", help="Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV)." + ) metrics_show_parser.add_argument( - '-t', - '--type', - help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV).') - metrics_show_parser.add_argument( - '-x', - '--xpath', - help='JSON/TSV/HTSV/CSV/HCSV path.') + "-x", "--xpath", help="JSON/TSV/HTSV/CSV/HCSV path." + ) metrics_show_group = metrics_show_parser.add_mutually_exclusive_group() + metrics_show_group.add_argument("--json-path", help="JSON path.") metrics_show_group.add_argument( - '--json-path', - help='JSON path.') + "--tsv-path", help="TSV path 'row,column' (e.g. '1,2')." + ) metrics_show_group.add_argument( - '--tsv-path', - help="TSV path 'row,column' (e.g. '1,2').") + "--htsv-path", help="Headed TSV path 'row,column (e.g. 'Name,3')." + ) metrics_show_group.add_argument( - '--htsv-path', - help="Headed TSV path 'row,column (e.g. 'Name,3').") + "--csv-path", help="CSV path 'row,column' (e.g. '1,2')." + ) metrics_show_group.add_argument( - '--csv-path', - help="CSV path 'row,column' (e.g. '1,2').") - metrics_show_group.add_argument( - '--hcsv-path', - help="Headed CSV path 'row,column' (e.g. 'Name,3').") + "--hcsv-path", help="Headed CSV path 'row,column' (e.g. 'Name,3')." + ) metrics_show_parser.add_argument( - '-a', - '--all-branches', - action='store_true', + "-a", + "--all-branches", + action="store_true", default=False, - help='Show metrics for all branches.') + help="Show metrics for all branches.", + ) metrics_show_parser.add_argument( - '-T', - '--all-tags', - action='store_true', + "-T", + "--all-tags", + action="store_true", default=False, - help='Show metrics for all tags.') + help="Show metrics for all tags.", + ) metrics_show_parser.set_defaults(func=CmdMetricsShow) - METRICS_ADD_HELP = 'Add metrics.' + METRICS_ADD_HELP = "Add metrics." metrics_add_parser = metrics_subparsers.add_parser( - 'add', + "add", parents=[parent_parser], description=METRICS_ADD_HELP, - help=METRICS_ADD_HELP) + help=METRICS_ADD_HELP, + ) metrics_add_parser.add_argument( - '-t', - '--type', - help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV).') + "-t", "--type", help="Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV)." + ) metrics_add_parser.add_argument( - '-x', - '--xpath', - help='JSON/TSV/HTSV/CSV/HCSV path.') - metrics_add_parser.add_argument( - 'path', - help='Path to metrics file.') + "-x", "--xpath", help="JSON/TSV/HTSV/CSV/HCSV path." + ) + metrics_add_parser.add_argument("path", help="Path to metrics file.") metrics_add_parser.set_defaults(func=CmdMetricsAdd) - METRICS_MODIFY_HELP = 'Modify metrics.' + METRICS_MODIFY_HELP = "Modify metrics." metrics_modify_parser = metrics_subparsers.add_parser( - 'modify', + "modify", parents=[parent_parser], description=METRICS_MODIFY_HELP, - help=METRICS_MODIFY_HELP) + help=METRICS_MODIFY_HELP, + ) metrics_modify_parser.add_argument( - '-t', - '--type', - help='Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV).') + "-t", "--type", help="Type of metrics(RAW/JSON/TSV/HTSV/CSV/HCSV)." + ) metrics_modify_parser.add_argument( - '-x', - '--xpath', - help='JSON/TSV/HTSV/CSV/HCSV path.') - metrics_modify_parser.add_argument( - 'path', - help='Metrics file.') + "-x", "--xpath", help="JSON/TSV/HTSV/CSV/HCSV path." + ) + metrics_modify_parser.add_argument("path", help="Metrics file.") metrics_modify_parser.set_defaults(func=CmdMetricsModify) - METRICS_REMOVE_HELP = 'Remove metrics.' + METRICS_REMOVE_HELP = "Remove metrics." metrics_remove_parser = metrics_subparsers.add_parser( - 'remove', + "remove", parents=[parent_parser], description=METRICS_REMOVE_HELP, - help=METRICS_REMOVE_HELP) - metrics_remove_parser.add_argument( - 'path', - help='Path to metrics file.') + help=METRICS_REMOVE_HELP, + ) + metrics_remove_parser.add_argument("path", help="Path to metrics file.") metrics_remove_parser.set_defaults(func=CmdMetricsRemove) diff --git a/dvc/command/move.py b/dvc/command/move.py index 606ed2430..2715cac3a 100644 --- a/dvc/command/move.py +++ b/dvc/command/move.py @@ -10,24 +10,22 @@ class CmdMove(CmdBase): try: self.project.move(self.args.src, self.args.dst) except DvcException: - msg = "failed to move '{}' -> '{}'".format(self.args.src, - self.args.dst) + msg = "failed to move '{}' -> '{}'".format(self.args.src, self.args.dst) logger.error(msg) return 1 return 0 def add_parser(subparsers, parent_parser): - description = "Rename or move a data file or a directory that "\ - "is under DVC control. It renames and modifies "\ - "the corresponding DVC file to reflect the changes." + description = ( + "Rename or move a data file or a directory that " + "is under DVC control. It renames and modifies " + "the corresponding DVC file to reflect the changes." + ) help = "Rename or move a DVC controlled data file or a directory." move_parser = subparsers.add_parser( - 'move', - parents=[parent_parser], - description=description, - help=help) - move_parser.add_argument('src', - help='Source path to a data file or directory.') - move_parser.add_argument('dst', help='Destination path.') + "move", parents=[parent_parser], description=description, help=help + ) + move_parser.add_argument("src", help="Source path to a data file or directory.") + move_parser.add_argument("dst", help="Destination path.") move_parser.set_defaults(func=CmdMove) diff --git a/dvc/command/pipeline.py b/dvc/command/pipeline.py index 3c2bfd0fd..8927d1986 100644 --- a/dvc/command/pipeline.py +++ b/dvc/command/pipeline.py @@ -16,7 +16,7 @@ class CmdPipelineShow(CmdBase): stage = Stage.load(self.project, target) G = self.project.graph()[0] - stages = networkx.get_node_attributes(G, 'stage') + stages = networkx.get_node_attributes(G, "stage") node = os.path.relpath(stage.path, self.project.root_dir) for n in networkx.dfs_postorder_nodes(G, node): @@ -35,12 +35,11 @@ class CmdPipelineShow(CmdBase): stage = Stage.load(self.project, target) node = os.path.relpath(stage.path, self.project.root_dir) - pipelines = list(filter(lambda g: node in g.nodes(), - self.project.pipelines())) + pipelines = list(filter(lambda g: node in g.nodes(), self.project.pipelines())) assert len(pipelines) == 1 G = pipelines[0] - stages = networkx.get_node_attributes(G, 'stage') + stages = networkx.get_node_attributes(G, "stage") nodes = [] for n in G.nodes(): @@ -66,8 +65,7 @@ class CmdPipelineShow(CmdBase): elif outs: for from_out in from_stage.outs: for to_out in to_stage.outs: - edges.append((str(from_out), - str(to_out))) + edges.append((str(from_out), str(to_out))) else: edges.append((from_stage.relpath, to_stage.relpath)) @@ -101,18 +99,13 @@ class CmdPipelineShow(CmdBase): for target in self.args.targets: try: if self.args.ascii: - self._show_ascii(target, - self.args.commands, - self.args.outs) + self._show_ascii(target, self.args.commands, self.args.outs) elif self.args.dot: - self.__write_dot(target, - self.args.commands, - self.args.outs, - self.args.dot) + self.__write_dot( + target, self.args.commands, self.args.outs, self.args.dot + ) else: - self._show(target, - self.args.commands, - self.args.outs) + self._show(target, self.args.commands, self.args.outs) except DvcException: msg = "failed to show pipeline for '{}'".format(target) logger.error(msg) @@ -126,67 +119,67 @@ class CmdPipelineList(CmdBase): pipelines = self.project.pipelines() for p in pipelines: - stages = networkx.get_node_attributes(p, 'stage') + stages = networkx.get_node_attributes(p, "stage") for stage in stages: logger.info(stage) if len(stages) != 0: - logger.info('=' * 80) - logger.info('{} pipeline(s) total'.format(len(pipelines))) + logger.info("=" * 80) + logger.info("{} pipeline(s) total".format(len(pipelines))) return 0 def add_parser(subparsers, parent_parser): - PIPELINE_HELP = 'Manage pipeline.' + PIPELINE_HELP = "Manage pipeline." pipeline_parser = subparsers.add_parser( - 'pipeline', + "pipeline", parents=[parent_parser], description=PIPELINE_HELP, - help=PIPELINE_HELP) + help=PIPELINE_HELP, + ) pipeline_subparsers = pipeline_parser.add_subparsers( - dest='cmd', - help="Use dvc pipeline CMD --help for command-specific help.") + dest="cmd", help="Use dvc pipeline CMD --help for command-specific help." + ) fix_subparsers(pipeline_subparsers) - PIPELINE_SHOW_HELP = 'Show pipeline.' + PIPELINE_SHOW_HELP = "Show pipeline." pipeline_show_parser = pipeline_subparsers.add_parser( - 'show', + "show", parents=[parent_parser], description=PIPELINE_SHOW_HELP, - help=PIPELINE_SHOW_HELP) + help=PIPELINE_SHOW_HELP, + ) pipeline_show_group = pipeline_show_parser.add_mutually_exclusive_group() pipeline_show_group.add_argument( - '-c', - '--commands', - action='store_true', + "-c", + "--commands", + action="store_true", default=False, - help='Print commands instead of paths to DVC files.') + help="Print commands instead of paths to DVC files.", + ) pipeline_show_group.add_argument( - '-o', - '--outs', - action='store_true', + "-o", + "--outs", + action="store_true", default=False, - help='Print output files instead of paths to DVC files.') + help="Print output files instead of paths to DVC files.", + ) pipeline_show_parser.add_argument( - '--ascii', - action='store_true', - default=False, - help='Output DAG as ASCII.') + "--ascii", action="store_true", default=False, help="Output DAG as ASCII." + ) + pipeline_show_parser.add_argument("--dot", help="Write DAG in .dot format.") pipeline_show_parser.add_argument( - '--dot', - help='Write DAG in .dot format.') - pipeline_show_parser.add_argument( - 'targets', - nargs='*', - help="DVC files. 'Dvcfile' by default.") + "targets", nargs="*", help="DVC files. 'Dvcfile' by default." + ) pipeline_show_parser.set_defaults(func=CmdPipelineShow) - PIPELINE_LIST_HELP = 'List pipelines.' + PIPELINE_LIST_HELP = "List pipelines." pipeline_list_parser = pipeline_subparsers.add_parser( - 'list', + "list", parents=[parent_parser], description=PIPELINE_LIST_HELP, - help=PIPELINE_LIST_HELP) + help=PIPELINE_LIST_HELP, + ) pipeline_list_parser.set_defaults(func=CmdPipelineList) diff --git a/dvc/command/remote.py b/dvc/command/remote.py index 48079cbd0..7e010ddf6 100644 --- a/dvc/command/remote.py +++ b/dvc/command/remote.py @@ -32,8 +32,7 @@ class CmdRemoteAdd(CmdConfig): def run(self): remote = _get({Config.SECTION_REMOTE_URL: self.args.url}) if remote == RemoteLOCAL: - self.args.url = self.resolve_path(self.args.url, - self.configobj.filename) + self.args.url = self.resolve_path(self.args.url, self.configobj.filename) section = Config.SECTION_REMOTE_FMT.format(self.args.name) ret = self._set(section, Config.SECTION_REMOTE_URL, self.args.url) @@ -43,9 +42,9 @@ class CmdRemoteAdd(CmdConfig): if self.args.default: msg = "Setting '{}' as a default remote.".format(self.args.name) logger.info(msg) - ret = self._set(Config.SECTION_CORE, - Config.SECTION_CORE_REMOTE, - self.args.name) + ret = self._set( + Config.SECTION_CORE, Config.SECTION_CORE_REMOTE, self.args.name + ) return ret @@ -61,9 +60,9 @@ class CmdRemoteRemove(CmdConfig): return 0 if default == self.args.name: - return self._unset(Config.SECTION_CORE, - opt=Config.SECTION_CORE_REMOTE, - configobj=config) + return self._unset( + Config.SECTION_CORE, opt=Config.SECTION_CORE_REMOTE, configobj=config + ) def run(self): section = Config.SECTION_REMOTE_FMT.format(self.args.name) @@ -71,10 +70,12 @@ class CmdRemoteRemove(CmdConfig): if ret != 0: return ret - for configobj in [self.config._local_config, - self.config._project_config, - self.config._global_config, - self.config._system_config]: + for configobj in [ + self.config._local_config, + self.config._project_config, + self.config._global_config, + self.config._system_config, + ]: self._remove_default(configobj) self.config.save(configobj) if configobj == self.configobj: @@ -86,14 +87,14 @@ class CmdRemoteRemove(CmdConfig): class CmdRemoteModify(CmdConfig): def run(self): section = Config.SECTION_REMOTE_FMT.format(self.args.name) - self.args.name = '{}.{}'.format(section, self.args.option) + self.args.name = "{}.{}".format(section, self.args.option) return super(CmdRemoteModify, self).run() class CmdRemoteDefault(CmdConfig): def run(self): self.args.value = self.args.name - self.args.name = 'core.remote' + self.args.name = "core.remote" return super(CmdRemoteDefault, self).run() @@ -102,108 +103,95 @@ class CmdRemoteList(CmdConfig): for section in self.configobj.keys(): r = re.match(Config.SECTION_REMOTE_REGEX, section) if r: - name = r.group('name') - url = self.configobj[section].get(Config.SECTION_REMOTE_URL, - '') - logger.info('{}\t{}'.format(name, url)) + name = r.group("name") + url = self.configobj[section].get(Config.SECTION_REMOTE_URL, "") + logger.info("{}\t{}".format(name, url)) return 0 def add_parser(subparsers, parent_parser): from dvc.command.config import parent_config_parser - REMOTE_HELP = 'Manage set of tracked repositories.' + REMOTE_HELP = "Manage set of tracked repositories." remote_parser = subparsers.add_parser( - 'remote', - parents=[parent_parser], - description=REMOTE_HELP, - help=REMOTE_HELP) + "remote", parents=[parent_parser], description=REMOTE_HELP, help=REMOTE_HELP + ) remote_subparsers = remote_parser.add_subparsers( - dest='cmd', - help='Use dvc remote CMD --help for ' - 'command-specific help.') + dest="cmd", help="Use dvc remote CMD --help for " "command-specific help." + ) fix_subparsers(remote_subparsers) - REMOTE_ADD_HELP = 'Add remote.' + REMOTE_ADD_HELP = "Add remote." remote_add_parser = remote_subparsers.add_parser( - 'add', + "add", parents=[parent_config_parser, parent_parser], description=REMOTE_ADD_HELP, - help=REMOTE_ADD_HELP) + help=REMOTE_ADD_HELP, + ) + remote_add_parser.add_argument("name", help="Name.") remote_add_parser.add_argument( - 'name', - help='Name.') + "url", + help="URL. See full list of supported urls at " + "https://dvc.org/doc/commands-reference/remote", + ) remote_add_parser.add_argument( - 'url', - help='URL. See full list of supported urls at ' - 'https://dvc.org/doc/commands-reference/remote') - remote_add_parser.add_argument( - '-d', - '--default', - action='store_true', + "-d", + "--default", + action="store_true", default=False, - help='Set as default remote.') + help="Set as default remote.", + ) remote_add_parser.set_defaults(func=CmdRemoteAdd) - REMOTE_DEFAULT_HELP = 'Set/unset default remote.' + REMOTE_DEFAULT_HELP = "Set/unset default remote." remote_default_parser = remote_subparsers.add_parser( - 'default', + "default", parents=[parent_config_parser, parent_parser], description=REMOTE_DEFAULT_HELP, - help=REMOTE_DEFAULT_HELP) + help=REMOTE_DEFAULT_HELP, + ) + remote_default_parser.add_argument("name", nargs="?", help="Name of the remote.") remote_default_parser.add_argument( - 'name', - nargs='?', - help='Name of the remote.') - remote_default_parser.add_argument( - '-u', - '--unset', - action='store_true', + "-u", + "--unset", + action="store_true", default=False, - help='Unset default remote.') + help="Unset default remote.", + ) remote_default_parser.set_defaults(func=CmdRemoteDefault) - REMOTE_REMOVE_HELP = 'Remove remote.' + REMOTE_REMOVE_HELP = "Remove remote." remote_remove_parser = remote_subparsers.add_parser( - 'remove', + "remove", parents=[parent_config_parser, parent_parser], description=REMOTE_REMOVE_HELP, - help=REMOTE_REMOVE_HELP) - remote_remove_parser.add_argument( - 'name', - help='Name') + help=REMOTE_REMOVE_HELP, + ) + remote_remove_parser.add_argument("name", help="Name") remote_remove_parser.set_defaults(func=CmdRemoteRemove) - REMOTE_MODIFY_HELP = 'Modify remote.' + REMOTE_MODIFY_HELP = "Modify remote." remote_modify_parser = remote_subparsers.add_parser( - 'modify', + "modify", parents=[parent_config_parser, parent_parser], description=REMOTE_MODIFY_HELP, - help=REMOTE_MODIFY_HELP) + help=REMOTE_MODIFY_HELP, + ) + remote_modify_parser.add_argument("name", help="Name.") + remote_modify_parser.add_argument("option", help="Option.") + remote_modify_parser.add_argument("value", nargs="?", help="Value.") remote_modify_parser.add_argument( - 'name', - help='Name.') - remote_modify_parser.add_argument( - 'option', - help='Option.') - remote_modify_parser.add_argument( - 'value', - nargs='?', - help='Value.') - remote_modify_parser.add_argument( - '-u', - '--unset', - default=False, - action='store_true', - help='Unset option.') + "-u", "--unset", default=False, action="store_true", help="Unset option." + ) remote_modify_parser.set_defaults(func=CmdRemoteModify) - REMOTE_LIST_HELP = 'List remotes.' + REMOTE_LIST_HELP = "List remotes." remote_list_parser = remote_subparsers.add_parser( - 'list', + "list", parents=[parent_config_parser, parent_parser], description=REMOTE_LIST_HELP, - help=REMOTE_LIST_HELP) + help=REMOTE_LIST_HELP, + ) remote_list_parser.set_defaults(func=CmdRemoteList) diff --git a/dvc/command/remove.py b/dvc/command/remove.py index 833d2134c..18f220aec 100644 --- a/dvc/command/remove.py +++ b/dvc/command/remove.py @@ -14,17 +14,13 @@ class CmdRemove(CmdBase): if self.args.force: return False - msg = ( - 'Are you sure you want to remove {} with its outputs?' - .format(target) - ) + msg = "Are you sure you want to remove {} with its outputs?".format(target) if prompt.confirm(msg): return False raise DvcException( - 'Cannot purge without a confirmation from the user.' - " Use '-f' to force." + "Cannot purge without a confirmation from the user." " Use '-f' to force." ) def run(self): @@ -33,39 +29,33 @@ class CmdRemove(CmdBase): outs_only = self._is_outs_only(target) self.project.remove(target, outs_only=outs_only) except DvcException: - logger.error('failed to remove {}'.format(target)) + logger.error("failed to remove {}".format(target)) return 1 return 0 def add_parser(subparsers, parent_parser): - REMOVE_HELP = 'Remove outputs of DVC file.' + REMOVE_HELP = "Remove outputs of DVC file." remove_parser = subparsers.add_parser( - 'remove', - parents=[parent_parser], - description=REMOVE_HELP, - help=REMOVE_HELP) + "remove", parents=[parent_parser], description=REMOVE_HELP, help=REMOVE_HELP + ) remove_parser_group = remove_parser.add_mutually_exclusive_group() remove_parser_group.add_argument( - '-o', - '--outs', - action='store_true', + "-o", + "--outs", + action="store_true", default=True, - help='Only remove DVC file outputs.(default)') + help="Only remove DVC file outputs.(default)", + ) remove_parser_group.add_argument( - '-p', - '--purge', - action='store_true', + "-p", + "--purge", + action="store_true", default=False, - help='Remove DVC file and all its outputs.') + help="Remove DVC file and all its outputs.", + ) remove_parser.add_argument( - '-f', - '--force', - action='store_true', - default=False, - help='Force purge.') - remove_parser.add_argument( - 'targets', - nargs='+', - help='DVC files.') + "-f", "--force", action="store_true", default=False, help="Force purge." + ) + remove_parser.add_argument("targets", nargs="+", help="DVC files.") remove_parser.set_defaults(func=CmdRemove) diff --git a/dvc/command/repro.py b/dvc/command/repro.py index 64be612ac..d2d32bbaf 100644 --- a/dvc/command/repro.py +++ b/dvc/command/repro.py @@ -25,14 +25,15 @@ class CmdRepro(CmdBase): for target in self.args.targets: try: stages = self.project.reproduce( - target, - recursive=recursive, - force=self.args.force, - dry=self.args.dry, - interactive=self.args.interactive, - pipeline=self.args.pipeline, - all_pipelines=self.args.all_pipelines, - ignore_build_cache=self.args.ignore_build_cache) + target, + recursive=recursive, + force=self.args.force, + dry=self.args.dry, + interactive=self.args.interactive, + pipeline=self.args.pipeline, + all_pipelines=self.args.all_pipelines, + ignore_build_cache=self.args.ignore_build_cache, + ) if len(stages) == 0: logger.info(CmdDataStatus.UP_TO_DATE_MSG) @@ -51,67 +52,70 @@ class CmdRepro(CmdBase): def add_parser(subparsers, parent_parser): REPRO_HELP = "Reproduce DVC file. Default file name - 'Dvcfile'." repro_parser = subparsers.add_parser( - 'repro', - parents=[parent_parser], - description=REPRO_HELP, - help=REPRO_HELP) + "repro", parents=[parent_parser], description=REPRO_HELP, help=REPRO_HELP + ) + repro_parser.add_argument("targets", nargs="*", help="DVC file to reproduce.") repro_parser.add_argument( - 'targets', - nargs='*', - help='DVC file to reproduce.') - repro_parser.add_argument( - '-f', - '--force', - action='store_true', + "-f", + "--force", + action="store_true", default=False, - help='Reproduce even if dependencies were not changed.') + help="Reproduce even if dependencies were not changed.", + ) repro_parser.add_argument( - '-s', - '--single-item', - action='store_true', + "-s", + "--single-item", + action="store_true", default=False, - help='Reproduce only single data item without recursive dependencies ' - 'check.') + help="Reproduce only single data item without recursive dependencies " "check.", + ) repro_parser.add_argument( - '-c', - '--cwd', + "-c", + "--cwd", default=os.path.curdir, - help='Directory within your project to reroduce from.') + help="Directory within your project to reroduce from.", + ) repro_parser.add_argument( - '-m', - '--metrics', - action='store_true', + "-m", + "--metrics", + action="store_true", default=False, - help='Show metrics after reproduction.') + help="Show metrics after reproduction.", + ) repro_parser.add_argument( - '--dry', - action='store_true', + "--dry", + action="store_true", default=False, - help='Only print the commands that would be executed without ' - 'actually executing.') + help="Only print the commands that would be executed without " + "actually executing.", + ) repro_parser.add_argument( - '-i', - '--interactive', - action='store_true', + "-i", + "--interactive", + action="store_true", default=False, - help='Ask for confirmation before reproducing each stage.') + help="Ask for confirmation before reproducing each stage.", + ) repro_parser.add_argument( - '-p', - '--pipeline', - action='store_true', + "-p", + "--pipeline", + action="store_true", default=False, - help='Reproduce the whole pipeline that the specified stage file ' - 'belongs to.') + help="Reproduce the whole pipeline that the specified stage file " + "belongs to.", + ) repro_parser.add_argument( - '-P', - '--all-pipelines', - action='store_true', + "-P", + "--all-pipelines", + action="store_true", default=False, - help='Reproduce all pipelines in the project.') + help="Reproduce all pipelines in the project.", + ) repro_parser.add_argument( - '--ignore-build-cache', - action='store_true', + "--ignore-build-cache", + action="store_true", default=False, help="Reproduce all descendants of a changed stage even if their " - "direct dependencies didn't change.") + "direct dependencies didn't change.", + ) repro_parser.set_defaults(func=CmdRepro) diff --git a/dvc/command/root.py b/dvc/command/root.py index ef8abedf6..8710f141d 100644 --- a/dvc/command/root.py +++ b/dvc/command/root.py @@ -18,8 +18,6 @@ class CmdRoot(CmdBase): def add_parser(subparsers, parent_parser): ROOT_HELP = "Relative path to project's directory." root_parser = subparsers.add_parser( - 'root', - parents=[parent_parser], - description=ROOT_HELP, - help=ROOT_HELP) + "root", parents=[parent_parser], description=ROOT_HELP, help=ROOT_HELP + ) root_parser.set_defaults(func=CmdRoot) diff --git a/dvc/command/run.py b/dvc/command/run.py index 5f3a7b08d..54eff2f9d 100644 --- a/dvc/command/run.py +++ b/dvc/command/run.py @@ -10,30 +10,33 @@ from dvc.exceptions import DvcException class CmdRun(CmdBase): def run(self): - overwrite = (self.args.yes or self.args.overwrite_dvcfile) + overwrite = self.args.yes or self.args.overwrite_dvcfile - if not any([self.args.deps, - self.args.outs, - self.args.outs_no_cache, - self.args.command]): # pragma: no cover - logger.error("too few arguments. Specify at least one: '-d', " - "'-o', '-O', 'command'.") + if not any( + [self.args.deps, self.args.outs, self.args.outs_no_cache, self.args.command] + ): # pragma: no cover + logger.error( + "too few arguments. Specify at least one: '-d', " + "'-o', '-O', 'command'." + ) return 1 try: - self.project.run(cmd=self._parsed_cmd(), - outs=self.args.outs, - outs_no_cache=self.args.outs_no_cache, - metrics_no_cache=self.args.metrics_no_cache, - deps=self.args.deps, - fname=self.args.file, - cwd=self.args.cwd, - no_exec=self.args.no_exec, - overwrite=overwrite, - ignore_build_cache=self.args.ignore_build_cache, - remove_outs=self.args.remove_outs) + self.project.run( + cmd=self._parsed_cmd(), + outs=self.args.outs, + outs_no_cache=self.args.outs_no_cache, + metrics_no_cache=self.args.metrics_no_cache, + deps=self.args.deps, + fname=self.args.file, + cwd=self.args.cwd, + no_exec=self.args.no_exec, + overwrite=overwrite, + ignore_build_cache=self.args.ignore_build_cache, + remove_outs=self.args.remove_outs, + ) except DvcException: - logger.error('failed to run command') + logger.error("failed to run command") return 1 return 0 @@ -49,97 +52,103 @@ class CmdRun(CmdBase): dvc run echo "foo bar" """ if len(self.args.command) < 2: - return ' '.join(self.args.command) + return " ".join(self.args.command) - return ' '.join(self._quote_argument(arg) for arg in self.args.command) + return " ".join(self._quote_argument(arg) for arg in self.args.command) def _quote_argument(self, argument): - if ' ' not in argument or '"' in argument: + if " " not in argument or '"' in argument: return argument return '"{}"'.format(argument) def add_parser(subparsers, parent_parser): - RUN_HELP = 'Generate a stage file from a given ' \ - 'command and execute the command.' + RUN_HELP = "Generate a stage file from a given " "command and execute the command." run_parser = subparsers.add_parser( - 'run', - parents=[parent_parser], - description=RUN_HELP, - help=RUN_HELP) + "run", parents=[parent_parser], description=RUN_HELP, help=RUN_HELP + ) run_parser.add_argument( - '-d', - '--deps', - action='append', + "-d", + "--deps", + action="append", default=[], - help='Declare dependencies for reproducible cmd.') + help="Declare dependencies for reproducible cmd.", + ) run_parser.add_argument( - '-o', - '--outs', - action='append', + "-o", + "--outs", + action="append", default=[], - help='Declare output data file or data directory.') + help="Declare output data file or data directory.", + ) run_parser.add_argument( - '-O', - '--outs-no-cache', - action='append', + "-O", + "--outs-no-cache", + action="append", default=[], - help='Declare output regular file or directory ' - '(sync to Git, not DVC cache).') + help="Declare output regular file or directory " + "(sync to Git, not DVC cache).", + ) run_parser.add_argument( - '-M', - '--metrics-no-cache', - action='append', + "-M", + "--metrics-no-cache", + action="append", default=[], - help='Declare output metric file or ' - 'directory (not cached by DVC).') + help="Declare output metric file or " "directory (not cached by DVC).", + ) run_parser.add_argument( - '-f', - '--file', + "-f", + "--file", help="Specify name of the stage file. It should be " - "either 'Dvcfile' or have a '.dvc' suffix (e.g. " - "'prepare.dvc', 'clean.dvc', etc) in order for " - "dvc to be able to find it later. By default " - "the first output basename + .dvc is used as " - "a stage filename.") + "either 'Dvcfile' or have a '.dvc' suffix (e.g. " + "'prepare.dvc', 'clean.dvc', etc) in order for " + "dvc to be able to find it later. By default " + "the first output basename + .dvc is used as " + "a stage filename.", + ) run_parser.add_argument( - '-c', - '--cwd', + "-c", + "--cwd", default=os.path.curdir, help="Directory within your project to run your command and place " - "stage file in.") + "stage file in.", + ) run_parser.add_argument( - '--no-exec', - action='store_true', + "--no-exec", + action="store_true", default=False, - help="Only create stage file without actually running it.") + help="Only create stage file without actually running it.", + ) run_parser.add_argument( - '-y', - '--yes', - action='store_true', + "-y", + "--yes", + action="store_true", default=False, help="(OBSOLETED, use --overwrite-dvcfile instead) Automatic 'yes' " - "answer to all prompts. E.g. when '.dvc' file exists and dvc " - "asks if you want to overwrite it.") + "answer to all prompts. E.g. when '.dvc' file exists and dvc " + "asks if you want to overwrite it.", + ) run_parser.add_argument( - '--overwrite-dvcfile', - action='store_true', + "--overwrite-dvcfile", + action="store_true", default=False, - help="Overwrite existing dvc file without asking for confirmation.") + help="Overwrite existing dvc file without asking for confirmation.", + ) run_parser.add_argument( - '--ignore-build-cache', - action='store_true', + "--ignore-build-cache", + action="store_true", default=False, help="Run this stage even if it has been already ran with the same " - "command/dependencies/outputs/etc before.") + "command/dependencies/outputs/etc before.", + ) run_parser.add_argument( - '--remove-outs', - action='store_true', + "--remove-outs", + action="store_true", default=False, - help="Remove outputs before running the command.") + help="Remove outputs before running the command.", + ) run_parser.add_argument( - 'command', - nargs=argparse.REMAINDER, - help='Command to execute.') + "command", nargs=argparse.REMAINDER, help="Command to execute." + ) run_parser.set_defaults(func=CmdRun) diff --git a/dvc/command/status.py b/dvc/command/status.py index 8c924de5c..5419e7166 100644 --- a/dvc/command/status.py +++ b/dvc/command/status.py @@ -6,36 +6,38 @@ from dvc.command.data_sync import CmdDataBase class CmdDataStatus(CmdDataBase): STATUS_LEN = 10 - STATUS_INDENT = '\t' + STATUS_INDENT = "\t" UP_TO_DATE_MSG = "Pipeline is up to date. Nothing to reproduce." def _normalize(self, s): - s += ':' + s += ":" assert len(s) < self.STATUS_LEN - return s + (self.STATUS_LEN - len(s))*' ' + return s + (self.STATUS_LEN - len(s)) * " " def _show(self, status, indent=0): ind = indent * self.STATUS_INDENT for key, value in status.items(): if isinstance(value, dict): - logger.info('{}{}'.format(ind, key)) - self._show(value, indent+1) + logger.info("{}{}".format(ind, key)) + self._show(value, indent + 1) else: - msg = '{}{}{}'.format(ind, self._normalize(value), key) + msg = "{}{}{}".format(ind, self._normalize(value), key) logger.info(msg) def do_run(self, target=None): indent = 1 if self.args.cloud else 0 try: - st = self.project.status(target=target, - jobs=self.args.jobs, - cloud=self.args.cloud, - show_checksums=self.args.show_checksums, - remote=self.args.remote, - all_branches=self.args.all_branches, - all_tags=self.args.all_tags, - with_deps=self.args.with_deps) + st = self.project.status( + target=target, + jobs=self.args.jobs, + cloud=self.args.cloud, + show_checksums=self.args.show_checksums, + remote=self.args.remote, + all_branches=self.args.all_branches, + all_tags=self.args.all_tags, + with_deps=self.args.with_deps, + ) if st: if self.args.quiet: return 1 @@ -45,6 +47,6 @@ class CmdDataStatus(CmdDataBase): logger.info(self.UP_TO_DATE_MSG) except Exception: - logger.error('failed to obtain data status') + logger.error("failed to obtain data status") return 1 return 0 diff --git a/dvc/command/unprotect.py b/dvc/command/unprotect.py index 3f4cd8705..cf158d587 100644 --- a/dvc/command/unprotect.py +++ b/dvc/command/unprotect.py @@ -18,14 +18,12 @@ class CmdUnprotect(CmdBase): def add_parser(subparsers, parent_parser): - UNPROTECT_HELP = 'Unprotect data file/directory.' + UNPROTECT_HELP = "Unprotect data file/directory." unprotect_parser = subparsers.add_parser( - 'unprotect', + "unprotect", parents=[parent_parser], description=UNPROTECT_HELP, - help=UNPROTECT_HELP) - unprotect_parser.add_argument( - 'targets', - nargs='+', - help='Data files/directory.') + help=UNPROTECT_HELP, + ) + unprotect_parser.add_argument("targets", nargs="+", help="Data files/directory.") unprotect_parser.set_defaults(func=CmdUnprotect) diff --git a/dvc/config.py b/dvc/config.py index c6941fcff..c37a2d937 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -20,10 +20,9 @@ class ConfigError(DvcException): msg (str): error message. ex (Exception): optional exception that has caused this error. """ + def __init__(self, msg, ex=None): - super(ConfigError, self).__init__( - 'config file error: {}'.format(msg), ex - ) + super(ConfigError, self).__init__("config file error: {}".format(msg), ex) def supported_cache_type(types): @@ -33,9 +32,9 @@ def supported_cache_type(types): types (list/string): type(s) of links that dvc should try out. """ if isinstance(types, str): - types = [typ.strip() for typ in types.split(',')] + types = [typ.strip() for typ in types.split(",")] for typ in types: - if typ not in ['reflink', 'hardlink', 'symlink', 'copy']: + if typ not in ["reflink", "hardlink", "symlink", "copy"]: return False return True @@ -46,7 +45,7 @@ def supported_loglevel(level): Args: level (str): log level name. """ - return level in ['info', 'debug', 'warning', 'error'] + return level in ["info", "debug", "warning", "error"] def supported_cloud(cloud): @@ -55,7 +54,7 @@ def supported_cloud(cloud): Args: cloud (str): cloud type name. """ - return cloud in ['aws', 'gcp', 'local', ''] + return cloud in ["aws", "gcp", "local", ""] def is_bool(val): @@ -67,7 +66,7 @@ def is_bool(val): Returns: bool: True if value stands for boolean, False otherwise. """ - return val.lower() in ['true', 'false'] + return val.lower() in ["true", "false"] def to_bool(val): @@ -79,7 +78,7 @@ def to_bool(val): Returns: bool: True if value.lower() == 'true', False otherwise. """ - return val.lower() == 'true' + return val.lower() == "true" def is_whole(val): @@ -120,36 +119,37 @@ class Config(object): # pylint: disable=too-many-instance-attributes Raises: ConfigError: thrown when config has an invalid format. """ - APPNAME = 'dvc' - APPAUTHOR = 'iterative' + + APPNAME = "dvc" + APPAUTHOR = "iterative" # NOTE: used internally in RemoteLOCAL to know config # location, that url should resolved relative to. - PRIVATE_CWD = '_cwd' + PRIVATE_CWD = "_cwd" - CONFIG = 'config' - CONFIG_LOCAL = 'config.local' + CONFIG = "config" + CONFIG_LOCAL = "config.local" - SECTION_CORE = 'core' - SECTION_CORE_LOGLEVEL = 'loglevel' + SECTION_CORE = "core" + SECTION_CORE_LOGLEVEL = "loglevel" SECTION_CORE_LOGLEVEL_SCHEMA = And(Use(str.lower), supported_loglevel) - SECTION_CORE_REMOTE = 'remote' + SECTION_CORE_REMOTE = "remote" SECTION_CORE_INTERACTIVE_SCHEMA = And(str, is_bool, Use(to_bool)) - SECTION_CORE_INTERACTIVE = 'interactive' - SECTION_CORE_ANALYTICS = 'analytics' + SECTION_CORE_INTERACTIVE = "interactive" + SECTION_CORE_ANALYTICS = "analytics" SECTION_CORE_ANALYTICS_SCHEMA = And(str, is_bool, Use(to_bool)) - SECTION_CACHE = 'cache' - SECTION_CACHE_DIR = 'dir' - SECTION_CACHE_TYPE = 'type' + SECTION_CACHE = "cache" + SECTION_CACHE_DIR = "dir" + SECTION_CACHE_TYPE = "type" SECTION_CACHE_TYPE_SCHEMA = supported_cache_type - SECTION_CACHE_PROTECTED = 'protected' - SECTION_CACHE_LOCAL = 'local' - SECTION_CACHE_S3 = 's3' - SECTION_CACHE_GS = 'gs' - SECTION_CACHE_SSH = 'ssh' - SECTION_CACHE_HDFS = 'hdfs' - SECTION_CACHE_AZURE = 'azure' + SECTION_CACHE_PROTECTED = "protected" + SECTION_CACHE_LOCAL = "local" + SECTION_CACHE_S3 = "s3" + SECTION_CACHE_GS = "gs" + SECTION_CACHE_SSH = "ssh" + SECTION_CACHE_HDFS = "hdfs" + SECTION_CACHE_AZURE = "azure" SECTION_CACHE_SCHEMA = { Optional(SECTION_CACHE_LOCAL): str, Optional(SECTION_CACHE_S3): str, @@ -157,92 +157,87 @@ class Config(object): # pylint: disable=too-many-instance-attributes Optional(SECTION_CACHE_HDFS): str, Optional(SECTION_CACHE_SSH): str, Optional(SECTION_CACHE_AZURE): str, - Optional(SECTION_CACHE_DIR): str, Optional(SECTION_CACHE_TYPE, default=None): SECTION_CACHE_TYPE_SCHEMA, - Optional(SECTION_CACHE_PROTECTED, - default=False): And(str, is_bool, Use(to_bool)), - + Optional(SECTION_CACHE_PROTECTED, default=False): And( + str, is_bool, Use(to_bool) + ), Optional(PRIVATE_CWD): str, } # backward compatibility - SECTION_CORE_CLOUD = 'cloud' + SECTION_CORE_CLOUD = "cloud" SECTION_CORE_CLOUD_SCHEMA = And(Use(str.lower), supported_cloud) - SECTION_CORE_STORAGEPATH = 'storagepath' + SECTION_CORE_STORAGEPATH = "storagepath" SECTION_CORE_SCHEMA = { - Optional(SECTION_CORE_LOGLEVEL, - default='info'): And(str, Use(str.lower), - SECTION_CORE_LOGLEVEL_SCHEMA), - Optional(SECTION_CORE_REMOTE, default=''): And(str, Use(str.lower)), - Optional(SECTION_CORE_INTERACTIVE, - default=False): SECTION_CORE_INTERACTIVE_SCHEMA, - Optional(SECTION_CORE_ANALYTICS, - default=True): SECTION_CORE_ANALYTICS_SCHEMA, - + Optional(SECTION_CORE_LOGLEVEL, default="info"): And( + str, Use(str.lower), SECTION_CORE_LOGLEVEL_SCHEMA + ), + Optional(SECTION_CORE_REMOTE, default=""): And(str, Use(str.lower)), + Optional( + SECTION_CORE_INTERACTIVE, default=False + ): SECTION_CORE_INTERACTIVE_SCHEMA, + Optional(SECTION_CORE_ANALYTICS, default=True): SECTION_CORE_ANALYTICS_SCHEMA, # backward compatibility - Optional(SECTION_CORE_CLOUD, default=''): SECTION_CORE_CLOUD_SCHEMA, - Optional(SECTION_CORE_STORAGEPATH, default=''): str, + Optional(SECTION_CORE_CLOUD, default=""): SECTION_CORE_CLOUD_SCHEMA, + Optional(SECTION_CORE_STORAGEPATH, default=""): str, } # backward compatibility - SECTION_AWS = 'aws' - SECTION_AWS_STORAGEPATH = 'storagepath' - SECTION_AWS_CREDENTIALPATH = 'credentialpath' - SECTION_AWS_ENDPOINT_URL = 'endpointurl' - SECTION_AWS_REGION = 'region' - SECTION_AWS_PROFILE = 'profile' - SECTION_AWS_USE_SSL = 'use_ssl' + SECTION_AWS = "aws" + SECTION_AWS_STORAGEPATH = "storagepath" + SECTION_AWS_CREDENTIALPATH = "credentialpath" + SECTION_AWS_ENDPOINT_URL = "endpointurl" + SECTION_AWS_REGION = "region" + SECTION_AWS_PROFILE = "profile" + SECTION_AWS_USE_SSL = "use_ssl" SECTION_AWS_SCHEMA = { SECTION_AWS_STORAGEPATH: str, Optional(SECTION_AWS_REGION): str, Optional(SECTION_AWS_PROFILE): str, Optional(SECTION_AWS_CREDENTIALPATH): str, Optional(SECTION_AWS_ENDPOINT_URL): str, - Optional(SECTION_AWS_USE_SSL, - default=True): And(str, is_bool, Use(to_bool)), + Optional(SECTION_AWS_USE_SSL, default=True): And(str, is_bool, Use(to_bool)), } # backward compatibility - SECTION_GCP = 'gcp' + SECTION_GCP = "gcp" SECTION_GCP_STORAGEPATH = SECTION_AWS_STORAGEPATH - SECTION_GCP_PROJECTNAME = 'projectname' + SECTION_GCP_PROJECTNAME = "projectname" SECTION_GCP_SCHEMA = { SECTION_GCP_STORAGEPATH: str, Optional(SECTION_GCP_PROJECTNAME): str, } # backward compatibility - SECTION_LOCAL = 'local' + SECTION_LOCAL = "local" SECTION_LOCAL_STORAGEPATH = SECTION_AWS_STORAGEPATH - SECTION_LOCAL_SCHEMA = { - SECTION_LOCAL_STORAGEPATH: str, - } + SECTION_LOCAL_SCHEMA = {SECTION_LOCAL_STORAGEPATH: str} - SECTION_AZURE_CONNECTION_STRING = 'connection_string' + SECTION_AZURE_CONNECTION_STRING = "connection_string" SECTION_REMOTE_REGEX = r'^\s*remote\s*"(?P.*)"\s*$' SECTION_REMOTE_FMT = 'remote "{}"' - SECTION_REMOTE_URL = 'url' - SECTION_REMOTE_USER = 'user' - SECTION_REMOTE_PORT = 'port' - SECTION_REMOTE_KEY_FILE = 'keyfile' - SECTION_REMOTE_TIMEOUT = 'timeout' - SECTION_REMOTE_PASSWORD = 'password' - SECTION_REMOTE_ASK_PASSWORD = 'ask_password' + SECTION_REMOTE_URL = "url" + SECTION_REMOTE_USER = "user" + SECTION_REMOTE_PORT = "port" + SECTION_REMOTE_KEY_FILE = "keyfile" + SECTION_REMOTE_TIMEOUT = "timeout" + SECTION_REMOTE_PASSWORD = "password" + SECTION_REMOTE_ASK_PASSWORD = "ask_password" SECTION_REMOTE_SCHEMA = { SECTION_REMOTE_URL: str, Optional(SECTION_AWS_REGION): str, Optional(SECTION_AWS_PROFILE): str, Optional(SECTION_AWS_CREDENTIALPATH): str, Optional(SECTION_AWS_ENDPOINT_URL): str, - Optional(SECTION_AWS_USE_SSL, - default=True): And(str, is_bool, Use(to_bool)), + Optional(SECTION_AWS_USE_SSL, default=True): And(str, is_bool, Use(to_bool)), Optional(SECTION_GCP_PROJECTNAME): str, Optional(SECTION_CACHE_TYPE): SECTION_CACHE_TYPE_SCHEMA, - Optional(SECTION_CACHE_PROTECTED, - default=False): And(str, is_bool, Use(to_bool)), + Optional(SECTION_CACHE_PROTECTED, default=False): And( + str, is_bool, Use(to_bool) + ), Optional(SECTION_REMOTE_USER): str, Optional(SECTION_REMOTE_PORT): Use(int), Optional(SECTION_REMOTE_KEY_FILE): str, @@ -250,13 +245,12 @@ class Config(object): # pylint: disable=too-many-instance-attributes Optional(SECTION_REMOTE_PASSWORD): str, Optional(SECTION_REMOTE_ASK_PASSWORD): And(str, is_bool, Use(to_bool)), Optional(SECTION_AZURE_CONNECTION_STRING): str, - Optional(PRIVATE_CWD): str, } - SECTION_STATE = 'state' - SECTION_STATE_ROW_LIMIT = 'row_limit' - SECTION_STATE_ROW_CLEANUP_QUOTA = 'row_cleanup_quota' + SECTION_STATE = "state" + SECTION_STATE_ROW_LIMIT = "row_limit" + SECTION_STATE_ROW_CLEANUP_QUOTA = "row_cleanup_quota" SECTION_STATE_SCHEMA = { Optional(SECTION_STATE_ROW_LIMIT): And(Use(int), is_whole), Optional(SECTION_STATE_ROW_CLEANUP_QUOTA): And(Use(int), is_percent), @@ -267,7 +261,6 @@ class Config(object): # pylint: disable=too-many-instance-attributes Optional(Regex(SECTION_REMOTE_REGEX)): SECTION_REMOTE_SCHEMA, Optional(SECTION_CACHE, default={}): SECTION_CACHE_SCHEMA, Optional(SECTION_STATE, default={}): SECTION_STATE_SCHEMA, - # backward compatibility Optional(SECTION_AWS, default={}): SECTION_AWS_SCHEMA, Optional(SECTION_GCP, default={}): SECTION_GCP_SCHEMA, @@ -275,10 +268,12 @@ class Config(object): # pylint: disable=too-many-instance-attributes } def __init__(self, dvc_dir=None, validate=True): - self.system_config_file = os.path.join(self.get_system_config_dir(), - self.CONFIG) - self.global_config_file = os.path.join(self.get_global_config_dir(), - self.CONFIG) + self.system_config_file = os.path.join( + self.get_system_config_dir(), self.CONFIG + ) + self.global_config_file = os.path.join( + self.get_global_config_dir(), self.CONFIG + ) if dvc_dir is not None: self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir)) @@ -306,8 +301,8 @@ class Config(object): # pylint: disable=too-many-instance-attributes str: path to the global config directory. """ from appdirs import user_config_dir - return user_config_dir(appname=Config.APPNAME, - appauthor=Config.APPAUTHOR) + + return user_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR) @staticmethod def get_system_config_dir(): @@ -317,8 +312,8 @@ class Config(object): # pylint: disable=too-many-instance-attributes str: path to the system config directory. """ from appdirs import site_config_dir - return site_config_dir(appname=Config.APPNAME, - appauthor=Config.APPAUTHOR) + + return site_config_dir(appname=Config.APPNAME, appauthor=Config.APPAUTHOR) @staticmethod def init(dvc_dir): @@ -331,7 +326,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes dvc.config.Config: config object. """ config_file = os.path.join(dvc_dir, Config.CONFIG) - open(config_file, 'w+').close() + open(config_file, "w+").close() return Config(dvc_dir) def _load(self): @@ -408,8 +403,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes self.config = Schema(self.SCHEMA).validate(self.config) # NOTE: now converting back to ConfigObj - self.config = configobj.ConfigObj(self.config, - write_empty_values=True) + self.config = configobj.ConfigObj(self.config, write_empty_values=True) self.config.filename = self.config_file self._resolve_paths(self.config, self.config_file) except Exception as ex: @@ -439,10 +433,12 @@ class Config(object): # pylint: disable=too-many-instance-attributes if config is not None: clist = [config] else: - clist = [self._system_config, - self._global_config, - self._project_config, - self._local_config] + clist = [ + self._system_config, + self._global_config, + self._project_config, + self._local_config, + ] for conf in clist: if conf.filename is None: @@ -478,8 +474,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes return if opt not in config[section].keys(): - raise ConfigError("option '{}.{}' doesn't exist".format(section, - opt)) + raise ConfigError("option '{}.{}' doesn't exist".format(section, opt)) del config[section][opt] if not config[section]: @@ -513,8 +508,7 @@ class Config(object): # pylint: disable=too-many-instance-attributes raise ConfigError("section '{}' doesn't exist".format(section)) if opt not in config[section].keys(): - raise ConfigError("option '{}.{}' doesn't exist".format(section, - opt)) + raise ConfigError("option '{}.{}' doesn't exist".format(section, opt)) logger.info(config[section][opt]) diff --git a/dvc/daemon.py b/dvc/daemon.py index 737af3953..48c197bb9 100644 --- a/dvc/daemon.py +++ b/dvc/daemon.py @@ -23,12 +23,14 @@ def _spawn_windows(cmd, env): startupinfo = STARTUPINFO() startupinfo.dwFlags |= STARTF_USESHOWWINDOW - Popen(cmd, - env=env, - close_fds=True, - shell=False, - creationflags=creationflags, - startupinfo=startupinfo).communicate() + Popen( + cmd, + env=env, + close_fds=True, + shell=False, + creationflags=creationflags, + startupinfo=startupinfo, + ).communicate() def _spawn_posix(cmd, env): @@ -71,18 +73,18 @@ def daemon(args): """ cmd = [sys.executable] if not is_binary(): - cmd += ['-m', 'dvc'] - cmd += ['daemon', '-q'] + args + cmd += ["-m", "dvc"] + cmd += ["daemon", "-q"] + args env = fix_env() file_path = os.path.abspath(inspect.stack()[0][1]) - env['PYTHONPATH'] = os.path.dirname(os.path.dirname(file_path)) + env["PYTHONPATH"] = os.path.dirname(os.path.dirname(file_path)) logger.debug("Trying to spawn '{}' with env '{}'".format(cmd, env)) - if os.name == 'nt': + if os.name == "nt": _spawn_windows(cmd, env) - elif os.name == 'posix': + elif os.name == "posix": _spawn_posix(cmd, env) else: raise NotImplementedError diff --git a/dvc/dagascii.py b/dvc/dagascii.py index 927a6acce..e8107b8e0 100644 --- a/dvc/dagascii.py +++ b/dvc/dagascii.py @@ -18,6 +18,7 @@ class VertexViewer(object): Args: name (str): name of the vertex. """ + HEIGHT = 3 # top and bottom box edges + text def __init__(self, name): @@ -43,6 +44,7 @@ class AsciiCanvas(object): cols (int): number of columns in the canvas. Should be > 1. lines (int): number of lines in the canvas. Should be > 1. """ + TIMEOUT = 10 def __init__(self, cols, lines): @@ -52,16 +54,17 @@ class AsciiCanvas(object): self.cols = cols self.lines = lines - self.canvas = [[' '] * cols for l in range(lines)] + self.canvas = [[" "] * cols for l in range(lines)] def draw(self): """Draws ASCII canvas on the screen.""" if sys.stdout.isatty(): # pragma: no cover from asciimatics.screen import Screen + Screen.wrapper(self._do_draw) else: for line in self.canvas: - print(''.join(line)) + print("".join(line)) def _do_draw(self, screen): # pragma: no cover # pylint: disable=too-many-locals @@ -93,13 +96,15 @@ class AsciiCanvas(object): line = [] for x in range(smaxcol + 1): x_index = offset_x + x - if len(self.canvas) > y_index \ - and len(self.canvas[y_index]) > x_index: + if ( + len(self.canvas) > y_index + and len(self.canvas[y_index]) > x_index + ): line.append(self.canvas[y_index][x_index]) else: - line.append(' ') + line.append(" ") assert len(line) == (smaxcol + 1) - screen.print_at(''.join(line), 0, y) + screen.print_at("".join(line), 0, y) screen.refresh() @@ -118,23 +123,23 @@ class AsciiCanvas(object): continue k = event.key_code - if k == screen.KEY_DOWN or k == ord('s'): + if k == screen.KEY_DOWN or k == ord("s"): offset_y += 1 - elif k == screen.KEY_PAGE_DOWN or k == ord('S'): + elif k == screen.KEY_PAGE_DOWN or k == ord("S"): offset_y += smaxrow - elif k == screen.KEY_UP or k == ord('w'): + elif k == screen.KEY_UP or k == ord("w"): offset_y -= 1 - elif k == screen.KEY_PAGE_UP or k == ord('W'): + elif k == screen.KEY_PAGE_UP or k == ord("W"): offset_y -= smaxrow - elif k == screen.KEY_RIGHT or k == ord('d'): + elif k == screen.KEY_RIGHT or k == ord("d"): offset_x += 1 - elif k == ord('D'): + elif k == ord("D"): offset_x += smaxcol - elif k == screen.KEY_LEFT or k == ord('a'): + elif k == screen.KEY_LEFT or k == ord("a"): offset_x -= 1 - elif k == ord('A'): + elif k == ord("A"): offset_x -= smaxcol - elif k == ord('q') or k == ord('Q'): + elif k == ord("q") or k == ord("Q"): break if offset_y > max_y: @@ -235,17 +240,17 @@ class AsciiCanvas(object): height -= 1 for x in range(x0, x0 + width): - self.point(x, y0, '-') - self.point(x, y0 + height, '-') + self.point(x, y0, "-") + self.point(x, y0 + height, "-") for y in range(y0, y0 + height): - self.point(x0, y, '|') - self.point(x0 + width, y, '|') + self.point(x0, y, "|") + self.point(x0 + width, y, "|") - self.point(x0, y0, '+') - self.point(x0 + width, y0, '+') - self.point(x0, y0 + height, '+') - self.point(x0 + width, y0 + height, '+') + self.point(x0, y0, "+") + self.point(x0 + width, y0, "+") + self.point(x0, y0 + height, "+") + self.point(x0 + width, y0 + height, "+") def _build_sugiyama_layout(vertexes, edges): @@ -306,8 +311,8 @@ def draw(vertexes, edges): for vertex in sug.g.sV: # NOTE: moving boxes w/2 to the left - Xs.append(vertex.view.xy[0] - vertex.view.w/2.0) - Xs.append(vertex.view.xy[0] + vertex.view.w/2.0) + Xs.append(vertex.view.xy[0] - vertex.view.w / 2.0) + Xs.append(vertex.view.xy[0] + vertex.view.w / 2.0) Ys.append(vertex.view.xy[1]) Ys.append(vertex.view.xy[1] + vertex.view.h) @@ -344,22 +349,17 @@ def draw(vertexes, edges): assert end_x >= 0 assert end_y >= 0 - canvas.line(start_x, start_y, - end_x, end_y, - '*') + canvas.line(start_x, start_y, end_x, end_y, "*") for vertex in sug.g.sV: # NOTE: moving boxes w/2 to the left - x = vertex.view.xy[0] - vertex.view.w/2.0 + x = vertex.view.xy[0] - vertex.view.w / 2.0 y = vertex.view.xy[1] - canvas.box(int(round(x - minx)), - int(round(y - miny)), - vertex.view.w, - vertex.view.h) + canvas.box( + int(round(x - minx)), int(round(y - miny)), vertex.view.w, vertex.view.h + ) - canvas.text(int(round(x - minx)) + 1, - int(round(y - miny)) + 1, - vertex.data) + canvas.text(int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data) canvas.draw() diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index d04a36f9c..a47190aa1 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -26,15 +26,16 @@ class DataCloud(object): Raises: config.ConfigError: thrown when config has invalid format. """ + CLOUD_MAP = { - 'aws': RemoteS3, - 'gcp': RemoteGS, - 'azure': RemoteAzure, - 'ssh': RemoteSSH, - 'hdfs': RemoteHDFS, - 'local': RemoteLOCAL, - 'http': RemoteHTTP, - 'https': RemoteHTTP, + "aws": RemoteS3, + "gcp": RemoteGS, + "azure": RemoteAzure, + "ssh": RemoteSSH, + "hdfs": RemoteHDFS, + "local": RemoteLOCAL, + "http": RemoteHTTP, + "https": RemoteHTTP, } def __init__(self, project, config=None): @@ -44,13 +45,13 @@ class DataCloud(object): @property def _cloud(self): - remote = self._core.get(Config.SECTION_CORE_REMOTE, '') - if remote != '': + remote = self._core.get(Config.SECTION_CORE_REMOTE, "") + if remote != "": return self._init_remote(remote) if self._core.get(Config.SECTION_CORE_CLOUD, None): # backward compatibility - msg = 'using obsoleted config format. Consider updating.' + msg = "using obsoleted config format. Consider updating." logger.warning(msg) return self._init_compat() @@ -66,8 +67,8 @@ class DataCloud(object): return Remote(self.project, cloud_config) def _init_compat(self): - name = self._core.get(Config.SECTION_CORE_CLOUD, '').strip().lower() - if name == '': + name = self._core.get(Config.SECTION_CORE_CLOUD, "").strip().lower() + if name == "": self._cloud = None return None @@ -91,7 +92,7 @@ class DataCloud(object): def _init_cloud(self, cloud_config, cloud_type): global_storage_path = self._core.get(Config.SECTION_CORE_STORAGEPATH) if global_storage_path: - logger.warning('using obsoleted config format. Consider updating.') + logger.warning("using obsoleted config format. Consider updating.") cloud = cloud_type(self.project, cloud_config) return cloud @@ -104,11 +105,10 @@ class DataCloud(object): return self._cloud raise ConfigError( - 'No remote repository specified. Setup default repository with\n' - ' dvc config core.remote \n' - 'or use:\n' - ' dvc {} -r \n' - .format(cmd) + "No remote repository specified. Setup default repository with\n" + " dvc config core.remote \n" + "or use:\n" + " dvc {} -r \n".format(cmd) ) def push(self, targets, jobs=None, remote=None, show_checksums=False): @@ -122,11 +122,12 @@ class DataCloud(object): show_checksums (bool): show checksums instead of file names in information messages. """ - return self.project.cache.local.push(targets, - jobs=jobs, - remote=self._get_cloud(remote, - 'push'), - show_checksums=show_checksums) + return self.project.cache.local.push( + targets, + jobs=jobs, + remote=self._get_cloud(remote, "push"), + show_checksums=show_checksums, + ) def pull(self, targets, jobs=None, remote=None, show_checksums=False): """Pull data items in a cloud-agnostic way. @@ -139,11 +140,12 @@ class DataCloud(object): show_checksums (bool): show checksums instead of file names in information messages. """ - return self.project.cache.local.pull(targets, - jobs=jobs, - remote=self._get_cloud(remote, - 'pull'), - show_checksums=show_checksums) + return self.project.cache.local.pull( + targets, + jobs=jobs, + remote=self._get_cloud(remote, "pull"), + show_checksums=show_checksums, + ) def status(self, targets, jobs=None, remote=None, show_checksums=False): """Check status of data items in a cloud-agnostic way. @@ -157,8 +159,7 @@ class DataCloud(object): show_checksums (bool): show checksums instead of file names in information messages. """ - cloud = self._get_cloud(remote, 'status') - return self.project.cache.local.status(targets, - jobs=jobs, - remote=cloud, - show_checksums=show_checksums) + cloud = self._get_cloud(remote, "status") + return self.project.cache.local.status( + targets, jobs=jobs, remote=cloud, show_checksums=show_checksums + ) diff --git a/dvc/dependency/__init__.py b/dvc/dependency/__init__.py index 9c84388dc..47e29dc10 100644 --- a/dvc/dependency/__init__.py +++ b/dvc/dependency/__init__.py @@ -26,13 +26,13 @@ DEPS = [ ] DEP_MAP = { - 'local': DependencyLOCAL, - 'ssh': DependencySSH, - 's3': DependencyS3, - 'gs': DependencyGS, - 'hdfs': DependencyHDFS, - 'http': DependencyHTTP, - 'https': DependencyHTTP, + "local": DependencyLOCAL, + "ssh": DependencySSH, + "s3": DependencyS3, + "gs": DependencyGS, + "hdfs": DependencyHDFS, + "http": DependencyHTTP, + "https": DependencyHTTP, } @@ -47,7 +47,7 @@ del SCHEMA[schema.Optional(OutputBase.PARAM_METRIC)] def _get(stage, p, info): parsed = urlparse(p) - if parsed.scheme == 'remote': + if parsed.scheme == "remote": name = Config.SECTION_REMOTE_FMT.format(parsed.netloc) sect = stage.project.config.config[name] remote = Remote(stage.project, sect) diff --git a/dvc/dependency/http.py b/dvc/dependency/http.py index 84691ca3f..c53db60c8 100644 --- a/dvc/dependency/http.py +++ b/dvc/dependency/http.py @@ -10,14 +10,8 @@ class DependencyHTTP(DependencyBase, OutputBase): REMOTE = RemoteHTTP def __init__(self, stage, path, info=None, remote=None): - super(DependencyHTTP, self).__init__(stage, - path, - info=info, - remote=remote) - if path.startswith('remote'): + super(DependencyHTTP, self).__init__(stage, path, info=info, remote=remote) + if path.startswith("remote"): path = urljoin(self.remote.cache_dir, urlparse(path).path) - self.path_info = { - 'scheme': urlparse(path).scheme, - 'path': path, - } + self.path_info = {"scheme": urlparse(path).scheme, "path": path} diff --git a/dvc/exceptions.py b/dvc/exceptions.py index cc4751b12..d5faf6268 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -14,6 +14,7 @@ class DvcException(Exception): msg (str): message for this exception. cause (Exception): optional cause exception. """ + def __init__(self, msg, cause=None): # NOTE: unlike python 3, python 2 doesn't have built-in support # for chained exceptions, so we are using our own implementation. @@ -35,15 +36,16 @@ class OutputDuplicationError(DvcException): output (str): path to the file/directory. stages (list): list of paths to stages. """ + def __init__(self, output, stages): assert isinstance(output, str) or isinstance(output, builtin_str) assert isinstance(stages, list) assert all( - isinstance(stage, str) or isinstance(stage, builtin_str) - for stage in stages + isinstance(stage, str) or isinstance(stage, builtin_str) for stage in stages ) - msg = ("file/directory '{}' is specified as an output in more than one" - "stage: {}").format(output, "\n ".join(stages)) + msg = ( + "file/directory '{}' is specified as an output in more than one" "stage: {}" + ).format(output, "\n ".join(stages)) super(OutputDuplicationError, self).__init__(msg) @@ -56,13 +58,15 @@ class WorkingDirectoryAsOutputError(DvcException): fname (str): path to the stage file that has cwd specified as an output. """ + def __init__(self, cwd, fname): assert isinstance(cwd, str) or isinstance(cwd, builtin_str) assert isinstance(fname, str) or isinstance(fname, builtin_str) msg = ( "current working directory '{cwd}' is specified as an output in" - " '{fname}'. Use another CWD to prevent any data removal." - .format(cwd=cwd, fname=fname) + " '{fname}'. Use another CWD to prevent any data removal.".format( + cwd=cwd, fname=fname + ) ) super(WorkingDirectoryAsOutputError, self).__init__(msg) @@ -74,12 +78,11 @@ class CircularDependencyError(DvcException): Args: dependency (str): path to the dependency. """ - def __init__(self, dependency): - assert (isinstance(dependency, str) or - isinstance(dependency, builtin_str)) - msg = ("file/directory '{}' is specified as an output and as a " - "dependency.") + def __init__(self, dependency): + assert isinstance(dependency, str) or isinstance(dependency, builtin_str) + + msg = "file/directory '{}' is specified as an output and as a " "dependency." super(CircularDependencyError, self).__init__(msg.format(dependency)) @@ -90,6 +93,7 @@ class ArgumentDuplicationError(DvcException): Args: path (str): path to the file/directory. """ + def __init__(self, path): assert isinstance(path, str) or isinstance(path, builtin_str) msg = "file '{}' is specified more than once." @@ -103,11 +107,14 @@ class MoveNotDataSourceError(DvcException): Args: path (str): path to the file/directory. """ + def __init__(self, path): - msg = "move is not permitted for stages that are not data sources. " \ - "You need to either move '{path}' to a new location and edit " \ - "it by hand, or remove '{path}' and create a new one at the " \ - "desired location." + msg = ( + "move is not permitted for stages that are not data sources. " + "You need to either move '{path}' to a new location and edit " + "it by hand, or remove '{path}' and create a new one at the " + "desired location." + ) super(MoveNotDataSourceError, self).__init__(msg.format(path=path)) @@ -117,6 +124,7 @@ class NotDvcProjectError(DvcException): Args: root (str): path to the directory. """ + def __init__(self, root): msg = "not a dvc repository (checked up to mount point '{}')" super(NotDvcProjectError, self).__init__(msg.format(root)) @@ -124,6 +132,7 @@ class NotDvcProjectError(DvcException): class DvcParserError(DvcException): """Base class for CLI parser errors.""" + def __init__(self): super(DvcParserError, self).__init__("parser error") @@ -131,12 +140,11 @@ class DvcParserError(DvcException): class CyclicGraphError(DvcException): def __init__(self, stages): assert isinstance(stages, list) - stages = '\n'.join('\t- {}'.format(stage) for stage in stages) + stages = "\n".join("\t- {}".format(stage) for stage in stages) msg = ( "you've introduced a cycle in your pipeline that involves" - ' the following stages:' - '\n' - '{stages}' - .format(stages=stages) + " the following stages:" + "\n" + "{stages}".format(stages=stages) ) super(CyclicGraphError, self).__init__(msg) diff --git a/dvc/istextfile.py b/dvc/istextfile.py index 8bd48ed06..c6159e991 100644 --- a/dvc/istextfile.py +++ b/dvc/istextfile.py @@ -18,7 +18,7 @@ def _int2byte(i): return chr(i) -TEXT_CHARS = (b''.join(_int2byte(i) for i in range(32, 127)) + b'\n\r\t\f\b') +TEXT_CHARS = b"".join(_int2byte(i) for i in range(32, 127)) + b"\n\r\t\f\b" def istextfile(fname, blocksize=512): @@ -27,14 +27,14 @@ def istextfile(fname, blocksize=512): If more than 30% of the chars in the block are non-text, or there are NUL ('\x00') bytes in the block, assume this is a binary file. """ - with open(fname, 'rb') as fobj: + with open(fname, "rb") as fobj: block = fobj.read(blocksize) if not block: # An empty file is considered a valid text file return True - if b'\x00' in block: + if b"\x00" in block: # Files with null bytes are binary return False diff --git a/dvc/lock.py b/dvc/lock.py index f66b162e8..8bd2a0fd9 100644 --- a/dvc/lock.py +++ b/dvc/lock.py @@ -21,7 +21,8 @@ class Lock(object): in. name (str): name of the lock file. """ - LOCK_FILE = 'lock' + + LOCK_FILE = "lock" TIMEOUT = 5 def __init__(self, dvc_dir, name=LOCK_FILE): @@ -32,8 +33,10 @@ class Lock(object): try: self._lock = zc.lockfile.LockFile(self.lock_file) except zc.lockfile.LockError: - raise LockError('cannot perform the cmd since DVC is busy and ' - 'locked. Please retry the cmd later.') + raise LockError( + "cannot perform the cmd since DVC is busy and " + "locked. Please retry the cmd later." + ) def lock(self): """Acquire lock for dvc project.""" diff --git a/dvc/logger.py b/dvc/logger.py index 11d24d8df..05b6b7a0a 100644 --- a/dvc/logger.py +++ b/dvc/logger.py @@ -21,34 +21,34 @@ def info(message): def debug(message): """Prints a debug message.""" - prefix = colorize('Debug', color='blue') + prefix = colorize("Debug", color="blue") - out = '{prefix}: {message}'.format(prefix=prefix, message=message) + out = "{prefix}: {message}".format(prefix=prefix, message=message) logger.debug(out) def warning(message): """Prints a warning message.""" - prefix = colorize('Warning', color='yellow') + prefix = colorize("Warning", color="yellow") - out = '{prefix}: {message}'.format(prefix=prefix, message=message) + out = "{prefix}: {message}".format(prefix=prefix, message=message) logger.warning(out) def error(message=None): """Prints an error message.""" - prefix = colorize('Error', color='red') + prefix = colorize("Error", color="red") exception, stack_trace = _parse_exc() out = ( - '{prefix}: {description}' - '\n' - '{stack_trace}' - '\n' - '{footer}'.format( + "{prefix}: {description}" + "\n" + "{stack_trace}" + "\n" + "{footer}".format( prefix=prefix, description=_description(message, exception), stack_trace=stack_trace, @@ -66,45 +66,41 @@ def box(message, border_color=None): message (str): message to print. border_color (str): name of a color to outline the box with. """ - lines = message.split('\n') + lines = message.split("\n") max_width = max(_visual_width(line) for line in lines) padding_horizontal = 5 padding_vertical = 1 - box_size_horizontal = (max_width + (padding_horizontal * 2)) + box_size_horizontal = max_width + (padding_horizontal * 2) - chars = { - 'corner': '+', - 'horizontal': '-', - 'vertical': '|', - 'empty': ' ', - } + chars = {"corner": "+", "horizontal": "-", "vertical": "|", "empty": " "} margin = "{corner}{line}{corner}\n".format( - corner=chars['corner'], - line=chars['horizontal'] * box_size_horizontal, + corner=chars["corner"], line=chars["horizontal"] * box_size_horizontal ) padding_lines = [ "{border}{space}{border}\n".format( - border=colorize(chars['vertical'], color=border_color), - space=chars['empty'] * box_size_horizontal, - ) * padding_vertical + border=colorize(chars["vertical"], color=border_color), + space=chars["empty"] * box_size_horizontal, + ) + * padding_vertical ] content_lines = [ "{border}{space}{content}{space}{border}\n".format( - border=colorize(chars['vertical'], color=border_color), - space=chars['empty'] * padding_horizontal, + border=colorize(chars["vertical"], color=border_color), + space=chars["empty"] * padding_horizontal, content=_visual_center(line, max_width), - ) for line in lines + ) + for line in lines ] box_str = "{margin}{padding}{content}{padding}{margin}".format( margin=colorize(margin, color=border_color), - padding=''.join(padding_lines), - content=''.join(content_lines), + padding="".join(padding_lines), + content="".join(content_lines), ) logger.info(box_str) @@ -126,11 +122,11 @@ def set_level(level_name): return levels = { - 'info': logging.INFO, - 'debug': logging.DEBUG, - 'warning': logging.WARNING, - 'error': logging.ERROR, - 'critical': logging.CRITICAL, + "info": logging.INFO, + "debug": logging.DEBUG, + "warning": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL, } logger.setLevel(levels.get(level_name)) @@ -182,16 +178,14 @@ def colorize(message, color=None): return message colors = { - 'green': colorama.Fore.GREEN, - 'yellow': colorama.Fore.YELLOW, - 'blue': colorama.Fore.BLUE, - 'red': colorama.Fore.RED, + "green": colorama.Fore.GREEN, + "yellow": colorama.Fore.YELLOW, + "blue": colorama.Fore.BLUE, + "red": colorama.Fore.RED, } - return '{color}{message}{nc}'.format( - color=colors.get(color, ''), - message=message, - nc=colorama.Fore.RESET, + return "{color}{message}{nc}".format( + color=colors.get(color, ""), message=message, nc=colorama.Fore.RESET ) @@ -205,7 +199,7 @@ def set_default_level(): def _add_handlers(): - formatter = '%(message)s' + formatter = "%(message)s" class _LogLevelFilter(logging.Filter): # pylint: disable=too-few-public-methods @@ -230,9 +224,9 @@ def _walk_exc(exc): tb_list = [traceback.format_exc()] # NOTE: parsing chained exceptions. See dvc/exceptions.py for more info. - while hasattr(exc, 'cause') and exc.cause is not None: + while hasattr(exc, "cause") and exc.cause is not None: exc_list.append(str(exc.cause)) - if hasattr(exc, 'cause_tb') and exc.cause_tb is not None: + if hasattr(exc, "cause_tb") and exc.cause_tb is not None: tb_list.insert(0, str(exc.cause_tb)) exc = exc.cause @@ -242,45 +236,44 @@ def _walk_exc(exc): def _parse_exc(): exc = sys.exc_info()[1] if not exc: - return (None, '') + return (None, "") exc_list, tb_list = _walk_exc(exc) - exception = ': '.join(exc_list) + exception = ": ".join(exc_list) if is_verbose(): - stack_trace = '{line}\n{stack_trace}{line}\n'.format( - line=colorize('-' * 60, color='red'), - stack_trace='\n'.join(tb_list), + stack_trace = "{line}\n{stack_trace}{line}\n".format( + line=colorize("-" * 60, color="red"), stack_trace="\n".join(tb_list) ) else: - stack_trace = '' + stack_trace = "" return (exception, stack_trace) def _description(message, exception): if exception and message: - description = '{message} - {exception}' + description = "{message} - {exception}" elif exception: - description = '{exception}' + description = "{exception}" elif message: - description = '{message}' + description = "{message}" return description.format(message=message, exception=exception) def _footer(): - return '{phrase} Hit us up at {url}, we are always happy to help!'.format( - phrase=colorize('Having any troubles?', 'yellow'), - url=colorize('https://dvc.org/support', 'blue'), + return "{phrase} Hit us up at {url}, we are always happy to help!".format( + phrase=colorize("Having any troubles?", "yellow"), + url=colorize("https://dvc.org/support", "blue"), ) def _visual_width(line): """Get the the number of columns required to display a string""" - return len(re.sub(colorama.ansitowin32.AnsiToWin32.ANSI_CSI_RE, '', line)) + return len(re.sub(colorama.ansitowin32.AnsiToWin32.ANSI_CSI_RE, "", line)) def _visual_center(line, width): @@ -288,12 +281,12 @@ def _visual_center(line, width): spaces = max(width - _visual_width(line), 0) left_padding = int(spaces / 2) - right_padding = (spaces - left_padding) + right_padding = spaces - left_padding - return (left_padding * ' ') + line + (right_padding * ' ') + return (left_padding * " ") + line + (right_padding * " ") -logger = logging.getLogger('dvc') # pylint: disable=invalid-name +logger = logging.getLogger("dvc") # pylint: disable=invalid-name set_default_level() _add_handlers() diff --git a/dvc/main.py b/dvc/main.py index 1fdd0943d..7ac37bcdb 100644 --- a/dvc/main.py +++ b/dvc/main.py @@ -39,7 +39,7 @@ def main(argv=None): except DvcParserError: ret = 254 except Exception: # pylint: disable=broad-except - logger.error('unexpected error') + logger.error("unexpected error") ret = 255 Analytics().send_cmd(cmd, args, ret) diff --git a/dvc/output/__init__.py b/dvc/output/__init__.py index 82f569141..e17b425b6 100644 --- a/dvc/output/__init__.py +++ b/dvc/output/__init__.py @@ -25,44 +25,41 @@ OUTS = [ # NOTE: OutputLOCAL is the default choice ] -OUTS_MAP = {'hdfs': OutputHDFS, - 's3': OutputS3, - 'gs': OutputGS, - 'ssh': OutputSSH, - 'local': OutputLOCAL} +OUTS_MAP = { + "hdfs": OutputHDFS, + "s3": OutputS3, + "gs": OutputGS, + "ssh": OutputSSH, + "local": OutputLOCAL, +} SCHEMA = { - OutputBase.PARAM_PATH: str, - - # NOTE: currently there are only 3 possible checksum names: - # - # 1) md5 (LOCAL, SSH, GS); - # 2) etag (S3); - # 3) checksum (HDFS); - # - # so when a few types of outputs share the same name, we only need - # specify it once. - schema.Optional(RemoteLOCAL.PARAM_CHECKSUM): schema.Or(str, None), - schema.Optional(RemoteS3.PARAM_CHECKSUM): schema.Or(str, None), - schema.Optional(RemoteHDFS.PARAM_CHECKSUM): schema.Or(str, None), - - schema.Optional(OutputBase.PARAM_CACHE): bool, - schema.Optional(OutputBase.PARAM_METRIC): OutputBase.METRIC_SCHEMA, + OutputBase.PARAM_PATH: str, + # NOTE: currently there are only 3 possible checksum names: + # + # 1) md5 (LOCAL, SSH, GS); + # 2) etag (S3); + # 3) checksum (HDFS); + # + # so when a few types of outputs share the same name, we only need + # specify it once. + schema.Optional(RemoteLOCAL.PARAM_CHECKSUM): schema.Or(str, None), + schema.Optional(RemoteS3.PARAM_CHECKSUM): schema.Or(str, None), + schema.Optional(RemoteHDFS.PARAM_CHECKSUM): schema.Or(str, None), + schema.Optional(OutputBase.PARAM_CACHE): bool, + schema.Optional(OutputBase.PARAM_METRIC): OutputBase.METRIC_SCHEMA, } def _get(stage, p, info, cache, metric): parsed = urlparse(p) - if parsed.scheme == 'remote': + if parsed.scheme == "remote": name = Config.SECTION_REMOTE_FMT.format(parsed.netloc) sect = stage.project.config.config[name] remote = Remote(stage.project, sect) - return OUTS_MAP[remote.scheme](stage, - p, - info, - cache=cache, - remote=remote, - metric=metric) + return OUTS_MAP[remote.scheme]( + stage, p, info, cache=cache, remote=remote, metric=metric + ) for o in OUTS: if o.supported(p): diff --git a/dvc/output/base.py b/dvc/output/base.py index 440f0d137..833513f74 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -30,26 +30,25 @@ class OutputBase(object): REMOTE = None - PARAM_PATH = 'path' - PARAM_CACHE = 'cache' - PARAM_METRIC = 'metric' - PARAM_METRIC_TYPE = 'type' - PARAM_METRIC_XPATH = 'xpath' + PARAM_PATH = "path" + PARAM_CACHE = "cache" + PARAM_METRIC = "metric" + PARAM_METRIC_TYPE = "type" + PARAM_METRIC_XPATH = "xpath" - METRIC_SCHEMA = Or(None, bool, - {Optional(PARAM_METRIC_TYPE): Or(str, None), - Optional(PARAM_METRIC_XPATH): Or(str, None)}) + METRIC_SCHEMA = Or( + None, + bool, + { + Optional(PARAM_METRIC_TYPE): Or(str, None), + Optional(PARAM_METRIC_XPATH): Or(str, None), + }, + ) DoesNotExistError = OutputDoesNotExistError IsNotFileOrDirError = OutputIsNotFileOrDirError - def __init__(self, - stage, - path, - info=None, - remote=None, - cache=True, - metric=False): + def __init__(self, stage, path, info=None, remote=None, cache=True, metric=False): self.stage = stage self.project = stage.project self.url = path @@ -58,15 +57,14 @@ class OutputBase(object): self.use_cache = False if self.IS_DEPENDENCY else cache self.metric = False if self.IS_DEPENDENCY else metric - if self.use_cache and getattr(self.project.cache, - self.REMOTE.scheme) is None: - raise DvcException("no cache location setup for '{}' outputs." - .format(self.REMOTE.scheme)) + if self.use_cache and getattr(self.project.cache, self.REMOTE.scheme) is None: + raise DvcException( + "no cache location setup for '{}' outputs.".format(self.REMOTE.scheme) + ) def __repr__(self): return "{class_name}: '{url}'".format( - class_name=type(self).__name__, - url=(self.url or 'No url') + class_name=type(self).__name__, url=(self.url or "No url") ) def __str__(self): @@ -92,11 +90,11 @@ class OutputBase(object): @property def path(self): - return self.path_info['path'] + return self.path_info["path"] @property def sep(self): - return '/' + return "/" @property def checksum(self): @@ -113,8 +111,9 @@ class OutputBase(object): if not self.use_cache: return self.info != self.remote.save_info(self.path_info) - return getattr(self.project.cache, self.scheme).changed(self.path_info, - self.info) + return getattr(self.project.cache, self.scheme).changed( + self.path_info, self.info + ) def changed_cache(self): if not self.use_cache or not self.checksum: @@ -127,15 +126,14 @@ class OutputBase(object): def status(self): if self.changed(): # FIXME better msgs - return {str(self): 'changed'} + return {str(self): "changed"} return {} def save(self): if not self.use_cache: self.info = self.remote.save_info(self.path_info) else: - self.info = getattr(self.project.cache, - self.scheme).save(self.path_info) + self.info = getattr(self.project.cache, self.scheme).save(self.path_info) def dumpd(self): ret = self.info.copy() @@ -147,8 +145,10 @@ class OutputBase(object): ret[self.PARAM_CACHE] = self.use_cache if isinstance(self.metric, dict): - if self.PARAM_METRIC_XPATH in self.metric \ - and not self.metric[self.PARAM_METRIC_XPATH]: + if ( + self.PARAM_METRIC_XPATH in self.metric + and not self.metric[self.PARAM_METRIC_XPATH] + ): del self.metric[self.PARAM_METRIC_XPATH] ret[self.PARAM_METRIC] = self.metric @@ -162,20 +162,20 @@ class OutputBase(object): if not self.use_cache: return - getattr(self.project.cache, self.scheme).checkout(self.path_info, - self.info, - force=force) + getattr(self.project.cache, self.scheme).checkout( + self.path_info, self.info, force=force + ) def remove(self, ignore_remove=False): self.remote.remove(self.path_info) - if self.scheme != 'local': + if self.scheme != "local": return if ignore_remove and self.use_cache and self.is_local: self.project.scm.ignore_remove(self.path) def move(self, out): - if self.scheme == 'local' and self.use_cache and self.is_local: + if self.scheme == "local" and self.use_cache and self.is_local: self.project.scm.ignore_remove(self.path) self.remote.move(self.path_info, out.path_info) @@ -183,5 +183,5 @@ class OutputBase(object): self.path_info = out.path_info self.save() - if self.scheme == 'local' and self.use_cache and self.is_local: + if self.scheme == "local" and self.use_cache and self.is_local: self.project.scm.ignore(self.path) diff --git a/dvc/output/hdfs.py b/dvc/output/hdfs.py index 9e35dd4a5..a162909d2 100644 --- a/dvc/output/hdfs.py +++ b/dvc/output/hdfs.py @@ -10,22 +10,11 @@ from dvc.remote.hdfs import RemoteHDFS class OutputHDFS(OutputBase): REMOTE = RemoteHDFS - def __init__(self, - stage, - path, - info=None, - remote=None, - cache=True, - metric=False): - super(OutputHDFS, self).__init__(stage, - path, - info=info, - remote=remote, - cache=cache, - metric=metric) + def __init__(self, stage, path, info=None, remote=None, cache=True, metric=False): + super(OutputHDFS, self).__init__( + stage, path, info=info, remote=remote, cache=cache, metric=metric + ) if remote: - path = posixpath.join(remote.url, urlparse(path).path.lstrip('/')) - user = remote.user if remote else self.group('user') - self.path_info = {'scheme': 'hdfs', - 'user': user, - 'path': path} + path = posixpath.join(remote.url, urlparse(path).path.lstrip("/")) + user = remote.user if remote else self.group("user") + self.path_info = {"scheme": "hdfs", "user": user, "path": path} diff --git a/dvc/output/local.py b/dvc/output/local.py index 5cf988359..8263680bd 100644 --- a/dvc/output/local.py +++ b/dvc/output/local.py @@ -13,22 +13,12 @@ from dvc.output.base import OutputBase, OutputAlreadyTrackedError class OutputLOCAL(OutputBase): REMOTE = RemoteLOCAL - def __init__(self, - stage, - path, - info=None, - remote=None, - cache=True, - metric=False): - super(OutputLOCAL, self).__init__(stage, - path, - info, - remote=remote, - cache=cache, - metric=metric) + def __init__(self, stage, path, info=None, remote=None, cache=True, metric=False): + super(OutputLOCAL, self).__init__( + stage, path, info, remote=remote, cache=cache, metric=metric + ) if remote: - p = os.path.join(remote.prefix, - urlparse(self.url).path.lstrip('/')) + p = os.path.join(remote.prefix, urlparse(self.url).path.lstrip("/")) else: p = path @@ -37,16 +27,14 @@ class OutputLOCAL(OutputBase): p = os.path.join(stage.cwd, p) p = os.path.abspath(os.path.normpath(p)) - self.path_info = {'scheme': 'local', - 'path': p} + self.path_info = {"scheme": "local", "path": p} def __str__(self): return self.rel_path @property def is_local(self): - return (urlparse(self.url).scheme != 'remote' - and not os.path.isabs(self.url)) + return urlparse(self.url).scheme != "remote" and not os.path.isabs(self.url) @property def sep(self): @@ -63,8 +51,7 @@ class OutputLOCAL(OutputBase): def dumpd(self): ret = super(OutputLOCAL, self).dumpd() if self.is_local: - path = self.remote.unixpath(os.path.relpath(self.path, - self.stage.cwd)) + path = self.remote.unixpath(os.path.relpath(self.path, self.stage.cwd)) else: path = self.url @@ -91,11 +78,12 @@ class OutputLOCAL(OutputBase): if not os.path.exists(self.path): raise self.DoesNotExistError(self.rel_path) - if (not os.path.isfile(self.path) and not os.path.isdir(self.path)): + if not os.path.isfile(self.path) and not os.path.isdir(self.path): raise self.IsNotFileOrDirError(self.rel_path) - if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or \ - (os.path.isdir(self.path) and len(os.listdir(self.path)) == 0): + if (os.path.isfile(self.path) and os.path.getsize(self.path) == 0) or ( + os.path.isdir(self.path) and len(os.listdir(self.path)) == 0 + ): msg = "file/directory '{}' is empty.".format(self.rel_path) logger.warning(msg) diff --git a/dvc/output/s3.py b/dvc/output/s3.py index 2434d2fc5..300ba1bb6 100644 --- a/dvc/output/s3.py +++ b/dvc/output/s3.py @@ -10,23 +10,12 @@ from dvc.output.base import OutputBase class OutputS3(OutputBase): REMOTE = RemoteS3 - def __init__(self, - stage, - path, - info=None, - remote=None, - cache=True, - metric=False): - super(OutputS3, self).__init__(stage, - path, - info=info, - remote=remote, - cache=cache, - metric=metric) + def __init__(self, stage, path, info=None, remote=None, cache=True, metric=False): + super(OutputS3, self).__init__( + stage, path, info=info, remote=remote, cache=cache, metric=metric + ) bucket = remote.bucket if remote else urlparse(path).netloc - path = urlparse(path).path.lstrip('/') + path = urlparse(path).path.lstrip("/") if remote: path = posixpath.join(remote.prefix, path) - self.path_info = {'scheme': self.scheme, - 'bucket': bucket, - 'path': path} + self.path_info = {"scheme": self.scheme, "bucket": bucket, "path": path} diff --git a/dvc/output/ssh.py b/dvc/output/ssh.py index fd02fdc73..2eac4c38b 100644 --- a/dvc/output/ssh.py +++ b/dvc/output/ssh.py @@ -11,34 +11,24 @@ from dvc.remote.ssh import RemoteSSH class OutputSSH(OutputBase): REMOTE = RemoteSSH - def __init__(self, - stage, - path, - info=None, - remote=None, - cache=True, - metric=False): - super(OutputSSH, self).__init__(stage, - path, - info=info, - remote=remote, - cache=cache, - metric=metric) + def __init__(self, stage, path, info=None, remote=None, cache=True, metric=False): + super(OutputSSH, self).__init__( + stage, path, info=info, remote=remote, cache=cache, metric=metric + ) parsed = urlparse(path) host = remote.host if remote else parsed.hostname - port = remote.port if remote else (parsed.port - or RemoteSSH.DEFAULT_PORT) - user = remote.user if remote else (parsed.username - or getpass.getuser()) + port = remote.port if remote else (parsed.port or RemoteSSH.DEFAULT_PORT) + user = remote.user if remote else (parsed.username or getpass.getuser()) if remote: - path = posixpath.join(remote.prefix, - urlparse(path).path.lstrip('/')) + path = posixpath.join(remote.prefix, urlparse(path).path.lstrip("/")) else: path = parsed.path - self.path_info = {'scheme': 'ssh', - 'host': host, - 'port': port, - 'user': user, - 'path': path} + self.path_info = { + "scheme": "ssh", + "host": host, + "port": port, + "user": user, + "path": path, + } diff --git a/dvc/progress.py b/dvc/progress.py index 4ac34132a..3851bf8a8 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -14,6 +14,7 @@ class Progress(object): """ Simple multi-target progress bar. """ + def __init__(self): self._n_total = 0 self._n_finished = 0 @@ -31,11 +32,11 @@ class Progress(object): return self._n_total == self._n_finished def _clearln(self): - self._print('\r\x1b[K', end='') + self._print("\r\x1b[K", end="") def _writeln(self, line): self._clearln() - self._print(line, end='') + self._print(line, end="") sys.stdout.flush() def refresh(self, line=None): @@ -83,7 +84,7 @@ class Progress(object): percent = "?% " else: total = int(total) - state = int((100 * current)/total) if current < total else 100 + state = int((100 * current) / total) if current < total else 100 percent = str(state) + "% " if self._n_total > 1: @@ -91,9 +92,9 @@ class Progress(object): else: num = "" - n_sh = int((state * bar_len)/100) + n_sh = int((state * bar_len) / 100) n_sp = bar_len - n_sh - pbar = "[" + '#'*n_sh + ' '*n_sp + "] " + pbar = "[" + "#" * n_sh + " " * n_sp + "] " return num + pbar + percent + target_name diff --git a/dvc/project.py b/dvc/project.py index f83fb7f82..8d9199dbc 100644 --- a/dvc/project.py +++ b/dvc/project.py @@ -7,9 +7,7 @@ import os import dvc.prompt as prompt import dvc.logger as logger -from dvc.exceptions import (DvcException, - MoveNotDataSourceError, - NotDvcProjectError) +from dvc.exceptions import DvcException, MoveNotDataSourceError, NotDvcProjectError class InitError(DvcException): @@ -25,7 +23,7 @@ class ReproductionError(DvcException): class Project(object): - DVC_DIR = '.dvc' + DVC_DIR = ".dvc" def __init__(self, root_dir=None): from dvc.config import Config @@ -91,11 +89,12 @@ class Project(object): if not self.files_to_git_add: return - logger.info('\n' - 'To track the changes with git run:\n' - '\n' - '\tgit add {files}' - .format(files=' '.join(self.files_to_git_add))) + logger.info( + "\n" + "To track the changes with git run:\n" + "\n" + "\tgit add {files}".format(files=" ".join(self.files_to_git_add)) + ) @staticmethod def init(root_dir=os.curdir, no_scm=False, force=False): @@ -127,15 +126,17 @@ class Project(object): if type(scm) == Base and not no_scm: raise InitError( "{project} is not tracked by any supported scm tool" - " (e.g. git). Use '--no-scm' if you don't want to use any scm." - .format(project=root_dir) + " (e.g. git). Use '--no-scm' if you don't want to use any scm.".format( + project=root_dir + ) ) if os.path.isdir(dvc_dir): if not force: raise InitError( - "'{project}' exists. Use '-f' to force." - .format(project=os.path.relpath(dvc_dir)) + "'{project}' exists. Use '-f' to force.".format( + project=os.path.relpath(dvc_dir) + ) ) shutil.rmtree(dvc_dir) @@ -149,7 +150,7 @@ class Project(object): if scm.ignore_file: scm.add([os.path.join(dvc_dir, scm.ignore_file)]) - logger.info('\nYou can now commit the changes to git.\n') + logger.info("\nYou can now commit the changes to git.\n") proj._welcome_message() @@ -212,9 +213,7 @@ class Project(object): self.files_to_git_add = [] with self.state: for f in fnames: - stage = Stage.create(project=self, - outs=[f], - add=True) + stage = Stage.create(project=self, outs=[f], add=True) if stage is None: stages.append(stage) @@ -277,19 +276,21 @@ class Project(object): import dvc.output as Output from dvc.stage import Stage - from_out = Output.loads_from(Stage(self, cwd=os.curdir), - [from_path])[0] + from_out = Output.loads_from(Stage(self, cwd=os.curdir), [from_path])[0] to_path = self._expand_target_path(from_path, to_path) try: - stage, out = next((stage, out) - for stage in self.stages() - for out in stage.outs - if from_out.path == out.path) + stage, out = next( + (stage, out) + for stage in self.stages() + for out in stage.outs + if from_out.path == out.path + ) except StopIteration: - raise DvcException("unable to find stage file with output '{path}'" - .format(path=from_path)) + raise DvcException( + "unable to find stage file with output '{path}'".format(path=from_path) + ) if not stage.is_data_source: raise MoveNotDataSourceError(stage.relpath) @@ -301,15 +302,14 @@ class Project(object): stage.path = os.path.join( os.path.dirname(to_path), - os.path.basename(to_path) + Stage.STAGE_FILE_SUFFIX + os.path.basename(to_path) + Stage.STAGE_FILE_SUFFIX, ) stage.cwd = os.path.join(self.root_dir, os.path.dirname(to_path)) - to_out = Output.loads_from(stage, - [os.path.basename(to_path)], - out.cache, - out.metric)[0] + to_out = Output.loads_from( + stage, [os.path.basename(to_path)], out.cache, out.metric + )[0] with self.state: out.move(to_out) @@ -327,15 +327,17 @@ class Project(object): if System.is_symlink(path) or System.is_hardlink(path): logger.debug("Unprotecting '{}'".format(path)) - tmp = os.path.join(os.path.dirname(path), '.' + str(uuid.uuid4())) + tmp = os.path.join(os.path.dirname(path), "." + str(uuid.uuid4())) move(path, tmp) copyfile(tmp, path) remove(tmp) else: - logger.debug("Skipping copying for '{}', since it is not " - "a symlink or a hardlink.".format(path)) + logger.debug( + "Skipping copying for '{}', since it is not " + "a symlink or a hardlink.".format(path) + ) os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE) @@ -347,42 +349,43 @@ class Project(object): def unprotect(self, path): if not os.path.exists(path): - raise DvcException( - "can't unprotect non-existing data '{}'" - .format(path) - ) + raise DvcException("can't unprotect non-existing data '{}'".format(path)) if os.path.isdir(path): self._unprotect_dir(path) else: self._unprotect_file(path) - def run(self, - cmd=None, - deps=[], - outs=[], - outs_no_cache=[], - metrics_no_cache=[], - fname=None, - cwd=os.curdir, - no_exec=False, - overwrite=False, - ignore_build_cache=False, - remove_outs=False): + def run( + self, + cmd=None, + deps=[], + outs=[], + outs_no_cache=[], + metrics_no_cache=[], + fname=None, + cwd=os.curdir, + no_exec=False, + overwrite=False, + ignore_build_cache=False, + remove_outs=False, + ): from dvc.stage import Stage with self.state: - stage = Stage.create(project=self, - fname=fname, - cmd=cmd, - cwd=cwd, - outs=outs, - outs_no_cache=outs_no_cache, - metrics_no_cache=metrics_no_cache, - deps=deps, - overwrite=overwrite, - ignore_build_cache=ignore_build_cache, - remove_outs=remove_outs) + stage = Stage.create( + project=self, + fname=fname, + cmd=cmd, + cwd=cwd, + outs=outs, + outs_no_cache=outs_no_cache, + metrics_no_cache=metrics_no_cache, + deps=deps, + overwrite=overwrite, + ignore_build_cache=ignore_build_cache, + remove_outs=remove_outs, + ) if stage is None: return None @@ -403,10 +406,7 @@ class Project(object): def imp(self, url, out): from dvc.stage import Stage - stage = Stage.create(project=self, - cmd=None, - deps=[url], - outs=[out]) + stage = Stage.create(project=self, cmd=None, deps=[url], outs=[out]) if stage is None: return None @@ -429,8 +429,7 @@ class Project(object): if stage.locked: logger.warning( "DVC file '{path}' is locked. Its dependencies are" - " not going to be reproduced." - .format(path=stage.relpath) + " not going to be reproduced.".format(path=stage.relpath) ) stage = stage.reproduce(force=force, dry=dry, interactive=interactive) @@ -442,15 +441,17 @@ class Project(object): return [stage] - def reproduce(self, - target=None, - recursive=True, - force=False, - dry=False, - interactive=False, - pipeline=False, - all_pipelines=False, - ignore_build_cache=False): + def reproduce( + self, + target=None, + recursive=True, + force=False, + dry=False, + interactive=False, + pipeline=False, + all_pipelines=False, + ignore_build_cache=False, + ): from dvc.stage import Stage if not target and not all_pipelines: @@ -482,68 +483,55 @@ class Project(object): ret = [] with self.state: for target in targets: - stages = self._reproduce(target, - recursive=recursive, - force=force, - dry=dry, - interactive=interactive, - ignore_build_cache=ignore_build_cache) + stages = self._reproduce( + target, + recursive=recursive, + force=force, + dry=dry, + interactive=interactive, + ignore_build_cache=ignore_build_cache, + ) ret.extend(stages) self._remind_to_git_add() return ret - def _reproduce(self, - target, - recursive=True, - force=False, - dry=False, - interactive=False, - ignore_build_cache=False): + def _reproduce( + self, + target, + recursive=True, + force=False, + dry=False, + interactive=False, + ignore_build_cache=False, + ): import networkx as nx from dvc.stage import Stage stage = Stage.load(self, target) G = self.graph()[1] - stages = nx.get_node_attributes(G, 'stage') + stages = nx.get_node_attributes(G, "stage") node = os.path.relpath(stage.path, self.root_dir) if recursive: - ret = self._reproduce_stages(G, - stages, - node, - force, - dry, - interactive, - ignore_build_cache) + ret = self._reproduce_stages( + G, stages, node, force, dry, interactive, ignore_build_cache + ) else: - ret = self._reproduce_stage(stages, - node, - force, - dry, - interactive) + ret = self._reproduce_stage(stages, node, force, dry, interactive) return ret - def _reproduce_stages(self, - G, - stages, - node, - force, - dry, - interactive, - ignore_build_cache): + def _reproduce_stages( + self, G, stages, node, force, dry, interactive, ignore_build_cache + ): import networkx as nx result = [] for n in nx.dfs_postorder_nodes(G, node): try: - ret = self._reproduce_stage(stages, - n, - force, - dry, - interactive) + ret = self._reproduce_stage(stages, n, force, dry, interactive) if len(ret) == 0 and ignore_build_cache: # NOTE: we are walking our pipeline from the top to the @@ -565,11 +553,7 @@ class Project(object): used.append(out.path) self.state.remove_unused_links(used) - def checkout(self, - target=None, - with_deps=False, - force=False, - recursive=False): + def checkout(self, target=None, with_deps=False, force=False, recursive=False): if target and not recursive: all_stages = self.active_stages() @@ -585,15 +569,13 @@ class Project(object): if stage.locked: logger.warning( "DVC file '{path}' is locked. Its dependencies are" - " not going to be checked out." - .format(path=stage.relpath) + " not going to be checked out.".format(path=stage.relpath) ) stage.checkout(force=force) def _get_pipeline(self, node): - pipelines = list(filter(lambda g: node in g.nodes(), - self.pipelines())) + pipelines = list(filter(lambda g: node in g.nodes(), self.pipelines())) assert len(pipelines) == 1 return pipelines[0] @@ -607,7 +589,7 @@ class Project(object): node = os.path.relpath(stage.path, self.root_dir) G = self._get_pipeline(node) - stages = nx.get_node_attributes(G, 'stage') + stages = nx.get_node_attributes(G, "stage") ret = [stage] for n in nx.dfs_postorder_nodes(G, node): @@ -615,12 +597,7 @@ class Project(object): return ret - def _collect_dir_cache(self, - out, - branch=None, - remote=None, - force=False, - jobs=None): + def _collect_dir_cache(self, out, branch=None, remote=None, force=False, jobs=None): info = out.dumpd() ret = [info] r = out.remote @@ -628,54 +605,50 @@ class Project(object): if self.cache.local.changed_cache_file(md5): try: - self.cloud.pull(ret, - jobs=jobs, - remote=remote, - show_checksums=False) + self.cloud.pull(ret, jobs=jobs, remote=remote, show_checksums=False) except DvcException as exc: msg = "Failed to pull cache for '{}': {}" logger.debug(msg.format(out, exc)) if self.cache.local.changed_cache_file(md5): - msg = "Missing cache for directory '{}'. " \ - "Cache for files inside will be lost. " \ - "Would you like to continue? Use '-f' to force. " + msg = ( + "Missing cache for directory '{}'. " + "Cache for files inside will be lost. " + "Would you like to continue? Use '-f' to force. " + ) if not force and not prompt.confirm(msg): raise DvcException( "unable to fully collect used cache" - " without cache for directory '{}'" - .format(out) + " without cache for directory '{}'".format(out) ) else: return ret for i in self.cache.local.load_dir_cache(md5): - i['branch'] = branch - i[r.PARAM_PATH] = os.path.join(info[r.PARAM_PATH], - i[r.PARAM_RELPATH]) + i["branch"] = branch + i[r.PARAM_PATH] = os.path.join(info[r.PARAM_PATH], i[r.PARAM_RELPATH]) ret.append(i) return ret - def _collect_used_cache(self, - out, - branch=None, - remote=None, - force=False, - jobs=None): + def _collect_used_cache( + self, out, branch=None, remote=None, force=False, jobs=None + ): if not out.use_cache or not out.info: if not out.info: - logger.warning("Output '{}'({}) is missing version " - "info. Cache for it will not be collected. " - "Use dvc repro to get your pipeline up to " - "date.".format(out, out.stage)) + logger.warning( + "Output '{}'({}) is missing version " + "info. Cache for it will not be collected. " + "Use dvc repro to get your pipeline up to " + "date.".format(out, out.stage) + ) return [] info = out.dumpd() - info['branch'] = branch + info["branch"] = branch ret = [info] - if out.scheme != 'local': + if out.scheme != "local": return ret md5 = info[out.remote.PARAM_CHECKSUM] @@ -683,38 +656,36 @@ class Project(object): if not out.remote.is_dir_cache(cache): return ret - return self._collect_dir_cache(out, - branch=branch, - remote=remote, - force=force, - jobs=jobs) + return self._collect_dir_cache( + out, branch=branch, remote=remote, force=force, jobs=jobs + ) - def _used_cache(self, - target=None, - all_branches=False, - active=True, - with_deps=False, - all_tags=False, - remote=None, - force=False, - jobs=None, - recursive=False): + def _used_cache( + self, + target=None, + all_branches=False, + active=True, + with_deps=False, + all_tags=False, + remote=None, + force=False, + jobs=None, + recursive=False, + ): cache = {} - cache['local'] = [] - cache['s3'] = [] - cache['gs'] = [] - cache['hdfs'] = [] - cache['ssh'] = [] - cache['azure'] = [] + cache["local"] = [] + cache["s3"] = [] + cache["gs"] = [] + cache["hdfs"] = [] + cache["ssh"] = [] + cache["azure"] = [] - for branch in self.scm.brancher(all_branches=all_branches, - all_tags=all_tags): + for branch in self.scm.brancher(all_branches=all_branches, all_tags=all_tags): if target: if recursive: stages = self.stages(target) else: - stages = self._collect(target, - with_deps=with_deps) + stages = self._collect(target, with_deps=with_deps) elif active: stages = self.active_stages() else: @@ -724,17 +695,16 @@ class Project(object): if active and not target and stage.locked: logger.warning( "DVC file '{path}' is locked. Its dependencies are" - " not going to be pushed/pulled/fetched." - .format(path=stage.relpath) + " not going to be pushed/pulled/fetched.".format( + path=stage.relpath + ) ) for out in stage.outs: - scheme = out.path_info['scheme'] - cache[scheme] += self._collect_used_cache(out, - branch=branch, - remote=remote, - force=force, - jobs=jobs) + scheme = out.path_info["scheme"] + cache[scheme] += self._collect_used_cache( + out, branch=branch, remote=remote, force=force, jobs=jobs + ) return cache @@ -751,27 +721,31 @@ class Project(object): return merged_cache @staticmethod - def load_all_used_cache(projects, - target=None, - all_branches=False, - active=True, - with_deps=False, - all_tags=False, - remote=None, - force=False, - jobs=None): + def load_all_used_cache( + projects, + target=None, + all_branches=False, + active=True, + with_deps=False, + all_tags=False, + remote=None, + force=False, + jobs=None, + ): clists = [] for project in projects: with project.state: - project_clist = project._used_cache(target=None, - all_branches=all_branches, - active=False, - with_deps=with_deps, - all_tags=all_tags, - remote=remote, - force=force, - jobs=jobs) + project_clist = project._used_cache( + target=None, + all_branches=all_branches, + active=False, + with_deps=with_deps, + all_tags=all_tags, + remote=remote, + force=force, + jobs=jobs, + ) clists.append(project_clist) @@ -782,30 +756,34 @@ class Project(object): if not removed: logger.info("No unused {} cache to remove.".format(typ)) - def gc(self, - all_branches=False, - cloud=False, - remote=None, - with_deps=False, - all_tags=False, - force=False, - jobs=None, - projects=None): + def gc( + self, + all_branches=False, + cloud=False, + remote=None, + with_deps=False, + all_tags=False, + force=False, + jobs=None, + projects=None, + ): all_projects = [self] if projects: all_projects.extend(Project(path) for path in projects) - all_clists = Project.load_all_used_cache(all_projects, - target=None, - all_branches=all_branches, - active=False, - with_deps=with_deps, - all_tags=all_tags, - remote=remote, - force=force, - jobs=jobs) + all_clists = Project.load_all_used_cache( + all_projects, + target=None, + all_branches=all_branches, + active=False, + with_deps=with_deps, + all_tags=all_tags, + remote=remote, + force=force, + jobs=jobs, + ) if len(all_clists) > 1: clist = Project.merge_cache_lists(all_clists) @@ -813,102 +791,105 @@ class Project(object): clist = all_clists[0] with self.state: - self._do_gc('local', self.cache.local.gc, clist) + self._do_gc("local", self.cache.local.gc, clist) if self.cache.s3: - self._do_gc('s3', self.cache.s3.gc, clist) + self._do_gc("s3", self.cache.s3.gc, clist) if self.cache.gs: - self._do_gc('gs', self.cache.gs.gc, clist) + self._do_gc("gs", self.cache.gs.gc, clist) if self.cache.ssh: - self._do_gc('ssh', self.cache.ssh.gc, clist) + self._do_gc("ssh", self.cache.ssh.gc, clist) if self.cache.hdfs: - self._do_gc('hdfs', self.cache.hdfs.gc, clist) + self._do_gc("hdfs", self.cache.hdfs.gc, clist) if self.cache.azure: - self._do_gc('azure', self.cache.azure.gc, clist) + self._do_gc("azure", self.cache.azure.gc, clist) if cloud: - self._do_gc('remote', self.cloud._get_cloud(remote, - 'gc -c').gc, clist) + self._do_gc("remote", self.cloud._get_cloud(remote, "gc -c").gc, clist) - def push(self, - target=None, - jobs=1, - remote=None, - all_branches=False, - show_checksums=False, - with_deps=False, - all_tags=False, - recursive=False): + def push( + self, + target=None, + jobs=1, + remote=None, + all_branches=False, + show_checksums=False, + with_deps=False, + all_tags=False, + recursive=False, + ): with self.state: - used = self._used_cache(target, - all_branches=all_branches, - all_tags=all_tags, - with_deps=with_deps, - force=True, - remote=remote, - jobs=jobs, - recursive=recursive)['local'] - self.cloud.push(used, - jobs, - remote=remote, - show_checksums=show_checksums) + used = self._used_cache( + target, + all_branches=all_branches, + all_tags=all_tags, + with_deps=with_deps, + force=True, + remote=remote, + jobs=jobs, + recursive=recursive, + )["local"] + self.cloud.push(used, jobs, remote=remote, show_checksums=show_checksums) - def fetch(self, - target=None, - jobs=1, - remote=None, - all_branches=False, - show_checksums=False, - with_deps=False, - all_tags=False, - recursive=False): + def fetch( + self, + target=None, + jobs=1, + remote=None, + all_branches=False, + show_checksums=False, + with_deps=False, + all_tags=False, + recursive=False, + ): with self.state: - used = self._used_cache(target, - all_branches=all_branches, - all_tags=all_tags, - with_deps=with_deps, - force=True, - remote=remote, - jobs=jobs, - recursive=recursive)['local'] - self.cloud.pull(used, - jobs, - remote=remote, - show_checksums=show_checksums) + used = self._used_cache( + target, + all_branches=all_branches, + all_tags=all_tags, + with_deps=with_deps, + force=True, + remote=remote, + jobs=jobs, + recursive=recursive, + )["local"] + self.cloud.pull(used, jobs, remote=remote, show_checksums=show_checksums) - def pull(self, - target=None, - jobs=1, - remote=None, - all_branches=False, - show_checksums=False, - with_deps=False, - all_tags=False, - force=False, - recursive=False): - self.fetch(target, - jobs, - remote=remote, - all_branches=all_branches, - all_tags=all_tags, - show_checksums=show_checksums, - with_deps=with_deps, - recursive=recursive) - self.checkout(target=target, - with_deps=with_deps, - force=force, - recursive=recursive) + def pull( + self, + target=None, + jobs=1, + remote=None, + all_branches=False, + show_checksums=False, + with_deps=False, + all_tags=False, + force=False, + recursive=False, + ): + self.fetch( + target, + jobs, + remote=remote, + all_branches=all_branches, + all_tags=all_tags, + show_checksums=show_checksums, + with_deps=with_deps, + recursive=recursive, + ) + self.checkout( + target=target, with_deps=with_deps, force=force, recursive=recursive + ) def _local_status(self, target=None, with_deps=False): status = {} if target: - stages = self._collect(target, - with_deps=with_deps) + stages = self._collect(target, with_deps=with_deps) else: stages = self.active_stages() @@ -916,72 +897,76 @@ class Project(object): if stage.locked: logger.warning( "DVC file '{path}' is locked. Its dependencies are" - " not going to be shown in the status output." - .format(path=stage.relpath) + " not going to be shown in the status output.".format( + path=stage.relpath + ) ) status.update(stage.status()) return status - def _cloud_status(self, - target=None, - jobs=1, - remote=None, - show_checksums=False, - all_branches=False, - with_deps=False, - all_tags=False): + def _cloud_status( + self, + target=None, + jobs=1, + remote=None, + show_checksums=False, + all_branches=False, + with_deps=False, + all_tags=False, + ): import dvc.remote.base as cloud - used = self._used_cache(target, - all_branches=all_branches, - all_tags=all_tags, - with_deps=with_deps, - force=True, - remote=remote, - jobs=jobs)['local'] + used = self._used_cache( + target, + all_branches=all_branches, + all_tags=all_tags, + with_deps=with_deps, + force=True, + remote=remote, + jobs=jobs, + )["local"] ret = {} - status_info = self.cloud.status(used, - jobs, - remote=remote, - show_checksums=show_checksums) + status_info = self.cloud.status( + used, jobs, remote=remote, show_checksums=show_checksums + ) for md5, info in status_info.items(): - name = info['name'] - status = info['status'] + name = info["name"] + status = info["status"] if status == cloud.STATUS_OK: continue - prefix_map = { - cloud.STATUS_DELETED: 'deleted', - cloud.STATUS_NEW: 'new', - } + prefix_map = {cloud.STATUS_DELETED: "deleted", cloud.STATUS_NEW: "new"} ret[name] = prefix_map[status] return ret - def status(self, - target=None, - jobs=1, - cloud=False, - remote=None, - show_checksums=False, - all_branches=False, - with_deps=False, - all_tags=False): + def status( + self, + target=None, + jobs=1, + cloud=False, + remote=None, + show_checksums=False, + all_branches=False, + with_deps=False, + all_tags=False, + ): with self.state: if cloud: - return self._cloud_status(target, - jobs, - remote=remote, - show_checksums=show_checksums, - all_branches=all_branches, - with_deps=with_deps, - all_tags=all_tags) - return self._local_status(target, - with_deps=with_deps) + return self._cloud_status( + target, + jobs, + remote=remote, + show_checksums=show_checksums, + all_branches=all_branches, + with_deps=with_deps, + all_tags=all_tags, + ) + return self._local_status(target, with_deps=with_deps) def _read_metric_json(self, fd, json_path): import json @@ -1002,7 +987,7 @@ class Project(object): def _read_metric_hxsv(self, fd, hxsv_path, delimiter): import csv - col, row = hxsv_path.split(',') + col, row = hxsv_path.split(",") row = int(row) reader = list(csv.DictReader(fd, delimiter=builtin_str(delimiter))) return self._do_read_metric_xsv(reader, row, col) @@ -1010,7 +995,7 @@ class Project(object): def _read_metric_xsv(self, fd, xsv_path, delimiter): import csv - col, row = xsv_path.split(',') + col, row = xsv_path.split(",") row = int(row) col = int(col) reader = list(csv.reader(fd, delimiter=builtin_str(delimiter))) @@ -1023,17 +1008,17 @@ class Project(object): return ret try: - with open(path, 'r') as fd: - if typ == 'json': + with open(path, "r") as fd: + if typ == "json": ret = self._read_metric_json(fd, xpath) - elif typ == 'csv': - ret = self._read_metric_xsv(fd, xpath, ',') - elif typ == 'tsv': - ret = self._read_metric_xsv(fd, xpath, '\t') - elif typ == 'hcsv': - ret = self._read_metric_hxsv(fd, xpath, ',') - elif typ == 'htsv': - ret = self._read_metric_hxsv(fd, xpath, '\t') + elif typ == "csv": + ret = self._read_metric_xsv(fd, xpath, ",") + elif typ == "tsv": + ret = self._read_metric_xsv(fd, xpath, "\t") + elif typ == "hcsv": + ret = self._read_metric_hxsv(fd, xpath, ",") + elif typ == "htsv": + ret = self._read_metric_hxsv(fd, xpath, "\t") else: ret = fd.read() except Exception: @@ -1056,27 +1041,25 @@ class Project(object): return matched[0] if matched else None - def metrics_show(self, - path=None, - typ=None, - xpath=None, - all_branches=False, - all_tags=False): + def metrics_show( + self, path=None, typ=None, xpath=None, all_branches=False, all_tags=False + ): res = {} - for branch in self.scm.brancher(all_branches=all_branches, - all_tags=all_tags): + for branch in self.scm.brancher(all_branches=all_branches, all_tags=all_tags): astages = self.active_stages() outs = [out for stage in astages for out in stage.outs] if path: out = self._find_output_by_path(path, outs=outs) stage = out.stage.path if out else None - if out and all([out.metric, - not typ, - isinstance(out.metric, dict)]): - entries = [(path, - out.metric.get(out.PARAM_METRIC_TYPE, None), - out.metric.get(out.PARAM_METRIC_XPATH, None))] + if out and all([out.metric, not typ, isinstance(out.metric, dict)]): + entries = [ + ( + path, + out.metric.get(out.PARAM_METRIC_TYPE, None), + out.metric.get(out.PARAM_METRIC_XPATH, None), + ) + ] else: entries = [(path, typ, xpath)] else: @@ -1097,9 +1080,7 @@ class Project(object): self.checkout(stage, force=True) rel = os.path.relpath(fname) - metric = self._read_metric(fname, - typ=t, - xpath=x) + metric = self._read_metric(fname, typ=t, xpath=x) if not metric: continue @@ -1110,9 +1091,9 @@ class Project(object): for branch, val in res.items(): if all_branches or all_tags: - logger.info('{}:'.format(branch)) + logger.info("{}:".format(branch)) for fname, metric in val.items(): - logger.info('\t{}: {}'.format(fname, metric)) + logger.info("\t{}: {}".format(fname, metric)) if res: return res @@ -1133,9 +1114,9 @@ class Project(object): msg = "unable to find file '{}' in the pipeline".format(path) raise DvcException(msg) - if out.scheme != 'local': + if out.scheme != "local": msg = "output '{}' scheme '{}' is not supported for metrics" - raise DvcException(msg.format(out.path, out.path_info['scheme'])) + raise DvcException(msg.format(out.path, out.path_info["scheme"])) if out.use_cache: msg = "cached output '{}' is not supported for metrics" @@ -1163,7 +1144,7 @@ class Project(object): def metrics_add(self, path, typ=None, xpath=None): if not typ: - typ = 'raw' + typ = "raw" self._metrics_modify(path, typ, xpath) def metrics_remove(self, path): @@ -1171,8 +1152,7 @@ class Project(object): def graph(self, stages=None, from_directory=None): import networkx as nx - from dvc.exceptions import (OutputDuplicationError, - WorkingDirectoryAsOutputError) + from dvc.exceptions import OutputDuplicationError, WorkingDirectoryAsOutputError G = nx.DiGraph() G_active = nx.DiGraph() @@ -1192,12 +1172,12 @@ class Project(object): for stage in stages: for out in outs: - overlaps = (stage.cwd == out.path - or stage.cwd.startswith(out.path + os.sep)) + overlaps = stage.cwd == out.path or stage.cwd.startswith( + out.path + os.sep + ) if overlaps: - raise WorkingDirectoryAsOutputError(stage.cwd, - stage.relpath) + raise WorkingDirectoryAsOutputError(stage.cwd, stage.relpath) # collect the whole DAG for stage in stages: @@ -1208,9 +1188,11 @@ class Project(object): for dep in stage.deps: for out in outs: - if out.path != dep.path \ - and not dep.path.startswith(out.path + out.sep) \ - and not out.path.startswith(dep.path + dep.sep): + if ( + out.path != dep.path + and not dep.path.startswith(out.path + out.sep) + and not out.path.startswith(dep.path + dep.sep) + ): continue dep_stage = out.stage @@ -1230,10 +1212,7 @@ class Project(object): G, G_active = self.graph(from_directory=from_directory) - return [ - G.subgraph(c).copy() - for c in nx.weakly_connected_components(G) - ] + return [G.subgraph(c).copy() for c in nx.weakly_connected_components(G)] def stages(self, from_directory=None): """ @@ -1280,7 +1259,7 @@ class Project(object): stages = [] for G in self.pipelines(from_directory): - stages.extend(list(nx.get_node_attributes(G, 'stage').values())) + stages.extend(list(nx.get_node_attributes(G, "stage").values())) return stages def _welcome_message(self): @@ -1289,12 +1268,10 @@ class Project(object): logger.box( "DVC has enabled anonymous aggregate usage analytics.\n" "Read the analytics documentation (and how to opt-out) here:\n" - "{blue}https://dvc.org/doc/user-guide/analytics{nc}" - .format( - blue=colorama.Fore.BLUE, - nc=colorama.Fore.RESET + "{blue}https://dvc.org/doc/user-guide/analytics{nc}".format( + blue=colorama.Fore.BLUE, nc=colorama.Fore.RESET ), - border_color='red' + border_color="red", ) logger.info( @@ -1302,10 +1279,11 @@ class Project(object): "{yellow}------------{nc}\n" "- Check out the documentation: {blue}https://dvc.org/doc{nc}\n" "- Get help and share ideas: {blue}https://dvc.org/chat{nc}\n" - "- Star us on GitHub: {blue}https://github.com/iterative/dvc{nc}" - .format(yellow=colorama.Fore.YELLOW, - blue=colorama.Fore.BLUE, - nc=colorama.Fore.RESET) + "- Star us on GitHub: {blue}https://github.com/iterative/dvc{nc}".format( + yellow=colorama.Fore.YELLOW, + blue=colorama.Fore.BLUE, + nc=colorama.Fore.RESET, + ) ) def _expand_target_path(self, from_path, to_path): diff --git a/dvc/prompt.py b/dvc/prompt.py index e88acd24d..3d07e08b9 100644 --- a/dvc/prompt.py +++ b/dvc/prompt.py @@ -14,7 +14,7 @@ def _ask(prompt, limited_to=None): return None while True: - answer = input(prompt + ' ').lower() + answer = input(prompt + " ").lower() if not limited_to: return answer @@ -22,8 +22,11 @@ def _ask(prompt, limited_to=None): if answer in limited_to: return answer - print("Your response must be one of: {options}. Please try again." - .format(options=limited_to)) + print( + "Your response must be one of: {options}. Please try again.".format( + options=limited_to + ) + ) def confirm(statement): @@ -35,9 +38,9 @@ def confirm(statement): Returns: bool: whether or not specified statement was confirmed. """ - prompt = '{statement} [y/n]'.format(statement=statement) - answer = _ask(prompt, limited_to=['yes', 'no', 'y', 'n']) - return answer and answer.startswith('y') + prompt = "{statement} [y/n]".format(statement=statement) + answer = _ask(prompt, limited_to=["yes", "no", "y", "n"]) + return answer and answer.startswith("y") def password(statement): @@ -49,5 +52,5 @@ def password(statement): Returns: str: password entered by the user. """ - prompt = '{statement}: '.format(statement=statement) + prompt = "{statement}: ".format(statement=statement) return getpass(prompt) diff --git a/dvc/remote/azure.py b/dvc/remote/azure.py index 9b3b1ac4e..0946cb235 100644 --- a/dvc/remote/azure.py +++ b/dvc/remote/azure.py @@ -25,71 +25,75 @@ class Callback(object): class RemoteAzure(RemoteBase): - scheme = 'azure' - REGEX = (r'azure://((?P[^=;]*)?|(' - # backward compatibility - r'(ContainerName=(?P[^;]+);?)?' - r'(?P.+)?)?)$') - REQUIRES = {'azure-storage-blob': BlockBlobService} - PARAM_CHECKSUM = 'etag' + scheme = "azure" + REGEX = ( + r"azure://((?P[^=;]*)?|(" + # backward compatibility + r"(ContainerName=(?P[^;]+);?)?" + r"(?P.+)?)?)$" + ) + REQUIRES = {"azure-storage-blob": BlockBlobService} + PARAM_CHECKSUM = "etag" COPY_POLL_SECONDS = 5 def __init__(self, project, config): super(RemoteAzure, self).__init__(project, config) - self.url = config.get(Config.SECTION_REMOTE_URL, 'azure://') + self.url = config.get(Config.SECTION_REMOTE_URL, "azure://") match = re.match(self.REGEX, self.url) # backward compatibility - path = match.group('path') + path = match.group("path") self.bucket = ( - urlparse(self.url if path else '').netloc - or match.group('container_name') # backward compatibility - or os.getenv('AZURE_STORAGE_CONTAINER_NAME')) + urlparse(self.url if path else "").netloc + or match.group("container_name") # backward compatibility + or os.getenv("AZURE_STORAGE_CONTAINER_NAME") + ) - self.prefix = urlparse(self.url).path.lstrip('/') if path else '' + self.prefix = urlparse(self.url).path.lstrip("/") if path else "" self.connection_string = ( config.get(Config.SECTION_AZURE_CONNECTION_STRING) - or match.group('connection_string') # backward compatibility - or os.getenv('AZURE_STORAGE_CONNECTION_STRING')) + or match.group("connection_string") # backward compatibility + or os.getenv("AZURE_STORAGE_CONNECTION_STRING") + ) if not self.bucket: - raise ValueError('azure storage container name missing') + raise ValueError("azure storage container name missing") if not self.connection_string: - raise ValueError('azure storage connection string missing') + raise ValueError("azure storage connection string missing") self.__blob_service = None - self.path_info = {'scheme': self.scheme, 'bucket': self.bucket} + self.path_info = {"scheme": self.scheme, "bucket": self.bucket} @property def blob_service(self): if self.__blob_service is None: - logger.debug('URL {}'.format(self.url)) - logger.debug('Connection string {}'.format(self.connection_string)) + logger.debug("URL {}".format(self.url)) + logger.debug("Connection string {}".format(self.connection_string)) self.__blob_service = BlockBlobService( - connection_string=self.connection_string) - logger.debug('Container name {}'.format(self.bucket)) + connection_string=self.connection_string + ) + logger.debug("Container name {}".format(self.bucket)) self.__blob_service.create_container(self.bucket) return self.__blob_service def remove(self, path_info): - if path_info['scheme'] != self.scheme: + if path_info["scheme"] != self.scheme: raise NotImplementedError - logger.debug('Removing azure://{}/{}'.format(path_info['bucket'], - path_info['path'])) + logger.debug( + "Removing azure://{}/{}".format(path_info["bucket"], path_info["path"]) + ) - self.blob_service.delete_blob(path_info['bucket'], path_info['path']) + self.blob_service.delete_blob(path_info["bucket"], path_info["path"]) def _list_paths(self, bucket, prefix): blob_service = self.blob_service next_marker = None while True: - blobs = blob_service.list_blobs(bucket, - prefix=prefix, - marker=next_marker) + blobs = blob_service.list_blobs(bucket, prefix=prefix, marker=next_marker) for blob in blobs: yield blob.name @@ -106,68 +110,68 @@ class RemoteAzure(RemoteBase): names = self._verify_path_args(to_infos, from_infos, names) for from_info, to_info, name in zip(from_infos, to_infos, names): - if to_info['scheme'] != self.scheme: + if to_info["scheme"] != self.scheme: raise NotImplementedError - if from_info['scheme'] != 'local': + if from_info["scheme"] != "local": raise NotImplementedError - bucket = to_info['bucket'] - path = to_info['path'] + bucket = to_info["bucket"] + path = to_info["path"] - logger.debug("Uploading '{}' to '{}/{}'".format( - from_info['path'], bucket, path)) + logger.debug( + "Uploading '{}' to '{}/{}'".format(from_info["path"], bucket, path) + ) if not name: - name = os.path.basename(from_info['path']) + name = os.path.basename(from_info["path"]) cb = Callback(name) try: self.blob_service.create_blob_from_path( - bucket, path, from_info['path'], progress_callback=cb) + bucket, path, from_info["path"], progress_callback=cb + ) except Exception: - msg = "failed to upload '{}'".format(from_info['path']) + msg = "failed to upload '{}'".format(from_info["path"]) logger.warning(msg) else: progress.finish_target(name) - def download(self, - from_infos, - to_infos, - no_progress_bar=False, - names=None): + def download(self, from_infos, to_infos, no_progress_bar=False, names=None): names = self._verify_path_args(from_infos, to_infos, names) for to_info, from_info, name in zip(to_infos, from_infos, names): - if from_info['scheme'] != self.scheme: + if from_info["scheme"] != self.scheme: raise NotImplementedError - if to_info['scheme'] != 'local': + if to_info["scheme"] != "local": raise NotImplementedError - bucket = from_info['bucket'] - path = from_info['path'] + bucket = from_info["bucket"] + path = from_info["path"] - logger.debug("Downloading '{}/{}' to '{}'".format( - bucket, path, to_info['path'])) + logger.debug( + "Downloading '{}/{}' to '{}'".format(bucket, path, to_info["path"]) + ) - tmp_file = self.tmp_file(to_info['path']) + tmp_file = self.tmp_file(to_info["path"]) if not name: - name = os.path.basename(to_info['path']) + name = os.path.basename(to_info["path"]) cb = None if no_progress_bar else Callback(name) - self._makedirs(to_info['path']) + self._makedirs(to_info["path"]) try: self.blob_service.get_blob_to_path( - bucket, path, tmp_file, progress_callback=cb) + bucket, path, tmp_file, progress_callback=cb + ) except Exception: msg = "failed to download '{}/{}'".format(bucket, path) logger.warning(msg) else: - os.rename(tmp_file, to_info['path']) + os.rename(tmp_file, to_info["path"]) if not no_progress_bar: progress.finish_target(name) diff --git a/dvc/remote/base.py b/dvc/remote/base.py index 977f6bb0e..6d2859c9c 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -30,8 +30,9 @@ STATUS_MAP = { class DataCloudError(DvcException): """ Data Cloud exception """ + def __init__(self, msg): - super(DataCloudError, self).__init__('Data sync error: {}'.format(msg)) + super(DataCloudError, self).__init__("Data sync error: {}".format(msg)) class RemoteBaseCmdError(DvcException): @@ -55,8 +56,7 @@ class RemoteBase(object): deps_ok = all(self.REQUIRES.values()) if not deps_ok: missing = [k for k, v in self.REQUIRES.items() if v is None] - url = config.get(Config.SECTION_REMOTE_URL, - '{}://'.format(self.scheme)) + url = config.get(Config.SECTION_REMOTE_URL, "{}://".format(self.scheme)) msg = ( "URL '{}' is supported but requires these missing " "dependencies: {}. If you have installed dvc using pip, " @@ -73,15 +73,15 @@ class RemoteBase(object): "\n" "If you have installed dvc from a binary package and you " "are still seeing this message, please report it to us " - "using https://github.com/iterative/dvc/issues. Thank you!" - .format(url, missing, " ".join(missing), self.scheme) + "using https://github.com/iterative/dvc/issues. Thank you!".format( + url, missing, " ".join(missing), self.scheme + ) ) raise RemoteMissingDepsError(msg) def __repr__(self): return "{class_name}: '{url}'".format( - class_name=type(self).__name__, - url=(self.url or 'No url') + class_name=type(self).__name__, url=(self.url or "No url") ) def compat_config(config): @@ -106,7 +106,8 @@ class RemoteBase(object): def tmp_file(fname): """ Temporary name for a partial download """ import uuid - return fname + '.' + str(uuid.uuid4()) + + return fname + "." + str(uuid.uuid4()) def save_info(self, path_info): raise NotImplementedError @@ -128,8 +129,9 @@ class RemoteBase(object): bool: True if data has changed, False otherwise. """ - logger.debug("checking if '{}'('{}') has changed." - .format(path_info, checksum_info)) + logger.debug( + "checking if '{}'('{}') has changed.".format(path_info, checksum_info) + ) if not self.exists(path_info): logger.debug("'{}' doesn't exist.".format(path_info)) @@ -141,14 +143,18 @@ class RemoteBase(object): return True if self.changed_cache(checksum): - logger.debug("cache for '{}'('{}') has changed." - .format(path_info, checksum)) + logger.debug( + "cache for '{}'('{}') has changed.".format(path_info, checksum) + ) return True actual = self.save_info(path_info)[self.PARAM_CHECKSUM] if checksum != actual: - logger.debug("checksum '{}'(actual '{}') for '{}' has changed." - .format(checksum, actual, path_info)) + logger.debug( + "checksum '{}'(actual '{}') for '{}' has changed.".format( + checksum, actual, path_info + ) + ) return True logger.debug("'{}' hasn't changed.".format(path_info)) @@ -211,14 +217,11 @@ class RemoteBase(object): def checksum_to_path_info(self, checksum): path_info = self.path_info.copy() - path_info['path'] = self.checksum_to_path(checksum) + path_info["path"] = self.checksum_to_path(checksum) return path_info def md5s_to_path_infos(self, md5s): - return [ - self.checksum_to_path_info(md5) - for md5 in md5s - ] + return [self.checksum_to_path_info(md5) for md5 in md5s] def list_cache_paths(self): raise NotImplementedError @@ -228,17 +231,14 @@ class RemoteBase(object): # is 32 bytes, so ~3200Mb list) and we don't really need all of it at # the same time, so it makes sense to use a generator to gradually # iterate over it, without keeping all of it in memory. - return ( - self.path_to_checksum(path) - for path in self.list_cache_paths() - ) + return (self.path_to_checksum(path) for path in self.list_cache_paths()) def gc(self, cinfos): from dvc.remote.local import RemoteLOCAL - used = [info[RemoteLOCAL.PARAM_CHECKSUM] for info in cinfos['local']] + used = [info[RemoteLOCAL.PARAM_CHECKSUM] for info in cinfos["local"]] - if self.scheme != '': + if self.scheme != "": used += [info[self.PARAM_CHECKSUM] for info in cinfos[self.scheme]] removed = False @@ -255,12 +255,11 @@ class RemoteBase(object): expected = {self.PARAM_CHECKSUM: checksum} actual = self.save_info(cache) - logger.debug("Cache '{}' actual '{}'.".format(str(expected), - str(actual))) + logger.debug("Cache '{}' actual '{}'.".format(str(expected), str(actual))) if expected != actual: if self.exists(cache): - msg = 'corrupted cache file {}' + msg = "corrupted cache file {}" logger.warning(msg.format(str(cache))) self.remove(cache) return True @@ -293,15 +292,13 @@ class RemoteBase(object): if not force and not self.already_cached(path_info): msg = ( "file '{}' is going to be removed." - ' Are you sure you want to proceed?' - .format(str(path_info)) + " Are you sure you want to proceed?".format(str(path_info)) ) if not prompt.confirm(msg): raise DvcException( "unable to remove '{}' without a confirmation" - " from the user. Use '-f' to force." - .format(str(path_info)) + " from the user. Use '-f' to force.".format(str(path_info)) ) self.remove(path_info) @@ -316,8 +313,8 @@ class RemoteBase(object): self.copy(from_info, path_info) def checkout(self, path_info, checksum_info, force=False): - scheme = path_info['scheme'] - if scheme not in ['', 'local'] and scheme != self.scheme: + scheme = path_info["scheme"] + if scheme not in ["", "local"] and scheme != self.scheme: raise NotImplementedError checksum = checksum_info.get(self.PARAM_CHECKSUM) diff --git a/dvc/remote/gs.py b/dvc/remote/gs.py index f486e5752..c4aa55c9a 100644 --- a/dvc/remote/gs.py +++ b/dvc/remote/gs.py @@ -16,29 +16,29 @@ from dvc.exceptions import DvcException class RemoteGS(RemoteBase): - scheme = 'gs' - REGEX = r'^gs://(?P.*)$' - REQUIRES = {'google.cloud.storage': storage} - PARAM_CHECKSUM = 'md5' + scheme = "gs" + REGEX = r"^gs://(?P.*)$" + REQUIRES = {"google.cloud.storage": storage} + PARAM_CHECKSUM = "md5" def __init__(self, project, config): super(RemoteGS, self).__init__(project, config) - storagepath = 'gs://' - storagepath += config.get(Config.SECTION_AWS_STORAGEPATH, '/') - storagepath.lstrip('/') + storagepath = "gs://" + storagepath += config.get(Config.SECTION_AWS_STORAGEPATH, "/") + storagepath.lstrip("/") self.url = config.get(Config.SECTION_REMOTE_URL, storagepath) self.projectname = config.get(Config.SECTION_GCP_PROJECTNAME, None) parsed = urlparse(self.url) self.bucket = parsed.netloc - self.prefix = parsed.path.lstrip('/') + self.prefix = parsed.path.lstrip("/") - self.path_info = {'scheme': 'gs', 'bucket': self.bucket} + self.path_info = {"scheme": "gs", "bucket": self.bucket} @staticmethod def compat_config(config): ret = config.copy() - url = 'gs://' + ret.pop(Config.SECTION_AWS_STORAGEPATH, '').lstrip('/') + url = "gs://" + ret.pop(Config.SECTION_AWS_STORAGEPATH, "").lstrip("/") ret[Config.SECTION_REMOTE_URL] = url return ret @@ -56,48 +56,50 @@ class RemoteGS(RemoteBase): b64_md5 = blob.md5_hash md5 = base64.b64decode(b64_md5) - return codecs.getencoder('hex')(md5)[0].decode('utf-8') + return codecs.getencoder("hex")(md5)[0].decode("utf-8") def save_info(self, path_info): - if path_info['scheme'] != 'gs': + if path_info["scheme"] != "gs": raise NotImplementedError - return {self.PARAM_CHECKSUM: self.get_md5(path_info['bucket'], - path_info['path'])} + return { + self.PARAM_CHECKSUM: self.get_md5(path_info["bucket"], path_info["path"]) + } def copy(self, from_info, to_info, gs=None): gs = gs if gs else self.gs - blob = gs.bucket(from_info['bucket']).get_blob(from_info['path']) + blob = gs.bucket(from_info["bucket"]).get_blob(from_info["path"]) if not blob: - msg = "'{}' doesn't exist in the cloud".format(from_info['path']) + msg = "'{}' doesn't exist in the cloud".format(from_info["path"]) raise DvcException(msg) - bucket = self.gs.bucket(to_info['bucket']) - bucket.copy_blob(blob, - self.gs.bucket(to_info['bucket']), - new_name=to_info['path']) + bucket = self.gs.bucket(to_info["bucket"]) + bucket.copy_blob( + blob, self.gs.bucket(to_info["bucket"]), new_name=to_info["path"] + ) def save(self, path_info): - if path_info['scheme'] != 'gs': + if path_info["scheme"] != "gs": raise NotImplementedError - md5 = self.get_md5(path_info['bucket'], path_info['path']) + md5 = self.get_md5(path_info["bucket"], path_info["path"]) path = self.checksum_to_path(md5) - to_info = {'scheme': 'gs', 'bucket': self.bucket, 'path': path} + to_info = {"scheme": "gs", "bucket": self.bucket, "path": path} self.copy(path_info, to_info) return {self.PARAM_CHECKSUM: md5} def remove(self, path_info): - if path_info['scheme'] != 'gs': + if path_info["scheme"] != "gs": raise NotImplementedError - logger.debug("Removing gs://{}/{}".format(path_info['bucket'], - path_info['path'])) + logger.debug( + "Removing gs://{}/{}".format(path_info["bucket"], path_info["path"]) + ) - blob = self.gs.bucket(path_info['bucket']).get_blob(path_info['path']) + blob = self.gs.bucket(path_info["bucket"]).get_blob(path_info["path"]) if not blob: return @@ -112,10 +114,10 @@ class RemoteGS(RemoteBase): def exists(self, path_info): assert not isinstance(path_info, list) - assert path_info['scheme'] == 'gs' + assert path_info["scheme"] == "gs" - paths = self._list_paths(path_info['bucket'], path_info['path']) - return any(path_info['path'] == path for path in paths) + paths = self._list_paths(path_info["bucket"], path_info["path"]) + return any(path_info["path"] == path for path in paths) def upload(self, from_infos, to_infos, names=None): names = self._verify_path_args(to_infos, from_infos, names) @@ -123,82 +125,80 @@ class RemoteGS(RemoteBase): gs = self.gs for from_info, to_info, name in zip(from_infos, to_infos, names): - if to_info['scheme'] != 'gs': + if to_info["scheme"] != "gs": raise NotImplementedError - if from_info['scheme'] != 'local': + if from_info["scheme"] != "local": raise NotImplementedError - logger.debug("Uploading '{}' to '{}/{}'".format(from_info['path'], - to_info['bucket'], - to_info['path'])) + logger.debug( + "Uploading '{}' to '{}/{}'".format( + from_info["path"], to_info["bucket"], to_info["path"] + ) + ) if not name: - name = os.path.basename(from_info['path']) + name = os.path.basename(from_info["path"]) progress.update_target(name, 0, None) try: - bucket = gs.bucket(to_info['bucket']) - blob = bucket.blob(to_info['path']) - blob.upload_from_filename(from_info['path']) + bucket = gs.bucket(to_info["bucket"]) + blob = bucket.blob(to_info["path"]) + blob.upload_from_filename(from_info["path"]) except Exception: msg = "failed to upload '{}' to '{}/{}'" - logger.error(msg.format(from_info['path'], - to_info['bucket'], - to_info['path'])) + logger.error( + msg.format(from_info["path"], to_info["bucket"], to_info["path"]) + ) continue progress.finish_target(name) - def download(self, - from_infos, - to_infos, - no_progress_bar=False, - names=None): + def download(self, from_infos, to_infos, no_progress_bar=False, names=None): names = self._verify_path_args(from_infos, to_infos, names) gs = self.gs for to_info, from_info, name in zip(to_infos, from_infos, names): - if from_info['scheme'] != 'gs': + if from_info["scheme"] != "gs": raise NotImplementedError - if to_info['scheme'] == 'gs': + if to_info["scheme"] == "gs": self.copy(from_info, to_info, gs=gs) continue - if to_info['scheme'] != 'local': + if to_info["scheme"] != "local": raise NotImplementedError - msg = "Downloading '{}/{}' to '{}'".format(from_info['bucket'], - from_info['path'], - to_info['path']) + msg = "Downloading '{}/{}' to '{}'".format( + from_info["bucket"], from_info["path"], to_info["path"] + ) logger.debug(msg) - tmp_file = self.tmp_file(to_info['path']) + tmp_file = self.tmp_file(to_info["path"]) if not name: - name = os.path.basename(to_info['path']) + name = os.path.basename(to_info["path"]) if not no_progress_bar: # percent_cb is not available for download_to_filename, so # lets at least update progress at pathpoints(start, finish) progress.update_target(name, 0, None) - self._makedirs(to_info['path']) + self._makedirs(to_info["path"]) try: - bucket = gs.bucket(from_info['bucket']) - blob = bucket.get_blob(from_info['path']) + bucket = gs.bucket(from_info["bucket"]) + blob = bucket.get_blob(from_info["path"]) blob.download_to_filename(tmp_file) except Exception: msg = "failed to download '{}/{}' to '{}'" - logger.error(msg.format(from_info['bucket'], - from_info['path'], - to_info['path'])) + logger.error( + msg.format(from_info["bucket"], from_info["path"], to_info["path"]) + ) continue - os.rename(tmp_file, to_info['path']) + os.rename(tmp_file, to_info["path"]) if not no_progress_bar: progress.finish_target(name) diff --git a/dvc/remote/hdfs.py b/dvc/remote/hdfs.py index f9c5acc65..300e2e5c2 100644 --- a/dvc/remote/hdfs.py +++ b/dvc/remote/hdfs.py @@ -17,43 +17,44 @@ class RemoteHDFSCmdError(RemoteBaseCmdError): class RemoteHDFS(RemoteBase): - scheme = 'hdfs' - REGEX = r'^hdfs://((?P.*)@)?.*$' - PARAM_CHECKSUM = 'checksum' + scheme = "hdfs" + REGEX = r"^hdfs://((?P.*)@)?.*$" + PARAM_CHECKSUM = "checksum" def __init__(self, project, config): super(RemoteHDFS, self).__init__(project, config) - self.url = config.get(Config.SECTION_REMOTE_URL, '/') + self.url = config.get(Config.SECTION_REMOTE_URL, "/") self.prefix = self.url - self.user = self.group('user') + self.user = self.group("user") if not self.user: - self.user = config.get(Config.SECTION_REMOTE_USER, - getpass.getuser()) + self.user = config.get(Config.SECTION_REMOTE_USER, getpass.getuser()) - self.path_info = {'scheme': 'hdfs', 'user': self.user} + self.path_info = {"scheme": "hdfs", "user": self.user} def hadoop_fs(self, cmd, user=None): - cmd = 'hadoop fs -' + cmd + cmd = "hadoop fs -" + cmd if user: - cmd = 'HADOOP_USER_NAME={} '.format(user) + cmd + cmd = "HADOOP_USER_NAME={} ".format(user) + cmd # NOTE: close_fds doesn't work with redirected stdin/stdout/stderr. # See https://github.com/iterative/dvc/issues/1197. - close_fds = (os.name != 'nt') + close_fds = os.name != "nt" - executable = os.getenv('SHELL') if os.name != 'nt' else None - p = Popen(cmd, - shell=True, - close_fds=close_fds, - executable=executable, - env=fix_env(os.environ), - stdin=PIPE, - stdout=PIPE, - stderr=PIPE) + executable = os.getenv("SHELL") if os.name != "nt" else None + p = Popen( + cmd, + shell=True, + close_fds=close_fds, + executable=executable, + env=fix_env(os.environ), + stdin=PIPE, + stdout=PIPE, + stderr=PIPE, + ) out, err = p.communicate() if p.returncode != 0: raise RemoteHDFSCmdError(cmd, p.returncode, err) - return out.decode('utf-8') + return out.decode("utf-8") @staticmethod def _group(regex, s, gname): @@ -62,60 +63,61 @@ class RemoteHDFS(RemoteBase): return match.group(gname) def checksum(self, path_info): - regex = r'.*\t.*\t(?P.*)' - stdout = self.hadoop_fs('checksum {}'.format(path_info['path']), - user=path_info['user']) - return self._group(regex, stdout, 'checksum') + regex = r".*\t.*\t(?P.*)" + stdout = self.hadoop_fs( + "checksum {}".format(path_info["path"]), user=path_info["user"] + ) + return self._group(regex, stdout, "checksum") def copy(self, from_info, to_info): - dname = posixpath.dirname(to_info['path']) - self.hadoop_fs('mkdir -p {}'.format(dname), user=to_info['user']) - self.hadoop_fs('cp -f {} {}'.format(from_info['path'], - to_info['path']), - user=to_info['user']) + dname = posixpath.dirname(to_info["path"]) + self.hadoop_fs("mkdir -p {}".format(dname), user=to_info["user"]) + self.hadoop_fs( + "cp -f {} {}".format(from_info["path"], to_info["path"]), + user=to_info["user"], + ) def rm(self, path_info): - self.hadoop_fs('rm {}'.format(path_info['path']), - user=path_info['user']) + self.hadoop_fs("rm {}".format(path_info["path"]), user=path_info["user"]) def save_info(self, path_info): - if path_info['scheme'] != 'hdfs': + if path_info["scheme"] != "hdfs": raise NotImplementedError - assert path_info.get('path') + assert path_info.get("path") return {self.PARAM_CHECKSUM: self.checksum(path_info)} def save(self, path_info): - if path_info['scheme'] != 'hdfs': + if path_info["scheme"] != "hdfs": raise NotImplementedError - assert path_info.get('path') + assert path_info.get("path") checksum = self.checksum(path_info) dest = path_info.copy() - dest['path'] = self.checksum_to_path(checksum) + dest["path"] = self.checksum_to_path(checksum) self.copy(path_info, dest) return {self.PARAM_CHECKSUM: checksum} def remove(self, path_info): - if path_info['scheme'] != 'hdfs': + if path_info["scheme"] != "hdfs": raise NotImplementedError - assert path_info.get('path') + assert path_info.get("path") - logger.debug('Removing {}'.format(path_info['path'])) + logger.debug("Removing {}".format(path_info["path"])) self.rm(path_info) def exists(self, path_info): assert not isinstance(path_info, list) - assert path_info['scheme'] == 'hdfs' + assert path_info["scheme"] == "hdfs" try: - self.hadoop_fs('test -e {}'.format(path_info['path'])) + self.hadoop_fs("test -e {}".format(path_info["path"])) return True except RemoteHDFSCmdError: return False @@ -124,56 +126,50 @@ class RemoteHDFS(RemoteBase): names = self._verify_path_args(to_infos, from_infos, names) for from_info, to_info, name in zip(from_infos, to_infos, names): - if to_info['scheme'] != 'hdfs': + if to_info["scheme"] != "hdfs": raise NotImplementedError - if from_info['scheme'] != 'local': + if from_info["scheme"] != "local": raise NotImplementedError - cmd = 'mkdir -p {}'.format(posixpath.dirname(to_info['path'])) - self.hadoop_fs(cmd, user=to_info['user']) + cmd = "mkdir -p {}".format(posixpath.dirname(to_info["path"])) + self.hadoop_fs(cmd, user=to_info["user"]) - cmd = 'copyFromLocal {} {}'.format(from_info['path'], - to_info['path']) - self.hadoop_fs(cmd, user=to_info['user']) + cmd = "copyFromLocal {} {}".format(from_info["path"], to_info["path"]) + self.hadoop_fs(cmd, user=to_info["user"]) - def download(self, - from_infos, - to_infos, - no_progress_bar=False, - names=None): + def download(self, from_infos, to_infos, no_progress_bar=False, names=None): names = self._verify_path_args(from_infos, to_infos, names) for to_info, from_info, name in zip(to_infos, from_infos, names): - if from_info['scheme'] != 'hdfs': + if from_info["scheme"] != "hdfs": raise NotImplementedError - if to_info['scheme'] == 'hdfs': + if to_info["scheme"] == "hdfs": self.copy(from_info, to_info) continue - if to_info['scheme'] != 'local': + if to_info["scheme"] != "local": raise NotImplementedError - dname = os.path.dirname(to_info['path']) + dname = os.path.dirname(to_info["path"]) if not os.path.exists(dname): os.makedirs(dname) - cmd = 'copyToLocal {} {}'.format(from_info['path'], - to_info['path']) - self.hadoop_fs(cmd, user=from_info['user']) + cmd = "copyToLocal {} {}".format(from_info["path"], to_info["path"]) + self.hadoop_fs(cmd, user=from_info["user"]) def list_cache_paths(self): try: - self.hadoop_fs('test -e {}'.format(self.prefix)) + self.hadoop_fs("test -e {}".format(self.prefix)) except RemoteHDFSCmdError: return [] - stdout = self.hadoop_fs('ls -R {}'.format(self.prefix)) - lines = stdout.split('\n') + stdout = self.hadoop_fs("ls -R {}".format(self.prefix)) + lines = stdout.split("\n") flist = [] for line in lines: - if not line.startswith('-'): + if not line.startswith("-"): continue flist.append(line.split()[-1]) return flist diff --git a/dvc/remote/http.py b/dvc/remote/http.py index 1e74d6bf7..c4f728596 100644 --- a/dvc/remote/http.py +++ b/dvc/remote/http.py @@ -25,47 +25,42 @@ class ProgressBarCallback(object): class RemoteHTTP(RemoteBase): - scheme = 'http' - REGEX = r'^https?://.*$' + scheme = "http" + REGEX = r"^https?://.*$" REQUEST_TIMEOUT = 10 CHUNK_SIZE = 1000000 # Megabyte - PARAM_CHECKSUM = 'etag' + PARAM_CHECKSUM = "etag" def __init__(self, project, config): super(RemoteHTTP, self).__init__(project, config) self.cache_dir = config.get(Config.SECTION_REMOTE_URL) self.url = self.cache_dir - self.path_info = {'scheme': 'http'} + self.path_info = {"scheme": "http"} @property def prefix(self): return self.cache_dir - def download(self, - from_infos, - to_infos, - no_progress_bar=False, - names=None): + def download(self, from_infos, to_infos, no_progress_bar=False, names=None): names = self._verify_path_args(to_infos, from_infos, names) for to_info, from_info, name in zip(to_infos, from_infos, names): - if from_info['scheme'] not in ['http', 'https']: + if from_info["scheme"] not in ["http", "https"]: raise NotImplementedError - if to_info['scheme'] != 'local': + if to_info["scheme"] != "local": raise NotImplementedError - msg = "Downloading '{}' to '{}'".format(from_info['path'], - to_info['path']) + msg = "Downloading '{}' to '{}'".format(from_info["path"], to_info["path"]) logger.debug(msg) - tmp_file = self.tmp_file(to_info['path']) + tmp_file = self.tmp_file(to_info["path"]) if not name: - name = os.path.basename(to_info['path']) + name = os.path.basename(to_info["path"]) - self._makedirs(to_info['path']) + self._makedirs(to_info["path"]) - total = self._content_length(from_info['path']) + total = self._content_length(from_info["path"]) if no_progress_bar or not total: cb = None @@ -73,50 +68,52 @@ class RemoteHTTP(RemoteBase): cb = ProgressBarCallback(name, total) try: - self._download_to(from_info['path'], tmp_file, callback=cb) + self._download_to(from_info["path"], tmp_file, callback=cb) except Exception: - msg = "failed to download '{}'".format(from_info['path']) + msg = "failed to download '{}'".format(from_info["path"]) logger.error(msg) continue - os.rename(tmp_file, to_info['path']) + os.rename(tmp_file, to_info["path"]) if not no_progress_bar: progress.finish_target(name) def exists(self, path_info): assert not isinstance(path_info, list) - assert path_info['scheme'] in ['http', 'https'] - return bool(self._request('HEAD', path_info.get('path'))) + assert path_info["scheme"] in ["http", "https"] + return bool(self._request("HEAD", path_info.get("path"))) def cache_exists(self, md5s): assert isinstance(md5s, list) def func(md5): - return bool(self._request('HEAD', self.checksum_to_path(md5))) + return bool(self._request("HEAD", self.checksum_to_path(md5))) return list(filter(func, md5s)) def save_info(self, path_info): - if path_info['scheme'] not in ['http', 'https']: + if path_info["scheme"] not in ["http", "https"]: raise NotImplementedError - return {self.PARAM_CHECKSUM: self._etag(path_info['path'])} + return {self.PARAM_CHECKSUM: self._etag(path_info["path"])} def _content_length(self, url): - return self._request('HEAD', url).headers.get('Content-Length') + return self._request("HEAD", url).headers.get("Content-Length") def _etag(self, url): - etag = (self._request('HEAD', url).headers.get('ETag') - or self._request('HEAD', url).headers.get('Content-MD5')) + etag = self._request("HEAD", url).headers.get("ETag") or self._request( + "HEAD", url + ).headers.get("Content-MD5") if not etag: raise DvcException( - "could not find an ETag or Content-MD5 header for '{url}'" - .format(url=url) + "could not find an ETag or Content-MD5 header for '{url}'".format( + url=url + ) ) - if etag.startswith('W/'): + if etag.startswith("W/"): raise DvcException( "Weak ETags are not supported." " (Etag: '{etag}', URL: '{url}')".format(etag=etag, url=url) @@ -125,9 +122,9 @@ class RemoteHTTP(RemoteBase): return etag def _download_to(self, url, file, callback=None): - r = self._request('GET', url, stream=True) + r = self._request("GET", url, stream=True) - with open(file, 'wb') as fd: + with open(file, "wb") as fd: bytes_transfered = 0 for chunk in r.iter_content(chunk_size=self.CHUNK_SIZE): @@ -138,13 +135,13 @@ class RemoteHTTP(RemoteBase): callback(bytes_transfered) def _request(self, method, url, **kwargs): - kwargs.setdefault('allow_redirects', True) - kwargs.setdefault('timeout', self.REQUEST_TIMEOUT) + kwargs.setdefault("allow_redirects", True) + kwargs.setdefault("timeout", self.REQUEST_TIMEOUT) try: return requests.request(method, url, **kwargs) except requests.exceptions.RequestException: - raise DvcException('could not perform a {} request'.format(method)) + raise DvcException("could not perform a {} request".format(method)) def gc(self): raise NotImplementedError diff --git a/dvc/remote/local.py b/dvc/remote/local.py index e455d4160..9521176bf 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -23,19 +23,19 @@ from concurrent.futures import ThreadPoolExecutor class RemoteLOCAL(RemoteBase): - scheme = 'local' - REGEX = r'^(?P.*)$' - PARAM_CHECKSUM = 'md5' - PARAM_PATH = 'path' - PARAM_RELPATH = 'relpath' - MD5_DIR_SUFFIX = '.dir' + scheme = "local" + REGEX = r"^(?P.*)$" + PARAM_CHECKSUM = "md5" + PARAM_PATH = "path" + PARAM_RELPATH = "relpath" + MD5_DIR_SUFFIX = ".dir" - CACHE_TYPES = ['reflink', 'hardlink', 'symlink', 'copy'] + CACHE_TYPES = ["reflink", "hardlink", "symlink", "copy"] CACHE_TYPE_MAP = { - 'copy': shutil.copyfile, - 'symlink': System.symlink, - 'hardlink': System.hardlink, - 'reflink': System.reflink, + "copy": shutil.copyfile, + "symlink": System.symlink, + "hardlink": System.hardlink, + "reflink": System.reflink, } def __init__(self, project, config): @@ -52,7 +52,7 @@ class RemoteLOCAL(RemoteBase): types = config.get(Config.SECTION_CACHE_TYPE, None) if types: if isinstance(types, str): - types = [t.strip() for t in types.split(',')] + types = [t.strip() for t in types.split(",")] self.cache_types = types else: self.cache_types = self.CACHE_TYPES @@ -60,12 +60,12 @@ class RemoteLOCAL(RemoteBase): if self.cache_dir is not None and not os.path.exists(self.cache_dir): os.mkdir(self.cache_dir) - self.path_info = {'scheme': 'local'} + self.path_info = {"scheme": "local"} @staticmethod def compat_config(config): ret = config.copy() - url = ret.pop(Config.SECTION_AWS_STORAGEPATH, '') + url = ret.pop(Config.SECTION_AWS_STORAGEPATH, "") ret[Config.SECTION_REMOTE_URL] = url return ret @@ -99,7 +99,7 @@ class RemoteLOCAL(RemoteBase): cache = self.get(md5) if self.state.changed(cache, md5=md5): if os.path.exists(cache): - msg = 'Corrupted cache file {}.' + msg = "Corrupted cache file {}." logger.warning(msg.format(os.path.relpath(cache))) remove(cache) return True @@ -107,8 +107,8 @@ class RemoteLOCAL(RemoteBase): def exists(self, path_info): assert not isinstance(path_info, list) - assert path_info['scheme'] == 'local' - return os.path.exists(path_info['path']) + assert path_info["scheme"] == "local" + return os.path.exists(path_info["path"]) def changed_cache(self, md5): cache = self.get(md5) @@ -141,7 +141,7 @@ class RemoteLOCAL(RemoteBase): # NOTE: just create an empty file for an empty cache if os.path.getsize(cache) == 0: - open(path, 'w+').close() + open(path, "w+").close() msg = "Created empty file: {} -> {}".format(cache, path) logger.debug(msg) @@ -156,8 +156,11 @@ class RemoteLOCAL(RemoteBase): os.chmod(path, stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH) msg = "Created {}'{}': {} -> {}".format( - 'protected ' if self.protected else '', - self.cache_types[0], cache, path) + "protected " if self.protected else "", + self.cache_types[0], + cache, + path, + ) logger.debug(msg) return @@ -168,17 +171,17 @@ class RemoteLOCAL(RemoteBase): del self.cache_types[0] i -= 1 - raise DvcException('no possible cache types left to try out.') + raise DvcException("no possible cache types left to try out.") @property def ospath(self): - if os.name == 'nt': + if os.name == "nt": return ntpath return posixpath @classmethod def to_ospath(cls, path): - if os.name == 'nt': + if os.name == "nt": return cls.ntpath(path) return cls.unixpath(path) @@ -186,13 +189,13 @@ class RemoteLOCAL(RemoteBase): def unixpath(path): assert not ntpath.isabs(path) assert not posixpath.isabs(path) - return path.replace('\\', '/') + return path.replace("\\", "/") @staticmethod def ntpath(path): assert not ntpath.isabs(path) assert not posixpath.isabs(path) - return path.replace('/', '\\') + return path.replace("/", "\\") def collect_dir_cache(self, dname): dir_info = [] @@ -201,8 +204,9 @@ class RemoteLOCAL(RemoteBase): bar = False if len(files) > LARGE_DIR_SIZE: - msg = "Computing md5 for a large directory {}. " \ - "This is only done once." + msg = ( + "Computing md5 for a large directory {}. " "This is only done once." + ) logger.info(msg.format(os.path.relpath(root))) bar = True title = os.path.relpath(root) @@ -219,8 +223,7 @@ class RemoteLOCAL(RemoteBase): processed += 1 md5 = self.state.update(path) - dir_info.append({self.PARAM_RELPATH: relpath, - self.PARAM_CHECKSUM: md5}) + dir_info.append({self.PARAM_RELPATH: relpath, self.PARAM_CHECKSUM: md5}) if bar: progress.finish_target(title) @@ -240,7 +243,7 @@ class RemoteLOCAL(RemoteBase): assert self.is_dir_cache(path) try: - with open(path, 'r') as fd: + with open(path, "r") as fd: d = json.load(fd) except Exception: msg = "Failed to load dir cache '{}'" @@ -253,7 +256,7 @@ class RemoteLOCAL(RemoteBase): return [] for info in d: - info['relpath'] = self.to_ospath(info['relpath']) + info["relpath"] = self.to_ospath(info["relpath"]) return d @@ -269,8 +272,8 @@ class RemoteLOCAL(RemoteBase): # NOTE: Writing first and renaming after that # to make sure that the operation is atomic. - tmp = '{}.{}'.format(path, str(uuid.uuid4())) - with open(tmp, 'w+') as fd: + tmp = "{}.{}".format(path, str(uuid.uuid4())) + with open(tmp, "w+") as fd: json.dump(dir_info, fd, sort_keys=True) move(tmp, path) @@ -279,7 +282,7 @@ class RemoteLOCAL(RemoteBase): return cache.endswith(cls.MD5_DIR_SUFFIX) def do_checkout(self, path_info, checksum, force=False): - path = path_info['path'] + path = path_info["path"] md5 = checksum cache = self.get(md5) @@ -309,7 +312,7 @@ class RemoteLOCAL(RemoteBase): p = os.path.join(path, relpath) c = self.get(m) - entry_info = {'scheme': path_info['scheme'], self.PARAM_PATH: p} + entry_info = {"scheme": path_info["scheme"], self.PARAM_PATH: p} entry_checksum_info = {self.PARAM_CHECKSUM: m} @@ -330,9 +333,9 @@ class RemoteLOCAL(RemoteBase): progress.finish_target(dir_relpath) def already_cached(self, path_info): - assert path_info['scheme'] in ['', 'local'] + assert path_info["scheme"] in ["", "local"] - current_md5 = self.state.update(path_info['path']) + current_md5 = self.state.update(path_info["path"]) if not current_md5: return False @@ -346,26 +349,23 @@ class RemoteLOCAL(RemoteBase): for file in files ) - cached_files = set( - os.path.join(path, file['relpath']) - for file in dir_info - ) + cached_files = set(os.path.join(path, file["relpath"]) for file in dir_info) delta = working_dir_files - cached_files for file in delta: - self.safe_remove({'scheme': 'local', 'path': file}, force=force) + self.safe_remove({"scheme": "local", "path": file}, force=force) def _move(self, inp, outp): # moving in two stages to make the whole operation atomic in # case inp and outp are in different filesystems and actual # physical copying of data is happening - tmp = '{}.{}'.format(outp, str(uuid.uuid4())) + tmp = "{}.{}".format(outp, str(uuid.uuid4())) move(inp, tmp) move(tmp, outp) def _save_file(self, path_info): - path = path_info['path'] + path = path_info["path"] md5 = self.state.update(path) assert md5 is not None @@ -382,7 +382,7 @@ class RemoteLOCAL(RemoteBase): return {self.PARAM_CHECKSUM: md5} def _save_dir(self, path_info): - path = path_info['path'] + path = path_info["path"] md5, dir_info = self.state.update_info(path) dir_relpath = os.path.relpath(path) dir_size = len(dir_info) @@ -414,14 +414,13 @@ class RemoteLOCAL(RemoteBase): return {self.PARAM_CHECKSUM: md5} def save(self, path_info): - if path_info['scheme'] != 'local': + if path_info["scheme"] != "local": raise NotImplementedError - path = path_info['path'] + path = path_info["path"] msg = "Saving '{}' to cache '{}'." - logger.info(msg.format(os.path.relpath(path), - os.path.relpath(self.cache_dir))) + logger.info(msg.format(os.path.relpath(path), os.path.relpath(self.cache_dir))) if os.path.isdir(path): return self._save_dir(path_info) @@ -429,22 +428,22 @@ class RemoteLOCAL(RemoteBase): return self._save_file(path_info) def save_info(self, path_info): - if path_info['scheme'] != 'local': + if path_info["scheme"] != "local": raise NotImplementedError - return {self.PARAM_CHECKSUM: self.state.update(path_info['path'])} + return {self.PARAM_CHECKSUM: self.state.update(path_info["path"])} def remove(self, path_info): - if path_info['scheme'] != 'local': + if path_info["scheme"] != "local": raise NotImplementedError - remove(path_info['path']) + remove(path_info["path"]) def move(self, from_info, to_info): - if from_info['scheme'] != 'local' or to_info['scheme'] != 'local': + if from_info["scheme"] != "local" or to_info["scheme"] != "local": raise NotImplementedError - move(from_info['path'], to_info['path']) + move(from_info["path"], to_info["path"]) def cache_exists(self, md5s): assert isinstance(md5s, list) @@ -454,64 +453,66 @@ class RemoteLOCAL(RemoteBase): names = self._verify_path_args(to_infos, from_infos, names) for from_info, to_info, name in zip(from_infos, to_infos, names): - if to_info['scheme'] != 'local': + if to_info["scheme"] != "local": raise NotImplementedError - if from_info['scheme'] != 'local': + if from_info["scheme"] != "local": raise NotImplementedError - logger.debug("Uploading '{}' to '{}'".format(from_info['path'], - to_info['path'])) + logger.debug( + "Uploading '{}' to '{}'".format(from_info["path"], to_info["path"]) + ) if not name: - name = os.path.basename(from_info['path']) + name = os.path.basename(from_info["path"]) - self._makedirs(to_info['path']) + self._makedirs(to_info["path"]) try: - copyfile(from_info['path'], to_info['path'], name=name) + copyfile(from_info["path"], to_info["path"], name=name) except Exception: logger.error( - "failed to upload '{}' to '{}'" - .format(from_info['path'], to_info['path']) + "failed to upload '{}' to '{}'".format( + from_info["path"], to_info["path"] + ) ) - def download(self, - from_infos, - to_infos, - no_progress_bar=False, - names=None): + def download(self, from_infos, to_infos, no_progress_bar=False, names=None): names = self._verify_path_args(from_infos, to_infos, names) for to_info, from_info, name in zip(to_infos, from_infos, names): - if from_info['scheme'] != 'local': + if from_info["scheme"] != "local": raise NotImplementedError - if to_info['scheme'] != 'local': + if to_info["scheme"] != "local": raise NotImplementedError - logger.debug("Downloading '{}' to '{}'".format(from_info['path'], - to_info['path'])) + logger.debug( + "Downloading '{}' to '{}'".format(from_info["path"], to_info["path"]) + ) if not name: - name = os.path.basename(to_info['path']) + name = os.path.basename(to_info["path"]) - self._makedirs(to_info['path']) - tmp_file = self.tmp_file(to_info['path']) + self._makedirs(to_info["path"]) + tmp_file = self.tmp_file(to_info["path"]) try: - copyfile(from_info['path'], - tmp_file, - no_progress_bar=no_progress_bar, - name=name) + copyfile( + from_info["path"], + tmp_file, + no_progress_bar=no_progress_bar, + name=name, + ) except Exception: logger.error( - "failed to download '{}' to '{}'" - .format(from_info['path'], to_info['path']) + "failed to download '{}' to '{}'".format( + from_info["path"], to_info["path"] + ) ) continue - os.rename(tmp_file, to_info['path']) + os.rename(tmp_file, to_info["path"]) def _group(self, checksum_infos, show_checksums=False): by_md5 = {} @@ -520,18 +521,18 @@ class RemoteLOCAL(RemoteBase): md5 = info[self.PARAM_CHECKSUM] if show_checksums: - by_md5[md5] = {'name': md5} + by_md5[md5] = {"name": md5} continue name = info[self.PARAM_PATH] - branch = info.get('branch') + branch = info.get("branch") if branch: - name += '({})'.format(branch) + name += "({})".format(branch) if md5 not in by_md5.keys(): - by_md5[md5] = {'name': name} + by_md5[md5] = {"name": name} else: - by_md5[md5]['name'] += ' ' + name + by_md5[md5]["name"] += " " + name return by_md5 @@ -560,8 +561,7 @@ class RemoteLOCAL(RemoteBase): progress.finish_target(title) for md5, info in ret.items(): - info['status'] = STATUS_MAP[(md5 in local_exists, - md5 in remote_exists)] + info["status"] = STATUS_MAP[(md5 in local_exists, md5 in remote_exists)] return ret @@ -576,10 +576,10 @@ class RemoteLOCAL(RemoteBase): path_infos = [] names = [] for md5, info in status_info.items(): - if info['status'] == status: + if info["status"] == status: cache.append(self.checksum_to_path_info(md5)) path_infos.append(remote.checksum_to_path_info(md5)) - names.append(info['name']) + names.append(info["name"]) current += 1 progress.update_target(title, current, total) @@ -594,20 +594,25 @@ class RemoteLOCAL(RemoteBase): to_infos = path_infos from_infos = cache - return list(zip(to_chunks(from_infos, jobs), - to_chunks(to_infos, jobs), - to_chunks(names, jobs))) + return list( + zip( + to_chunks(from_infos, jobs), + to_chunks(to_infos, jobs), + to_chunks(names, jobs), + ) + ) - def _process(self, - checksum_infos, - remote, - jobs=None, - show_checksums=False, - download=False): + def _process( + self, checksum_infos, remote, jobs=None, show_checksums=False, download=False + ): msg = "Preparing to {} data {} '{}'" - logger.info(msg.format('download' if download else 'upload', - 'from' if download else 'to', - remote.url)) + logger.info( + msg.format( + "download" if download else "upload", + "from" if download else "to", + remote.url, + ) + ) if download: func = remote.download @@ -619,10 +624,9 @@ class RemoteLOCAL(RemoteBase): if jobs is None: jobs = remote.JOBS - status_info = self.status(checksum_infos, - remote, - jobs=jobs, - show_checksums=show_checksums) + status_info = self.status( + checksum_infos, remote, jobs=jobs, show_checksums=show_checksums + ) chunks = self._get_chunks(download, remote, status_info, status, jobs) @@ -632,33 +636,26 @@ class RemoteLOCAL(RemoteBase): futures = [] with ThreadPoolExecutor(max_workers=jobs) as executor: for from_infos, to_infos, names in chunks: - res = executor.submit(func, - from_infos, - to_infos, - names=names) + res = executor.submit(func, from_infos, to_infos, names=names) futures.append(res) for f in futures: f.result() - def push(self, - checksum_infos, - remote, - jobs=None, - show_checksums=False): - self._process(checksum_infos, - remote, - jobs=jobs, - show_checksums=show_checksums, - download=False) + def push(self, checksum_infos, remote, jobs=None, show_checksums=False): + self._process( + checksum_infos, + remote, + jobs=jobs, + show_checksums=show_checksums, + download=False, + ) - def pull(self, - checksum_infos, - remote, - jobs=None, - show_checksums=False): - self._process(checksum_infos, - remote, - jobs=jobs, - show_checksums=show_checksums, - download=True) + def pull(self, checksum_infos, remote, jobs=None, show_checksums=False): + self._process( + checksum_infos, + remote, + jobs=jobs, + show_checksums=show_checksums, + download=True, + ) diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index ee0c5e127..dd3658537 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -30,28 +30,26 @@ class Callback(object): class RemoteS3(RemoteBase): - scheme = 's3' - REGEX = r'^s3://(?P.*)$' - REQUIRES = {'boto3': boto3} - PARAM_CHECKSUM = 'etag' + scheme = "s3" + REGEX = r"^s3://(?P.*)$" + REQUIRES = {"boto3": boto3} + PARAM_CHECKSUM = "etag" def __init__(self, project, config): super(RemoteS3, self).__init__(project, config) - storagepath = 's3://{}'.format( - config.get(Config.SECTION_AWS_STORAGEPATH, '').lstrip('/') + storagepath = "s3://{}".format( + config.get(Config.SECTION_AWS_STORAGEPATH, "").lstrip("/") ) self.url = config.get(Config.SECTION_REMOTE_URL, storagepath) - self.region = ( - os.environ.get('AWS_DEFAULT_REGION') or - config.get(Config.SECTION_AWS_REGION) + self.region = os.environ.get("AWS_DEFAULT_REGION") or config.get( + Config.SECTION_AWS_REGION ) - self.profile = ( - os.environ.get('AWS_PROFILE') or - config.get(Config.SECTION_AWS_PROFILE) + self.profile = os.environ.get("AWS_PROFILE") or config.get( + Config.SECTION_AWS_PROFILE ) self.endpoint_url = config.get(Config.SECTION_AWS_ENDPOINT_URL) @@ -60,108 +58,107 @@ class RemoteS3(RemoteBase): shared_creds = config.get(Config.SECTION_AWS_CREDENTIALPATH) if shared_creds: - os.environ.setdefault('AWS_SHARED_CREDENTIALS_FILE', shared_creds) + os.environ.setdefault("AWS_SHARED_CREDENTIALS_FILE", shared_creds) parsed = urlparse(self.url) self.bucket = parsed.netloc - self.prefix = parsed.path.lstrip('/') + self.prefix = parsed.path.lstrip("/") - self.path_info = {'scheme': self.scheme, 'bucket': self.bucket} + self.path_info = {"scheme": self.scheme, "bucket": self.bucket} @staticmethod def compat_config(config): ret = config.copy() - url = 's3://' + ret.pop(Config.SECTION_AWS_STORAGEPATH, '').lstrip('/') + url = "s3://" + ret.pop(Config.SECTION_AWS_STORAGEPATH, "").lstrip("/") ret[Config.SECTION_REMOTE_URL] = url return ret @property def s3(self): - session = boto3.session.Session(profile_name=self.profile, - region_name=self.region) + session = boto3.session.Session( + profile_name=self.profile, region_name=self.region + ) - return session.client('s3', - endpoint_url=self.endpoint_url, - use_ssl=self.use_ssl) + return session.client( + "s3", endpoint_url=self.endpoint_url, use_ssl=self.use_ssl + ) def get_etag(self, bucket, path): try: obj = self.s3.head_object(Bucket=bucket, Key=path) except Exception: - raise DvcException('s3://{}/{} does not exist'.format(bucket, - path)) + raise DvcException("s3://{}/{} does not exist".format(bucket, path)) - return obj['ETag'].strip('"') + return obj["ETag"].strip('"') def save_info(self, path_info): - if path_info['scheme'] != 's3': + if path_info["scheme"] != "s3": raise NotImplementedError - return {self.PARAM_CHECKSUM: self.get_etag(path_info['bucket'], - path_info['path'])} + return { + self.PARAM_CHECKSUM: self.get_etag(path_info["bucket"], path_info["path"]) + } def copy(self, from_info, to_info, s3=None): s3 = s3 if s3 else self.s3 - source = {'Bucket': from_info['bucket'], - 'Key': from_info['path']} - self.s3.copy(source, to_info['bucket'], to_info['path']) + source = {"Bucket": from_info["bucket"], "Key": from_info["path"]} + self.s3.copy(source, to_info["bucket"], to_info["path"]) def save(self, path_info): - if path_info['scheme'] != 's3': + if path_info["scheme"] != "s3": raise NotImplementedError - etag = self.get_etag(path_info['bucket'], path_info['path']) + etag = self.get_etag(path_info["bucket"], path_info["path"]) path = self.checksum_to_path(etag) - to_info = {'scheme': 's3', 'bucket': self.bucket, 'path': path} + to_info = {"scheme": "s3", "bucket": self.bucket, "path": path} self.copy(path_info, to_info) return {self.PARAM_CHECKSUM: etag} def remove(self, path_info): - if path_info['scheme'] != 's3': + if path_info["scheme"] != "s3": raise NotImplementedError - logger.debug('Removing s3://{}/{}'.format(path_info['bucket'], - path_info['path'])) + logger.debug( + "Removing s3://{}/{}".format(path_info["bucket"], path_info["path"]) + ) - self.s3.delete_object(Bucket=path_info['bucket'], - Key=path_info['path']) + self.s3.delete_object(Bucket=path_info["bucket"], Key=path_info["path"]) def _list_paths(self, bucket, prefix): s3 = self.s3 - kwargs = {'Bucket': bucket, - 'Prefix': prefix} + kwargs = {"Bucket": bucket, "Prefix": prefix} while True: # NOTE: list_objects_v2() is 90% faster than head_object [1] # # [1] https://www.peterbe.com/plog/ # fastest-way-to-find-out-if-a-file-exists-in-s3 resp = s3.list_objects_v2(**kwargs) - contents = resp.get('Contents', None) + contents = resp.get("Contents", None) if not contents: break for obj in contents: - yield obj['Key'] + yield obj["Key"] - token = resp.get('NextContinuationToken', None) + token = resp.get("NextContinuationToken", None) if not token: break - kwargs['ContinuationToken'] = token + kwargs["ContinuationToken"] = token def list_cache_paths(self): return self._list_paths(self.bucket, self.prefix) def exists(self, path_info): assert not isinstance(path_info, list) - assert path_info['scheme'] == 's3' + assert path_info["scheme"] == "s3" - paths = self._list_paths(path_info['bucket'], path_info['path']) - return any(path_info['path'] == path for path in paths) + paths = self._list_paths(path_info["bucket"], path_info["path"]) + return any(path_info["path"] == path for path in paths) def upload(self, from_infos, to_infos, names=None): names = self._verify_path_args(to_infos, from_infos, names) @@ -169,85 +166,82 @@ class RemoteS3(RemoteBase): s3 = self.s3 for from_info, to_info, name in zip(from_infos, to_infos, names): - if to_info['scheme'] != 's3': + if to_info["scheme"] != "s3": raise NotImplementedError - if from_info['scheme'] != 'local': + if from_info["scheme"] != "local": raise NotImplementedError - logger.debug("Uploading '{}' to '{}/{}'".format(from_info['path'], - to_info['bucket'], - to_info['path'])) + logger.debug( + "Uploading '{}' to '{}/{}'".format( + from_info["path"], to_info["bucket"], to_info["path"] + ) + ) if not name: - name = os.path.basename(from_info['path']) + name = os.path.basename(from_info["path"]) - total = os.path.getsize(from_info['path']) + total = os.path.getsize(from_info["path"]) cb = Callback(name, total) try: - s3.upload_file(from_info['path'], - to_info['bucket'], - to_info['path'], - Callback=cb) + s3.upload_file( + from_info["path"], to_info["bucket"], to_info["path"], Callback=cb + ) except Exception: - msg = "failed to upload '{}'".format(from_info['path']) + msg = "failed to upload '{}'".format(from_info["path"]) logger.error(msg) continue progress.finish_target(name) - def download(self, - from_infos, - to_infos, - no_progress_bar=False, - names=None): + def download(self, from_infos, to_infos, no_progress_bar=False, names=None): names = self._verify_path_args(from_infos, to_infos, names) s3 = self.s3 for to_info, from_info, name in zip(to_infos, from_infos, names): - if from_info['scheme'] != 's3': + if from_info["scheme"] != "s3": raise NotImplementedError - if to_info['scheme'] == 's3': + if to_info["scheme"] == "s3": self.copy(from_info, to_info, s3=s3) continue - if to_info['scheme'] != 'local': + if to_info["scheme"] != "local": raise NotImplementedError - msg = "Downloading '{}/{}' to '{}'".format(from_info['bucket'], - from_info['path'], - to_info['path']) + msg = "Downloading '{}/{}' to '{}'".format( + from_info["bucket"], from_info["path"], to_info["path"] + ) logger.debug(msg) - tmp_file = self.tmp_file(to_info['path']) + tmp_file = self.tmp_file(to_info["path"]) if not name: - name = os.path.basename(to_info['path']) + name = os.path.basename(to_info["path"]) - self._makedirs(to_info['path']) + self._makedirs(to_info["path"]) try: if no_progress_bar: cb = None else: total = s3.head_object( - Bucket=from_info['bucket'], - Key=from_info['path'])['ContentLength'] + Bucket=from_info["bucket"], Key=from_info["path"] + )["ContentLength"] cb = Callback(name, total) - s3.download_file(from_info['bucket'], - from_info['path'], - tmp_file, - Callback=cb) + s3.download_file( + from_info["bucket"], from_info["path"], tmp_file, Callback=cb + ) except Exception: - msg = "failed to download '{}/{}'".format(from_info['bucket'], - from_info['path']) + msg = "failed to download '{}/{}'".format( + from_info["bucket"], from_info["path"] + ) logger.error(msg) continue - os.rename(tmp_file, to_info['path']) + os.rename(tmp_file, to_info["path"]) if not no_progress_bar: progress.finish_target(name) diff --git a/dvc/remote/ssh.py b/dvc/remote/ssh.py index bd6ecf778..8f3563510 100644 --- a/dvc/remote/ssh.py +++ b/dvc/remote/ssh.py @@ -19,26 +19,28 @@ from dvc.exceptions import DvcException from dvc.remote.base import RemoteBase, RemoteBaseCmdError -def sizeof_fmt(num, suffix='B'): +def sizeof_fmt(num, suffix="B"): """ Convert number of bytes to human-readable string """ - for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']: + for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]: if abs(num) < 1024.0: return "%3.1f%s%s" % (num, unit, suffix) num /= 1024.0 - return "%.1f%s%s" % (num, 'Y', suffix) + return "%.1f%s%s" % (num, "Y", suffix) def percent_cb(name, complete, total): """ Callback for updating target progress """ - logger.debug('{}: {} transferred out of {}'.format(name, - sizeof_fmt(complete), - sizeof_fmt(total))) + logger.debug( + "{}: {} transferred out of {}".format( + name, sizeof_fmt(complete), sizeof_fmt(total) + ) + ) progress.update_target(name, complete, total) def create_cb(name): """ Create callback function for multipart object """ - return (lambda cur, tot: percent_cb(name, cur, tot)) + return lambda cur, tot: percent_cb(name, cur, tot) class RemoteSSHCmdError(RemoteBaseCmdError): @@ -46,48 +48,50 @@ class RemoteSSHCmdError(RemoteBaseCmdError): class RemoteSSH(RemoteBase): - scheme = 'ssh' + scheme = "ssh" # NOTE: we support both URL-like (ssh://[user@]host.xz[:port]/path) and # SCP-like (ssh://[user@]host.xz:path) urls. - REGEX = r'^ssh://.*$' + REGEX = r"^ssh://.*$" - REQUIRES = {'paramiko': paramiko} + REQUIRES = {"paramiko": paramiko} JOBS = 4 - PARAM_CHECKSUM = 'md5' + PARAM_CHECKSUM = "md5" DEFAULT_PORT = 22 TIMEOUT = 1800 def __init__(self, project, config): super(RemoteSSH, self).__init__(project, config) - self.url = config.get(Config.SECTION_REMOTE_URL, 'ssh://') + self.url = config.get(Config.SECTION_REMOTE_URL, "ssh://") parsed = urlparse(self.url) self.host = parsed.hostname - self.user = (config.get(Config.SECTION_REMOTE_USER) - or parsed.username - or getpass.getuser()) + self.user = ( + config.get(Config.SECTION_REMOTE_USER) + or parsed.username + or getpass.getuser() + ) self.prefix = parsed.path - self.port = (config.get(Config.SECTION_REMOTE_PORT) - or parsed.port - or self.DEFAULT_PORT) + self.port = ( + config.get(Config.SECTION_REMOTE_PORT) or parsed.port or self.DEFAULT_PORT + ) self.keyfile = config.get(Config.SECTION_REMOTE_KEY_FILE, None) self.timeout = config.get(Config.SECTION_REMOTE_TIMEOUT, self.TIMEOUT) self.password = config.get(Config.SECTION_REMOTE_PASSWORD, None) - self.ask_password = config.get(Config.SECTION_REMOTE_ASK_PASSWORD, - False) + self.ask_password = config.get(Config.SECTION_REMOTE_ASK_PASSWORD, False) - self.path_info = {'scheme': 'ssh', - 'host': self.host, - 'user': self.user, - 'port': self.port} + self.path_info = { + "scheme": "ssh", + "host": self.host, + "user": self.user, + "port": self.port, + } def ssh(self, host=None, user=None, port=None): - msg = "Establishing ssh connection with '{}' " \ - "through port '{}' as user '{}'" + msg = "Establishing ssh connection with '{}' " "through port '{}' as user '{}'" logger.debug(msg.format(host, port, user)) ssh = paramiko.SSHClient() @@ -96,28 +100,30 @@ class RemoteSSH(RemoteBase): ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) if self.ask_password and self.password is None: - msg = ("Enter a private key passphrase or a password for " - "host '{}' port '{}' user '{}'").format(host, port, user) + msg = ( + "Enter a private key passphrase or a password for " + "host '{}' port '{}' user '{}'" + ).format(host, port, user) self.password = prompt.password(msg) - ssh.connect(host, - username=user, - port=port, - key_filename=self.keyfile, - timeout=self.timeout, - password=self.password) + ssh.connect( + host, + username=user, + port=port, + key_filename=self.keyfile, + timeout=self.timeout, + password=self.password, + ) return ssh def exists(self, path_info): assert not isinstance(path_info, list) - assert path_info['scheme'] == 'ssh' + assert path_info["scheme"] == "ssh" - with self.ssh(path_info['host'], - path_info['user'], - path_info['port']) as ssh: + with self.ssh(path_info["host"], path_info["user"], path_info["port"]) as ssh: try: - self._exec(ssh, 'test -e {}'.format(path_info['path'])) + self._exec(ssh, "test -e {}".format(path_info["path"])) exists = True except RemoteSSHCmdError: exists = False @@ -133,10 +139,9 @@ class RemoteSSH(RemoteBase): stdout_chunks = [] stderr_chunks = [] - while (not channel.closed - or channel.recv_ready() - or channel.recv_stderr_ready()): + while not channel.closed or channel.recv_ready() or channel.recv_stderr_ready(): import select + got_chunk = False readq, _, _ = select.select([stdout.channel], [], [], self.timeout) for c in readq: @@ -150,10 +155,12 @@ class RemoteSSH(RemoteBase): stderr_chunks.append(s) got_chunk = True - if not got_chunk \ - and stdout.channel.exit_status_ready() \ - and not stderr.channel.recv_stderr_ready() \ - and not stdout.channel.recv_ready(): + if ( + not got_chunk + and stdout.channel.exit_status_ready() + and not stderr.channel.recv_stderr_ready() + and not stdout.channel.recv_ready() + ): stdout.channel.shutdown_read() stdout.channel.close() break @@ -163,32 +170,32 @@ class RemoteSSH(RemoteBase): ret = stdout.channel.recv_exit_status() if ret != 0: - err = b''.join(stderr_chunks).decode('utf-8') + err = b"".join(stderr_chunks).decode("utf-8") raise RemoteSSHCmdError(cmd, ret, err) - return b''.join(stdout_chunks).decode('utf-8') + return b"".join(stdout_chunks).decode("utf-8") def md5(self, path_info): - if path_info['scheme'] != 'ssh': + if path_info["scheme"] != "ssh": raise NotImplementedError - ssh = self.ssh(host=path_info['host'], - user=path_info['user'], - port=path_info['port']) + ssh = self.ssh( + host=path_info["host"], user=path_info["user"], port=path_info["port"] + ) # Use different md5 commands depending on os - stdout = self._exec(ssh, 'uname').strip() - if stdout == 'Darwin': - md5cmd = 'md5' + stdout = self._exec(ssh, "uname").strip() + if stdout == "Darwin": + md5cmd = "md5" index = -1 - elif stdout == 'Linux': - md5cmd = 'md5sum' + elif stdout == "Linux": + md5cmd = "md5sum" index = 0 else: msg = "'{}' is not supported as a remote".format(stdout) raise DvcException(msg) - stdout = self._exec(ssh, '{} {}'.format(md5cmd, path_info['path'])) + stdout = self._exec(ssh, "{} {}".format(md5cmd, path_info["path"])) md5 = stdout.split()[index] ssh.close() @@ -197,102 +204,106 @@ class RemoteSSH(RemoteBase): return md5 def copy(self, from_info, to_info, ssh=None): - if from_info['scheme'] != 'ssh' or to_info['scheme'] != 'ssh': + if from_info["scheme"] != "ssh" or to_info["scheme"] != "ssh": raise NotImplementedError - assert from_info['host'] == to_info['host'] - assert from_info['user'] == to_info['user'] + assert from_info["host"] == to_info["host"] + assert from_info["user"] == to_info["user"] - s = ssh if ssh else self.ssh(host=from_info['host'], - user=from_info['user'], - port=from_info['port']) + s = ( + ssh + if ssh + else self.ssh( + host=from_info["host"], user=from_info["user"], port=from_info["port"] + ) + ) - dname = posixpath.dirname(to_info['path']) - self._exec(s, 'mkdir -p {}'.format(dname)) - self._exec(s, 'cp {} {}'.format(from_info['path'], to_info['path'])) + dname = posixpath.dirname(to_info["path"]) + self._exec(s, "mkdir -p {}".format(dname)) + self._exec(s, "cp {} {}".format(from_info["path"], to_info["path"])) if not ssh: s.close() def save_info(self, path_info): - if path_info['scheme'] != 'ssh': + if path_info["scheme"] != "ssh": raise NotImplementedError return {self.PARAM_CHECKSUM: self.md5(path_info)} def save(self, path_info): - if path_info['scheme'] != 'ssh': + if path_info["scheme"] != "ssh": raise NotImplementedError md5 = self.md5(path_info) dest = path_info.copy() - dest['path'] = self.checksum_to_path(md5) + dest["path"] = self.checksum_to_path(md5) self.copy(path_info, dest) return {self.PARAM_CHECKSUM: md5} def remove(self, path_info): - if path_info['scheme'] != 'ssh': + if path_info["scheme"] != "ssh": raise NotImplementedError - logger.debug('Removing ssh://{}@{}/{}'.format(path_info['user'], - path_info['host'], - path_info['path'])) + logger.debug( + "Removing ssh://{}@{}/{}".format( + path_info["user"], path_info["host"], path_info["path"] + ) + ) - ssh = self.ssh(host=path_info['host'], - user=path_info['user'], - port=path_info['port']) - ssh.open_sftp().remove(path_info['path']) + ssh = self.ssh( + host=path_info["host"], user=path_info["user"], port=path_info["port"] + ) + ssh.open_sftp().remove(path_info["path"]) ssh.close() - def download(self, - from_infos, - to_infos, - no_progress_bar=False, - names=None): + def download(self, from_infos, to_infos, no_progress_bar=False, names=None): names = self._verify_path_args(from_infos, to_infos, names) - ssh = self.ssh(host=from_infos[0]['host'], - user=from_infos[0]['user'], - port=from_infos[0]['port']) + ssh = self.ssh( + host=from_infos[0]["host"], + user=from_infos[0]["user"], + port=from_infos[0]["port"], + ) for to_info, from_info, name in zip(to_infos, from_infos, names): - if from_info['scheme'] != 'ssh': + if from_info["scheme"] != "ssh": raise NotImplementedError - if to_info['scheme'] == 'ssh': - assert from_info['host'] == to_info['host'] - assert from_info['port'] == to_info['port'] - assert from_info['user'] == to_info['user'] + if to_info["scheme"] == "ssh": + assert from_info["host"] == to_info["host"] + assert from_info["port"] == to_info["port"] + assert from_info["user"] == to_info["user"] self.copy(from_info, to_info, ssh=ssh) continue - if to_info['scheme'] != 'local': + if to_info["scheme"] != "local": raise NotImplementedError - msg = "Downloading '{}/{}' to '{}'".format(from_info['host'], - from_info['path'], - to_info['path']) + msg = "Downloading '{}/{}' to '{}'".format( + from_info["host"], from_info["path"], to_info["path"] + ) logger.debug(msg) if not name: - name = os.path.basename(to_info['path']) + name = os.path.basename(to_info["path"]) - self._makedirs(to_info['path']) - tmp_file = self.tmp_file(to_info['path']) + self._makedirs(to_info["path"]) + tmp_file = self.tmp_file(to_info["path"]) try: - ssh.open_sftp().get(from_info['path'], - tmp_file, - callback=create_cb(name)) + ssh.open_sftp().get( + from_info["path"], tmp_file, callback=create_cb(name) + ) except Exception: msg = "failed to download '{}/{}' to '{}'" - logger.error(msg.format(from_info['host'], - from_info['path'], - to_info['path'])) + logger.error( + msg.format(from_info["host"], from_info["path"], to_info["path"]) + ) continue - os.rename(tmp_file, to_info['path']) + os.rename(tmp_file, to_info["path"]) progress.finish_target(name) ssh.close() @@ -300,37 +311,37 @@ class RemoteSSH(RemoteBase): def upload(self, from_infos, to_infos, names=None): names = self._verify_path_args(to_infos, from_infos, names) - ssh = self.ssh(host=to_infos[0]['host'], - user=to_infos[0]['user'], - port=to_infos[0]['port']) + ssh = self.ssh( + host=to_infos[0]["host"], user=to_infos[0]["user"], port=to_infos[0]["port"] + ) sftp = ssh.open_sftp() for from_info, to_info, name in zip(from_infos, to_infos, names): - if to_info['scheme'] != 'ssh': + if to_info["scheme"] != "ssh": raise NotImplementedError - if from_info['scheme'] != 'local': + if from_info["scheme"] != "local": raise NotImplementedError - logger.debug("Uploading '{}' to '{}/{}'".format(from_info['path'], - to_info['host'], - to_info['path'])) + logger.debug( + "Uploading '{}' to '{}/{}'".format( + from_info["path"], to_info["host"], to_info["path"] + ) + ) if not name: - name = os.path.basename(from_info['path']) + name = os.path.basename(from_info["path"]) - dname = posixpath.dirname(to_info['path']) - self._exec(ssh, 'mkdir -p {}'.format(dname)) + dname = posixpath.dirname(to_info["path"]) + self._exec(ssh, "mkdir -p {}".format(dname)) try: - sftp.put(from_info['path'], - to_info['path'], - callback=create_cb(name)) + sftp.put(from_info["path"], to_info["path"], callback=create_cb(name)) except Exception: msg = "failed to upload '{}' to '{}/{}'" - logger.error(msg.format(from_info['path'], - to_info['host'], - to_info['path'])) + logger.error( + msg.format(from_info["path"], to_info["host"], to_info["path"]) + ) continue progress.finish_target(name) @@ -339,10 +350,8 @@ class RemoteSSH(RemoteBase): ssh.close() def list_cache_paths(self): - ssh = self.ssh(host=self.host, - user=self.user, - port=self.port) - cmd = 'find {} -type f -follow -print'.format(self.prefix) + ssh = self.ssh(host=self.host, user=self.user, port=self.port) + cmd = "find {} -type f -follow -print".format(self.prefix) stdout = self._exec(ssh, cmd) flist = stdout.split() ssh.close() diff --git a/dvc/scm/base.py b/dvc/scm/base.py index d7c83e2bf..f39f133cc 100644 --- a/dvc/scm/base.py +++ b/dvc/scm/base.py @@ -19,14 +19,14 @@ class FileNotInRepoError(DvcException): class Base(object): """Base class for source control management driver implementations.""" + def __init__(self, root_dir=os.curdir, project=None): self.project = project self.root_dir = root_dir def __repr__(self): return "{class_name}: '{directory}'".format( - class_name=type(self).__name__, - directory=self.dir + class_name=type(self).__name__, directory=self.dir ) @property @@ -76,11 +76,7 @@ class Base(object): def tag(self, tag): """Makes SCM create a tag with a specified name.""" - def brancher(self, - branches=None, - all_branches=False, - tags=None, - all_tags=False): + def brancher(self, branches=None, all_branches=False, tags=None, all_tags=False): """Generator that iterates over specified revisions. Args: @@ -92,9 +88,8 @@ class Base(object): Yields: str: the current revision. """ - if not branches and not all_branches \ - and not tags and not all_tags: - yield '' + if not branches and not all_branches and not tags and not all_tags: + yield "" return saved = self.active_branch() @@ -130,7 +125,7 @@ class Base(object): def active_branch(self): # pylint: disable=no-self-use """Returns current branch in the project.""" - return '' + return "" def list_branches(self): # pylint: disable=no-self-use """Returns a list of available branches in the project.""" diff --git a/dvc/scm/git.py b/dvc/scm/git.py index 0b38aeabc..cb161fb04 100644 --- a/dvc/scm/git.py +++ b/dvc/scm/git.py @@ -17,24 +17,26 @@ from dvc.scm.base import Base, SCMError, FileNotInRepoError class Git(Base): """Class for managing Git.""" - GITIGNORE = '.gitignore' - GIT_DIR = '.git' + + GITIGNORE = ".gitignore" + GIT_DIR = ".git" def __init__(self, root_dir=os.curdir, project=None): super(Git, self).__init__(root_dir, project=project) import git from git.exc import InvalidGitRepositoryError + try: self.repo = git.Repo(root_dir) except InvalidGitRepositoryError: - msg = '{} is not a git repository' + msg = "{} is not a git repository" raise SCMError(msg.format(root_dir)) # NOTE: fixing LD_LIBRARY_PATH for binary built by PyInstaller. # http://pyinstaller.readthedocs.io/en/stable/runtime-information.html env = fix_env(None) - libpath = env.get('LD_LIBRARY_PATH', None) + libpath = env.get("LD_LIBRARY_PATH", None) self.repo.git.update_environment(LD_LIBRARY_PATH=libpath) @staticmethod @@ -60,7 +62,7 @@ class Git(Base): def _get_gitignore(self, path): assert os.path.isabs(path) # NOTE: using '/' prefix to make path unambiguous - entry = '/' + os.path.basename(path) + entry = "/" + os.path.basename(path) gitignore = os.path.join(os.path.dirname(path), self.GITIGNORE) if not gitignore.startswith(self.root_dir): @@ -73,21 +75,21 @@ class Git(Base): ignore_list = [] if os.path.exists(gitignore): - ignore_list = open(gitignore, 'r').readlines() - filtered = list(filter(lambda x: x.strip() == entry.strip(), - ignore_list)) + ignore_list = open(gitignore, "r").readlines() + filtered = list(filter(lambda x: x.strip() == entry.strip(), ignore_list)) if filtered: return - msg = "Adding '{}' to '{}'.".format(os.path.relpath(path), - os.path.relpath(gitignore)) + msg = "Adding '{}' to '{}'.".format( + os.path.relpath(path), os.path.relpath(gitignore) + ) logger.info(msg) content = entry if ignore_list: - content = '\n' + content + content = "\n" + content - with open(gitignore, 'a') as fobj: + with open(gitignore, "a") as fobj: fobj.write(content) if self.project is not None: @@ -99,12 +101,12 @@ class Git(Base): if not os.path.exists(gitignore): return - with open(gitignore, 'r') as fobj: + with open(gitignore, "r") as fobj: lines = fobj.readlines() filtered = list(filter(lambda x: x.strip() != entry.strip(), lines)) - with open(gitignore, 'w') as fobj: + with open(gitignore, "w") as fobj: fobj.writelines(filtered) if self.project is not None: @@ -120,8 +122,7 @@ class Git(Base): "failed to add '{}' to git. You can add those files" " manually using 'git add'." " See 'https://github.com/iterative/dvc/issues/610'" - " for more details." - .format(str(paths)) + " for more details.".format(str(paths)) ) logger.error(msg) @@ -131,7 +132,7 @@ class Git(Base): def checkout(self, branch, create_new=False): if create_new: - self.repo.git.checkout('HEAD', b=branch) + self.repo.git.checkout("HEAD", b=branch) else: self.repo.git.checkout(branch) @@ -158,17 +159,14 @@ class Git(Base): return [t.name for t in self.repo.tags] def _install_hook(self, name, cmd): - hook = os.path.join(self.root_dir, - self.GIT_DIR, - 'hooks', - name) + hook = os.path.join(self.root_dir, self.GIT_DIR, "hooks", name) if os.path.isfile(hook): msg = "git hook '{}' already exists." raise SCMError(msg.format(os.path.relpath(hook))) - with open(hook, 'w+') as fobj: - fobj.write('#!/bin/sh\nexec dvc {}\n'.format(cmd)) + with open(hook, "w+") as fobj: + fobj.write("#!/bin/sh\nexec dvc {}\n".format(cmd)) os.chmod(hook, 0o777) def install(self): - self._install_hook('post-checkout', 'checkout') - self._install_hook('pre-commit', 'status') + self._install_hook("post-checkout", "checkout") + self._install_hook("pre-commit", "status") diff --git a/dvc/stage.py b/dvc/stage.py index 426376a88..c55b29e4c 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -72,11 +72,11 @@ class MissingDep(DvcException): assert len(deps) > 0 if len(deps) > 1: - dep = 'dependencies' + dep = "dependencies" else: - dep = 'dependency' + dep = "dependency" - msg = 'missing {}: {}'.format(dep, ', '.join(map(str, deps))) + msg = "missing {}: {}".format(dep, ", ".join(map(str, deps))) super(MissingDep, self).__init__(msg) @@ -84,23 +84,23 @@ class MissingDataSource(DvcException): def __init__(self, missing_files): assert len(missing_files) > 0 - source = 'source' + source = "source" if len(missing_files) > 1: - source += 's' + source += "s" - msg = 'missing data {}: {}'.format(source, ', '.join(missing_files)) + msg = "missing data {}: {}".format(source, ", ".join(missing_files)) super(MissingDataSource, self).__init__(msg) class Stage(object): - STAGE_FILE = 'Dvcfile' - STAGE_FILE_SUFFIX = '.dvc' + STAGE_FILE = "Dvcfile" + STAGE_FILE_SUFFIX = ".dvc" - PARAM_MD5 = 'md5' - PARAM_CMD = 'cmd' - PARAM_DEPS = 'deps' - PARAM_OUTS = 'outs' - PARAM_LOCKED = 'locked' + PARAM_MD5 = "md5" + PARAM_CMD = "cmd" + PARAM_DEPS = "deps" + PARAM_OUTS = "outs" + PARAM_LOCKED = "locked" SCHEMA = { Optional(PARAM_MD5): Or(str, None), @@ -110,15 +110,17 @@ class Stage(object): Optional(PARAM_LOCKED): bool, } - def __init__(self, - project, - path=None, - cmd=None, - cwd=os.curdir, - deps=[], - outs=[], - md5=None, - locked=False): + def __init__( + self, + project, + path=None, + cmd=None, + cwd=os.curdir, + deps=[], + outs=[], + md5=None, + locked=False, + ): self.project = project self.path = path self.cmd = cmd @@ -129,9 +131,7 @@ class Stage(object): self.locked = locked def __repr__(self): - return "Stage: '{path}'".format( - path=self.relpath if self.path else 'No path' - ) + return "Stage: '{path}'".format(path=self.relpath if self.path else "No path") @property def relpath(self): @@ -144,8 +144,10 @@ class Stage(object): @staticmethod def is_valid_filename(path): - return (path.endswith(Stage.STAGE_FILE_SUFFIX) - or os.path.basename(path) == Stage.STAGE_FILE) + return ( + path.endswith(Stage.STAGE_FILE_SUFFIX) + or os.path.basename(path) == Stage.STAGE_FILE + ) @staticmethod def is_stage_file(path): @@ -165,9 +167,7 @@ class Stage(object): @property def is_import(self): """Whether the stage file was created with `dvc import`.""" - return (not self.cmd - and len(self.deps) == 1 - and len(self.outs) == 1) + return not self.cmd and len(self.deps) == 1 and len(self.outs) == 1 def _changed_deps(self): if self.locked: @@ -176,15 +176,19 @@ class Stage(object): if self.is_callback: logger.warning( "Dvc file '{fname}' is a 'callback' stage (has a command and" - " no dependencies) and thus always considered as changed." - .format(fname=self.relpath) + " no dependencies) and thus always considered as changed.".format( + fname=self.relpath + ) ) return True for dep in self.deps: if dep.changed(): - logger.warning("Dependency '{dep}' of '{stage}' changed." - .format(dep=dep, stage=self.relpath)) + logger.warning( + "Dependency '{dep}' of '{stage}' changed.".format( + dep=dep, stage=self.relpath + ) + ) return True return False @@ -193,8 +197,9 @@ class Stage(object): for out in self.outs: if out.changed(): logger.warning( - "Output '{out}' of '{stage}' changed." - .format(out=out, stage=self.relpath) + "Output '{out}' of '{stage}' changed.".format( + out=out, stage=self.relpath + ) ) return True @@ -207,16 +212,14 @@ class Stage(object): return False def changed(self): - ret = any([self._changed_deps(), - self._changed_outs(), - self._changed_md5()]) + ret = any([self._changed_deps(), self._changed_outs(), self._changed_md5()]) if ret: msg = "Stage '{}' changed.".format(self.relpath) - color = 'yellow' + color = "yellow" else: msg = "Stage '{}' didn't change.".format(self.relpath) - color = 'green' + color = "green" logger.info(logger.colorize(msg, color)) @@ -231,7 +234,7 @@ class Stage(object): def unprotect_outs(self): for out in self.outs: - if out.scheme != 'local' or not out.exists: + if out.scheme != "local" or not out.exists: continue self.project.unprotect(out.path) @@ -247,13 +250,12 @@ class Stage(object): # Removing outputs only if we actually have command to reproduce self.remove_outs(ignore_remove=False) - msg = ( - "Going to reproduce '{stage}'. Are you sure you want to continue?" - .format(stage=self.relpath) + msg = "Going to reproduce '{stage}'. Are you sure you want to continue?".format( + stage=self.relpath ) if interactive and not prompt.confirm(msg): - raise DvcException('reproduction aborted by the user') + raise DvcException("reproduction aborted by the user") logger.info("Reproducing '{stage}'".format(stage=self.relpath)) @@ -281,7 +283,7 @@ class Stage(object): return (cls.STAGE_FILE, cwd if cwd else os.getcwd()) out = outs[0] - if out.scheme == 'local': + if out.scheme == "local": path = os.path else: path = posixpath @@ -333,29 +335,29 @@ class Stage(object): return old_d == new_d @staticmethod - def create(project=None, - cmd=None, - deps=[], - outs=[], - outs_no_cache=[], - metrics_no_cache=[], - fname=None, - cwd=os.curdir, - locked=False, - add=False, - overwrite=True, - ignore_build_cache=False, - remove_outs=False): + def create( + project=None, + cmd=None, + deps=[], + outs=[], + outs_no_cache=[], + metrics_no_cache=[], + fname=None, + cwd=os.curdir, + locked=False, + add=False, + overwrite=True, + ignore_build_cache=False, + remove_outs=False, + ): - stage = Stage(project=project, - cwd=cwd, - cmd=cmd, - locked=locked) + stage = Stage(project=project, cwd=cwd, cmd=cmd, locked=locked) stage.outs = output.loads_from(stage, outs, use_cache=True) stage.outs += output.loads_from(stage, outs_no_cache, use_cache=False) - stage.outs += output.loads_from(stage, metrics_no_cache, - use_cache=False, metric=True) + stage.outs += output.loads_from( + stage, metrics_no_cache, use_cache=False, metric=True + ) stage.deps = dependency.loads_from(stage, deps) stage._check_circular_dependency() @@ -364,8 +366,9 @@ class Stage(object): if fname is not None and os.path.basename(fname) != fname: raise StageFileBadNameError( "stage file name '{fname}' should not contain subdirectories." - " Use '-c|--cwd' to change location of the stage file." - .format(fname=fname) + " Use '-c|--cwd' to change location of the stage file.".format( + fname=fname + ) ) fname, cwd = Stage._stage_fname_cwd(fname, cwd, stage.outs, add=add) @@ -388,11 +391,13 @@ class Stage(object): if os.path.exists(path): if not ignore_build_cache and stage.is_cached: - logger.info('Stage is cached, skipping.') + logger.info("Stage is cached, skipping.") return None - msg = "'{}' already exists. Do you wish to run the command and " \ - "overwrite it?".format(stage.relpath) + msg = ( + "'{}' already exists. Do you wish to run the command and " + "overwrite it?".format(stage.relpath) + ) if not overwrite and not prompt.confirm(msg): raise StageFileAlreadyExistsError(stage.relpath) @@ -406,8 +411,9 @@ class Stage(object): if not Stage.is_valid_filename(fname): raise StageFileBadNameError( "bad stage filename '{}'. Stage files should be named" - " 'Dvcfile' or have a '.dvc' suffix(e.g. '{}.dvc')." - .format(os.path.relpath(fname), os.path.basename(fname)) + " 'Dvcfile' or have a '.dvc' suffix(e.g. '{}.dvc').".format( + os.path.relpath(fname), os.path.basename(fname) + ) ) @staticmethod @@ -423,17 +429,19 @@ class Stage(object): if not Stage.is_stage_file(fname): raise StageFileIsNotDvcFileError(fname) - with open(fname, 'r') as fd: + with open(fname, "r") as fd: d = yaml.safe_load(fd) or {} Stage.validate(d, fname=os.path.relpath(fname)) - stage = Stage(project=project, - path=os.path.abspath(fname), - cwd=os.path.dirname(os.path.abspath(fname)), - cmd=d.get(Stage.PARAM_CMD), - md5=d.get(Stage.PARAM_MD5), - locked=d.get(Stage.PARAM_LOCKED, False)) + stage = Stage( + project=project, + path=os.path.abspath(fname), + cwd=os.path.dirname(os.path.abspath(fname)), + cmd=d.get(Stage.PARAM_CMD), + md5=d.get(Stage.PARAM_MD5), + locked=d.get(Stage.PARAM_LOCKED, False), + ) stage.deps = dependency.loadd_from(stage, d.get(Stage.PARAM_DEPS, [])) stage.outs = output.loadd_from(stage, d.get(Stage.PARAM_OUTS, [])) @@ -458,10 +466,11 @@ class Stage(object): self._check_dvc_filename(fname) - logger.info("Saving information to '{file}'." - .format(file=os.path.relpath(fname))) + logger.info( + "Saving information to '{file}'.".format(file=os.path.relpath(fname)) + ) - with open(fname, 'w') as fd: + with open(fname, "w") as fd: yaml.safe_dump(self.dumpd(), fd, default_flow_style=False) self.project.files_to_git_add.append(os.path.relpath(fname)) @@ -478,8 +487,7 @@ class Stage(object): # NOTE: excluding parameters that don't affect the state of the # pipeline. Not excluding `OutputLOCAL.PARAM_CACHE`, because if # it has changed, we might not have that output in our cache. - return dict_md5(d, exclude=[self.PARAM_LOCKED, - OutputLOCAL.PARAM_METRIC]) + return dict_md5(d, exclude=[self.PARAM_LOCKED, OutputLOCAL.PARAM_METRIC]) def save(self): for dep in self.deps: @@ -497,8 +505,10 @@ class Stage(object): raise MissingDep(missing) def _warn_if_fish(self, executable): # pragma: no cover - if executable is None \ - or os.path.basename(os.path.realpath(executable)) != 'fish': + if ( + executable is None + or os.path.basename(os.path.realpath(executable)) != "fish" + ): return logger.warning( @@ -512,9 +522,8 @@ class Stage(object): def _check_circular_dependency(self): from dvc.exceptions import CircularDependencyError - circular_dependencies = ( - set(file.path for file in self.deps) & - set(file.path for file in self.outs) + circular_dependencies = set(file.path for file in self.deps) & set( + file.path for file in self.outs ) if circular_dependencies: @@ -532,14 +541,16 @@ class Stage(object): def _run(self): self._check_missing_deps() - executable = os.getenv('SHELL') if os.name != 'nt' else None + executable = os.getenv("SHELL") if os.name != "nt" else None self._warn_if_fish(executable) - p = subprocess.Popen(self.cmd, - cwd=self.cwd, - shell=True, - env=fix_env(os.environ), - executable=executable) + p = subprocess.Popen( + self.cmd, + cwd=self.cwd, + shell=True, + env=fix_env(os.environ), + executable=executable, + ) p.communicate() if p.returncode != 0: @@ -547,14 +558,18 @@ class Stage(object): def run(self, dry=False): if self.locked: - logger.info("Verifying outputs in locked stage '{stage}'" - .format(stage=self.relpath)) + logger.info( + "Verifying outputs in locked stage '{stage}'".format(stage=self.relpath) + ) if not dry: self.check_missing_outputs() elif self.is_import: - logger.info("Importing '{dep}' -> '{out}'" - .format(dep=self.deps[0].path, out=self.outs[0].path)) + logger.info( + "Importing '{dep}' -> '{out}'".format( + dep=self.deps[0].path, out=self.outs[0].path + ) + ) if not dry: if self._already_cached(): self.outs[0].checkout() @@ -568,7 +583,7 @@ class Stage(object): self.check_missing_outputs() else: - logger.info('Running command:\n\t{}'.format(self.cmd)) + logger.info("Running command:\n\t{}".format(self.cmd)) if not dry: if self._already_cached(): self.checkout() @@ -580,7 +595,7 @@ class Stage(object): def check_missing_outputs(self): paths = [ - out.path if out.scheme != 'local' else out.rel_path + out.path if out.scheme != "local" else out.rel_path for out in self.outs if not out.exists ] @@ -607,9 +622,9 @@ class Stage(object): ret = {} if not self.locked: - ret.update(self._status(self.deps, 'deps')) + ret.update(self._status(self.deps, "deps")) - ret.update(self._status(self.outs, 'outs')) + ret.update(self._status(self.outs, "outs")) if ret or self.changed_md5() or self.is_callback: return {self.relpath: ret} @@ -617,9 +632,11 @@ class Stage(object): return {} def _already_cached(self): - return (not self.changed_md5() - and all(not dep.changed() for dep in self.deps) - and all(not out.changed_cache() - if out.use_cache - else not out.changed() - for out in self.outs)) + return ( + not self.changed_md5() + and all(not dep.changed() for dep in self.deps) + and all( + not out.changed_cache() if out.use_cache else not out.changed() + for out in self.outs + ) + ) diff --git a/dvc/state.py b/dvc/state.py index 4ed5a20f1..c166cc48f 100644 --- a/dvc/state.py +++ b/dvc/state.py @@ -18,15 +18,16 @@ from dvc.exceptions import DvcException class StateVersionTooNewError(DvcException): """Thrown when dvc version is older than the state database version.""" + def __init__(self, dvc_version, expected, actual): super(StateVersionTooNewError, self).__init__( "you are using an old version '{dvc_version}' of dvc that is " "using state file version '{expected}' which is not compatible " "with the state file version '{actual}' that is used in this " - "projet. Please upgrade right now!" - .format(dvc_version=dvc_version, - expected=expected, - actual=actual)) + "projet. Please upgrade right now!".format( + dvc_version=dvc_version, expected=expected, actual=actual + ) + ) class State(object): # pylint: disable=too-many-instance-attributes @@ -41,29 +42,32 @@ class State(object): # pylint: disable=too-many-instance-attributes StateVersionTooNewError: thrown when dvc version is older than the state database version. """ - VERSION = 3 - STATE_FILE = 'state' - STATE_TABLE = 'state' - STATE_TABLE_LAYOUT = ("inode INTEGER PRIMARY KEY, " - "mtime TEXT NOT NULL, " - "size TEXT NOT NULL, " - "md5 TEXT NOT NULL, " - "timestamp TEXT NOT NULL") - STATE_INFO_TABLE = 'state_info' - STATE_INFO_TABLE_LAYOUT = 'count INTEGER' + VERSION = 3 + STATE_FILE = "state" + STATE_TABLE = "state" + STATE_TABLE_LAYOUT = ( + "inode INTEGER PRIMARY KEY, " + "mtime TEXT NOT NULL, " + "size TEXT NOT NULL, " + "md5 TEXT NOT NULL, " + "timestamp TEXT NOT NULL" + ) + + STATE_INFO_TABLE = "state_info" + STATE_INFO_TABLE_LAYOUT = "count INTEGER" STATE_INFO_ROW = 1 - LINK_STATE_TABLE = 'link_state' - LINK_STATE_TABLE_LAYOUT = ("path TEXT PRIMARY KEY, " - "inode INTEGER NOT NULL, " - "mtime TEXT NOT NULL") + LINK_STATE_TABLE = "link_state" + LINK_STATE_TABLE_LAYOUT = ( + "path TEXT PRIMARY KEY, " "inode INTEGER NOT NULL, " "mtime TEXT NOT NULL" + ) STATE_ROW_LIMIT = 100000000 STATE_ROW_CLEANUP_QUOTA = 50 - MAX_INT = 2**63 - 1 - MAX_UINT = 2**64 - 2 + MAX_INT = 2 ** 63 - 1 + MAX_UINT = 2 ** 64 - 2 def __init__(self, project, config): self.project = project @@ -74,11 +78,12 @@ class State(object): # pylint: disable=too-many-instance-attributes self.row_cleanup_quota = 50 state_config = config.get(Config.SECTION_STATE, {}) - self.row_limit = state_config.get(Config.SECTION_STATE_ROW_LIMIT, - self.STATE_ROW_LIMIT) + self.row_limit = state_config.get( + Config.SECTION_STATE_ROW_LIMIT, self.STATE_ROW_LIMIT + ) self.row_cleanup_quota = state_config.get( - Config.SECTION_STATE_ROW_CLEANUP_QUOTA, - self.STATE_ROW_CLEANUP_QUOTA) + Config.SECTION_STATE_ROW_CLEANUP_QUOTA, self.STATE_ROW_CLEANUP_QUOTA + ) if not self.dvc_dir: self.state_file = None @@ -87,10 +92,7 @@ class State(object): # pylint: disable=too-many-instance-attributes self.state_file = os.path.join(self.dvc_dir, self.STATE_FILE) # https://www.sqlite.org/tempfiles.html - self.temp_files = [ - self.state_file + '-journal', - self.state_file + '-wal', - ] + self.temp_files = [self.state_file + "-journal", self.state_file + "-wal"] self.database = None self.cursor = None @@ -125,7 +127,7 @@ class State(object): # pylint: disable=too-many-instance-attributes if not md5 or not actual: return True - return actual.split('.')[0] != md5.split('.')[0] + return actual.split(".")[0] != md5.split(".")[0] def _execute(self, cmd): logger.debug(cmd) @@ -171,8 +173,10 @@ class State(object): # pylint: disable=too-many-instance-attributes if version > self.VERSION: raise StateVersionTooNewError(VERSION, self.VERSION, version) elif version < self.VERSION: - msg = "State file version '{}' is too old. " \ - "Reformatting to the current version '{}'." + msg = ( + "State file version '{}' is too old. " + "Reformatting to the current version '{}'." + ) logger.warning(msg.format(version, self.VERSION)) cmd = "DROP TABLE IF EXISTS {};" self._execute(cmd.format(self.STATE_TABLE)) @@ -181,17 +185,15 @@ class State(object): # pylint: disable=too-many-instance-attributes # Check that the state file is indeed a database cmd = "CREATE TABLE IF NOT EXISTS {} ({})" - self._execute(cmd.format(self.STATE_TABLE, - self.STATE_TABLE_LAYOUT)) - self._execute(cmd.format(self.STATE_INFO_TABLE, - self.STATE_INFO_TABLE_LAYOUT)) - self._execute(cmd.format(self.LINK_STATE_TABLE, - self.LINK_STATE_TABLE_LAYOUT)) + self._execute(cmd.format(self.STATE_TABLE, self.STATE_TABLE_LAYOUT)) + self._execute(cmd.format(self.STATE_INFO_TABLE, self.STATE_INFO_TABLE_LAYOUT)) + self._execute(cmd.format(self.LINK_STATE_TABLE, self.LINK_STATE_TABLE_LAYOUT)) - cmd = "INSERT OR IGNORE INTO {} (count) SELECT 0 " \ - "WHERE NOT EXISTS (SELECT * FROM {})" - self._execute(cmd.format(self.STATE_INFO_TABLE, - self.STATE_INFO_TABLE)) + cmd = ( + "INSERT OR IGNORE INTO {} (count) SELECT 0 " + "WHERE NOT EXISTS (SELECT * FROM {})" + ) + self._execute(cmd.format(self.STATE_INFO_TABLE, self.STATE_INFO_TABLE)) cmd = "PRAGMA user_version = {};" self._execute(cmd.format(self.VERSION)) @@ -228,15 +230,14 @@ class State(object): # pylint: disable=too-many-instance-attributes # NOTE: see https://bugs.python.org/issue28518 self.database.isolation_level = None self._execute("VACUUM") - self.database.isolation_level = '' + self.database.isolation_level = "" def dump(self): """Saves state database.""" assert self.database is not None cmd = "SELECT count from {} WHERE rowid={}" - self._execute(cmd.format(self.STATE_INFO_TABLE, - self.STATE_INFO_ROW)) + self._execute(cmd.format(self.STATE_INFO_TABLE, self.STATE_INFO_ROW)) ret = self._fetchall() assert len(ret) == 1 assert len(ret[0]) == 1 @@ -246,13 +247,13 @@ class State(object): # pylint: disable=too-many-instance-attributes msg = "cleaning up state, this might take a while." logger.warning(msg) - delete = (count - self.row_limit) - delete += int(self.row_limit * (self.row_cleanup_quota/100.)) - cmd = "DELETE FROM {} WHERE timestamp IN (" \ - "SELECT timestamp FROM {} ORDER BY timestamp ASC LIMIT {});" - self._execute(cmd.format(self.STATE_TABLE, - self.STATE_TABLE, - delete)) + delete = count - self.row_limit + delete += int(self.row_limit * (self.row_cleanup_quota / 100.0)) + cmd = ( + "DELETE FROM {} WHERE timestamp IN (" + "SELECT timestamp FROM {} ORDER BY timestamp ASC LIMIT {});" + ) + self._execute(cmd.format(self.STATE_TABLE, self.STATE_TABLE, delete)) self._vacuum() @@ -265,9 +266,11 @@ class State(object): # pylint: disable=too-many-instance-attributes count = ret[0][0] cmd = "UPDATE {} SET count = {} WHERE rowid = {}" - self._execute(cmd.format(self.STATE_INFO_TABLE, - self._to_sqlite(count), - self.STATE_INFO_ROW)) + self._execute( + cmd.format( + self.STATE_INFO_TABLE, self._to_sqlite(count), self.STATE_INFO_ROW + ) + ) self.database.commit() self.cursor.close() @@ -297,7 +300,7 @@ class State(object): # pylint: disable=too-many-instance-attributes @staticmethod def _inode(path): - logger.debug('Path {} inode {}'.format(path, System.inode(path))) + logger.debug("Path {} inode {}".format(path, System.inode(path))) return System.inode(path) def _do_update(self, path): @@ -310,21 +313,28 @@ class State(object): # pylint: disable=too-many-instance-attributes actual_mtime, actual_size = self._mtime_and_size(path) actual_inode = self._inode(path) - cmd = ('SELECT * from {} WHERE inode={}' - .format(self.STATE_TABLE, self._to_sqlite(actual_inode))) + cmd = "SELECT * from {} WHERE inode={}".format( + self.STATE_TABLE, self._to_sqlite(actual_inode) + ) self._execute(cmd) ret = self._fetchall() if not ret: md5, info = self._collect(path) - cmd = 'INSERT INTO {}(inode, mtime, size, md5, timestamp) ' \ - 'VALUES ({}, "{}", "{}", "{}", "{}")' - self._execute(cmd.format(self.STATE_TABLE, - self._to_sqlite(actual_inode), - actual_mtime, - actual_size, - md5, - int(nanotime.timestamp(time.time())))) + cmd = ( + "INSERT INTO {}(inode, mtime, size, md5, timestamp) " + 'VALUES ({}, "{}", "{}", "{}", "{}")' + ) + self._execute( + cmd.format( + self.STATE_TABLE, + self._to_sqlite(actual_inode), + actual_mtime, + actual_size, + md5, + int(nanotime.timestamp(time.time())), + ) + ) self.inserts += 1 else: assert len(ret) == 1 @@ -334,26 +344,38 @@ class State(object): # pylint: disable=too-many-instance-attributes assert inode == actual_inode logger.debug( "Inode '{}', mtime '{}', actual mtime '{}', size '{}', " - "actual size '{}'." - .format(inode, mtime, actual_mtime, size, actual_size)) + "actual size '{}'.".format( + inode, mtime, actual_mtime, size, actual_size + ) + ) if actual_mtime != mtime or actual_size != size: md5, info = self._collect(path) - cmd = ('UPDATE {} SET ' - 'mtime = "{}", size = "{}", ' - 'md5 = "{}", timestamp = "{}" ' - 'WHERE inode = {}') - self._execute(cmd.format(self.STATE_TABLE, - actual_mtime, - actual_size, - md5, - int(nanotime.timestamp(time.time())), - self._to_sqlite(actual_inode))) + cmd = ( + "UPDATE {} SET " + 'mtime = "{}", size = "{}", ' + 'md5 = "{}", timestamp = "{}" ' + "WHERE inode = {}" + ) + self._execute( + cmd.format( + self.STATE_TABLE, + actual_mtime, + actual_size, + md5, + int(nanotime.timestamp(time.time())), + self._to_sqlite(actual_inode), + ) + ) else: info = None cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}' - self._execute(cmd.format(self.STATE_TABLE, - int(nanotime.timestamp(time.time())), - self._to_sqlite(actual_inode))) + self._execute( + cmd.format( + self.STATE_TABLE, + int(nanotime.timestamp(time.time())), + self._to_sqlite(actual_inode), + ) + ) return (md5, info) @@ -401,11 +423,9 @@ class State(object): # pylint: disable=too-many-instance-attributes inode = self._inode(path) relpath = os.path.relpath(path, self.root_dir) - cmd = 'REPLACE INTO {}(path, inode, mtime) ' \ - 'VALUES ("{}", {}, "{}")'.format(self.LINK_STATE_TABLE, - relpath, - self._to_sqlite(inode), - mtime) + cmd = "REPLACE INTO {}(path, inode, mtime) " 'VALUES ("{}", {}, "{}")'.format( + self.LINK_STATE_TABLE, relpath, self._to_sqlite(inode), mtime + ) self._execute(cmd) def remove_unused_links(self, used): @@ -416,7 +436,7 @@ class State(object): # pylint: disable=too-many-instance-attributes """ unused = [] - self._execute('SELECT * FROM {}'.format(self.LINK_STATE_TABLE)) + self._execute("SELECT * FROM {}".format(self.LINK_STATE_TABLE)) for row in self.cursor: relpath, inode, mtime = row inode = self._from_sqlite(inode) diff --git a/dvc/system.py b/dvc/system.py index 72b5a2d62..f27ea90bd 100644 --- a/dvc/system.py +++ b/dvc/system.py @@ -8,7 +8,7 @@ import os class System(object): @staticmethod def is_unix(): - return os.name != 'nt' + return os.name != "nt" @staticmethod def hardlink(source, link_name): @@ -20,17 +20,15 @@ class System(object): os.link(source, link_name) return except Exception as exc: - raise DvcException('link', cause=exc) + raise DvcException("link", cause=exc) CreateHardLink = ctypes.windll.kernel32.CreateHardLinkW - CreateHardLink.argtypes = [ctypes.c_wchar_p, - ctypes.c_wchar_p, - ctypes.c_void_p] + CreateHardLink.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_void_p] CreateHardLink.restype = ctypes.wintypes.BOOL res = CreateHardLink(link_name, source, None) if res == 0: - raise DvcException('CreateHardLinkW', cause=ctypes.WinError()) + raise DvcException("CreateHardLinkW", cause=ctypes.WinError()) @staticmethod def symlink(source, link_name): @@ -54,25 +52,25 @@ class System(object): func.restype = ctypes.c_ubyte if func(link_name, source, flags) == 0: - raise DvcException('CreateSymbolicLinkW', cause=ctypes.WinError()) + raise DvcException("CreateSymbolicLinkW", cause=ctypes.WinError()) @staticmethod def _reflink_darwin(src, dst): import ctypes - clib = ctypes.CDLL('libc.dylib') - if not hasattr(clib, 'clonefile'): + clib = ctypes.CDLL("libc.dylib") + if not hasattr(clib, "clonefile"): return -1 clonefile = clib.clonefile - clonefile.argtypes = [ctypes.c_char_p, - ctypes.c_char_p, - ctypes.c_int] + clonefile.argtypes = [ctypes.c_char_p, ctypes.c_char_p, ctypes.c_int] clonefile.restype = ctypes.c_int - return clonefile(ctypes.c_char_p(src.encode('utf-8')), - ctypes.c_char_p(dst.encode('utf-8')), - ctypes.c_int(0)) + return clonefile( + ctypes.c_char_p(src.encode("utf-8")), + ctypes.c_char_p(dst.encode("utf-8")), + ctypes.c_int(0), + ) @staticmethod def _reflink_windows(src, dst): @@ -85,8 +83,8 @@ class System(object): FICLONE = 0x40049409 - s = open(src, 'r') - d = open(dst, 'w+') + s = open(src, "r") + d = open(dst, "w+") try: ret = fcntl.ioctl(d.fileno(), FICLONE, s.fileno()) @@ -111,11 +109,11 @@ class System(object): system = platform.system() try: - if system == 'Windows': + if system == "Windows": ret = System._reflink_windows(source, link_name) - elif system == 'Darwin': + elif system == "Darwin": ret = System._reflink_darwin(source, link_name) - elif system == 'Linux': + elif system == "Linux": ret = System._reflink_linux(source, link_name) else: ret = -1 @@ -123,7 +121,7 @@ class System(object): ret = -1 if ret != 0: - raise DvcException('reflink is not supported') + raise DvcException("reflink is not supported") @staticmethod def getdirinfo(path): @@ -141,40 +139,29 @@ class System(object): OPEN_EXISTING = 3 class FILETIME(Structure): - _fields_ = [("dwLowDateTime", DWORD), - ("dwHighDateTime", DWORD)] + _fields_ = [("dwLowDateTime", DWORD), ("dwHighDateTime", DWORD)] class BY_HANDLE_FILE_INFORMATION(Structure): - _fields_ = [("dwFileAttributes", DWORD), - ("ftCreationTime", FILETIME), - ("ftLastAccessTime", FILETIME), - ("ftLastWriteTime", FILETIME), - ("dwVolumeSerialNumber", DWORD), - ("nFileSizeHigh", DWORD), - ("nFileSizeLow", DWORD), - ("nNumberOfLinks", DWORD), - ("nFileIndexHigh", DWORD), - ("nFileIndexLow", DWORD)] + _fields_ = [ + ("dwFileAttributes", DWORD), + ("ftCreationTime", FILETIME), + ("ftLastAccessTime", FILETIME), + ("ftLastWriteTime", FILETIME), + ("dwVolumeSerialNumber", DWORD), + ("nFileSizeHigh", DWORD), + ("nFileSizeLow", DWORD), + ("nNumberOfLinks", DWORD), + ("nFileIndexHigh", DWORD), + ("nFileIndexLow", DWORD), + ] flags = FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT func = ctypes.windll.kernel32.CreateFileW - func.argtypes = [c_wchar_p, - DWORD, - DWORD, - c_void_p, - DWORD, - DWORD, - HANDLE] + func.argtypes = [c_wchar_p, DWORD, DWORD, c_void_p, DWORD, DWORD, HANDLE] func.restype = HANDLE - hfile = func(path, - 0, - FILE_SHARE_READ, - None, - OPEN_EXISTING, - flags, - None) + hfile = func(path, 0, FILE_SHARE_READ, None, OPEN_EXISTING, flags, None) if hfile is None: raise WinError() @@ -200,6 +187,7 @@ class System(object): def inode(path): if System.is_unix(): import ctypes + inode = os.lstat(path).st_ino # NOTE: See https://bugs.python.org/issue29619 and # https://stackoverflow.com/questions/34643289/ @@ -208,11 +196,13 @@ class System(object): else: # getdirinfo from ntfsutils works on both files and dirs info = System.getdirinfo(path) - inode = abs(hash((info.dwVolumeSerialNumber, - info.nFileIndexHigh, - info.nFileIndexLow))) + inode = abs( + hash( + (info.dwVolumeSerialNumber, info.nFileIndexHigh, info.nFileIndexLow) + ) + ) assert inode >= 0 - assert inode < 2**64 + assert inode < 2 ** 64 return inode @staticmethod @@ -238,6 +228,7 @@ class System(object): def _wait_for_input_posix(timeout): import sys import select + try: select.select([sys.stdin], [], [], timeout) except select.error: diff --git a/dvc/updater.py b/dvc/updater.py index 62343977b..cefc6f14a 100644 --- a/dvc/updater.py +++ b/dvc/updater.py @@ -13,20 +13,20 @@ from dvc.utils import is_binary class Updater(object): # pragma: no cover - URL = 'https://updater.dvc.org' - UPDATER_FILE = 'updater' + URL = "https://updater.dvc.org" + UPDATER_FILE = "updater" TIMEOUT = 24 * 60 * 60 # every day TIMEOUT_GET = 10 def __init__(self, dvc_dir): self.dvc_dir = dvc_dir self.updater_file = os.path.join(dvc_dir, self.UPDATER_FILE) - self.lock = Lock(dvc_dir, self.updater_file + '.lock') + self.lock = Lock(dvc_dir, self.updater_file + ".lock") self.current = VERSION_BASE def _is_outdated_file(self): ctime = os.path.getmtime(self.updater_file) - outdated = (time.time() - ctime >= self.TIMEOUT) + outdated = time.time() - ctime >= self.TIMEOUT if outdated: logger.debug("'{}' is outdated(".format(self.updater_file)) return outdated @@ -40,22 +40,22 @@ class Updater(object): # pragma: no cover logger.debug(msg.format(self.lock.lock_file, action)) def check(self): - if os.getenv('CI') or os.getenv('DVC_TEST'): + if os.getenv("CI") or os.getenv("DVC_TEST"): return - self._with_lock(self._check, 'checking') + self._with_lock(self._check, "checking") def _check(self): if not os.path.exists(self.updater_file) or self._is_outdated_file(): self.fetch() return - with open(self.updater_file, 'r') as fobj: + with open(self.updater_file, "r") as fobj: import json try: info = json.load(fobj) - self.latest = info['version'] + self.latest = info["version"] except Exception as exc: msg = "'{}' is not a valid json: {}" logger.debug(msg.format(self.updater_file, exc)) @@ -69,10 +69,10 @@ class Updater(object): # pragma: no cover from dvc.daemon import daemon if detach: - daemon(['updater']) + daemon(["updater"]) return - self._with_lock(self._get_latest_version, 'fetching') + self._with_lock(self._get_latest_version, "fetching") def _get_latest_version(self): import json @@ -85,12 +85,12 @@ class Updater(object): # pragma: no cover logger.debug(msg.format(exc)) return - with open(self.updater_file, 'w+') as fobj: + with open(self.updater_file, "w+") as fobj: json.dump(info, fobj) def _is_outdated(self): - l_major, l_minor, l_patch = [int(x) for x in self.latest.split('.')] - c_major, c_minor, c_patch = [int(x) for x in self.current.split('.')] + l_major, l_minor, l_patch = [int(x) for x in self.latest.split(".")] + c_major, c_minor, c_patch = [int(x) for x in self.current.split(".")] if l_major != c_major: return l_major > c_major @@ -105,35 +105,42 @@ class Updater(object): # pragma: no cover return message = ( - 'Update available {red}{current}{reset} -> {green}{latest}{reset}' - + '\n' + "Update available {red}{current}{reset} -> {green}{latest}{reset}" + + "\n" + self._get_update_instructions() - ).format(red=colorama.Fore.RED, - reset=colorama.Fore.RESET, - green=colorama.Fore.GREEN, - yellow=colorama.Fore.YELLOW, - blue=colorama.Fore.BLUE, - current=self.current, - latest=self.latest) + ).format( + red=colorama.Fore.RED, + reset=colorama.Fore.RESET, + green=colorama.Fore.GREEN, + yellow=colorama.Fore.YELLOW, + blue=colorama.Fore.BLUE, + current=self.current, + latest=self.latest, + ) - logger.box(message, border_color='yellow') + logger.box(message, border_color="yellow") def _get_update_instructions(self): instructions = { - 'pip': 'Run {yellow}pip{reset} install dvc {blue}--upgrade{reset}', - 'yum': 'Run {yellow}yum{reset} update dvc', - 'yay': 'Run {yellow}yay{reset} {blue}-S{reset} dvc', - 'formula': 'Run {yellow}brew{reset} upgrade dvc', - 'cask': 'Run {yellow}brew cask{reset} upgrade dvc', - 'apt': ('Run {yellow}apt-get{reset} install' - ' {blue}--only-upgrade{reset} dvc'), - 'binary': ('To upgrade follow this steps:\n' - '1. Uninstall dvc binary\n' - '2. Go to {blue}https://dvc.org{reset}\n' - '3. Download and install new binary'), - None: ('Find the latest release at\n{blue}' - 'https://github.com/iterative/dvc/releases/latest' - '{reset}'), + "pip": "Run {yellow}pip{reset} install dvc {blue}--upgrade{reset}", + "yum": "Run {yellow}yum{reset} update dvc", + "yay": "Run {yellow}yay{reset} {blue}-S{reset} dvc", + "formula": "Run {yellow}brew{reset} upgrade dvc", + "cask": "Run {yellow}brew cask{reset} upgrade dvc", + "apt": ( + "Run {yellow}apt-get{reset} install" " {blue}--only-upgrade{reset} dvc" + ), + "binary": ( + "To upgrade follow this steps:\n" + "1. Uninstall dvc binary\n" + "2. Go to {blue}https://dvc.org{reset}\n" + "3. Download and install new binary" + ), + None: ( + "Find the latest release at\n{blue}" + "https://github.com/iterative/dvc/releases/latest" + "{reset}" + ), } package_manager = self._get_package_manager() @@ -144,47 +151,47 @@ class Updater(object): # pragma: no cover import distro if not is_binary(): - return 'pip' + return "pip" package_managers = { - 'rhel': 'yum', - 'centos': 'yum', - 'fedora': 'yum', - 'amazon': 'yum', - 'opensuse': 'yum', - 'ubuntu': 'apt', - 'debian': 'apt', + "rhel": "yum", + "centos": "yum", + "fedora": "yum", + "amazon": "yum", + "opensuse": "yum", + "ubuntu": "apt", + "debian": "apt", } return package_managers.get(distro.id()) def _get_darwin(self): if not is_binary(): - if __file__.startswith('/usr/local/Cellar'): - return 'formula' + if __file__.startswith("/usr/local/Cellar"): + return "formula" else: - return 'pip' + return "pip" # NOTE: both pkg and cask put dvc binary into /usr/local/bin, # so in order to know which method of installation was used, # we need to actually call `brew cask` - ret = os.system('brew cask ls dvc') + ret = os.system("brew cask ls dvc") if ret == 0: - return 'cask' + return "cask" return None def _get_windows(self): - return None if is_binary() else 'pip' + return None if is_binary() else "pip" def _get_package_manager(self): import platform from dvc.exceptions import DvcException m = { - 'Windows': self._get_windows, - 'Darwin': self._get_darwin, - 'Linux': self._get_linux, + "Windows": self._get_windows, + "Darwin": self._get_darwin, + "Linux": self._get_linux, } system = platform.system() diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index 7d8a243fd..c9627022e 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -11,13 +11,13 @@ import json import shutil import hashlib -LOCAL_CHUNK_SIZE = 1024*1024 -LARGE_FILE_SIZE = 1024*1024*1024 +LOCAL_CHUNK_SIZE = 1024 * 1024 +LARGE_FILE_SIZE = 1024 * 1024 * 1024 LARGE_DIR_SIZE = 100 def dos2unix(data): - return data.replace(b'\r\n', b'\n') + return data.replace(b"\r\n", b"\n") def file_md5(fname): @@ -38,7 +38,7 @@ def file_md5(fname): name = os.path.relpath(fname) total = 0 - with open(fname, 'rb') as fobj: + with open(fname, "rb") as fobj: while True: data = fobj.read(LOCAL_CHUNK_SIZE) if not data: @@ -96,7 +96,7 @@ def dict_filter(d, exclude=[]): def dict_md5(d, exclude=[]): filtered = dict_filter(d, exclude) - byts = json.dumps(filtered, sort_keys=True).encode('utf-8') + byts = json.dumps(filtered, sort_keys=True).encode("utf-8") return bytes_md5(byts) @@ -108,12 +108,12 @@ def copyfile(src, dest, no_progress_bar=False, name=None): name = name if name else os.path.basename(dest) total = os.stat(src).st_size - fsrc = open(src, 'rb') + fsrc = open(src, "rb") if os.path.isdir(dest): - fdest = open(os.path.join(dest, os.path.basename(src)), 'wb+') + fdest = open(os.path.join(dest, os.path.basename(src)), "wb+") else: - fdest = open(dest, 'wb+') + fdest = open(dest, "wb+") while True: buf = fsrc.read(LOCAL_CHUNK_SIZE) @@ -174,13 +174,13 @@ def to_chunks(l, jobs): if n == 0: n = 1 - return [l[x:x+n] for x in range(0, len(l), n)] + return [l[x : x + n] for x in range(0, len(l), n)] # NOTE: Check if we are in a bundle # https://pythonhosted.org/PyInstaller/runtime-information.html def is_binary(): - return getattr(sys, 'frozen', False) + return getattr(sys, "frozen", False) # NOTE: Fix env variables modified by PyInstaller @@ -192,8 +192,8 @@ def fix_env(env=None): env = env.copy() if is_binary(): - lp_key = 'LD_LIBRARY_PATH' - lp_orig = env.get(lp_key + '_ORIG', None) + lp_key = "LD_LIBRARY_PATH" + lp_orig = env.get(lp_key + "_ORIG", None) if lp_orig is not None: env[lp_key] = lp_orig else: diff --git a/dvc/utils/compat.py b/dvc/utils/compat.py index 01b80d1a1..7379cf2c4 100644 --- a/dvc/utils/compat.py +++ b/dvc/utils/compat.py @@ -7,39 +7,39 @@ import sys _ver = sys.version_info #: Python 2.x? -is_py2 = (_ver[0] == 2) +is_py2 = _ver[0] == 2 #: Python 3.x? -is_py3 = (_ver[0] == 3) +is_py3 = _ver[0] == 3 if is_py2: - from urlparse import urlparse, urljoin # noqa: F401 - from StringIO import StringIO # noqa: F401 - from BaseHTTPServer import HTTPServer # noqa: F401 + from urlparse import urlparse, urljoin # noqa: F401 + from StringIO import StringIO # noqa: F401 + from BaseHTTPServer import HTTPServer # noqa: F401 from SimpleHTTPServer import SimpleHTTPRequestHandler # noqa: F401 - import ConfigParser # noqa: F401 + import ConfigParser # noqa: F401 - builtin_str = str # noqa: F821 - bytes = str # noqa: F821 - str = unicode # noqa: F821 - basestring = basestring # noqa: F821 + builtin_str = str # noqa: F821 + bytes = str # noqa: F821 + str = unicode # noqa: F821 + basestring = basestring # noqa: F821 numeric_types = (int, long, float) # noqa: F821 - integer_types = (int, long) # noqa: F821 - input = raw_input # noqa: F821 + integer_types = (int, long) # noqa: F821 + input = raw_input # noqa: F821 elif is_py3: from urllib.parse import urlparse, urljoin # noqa: F401 - from io import StringIO # noqa: F401 - from http.server import ( # noqa: F401 - HTTPServer, # noqa: F401 - SimpleHTTPRequestHandler # noqa: F401 - ) # noqa: F401 - import configparser as ConfigParser # noqa: F401 + from io import StringIO # noqa: F401 + from http.server import ( # noqa: F401 + HTTPServer, # noqa: F401 + SimpleHTTPRequestHandler, # noqa: F401 + ) # noqa: F401 + import configparser as ConfigParser # noqa: F401 - builtin_str = str # noqa: F821 - str = str # noqa: F821 - bytes = bytes # noqa: F821 - basestring = (str, bytes) # noqa: F821 + builtin_str = str # noqa: F821 + str = str # noqa: F821 + bytes = bytes # noqa: F821 + basestring = (str, bytes) # noqa: F821 numeric_types = (int, float) # noqa: F821 - integer_types = (int,) # noqa: F821 - input = input # noqa: F821 + integer_types = (int,) # noqa: F821 + input = input # noqa: F821 diff --git a/scripts/ci/requirements.txt b/scripts/ci/requirements.txt new file mode 100644 index 000000000..928b3b946 --- /dev/null +++ b/scripts/ci/requirements.txt @@ -0,0 +1,2 @@ +-r ../../test-style-requirements.txt +gitpython # used by dvc/__init__.py to dynamically create dvc version diff --git a/test-requirements.txt b/test-requirements.txt index 13e670b02..6f1750217 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -7,5 +7,3 @@ xmltodict>=0.11.0 awscli>=1.14.18 google-compute-engine pywin32; sys_platform == 'win32' -Pygments -collective.checkdocs diff --git a/test-style-requirements.txt b/test-style-requirements.txt new file mode 100644 index 000000000..0e5051ad0 --- /dev/null +++ b/test-style-requirements.txt @@ -0,0 +1,3 @@ +Pygments # required by collective.checkdocs +collective.checkdocs +black diff --git a/tests/__init__.py b/tests/__init__.py index 3926fc5ff..25ff0c906 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,8 +1,11 @@ import os + # Increasing fd ulimit for tests -if os.name == 'nt': +if os.name == "nt": import win32file + win32file._setmaxstdio(2048) else: import resource + resource.setrlimit(resource.RLIMIT_NOFILE, (2048, 2048)) diff --git a/tests/__main__.py b/tests/__main__.py index e44c95c4b..279924954 100644 --- a/tests/__main__.py +++ b/tests/__main__.py @@ -4,11 +4,12 @@ from subprocess import check_call REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) os.chdir(REPO_ROOT) -os.putenv('PATH', '{}:{}'.format(os.path.join(REPO_ROOT, 'bin'), - os.getenv('PATH'))) -os.putenv('DVC_HOME', REPO_ROOT) -os.putenv('DVC_TEST', 'true') +os.putenv("PATH", "{}:{}".format(os.path.join(REPO_ROOT, "bin"), os.getenv("PATH"))) +os.putenv("DVC_HOME", REPO_ROOT) +os.putenv("DVC_TEST", "true") -cmd = 'nosetests -v --processes=-1 --process-timeout=500 --cover-inclusive ' \ - '--cover-erase --cover-package=dvc --with-coverage --all-modules' +cmd = ( + "nosetests -v --processes=-1 --process-timeout=500 --cover-inclusive " + "--cover-erase --cover-package=dvc --with-coverage --all-modules" +) check_call(cmd, shell=True) diff --git a/tests/basic_env.py b/tests/basic_env.py index ce007e8c5..52a775343 100644 --- a/tests/basic_env.py +++ b/tests/basic_env.py @@ -9,18 +9,16 @@ from dvc.project import Project class TestDir(TestCase): - GCP_CREDS_FILE = os.path.abspath(os.path.join('scripts', - 'ci', - 'gcp-creds.json')) - DATA_DIR = 'data_dir' - DATA_SUB_DIR = os.path.join(DATA_DIR, 'data_sub_dir') - DATA = os.path.join(DATA_DIR, 'data') - DATA_SUB = os.path.join(DATA_SUB_DIR, 'data_sub') + GCP_CREDS_FILE = os.path.abspath(os.path.join("scripts", "ci", "gcp-creds.json")) + DATA_DIR = "data_dir" + DATA_SUB_DIR = os.path.join(DATA_DIR, "data_sub_dir") + DATA = os.path.join(DATA_DIR, "data") + DATA_SUB = os.path.join(DATA_SUB_DIR, "data_sub") DATA_CONTENTS = DATA DATA_SUB_CONTENTS = DATA_SUB - FOO = 'foo' + FOO = "foo" FOO_CONTENTS = FOO - BAR = 'bar' + BAR = "bar" # NOTE: len(FOO_CONTENTS) must be != len(BAR_CONTENTS) # # Our state database relies on file mtime and file size to determine @@ -30,10 +28,11 @@ class TestDir(TestCase): # being able to detect that, thus causing tests to fail. Usually, # in tests, we replace foo with bar, so we need to make sure that when we # modify a file in our tests, its content length changes. - BAR_CONTENTS = BAR + 'r' - CODE = 'code.py' - CODE_CONTENTS = ('import sys\nimport shutil\n' - 'shutil.copyfile(sys.argv[1], sys.argv[2])') + BAR_CONTENTS = BAR + "r" + CODE = "code.py" + CODE_CONTENTS = ( + "import sys\nimport shutil\n" "shutil.copyfile(sys.argv[1], sys.argv[2])" + ) def _pushd(self, d): self._saved_dir = os.path.realpath(os.curdir) @@ -48,13 +47,13 @@ class TestDir(TestCase): if len(dname) > 0 and not os.path.isdir(dname): os.makedirs(dname) - with open(name, 'a') as f: + with open(name, "a") as f: f.write(contents) @staticmethod def mkdtemp(): - prefix = 'dvc-test.{}.'.format(os.getpid()) - suffix = '.{}'.format(uuid.uuid4()) + prefix = "dvc-test.{}.".format(os.getpid()) + suffix = ".{}".format(uuid.uuid4()) return tempfile.mkdtemp(prefix=prefix, suffix=suffix) def setUp(self): @@ -78,14 +77,14 @@ class TestGit(TestDir): super(TestGit, self).setUp() self.git = Repo.init() self.git.index.add([self.CODE]) - self.git.index.commit('add code') + self.git.index.commit("add code") class TestGitSubmodule(TestGit): def setUp(self): super(TestGitSubmodule, self).setUp() subrepo = Repo.init() - subrepo_path = 'subrepo' + subrepo_path = "subrepo" self.git.create_submodule(subrepo_path, subrepo_path, subrepo.git_dir) self._pushd(subrepo_path) diff --git a/tests/test_add.py b/tests/test_add.py index 3fe7e7a4a..7ccc109a7 100644 --- a/tests/test_add.py +++ b/tests/test_add.py @@ -30,20 +30,20 @@ class TestAdd(TestDvc): self.assertEqual(len(stage.outs), 1) self.assertEqual(len(stage.deps), 0) self.assertEqual(stage.cmd, None) - self.assertEqual(stage.outs[0].info['md5'], md5) + self.assertEqual(stage.outs[0].info["md5"], md5) class TestAddUnupportedFile(TestDvc): def test(self): with self.assertRaises(DvcException): - self.dvc.add('unsupported://unsupported') + self.dvc.add("unsupported://unsupported") class TestAddDirectory(TestDvc): def test(self): - dname = 'directory' + dname = "directory" os.mkdir(dname) - self.create(os.path.join(dname, 'file'), 'file') + self.create(os.path.join(dname, "file"), "file") stages = self.dvc.add(dname) self.assertEqual(len(stages), 1) stage = stages[0] @@ -51,11 +51,11 @@ class TestAddDirectory(TestDvc): self.assertEqual(len(stage.deps), 0) self.assertEqual(len(stage.outs), 1) - md5 = stage.outs[0].info['md5'] + md5 = stage.outs[0].info["md5"] dir_info = self.dvc.cache.local.load_dir_cache(md5) for info in dir_info: - self.assertTrue('\\' not in info['relpath']) + self.assertTrue("\\" not in info["relpath"]) class TestAddDirectoryRecursive(TestDvc): @@ -66,30 +66,28 @@ class TestAddDirectoryRecursive(TestDvc): class TestAddCmdDirectoryRecursive(TestDvc): def test(self): - ret = main(['add', - '--recursive', - self.DATA_DIR]) + ret = main(["add", "--recursive", self.DATA_DIR]) self.assertEqual(ret, 0) class TestAddDirectoryWithForwardSlash(TestDvc): def test(self): - dname = 'directory/' + dname = "directory/" os.mkdir(dname) - self.create(os.path.join(dname, 'file'), 'file') + self.create(os.path.join(dname, "file"), "file") stages = self.dvc.add(dname) self.assertEqual(len(stages), 1) stage = stages[0] self.assertTrue(stage is not None) - self.assertEqual(os.path.abspath('directory.dvc'), stage.path) + self.assertEqual(os.path.abspath("directory.dvc"), stage.path) class TestAddTrackedFile(TestDvc): def test(self): - fname = 'tracked_file' - self.create(fname, 'tracked file contents') + fname = "tracked_file" + self.create(fname, "tracked file contents") self.dvc.scm.add([fname]) - self.dvc.scm.commit('add {}'.format(fname)) + self.dvc.scm.commit("add {}".format(fname)) with self.assertRaises(OutputAlreadyTrackedError): self.dvc.add(fname) @@ -97,8 +95,8 @@ class TestAddTrackedFile(TestDvc): class TestAddDirWithExistingCache(TestDvc): def test(self): - dname = 'a' - fname = os.path.join(dname, 'b') + dname = "a" + fname = os.path.join(dname, "b") os.mkdir(dname) shutil.copyfile(self.FOO, fname) @@ -132,13 +130,13 @@ class TestAddFileInDir(TestDvc): self.assertNotEqual(stage, None) self.assertEqual(len(stage.deps), 0) self.assertEqual(len(stage.outs), 1) - self.assertEqual(stage.relpath, self.DATA_SUB + '.dvc') + self.assertEqual(stage.relpath, self.DATA_SUB + ".dvc") class TestAddExternalLocalFile(TestDvc): def test(self): dname = TestDvc.mkdtemp() - fname = os.path.join(dname, 'foo') + fname = os.path.join(dname, "foo") shutil.copyfile(self.FOO, fname) stages = self.dvc.add(fname) @@ -147,10 +145,10 @@ class TestAddExternalLocalFile(TestDvc): self.assertNotEqual(stage, None) self.assertEqual(len(stage.deps), 0) self.assertEqual(len(stage.outs), 1) - self.assertEqual(stage.relpath, 'foo.dvc') + self.assertEqual(stage.relpath, "foo.dvc") self.assertEqual(len(os.listdir(dname)), 1) self.assertTrue(os.path.isfile(fname)) - self.assertTrue(filecmp.cmp(fname, 'foo', shallow=False)) + self.assertTrue(filecmp.cmp(fname, "foo", shallow=False)) class TestAddLocalRemoteFile(TestDvc): @@ -159,36 +157,34 @@ class TestAddLocalRemoteFile(TestDvc): Making sure that 'remote' syntax is handled properly for local outs. """ cwd = os.getcwd() - remote = 'myremote' + remote = "myremote" - ret = main(['remote', 'add', remote, cwd]) + ret = main(["remote", "add", remote, cwd]) self.assertEqual(ret, 0) self.dvc = Project() - foo = 'remote://{}/{}'.format(remote, self.FOO) - ret = main(['add', foo]) + foo = "remote://{}/{}".format(remote, self.FOO) + ret = main(["add", foo]) self.assertEqual(ret, 0) - with open('foo.dvc', 'r') as fobj: + with open("foo.dvc", "r") as fobj: d = yaml.safe_load(fobj) - self.assertEqual(d['outs'][0]['path'], foo) + self.assertEqual(d["outs"][0]["path"], foo) bar = os.path.join(cwd, self.BAR) - ret = main(['add', bar]) + ret = main(["add", bar]) self.assertEqual(ret, 0) - with open('bar.dvc', 'r') as fobj: + with open("bar.dvc", "r") as fobj: d = yaml.safe_load(fobj) - self.assertEqual(d['outs'][0]['path'], bar) + self.assertEqual(d["outs"][0]["path"], bar) class TestCmdAdd(TestDvc): def test(self): - ret = main(['add', - self.FOO]) + ret = main(["add", self.FOO]) self.assertEqual(ret, 0) - ret = main(['add', - 'non-existing-file']) + ret = main(["add", "non-existing-file"]) self.assertNotEqual(ret, 0) diff --git a/tests/test_analytics.py b/tests/test_analytics.py index a0666a5f7..02ce5a756 100644 --- a/tests/test_analytics.py +++ b/tests/test_analytics.py @@ -11,7 +11,7 @@ def _clean_getenv(key, default=None): """ Remove env vars that affect dvc behavior in tests """ - if key in ['DVC_TEST', 'CI']: + if key in ["DVC_TEST", "CI"]: return None return os.environ.get(key, default) @@ -26,19 +26,20 @@ class TestAnalytics(TestDir): self.assertTrue(a.PARAM_SYSTEM_INFO in a.info.keys()) self.assertNotEqual(a.info[a.PARAM_SYSTEM_INFO], {}) - @mock.patch.object(os, 'getenv', new=_clean_getenv) - @mock.patch('requests.post') + @mock.patch.object(os, "getenv", new=_clean_getenv) + @mock.patch("requests.post") def test_send(self, mockpost): - ret = main(['daemon', 'analytics', Analytics().dump(), '-v']) + ret = main(["daemon", "analytics", Analytics().dump(), "-v"]) self.assertEqual(ret, 0) self.assertTrue(mockpost.called) - @mock.patch.object(os, 'getenv', new=_clean_getenv) - @mock.patch.object(requests, 'post', - side_effect=requests.exceptions.RequestException()) + @mock.patch.object(os, "getenv", new=_clean_getenv) + @mock.patch.object( + requests, "post", side_effect=requests.exceptions.RequestException() + ) def test_send_failed(self, mockpost): - ret = main(['daemon', 'analytics', Analytics().dump(), '-v']) + ret = main(["daemon", "analytics", Analytics().dump(), "-v"]) self.assertEqual(ret, 0) self.assertTrue(mockpost.called) @@ -49,13 +50,13 @@ class TestAnalyticsGit(TestAnalytics, TestGit): class TestAnalyticsDvc(TestAnalytics, TestDvc): - @mock.patch('requests.post') + @mock.patch("requests.post") def test_send_disabled(self, mockpost): - ret = main(['config', 'core.analytics', 'false']) + ret = main(["config", "core.analytics", "false"]) self.assertEqual(ret, 0) - with mock.patch.object(os, 'getenv', new=_clean_getenv): - ret = main(['daemon', 'analytics', Analytics().dump(), '-v']) + with mock.patch.object(os, "getenv", new=_clean_getenv): + ret = main(["daemon", "analytics", Analytics().dump(), "-v"]) self.assertEqual(ret, 0) self.assertFalse(mockpost.called) diff --git a/tests/test_cache.py b/tests/test_cache.py index 7c39e35a8..e4568c5c1 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -11,16 +11,16 @@ from tests.basic_env import TestDvc, TestDir class TestCache(TestDvc): def setUp(self): super(TestCache, self).setUp() - self.cache1_md5 = '123' - self.cache2_md5 = '234' - self.cache1 = os.path.join(self.dvc.cache.local.cache_dir, - self.cache1_md5[0:2], - self.cache1_md5[2:]) - self.cache2 = os.path.join(self.dvc.cache.local.cache_dir, - self.cache2_md5[0:2], - self.cache2_md5[2:]) - self.create(self.cache1, '1') - self.create(self.cache2, '2') + self.cache1_md5 = "123" + self.cache2_md5 = "234" + self.cache1 = os.path.join( + self.dvc.cache.local.cache_dir, self.cache1_md5[0:2], self.cache1_md5[2:] + ) + self.cache2 = os.path.join( + self.dvc.cache.local.cache_dir, self.cache2_md5[0:2], self.cache2_md5[2:] + ) + self.create(self.cache1, "1") + self.create(self.cache2, "2") def test_all(self): md5_list = list(Cache(self.dvc).local.all()) @@ -39,11 +39,11 @@ class TestCacheLoadBadDirCache(TestDvc): self.assertEqual(len(ret), 0) def test(self): - fname = self.dvc.cache.local.get('123.dir') - self.create(fname, 'not,json') + fname = self.dvc.cache.local.get("123.dir") + self.create(fname, "not,json") self._do_test(self.dvc.cache.local.load_dir_cache(fname)) - fname = self.dvc.cache.local.get('234.dir') + fname = self.dvc.cache.local.get("234.dir") self.create(fname, '{"a": "b"}') self._do_test(self.dvc.cache.local.load_dir_cache(fname)) @@ -52,93 +52,93 @@ class TestExternalCacheDir(TestDvc): def test(self): cache_dir = TestDvc.mkdtemp() - ret = main(['config', 'cache.dir', cache_dir]) + ret = main(["config", "cache.dir", cache_dir]) self.assertEqual(ret, 0) - shutil.rmtree('.dvc/cache') + shutil.rmtree(".dvc/cache") - ret = main(['add', self.FOO]) + ret = main(["add", self.FOO]) self.assertEqual(ret, 0) - ret = main(['add', self.DATA_DIR]) + ret = main(["add", self.DATA_DIR]) self.assertEqual(ret, 0) - self.assertFalse(os.path.exists('.dvc/cache')) + self.assertFalse(os.path.exists(".dvc/cache")) self.assertNotEqual(len(os.listdir(cache_dir)), 0) class TestSharedCacheDir(TestDir): def test(self): - cache_dir = os.path.abspath(os.path.join(os.curdir, 'cache')) - for d in ['dir1', 'dir2']: + cache_dir = os.path.abspath(os.path.join(os.curdir, "cache")) + for d in ["dir1", "dir2"]: os.mkdir(d) os.chdir(d) - ret = main(['init', '--no-scm']) + ret = main(["init", "--no-scm"]) self.assertEqual(ret, 0) - ret = main(['config', 'cache.dir', cache_dir]) + ret = main(["config", "cache.dir", cache_dir]) self.assertEqual(ret, 0) - shutil.rmtree(os.path.join('.dvc', 'cache')) + shutil.rmtree(os.path.join(".dvc", "cache")) - with open('common', 'w+') as fd: - fd.write('common') + with open("common", "w+") as fd: + fd.write("common") - with open('unique', 'w+') as fd: + with open("unique", "w+") as fd: fd.write(d) - ret = main(['add', 'common', 'unique']) + ret = main(["add", "common", "unique"]) self.assertEqual(ret, 0) - os.chdir('..') + os.chdir("..") - self.assertFalse(os.path.exists(os.path.join('dir1', '.dvc', 'cache'))) - self.assertFalse(os.path.exists(os.path.join('dir2', '.dvc', 'cache'))) + self.assertFalse(os.path.exists(os.path.join("dir1", ".dvc", "cache"))) + self.assertFalse(os.path.exists(os.path.join("dir2", ".dvc", "cache"))) - subdirs = list(filter( - lambda x: os.path.isdir(os.path.join(cache_dir, x)), - os.listdir(cache_dir))) + subdirs = list( + filter( + lambda x: os.path.isdir(os.path.join(cache_dir, x)), + os.listdir(cache_dir), + ) + ) self.assertEqual(len(subdirs), 3) - self.assertEqual(len(os.listdir(os.path.join(cache_dir, - subdirs[0]))), 1) - self.assertEqual(len(os.listdir(os.path.join(cache_dir, - subdirs[1]))), 1) - self.assertEqual(len(os.listdir(os.path.join(cache_dir, - subdirs[2]))), 1) + self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[0]))), 1) + self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[1]))), 1) + self.assertEqual(len(os.listdir(os.path.join(cache_dir, subdirs[2]))), 1) class TestCacheLinkType(TestDvc): def test(self): - ret = main(['config', 'cache.type', 'reflink,copy']) + ret = main(["config", "cache.type", "reflink,copy"]) self.assertEqual(ret, 0) - ret = main(['add', self.FOO]) + ret = main(["add", self.FOO]) self.assertEqual(ret, 0) class TestCmdCacheDir(TestDvc): def test_abs_path(self): - dname = os.path.join(os.path.dirname(self._root_dir), 'dir') - ret = main(['cache', 'dir', dname]) + dname = os.path.join(os.path.dirname(self._root_dir), "dir") + ret = main(["cache", "dir", dname]) self.assertEqual(ret, 0) config = configobj.ConfigObj(self.dvc.config.config_file) - self.assertEqual(config['cache']['dir'], dname) + self.assertEqual(config["cache"]["dir"], dname) def test_relative_path(self): tmpdir = self.mkdtemp() dname = os.path.relpath(tmpdir) - ret = main(['cache', 'dir', dname]) + ret = main(["cache", "dir", dname]) self.assertEqual(ret, 0) # NOTE: we are in the project's root and config is in .dvc/, so # dir path written to config should be just one level above. - rel = os.path.join('..', dname) + rel = os.path.join("..", dname) config = configobj.ConfigObj(self.dvc.config.config_file) - self.assertEqual(config['cache']['dir'], rel) + self.assertEqual(config["cache"]["dir"], rel) - ret = main(['add', self.FOO]) + ret = main(["add", self.FOO]) self.assertEqual(ret, 0) subdirs = os.listdir(tmpdir) diff --git a/tests/test_checkout.py b/tests/test_checkout.py index 2fe43da65..a1bae46ec 100644 --- a/tests/test_checkout.py +++ b/tests/test_checkout.py @@ -25,11 +25,11 @@ class TestCheckout(TestRepro): self.data_dir_stage = stages[0] self.assertTrue(self.data_dir_stage is not None) - self.orig = 'orig' + self.orig = "orig" shutil.copy(self.FOO, self.orig) os.unlink(self.FOO) - self.orig_dir = 'orig_dir' + self.orig_dir = "orig_dir" shutil.copytree(self.DATA_DIR, self.orig_dir) shutil.rmtree(self.DATA_DIR) @@ -44,10 +44,10 @@ class TestCheckout(TestRepro): class TestCheckoutSingleStage(TestCheckout): def test(self): - ret = main(['checkout', '--force', self.foo_stage.path]) + ret = main(["checkout", "--force", self.foo_stage.path]) self.assertEqual(ret, 0) - ret = main(['checkout', '--force', self.data_dir_stage.path]) + ret = main(["checkout", "--force", self.data_dir_stage.path]) self.assertEqual(ret, 0) self._test_checkout() @@ -57,8 +57,8 @@ class TestCheckoutCorruptedCacheFile(TestRepro): def test(self): cache = self.foo_stage.outs[0].cache - with open(cache, 'a') as fd: - fd.write('1') + with open(cache, "a") as fd: + fd.write("1") self.dvc.checkout(force=True) @@ -69,10 +69,10 @@ class TestCheckoutCorruptedCacheFile(TestRepro): class TestCheckoutCorruptedCacheDir(TestDvc): def test(self): # NOTE: using 'copy' so that cache and link don't have same inode - ret = main(['config', 'cache.type', 'copy']) + ret = main(["config", "cache.type", "copy"]) self.assertEqual(ret, 0) - self.dvc = Project('.') + self.dvc = Project(".") stages = self.dvc.add(self.DATA_DIR) self.assertEqual(len(stages), 1) self.assertEqual(len(stages[0].outs), 1) @@ -84,8 +84,8 @@ class TestCheckoutCorruptedCacheDir(TestDvc): checksum = entry[self.dvc.cache.local.PARAM_CHECKSUM] cache = self.dvc.cache.local.get(checksum) - with open(cache, 'w+') as fobj: - fobj.write('1') + with open(cache, "w+") as fobj: + fobj.write("1") self.dvc.checkout(force=True) @@ -94,29 +94,28 @@ class TestCheckoutCorruptedCacheDir(TestDvc): class TestCmdCheckout(TestCheckout): def test(self): - ret = main(['checkout', '--force']) + ret = main(["checkout", "--force"]) self.assertEqual(ret, 0) self._test_checkout() class CheckoutBase(TestDvc): - GIT_IGNORE = '.gitignore' + GIT_IGNORE = ".gitignore" - def commit_data_file(self, fname, content='random text'): - with open(fname, 'w') as fd: + def commit_data_file(self, fname, content="random text"): + with open(fname, "w") as fd: fd.write(content) stages = self.dvc.add(fname) self.assertEqual(len(stages), 1) self.assertTrue(stages[0] is not None) - self.dvc.scm.add([fname + '.dvc', '.gitignore']) - self.dvc.scm.commit('adding ' + fname) + self.dvc.scm.add([fname + ".dvc", ".gitignore"]) + self.dvc.scm.commit("adding " + fname) def read_ignored(self): - return list(map(lambda s: s.strip('\n'), - open(self.GIT_IGNORE).readlines())) + return list(map(lambda s: s.strip("\n"), open(self.GIT_IGNORE).readlines())) def outs_info(self, stage): - FileInfo = collections.namedtuple('FileInfo', 'path inode') + FileInfo = collections.namedtuple("FileInfo", "path inode") paths = [ os.path.join(root, file) @@ -125,24 +124,21 @@ class CheckoutBase(TestDvc): for file in files ] - return [ - FileInfo(path=path, inode=System.inode(path)) - for path in paths - ] + return [FileInfo(path=path, inode=System.inode(path)) for path in paths] class TestRemoveFilesWhenCheckout(CheckoutBase): def test(self): - fname = 'file_in_a_branch' - branch_master = 'master' - branch_1 = 'b1' + fname = "file_in_a_branch" + branch_master = "master" + branch_1 = "b1" self.dvc.scm.add(self.dvc.scm.untracked_files()) - self.dvc.scm.commit('add all files') + self.dvc.scm.commit("add all files") # add the file into a separate branch self.dvc.scm.checkout(branch_1, True) - ret = main(['checkout', '--force']) + ret = main(["checkout", "--force"]) self.assertEqual(ret, 0) self.commit_data_file(fname) @@ -152,28 +148,28 @@ class TestRemoveFilesWhenCheckout(CheckoutBase): # Make sure `dvc checkout` removes the file # self.dvc.checkout() - ret = main(['checkout', '--force']) + ret = main(["checkout", "--force"]) self.assertEqual(ret, 0) self.assertFalse(os.path.exists(fname)) class TestCheckoutCleanWorkingDir(CheckoutBase): - @patch('dvc.prompt.confirm', return_value=True) + @patch("dvc.prompt.confirm", return_value=True) def test(self, mock_prompt): mock_prompt.return_value = True stages = self.dvc.add(self.DATA_DIR) stage = stages[0] - working_dir_change = os.path.join(self.DATA_DIR, 'not_cached.txt') - with open(working_dir_change, 'w') as f: - f.write('not_cached') + working_dir_change = os.path.join(self.DATA_DIR, "not_cached.txt") + with open(working_dir_change, "w") as f: + f.write("not_cached") - ret = main(['checkout', stage.relpath]) + ret = main(["checkout", stage.relpath]) self.assertEqual(ret, 0) self.assertFalse(os.path.exists(working_dir_change)) - @patch('dvc.prompt.confirm', return_value=False) + @patch("dvc.prompt.confirm", return_value=False) def test_force(self, mock_prompt): mock_prompt.return_value = False @@ -181,11 +177,11 @@ class TestCheckoutCleanWorkingDir(CheckoutBase): self.assertEqual(len(stages), 1) stage = stages[0] - working_dir_change = os.path.join(self.DATA_DIR, 'not_cached.txt') - with open(working_dir_change, 'w') as f: - f.write('not_cached') + working_dir_change = os.path.join(self.DATA_DIR, "not_cached.txt") + with open(working_dir_change, "w") as f: + f.write("not_cached") - ret = main(['checkout', stage.relpath]) + ret = main(["checkout", stage.relpath]) self.assertNotEqual(ret, 0) mock_prompt.assert_called() @@ -196,7 +192,7 @@ class TestCheckoutCleanWorkingDir(CheckoutBase): class TestCheckoutSelectiveRemove(CheckoutBase): def test(self): # Use copy to test for changes in the inodes - ret = main(['config', 'cache.type', 'copy']) + ret = main(["config", "cache.type", "copy"]) self.assertEqual(ret, 0) stages = self.dvc.add(self.DATA_DIR) @@ -206,7 +202,7 @@ class TestCheckoutSelectiveRemove(CheckoutBase): os.remove(staged_files[0].path) - ret = main(['checkout', '--force', stage.relpath]) + ret = main(["checkout", "--force", stage.relpath]) self.assertEqual(ret, 0) checkedout_files = self.outs_info(stage) @@ -219,20 +215,22 @@ class TestCheckoutSelectiveRemove(CheckoutBase): class TestGitIgnoreBasic(CheckoutBase): def test(self): - fname1 = 'file_1' - fname2 = 'file_2' - fname3 = 'file_3' + fname1 = "file_1" + fname2 = "file_2" + fname3 = "file_3" self.dvc.scm.add(self.dvc.scm.untracked_files()) - self.dvc.scm.commit('add all files') + self.dvc.scm.commit("add all files") self.assertFalse(os.path.exists(self.GIT_IGNORE)) self.commit_data_file(fname1) self.commit_data_file(fname2) - self.dvc.run(cmd='python {} {} {}'.format(self.CODE, self.FOO, fname3), - deps=[self.CODE, self.FOO], - outs_no_cache=[fname3]) + self.dvc.run( + cmd="python {} {} {}".format(self.CODE, self.FOO, fname3), + deps=[self.CODE, self.FOO], + outs_no_cache=[fname3], + ) self.assertTrue(os.path.exists(self.GIT_IGNORE)) @@ -240,51 +238,51 @@ class TestGitIgnoreBasic(CheckoutBase): self.assertEqual(len(ignored), 2) - self.assertIn('/' + fname1, ignored) - self.assertIn('/' + fname2, ignored) + self.assertIn("/" + fname1, ignored) + self.assertIn("/" + fname2, ignored) class TestGitIgnoreWhenCheckout(CheckoutBase): def test(self): - fname_master = 'file_in_a_master' - branch_master = 'master' - fname_branch = 'file_in_a_branch' - branch_1 = 'b1' + fname_master = "file_in_a_master" + branch_master = "master" + fname_branch = "file_in_a_branch" + branch_1 = "b1" self.dvc.scm.add(self.dvc.scm.untracked_files()) - self.dvc.scm.commit('add all files') + self.dvc.scm.commit("add all files") self.commit_data_file(fname_master) self.dvc.scm.checkout(branch_1, True) - ret = main(['checkout', '--force']) + ret = main(["checkout", "--force"]) self.assertEqual(ret, 0) self.commit_data_file(fname_branch) self.dvc.scm.checkout(branch_master) - ret = main(['checkout', '--force']) + ret = main(["checkout", "--force"]) self.assertEqual(ret, 0) ignored = self.read_ignored() self.assertEqual(len(ignored), 1) - self.assertIn('/' + fname_master, ignored) + self.assertIn("/" + fname_master, ignored) self.dvc.scm.checkout(branch_1) - ret = main(['checkout', '--force']) + ret = main(["checkout", "--force"]) self.assertEqual(ret, 0) ignored = self.read_ignored() - self.assertIn('/' + fname_branch, ignored) + self.assertIn("/" + fname_branch, ignored) class TestCheckoutMissingMd5InStageFile(TestRepro): def test(self): - with open(self.file1_stage, 'r') as fd: + with open(self.file1_stage, "r") as fd: d = yaml.load(fd) - del(d[Stage.PARAM_OUTS][0][RemoteLOCAL.PARAM_CHECKSUM]) - del(d[Stage.PARAM_DEPS][0][RemoteLOCAL.PARAM_CHECKSUM]) + del (d[Stage.PARAM_OUTS][0][RemoteLOCAL.PARAM_CHECKSUM]) + del (d[Stage.PARAM_DEPS][0][RemoteLOCAL.PARAM_CHECKSUM]) - with open(self.file1_stage, 'w') as fd: + with open(self.file1_stage, "w") as fd: yaml.dump(d, fd) self.dvc.checkout(force=True) @@ -292,7 +290,7 @@ class TestCheckoutMissingMd5InStageFile(TestRepro): class TestCheckoutEmptyDir(TestDvc): def test(self): - dname = 'empty_dir' + dname = "empty_dir" os.mkdir(dname) stages = self.dvc.add(dname) @@ -312,12 +310,10 @@ class TestCheckoutEmptyDir(TestDvc): class TestCheckoutNotCachedFile(TestDvc): def test(self): - cmd = 'python {} {} {}'.format(self.CODE, self.FOO, 'out') + cmd = "python {} {} {}".format(self.CODE, self.FOO, "out") self.dvc.add(self.FOO) - stage = self.dvc.run(cmd=cmd, - deps=[self.FOO, self.CODE], - outs_no_cache=['out']) + stage = self.dvc.run(cmd=cmd, deps=[self.FOO, self.CODE], outs_no_cache=["out"]) self.assertTrue(stage is not None) self.dvc.checkout(force=True) @@ -331,7 +327,7 @@ class TestCheckoutWithDeps(TestRepro): self.assertFalse(os.path.exists(self.FOO)) self.assertFalse(os.path.exists(self.file1)) - ret = main(['checkout', '--force', self.file1_stage, '--with-deps']) + ret = main(["checkout", "--force", self.file1_stage, "--with-deps"]) self.assertEqual(ret, 0) self.assertTrue(os.path.exists(self.FOO)) @@ -345,7 +341,7 @@ class TestCheckoutDirectory(TestRepro): shutil.rmtree(self.DATA_DIR) self.assertFalse(os.path.exists(self.DATA_DIR)) - ret = main(['checkout', stage.path]) + ret = main(["checkout", stage.path]) self.assertEqual(ret, 0) self.assertTrue(os.path.exists(self.DATA_DIR)) diff --git a/tests/test_cli.py b/tests/test_cli.py index 7d19b4783..5ecb9e2cd 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -19,38 +19,53 @@ from tests.basic_env import TestDvc class TestArgParse(TestDvc): def test(self): - args = parse_args(['init']) + args = parse_args(["init"]) self.assertIsInstance(args.func(args), CmdInit) class TestRun(TestDvc): def test(self): - dep1 = 'dep1' - dep2 = 'dep2' + dep1 = "dep1" + dep2 = "dep2" - out1 = 'out1' - out2 = 'out2' + out1 = "out1" + out2 = "out2" - out_no_cache1 = 'out_no_cache1' - out_no_cache2 = 'out_no_cache2' + out_no_cache1 = "out_no_cache1" + out_no_cache2 = "out_no_cache2" - fname = 'dvc.dvc' + fname = "dvc.dvc" cwd = os.curdir - cmd = 'cmd' - arg1 = 'arg1' - arg2 = 'arg2' + cmd = "cmd" + arg1 = "arg1" + arg2 = "arg2" - args = parse_args(['run', - '-d', dep1, - '--deps', dep2, - '-o', out1, - '--outs', out2, - '-O', out_no_cache1, - '--outs-no-cache', out_no_cache2, - '-f', fname, - '--file', fname, - '-c', cwd, - cmd, arg1, arg2]) + args = parse_args( + [ + "run", + "-d", + dep1, + "--deps", + dep2, + "-o", + out1, + "--outs", + out2, + "-O", + out_no_cache1, + "--outs-no-cache", + out_no_cache2, + "-f", + fname, + "--file", + fname, + "-c", + cwd, + cmd, + arg1, + arg2, + ] + ) self.assertIsInstance(args.func(args), CmdRun) self.assertEqual(args.deps, [dep1, dep2]) @@ -63,32 +78,30 @@ class TestRun(TestDvc): class TestPull(TestDvc): def test(self): - args = parse_args(['pull']) + args = parse_args(["pull"]) self.assertIsInstance(args.func(args), CmdDataPull) class TestPush(TestDvc): def test(self): - args = parse_args(['push']) + args = parse_args(["push"]) self.assertIsInstance(args.func(args), CmdDataPush) class TestStatus(TestDvc): def test(self): - args = parse_args(['status']) + args = parse_args(["status"]) self.assertIsInstance(args.func(args), CmdDataStatus) class TestRepro(TestDvc): def test(self): - target1 = '1' - target2 = '2' + target1 = "1" + target2 = "2" - args = parse_args(['repro', - target1, - target2, - '-f', '--force', - '-s', '--single-item']) + args = parse_args( + ["repro", target1, target2, "-f", "--force", "-s", "--single-item"] + ) self.assertIsInstance(args.func(args), CmdRepro) self.assertEqual(args.targets, [target1, target2]) @@ -98,12 +111,10 @@ class TestRepro(TestDvc): class TestRemove(TestDvc): def test(self): - target1 = '1' - target2 = '2' + target1 = "1" + target2 = "2" - args = parse_args(['remove', - target1, - target2]) + args = parse_args(["remove", target1, target2]) self.assertIsInstance(args.func(args), CmdRemove) self.assertEqual(args.targets, [target1, target2]) @@ -111,12 +122,10 @@ class TestRemove(TestDvc): class TestAdd(TestDvc): def test(self): - target1 = '1' - target2 = '2' + target1 = "1" + target2 = "2" - args = parse_args(['add', - target1, - target2]) + args = parse_args(["add", target1, target2]) self.assertIsInstance(args.func(args), CmdAdd) self.assertEqual(args.targets, [target1, target2]) @@ -124,33 +133,25 @@ class TestAdd(TestDvc): class TestGC(TestDvc): def test(self): - args = parse_args(['gc']) + args = parse_args(["gc"]) self.assertIsInstance(args.func(args), CmdGC) class TestGCMultipleProjects(TestDvc): def test(self): - args = parse_args([ - 'gc', - '-p', - '/tmp/asdf', - '/tmp/xyz' - ]) + args = parse_args(["gc", "-p", "/tmp/asdf", "/tmp/xyz"]) self.assertIsInstance(args.func(args), CmdGC) - self.assertEqual(args.projects, ['/tmp/asdf', '/tmp/xyz']) + self.assertEqual(args.projects, ["/tmp/asdf", "/tmp/xyz"]) class TestConfig(TestDvc): def test(self): - name = 'param' - value = '1' + name = "param" + value = "1" - args = parse_args(['config', - '-u', '--unset', - name, - value]) + args = parse_args(["config", "-u", "--unset", name, value]) self.assertIsInstance(args.func(args), CmdConfig) self.assertEqual(args.unset, True) @@ -160,7 +161,7 @@ class TestConfig(TestDvc): class TestCheckout(TestDvc): def test(self): - args = parse_args(['checkout']) + args = parse_args(["checkout"]) self.assertIsInstance(args.func(args), CmdCheckout) diff --git a/tests/test_config.py b/tests/test_config.py index 8d2c02bcf..0de3bafd9 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -25,24 +25,24 @@ class TestConfigCLI(TestDvc): return True def test_root(self): - ret = main(['root']) + ret = main(["root"]) self.assertEqual(ret, 0) # NOTE: check that `dvc root` is not blocked with dvc lock with self.dvc.lock: - ret = main(['root']) + ret = main(["root"]) self.assertEqual(ret, 0) def _do_test(self, local=False): - section = 'setsection' - field = 'setfield' - section_field = '{}.{}'.format(section, field) - value = 'setvalue' - newvalue = 'setnewvalue' + section = "setsection" + field = "setfield" + section_field = "{}.{}".format(section, field) + value = "setvalue" + newvalue = "setnewvalue" - base = ['config'] + base = ["config"] if local: - base.append('--local') + base.append("--local") ret = main(base + [section_field, value]) self.assertEqual(ret, 0) @@ -56,7 +56,7 @@ class TestConfigCLI(TestDvc): self.assertTrue(self._contains(section, field, newvalue, local)) self.assertFalse(self._contains(section, field, value, local)) - ret = main(base + [section_field, '--unset']) + ret = main(base + [section_field, "--unset"]) self.assertEqual(ret, 0) self.assertFalse(self._contains(section, field, value, local)) @@ -69,22 +69,22 @@ class TestConfigCLI(TestDvc): def test_non_existing(self): # FIXME check stdout/err - ret = main(['config', 'non_existing_section.field']) + ret = main(["config", "non_existing_section.field"]) self.assertEqual(ret, 1) - ret = main(['config', 'global.non_existing_field']) + ret = main(["config", "global.non_existing_field"]) self.assertEqual(ret, 1) - ret = main(['config', 'non_existing_section.field', '-u']) + ret = main(["config", "non_existing_section.field", "-u"]) self.assertEqual(ret, 1) - ret = main(['config', 'global.non_existing_field', '-u']) + ret = main(["config", "global.non_existing_field", "-u"]) self.assertEqual(ret, 1) - ret = main(['config', 'core.remote', 'myremote']) + ret = main(["config", "core.remote", "myremote"]) self.assertEqual(ret, 0) - ret = main(['config', 'core.non_existing_field', '-u']) + ret = main(["config", "core.non_existing_field", "-u"]) self.assertEqual(ret, 1) def test_failed_write(self): @@ -92,8 +92,8 @@ class TestConfigCLI(TestDvc): system = False glob = False local = False - name = 'core.remote' - value = 'myremote' + name = "core.remote" + value = "myremote" unset = False args = A() diff --git a/tests/test_data_cloud.py b/tests/test_data_cloud.py index 8aae99606..7a892d894 100644 --- a/tests/test_data_cloud.py +++ b/tests/test_data_cloud.py @@ -14,8 +14,16 @@ import dvc.logger as logger from dvc.utils.compat import str from dvc.main import main from dvc.config import Config -from dvc.data_cloud import (DataCloud, RemoteS3, RemoteGS, RemoteAzure, - RemoteLOCAL, RemoteSSH, RemoteHDFS, RemoteHTTP) +from dvc.data_cloud import ( + DataCloud, + RemoteS3, + RemoteGS, + RemoteAzure, + RemoteLOCAL, + RemoteSSH, + RemoteHDFS, + RemoteHTTP, +) from dvc.remote.base import STATUS_OK, STATUS_NEW, STATUS_DELETED from dvc.utils import file_md5 @@ -24,13 +32,15 @@ from tests.utils import spy from dvc.utils.compat import StringIO -TEST_REMOTE = 'upstream' +TEST_REMOTE = "upstream" TEST_SECTION = 'remote "{}"'.format(TEST_REMOTE) -TEST_CONFIG = {Config.SECTION_CORE: {Config.SECTION_CORE_REMOTE: TEST_REMOTE}, - TEST_SECTION: {Config.SECTION_REMOTE_URL: ''}} +TEST_CONFIG = { + Config.SECTION_CORE: {Config.SECTION_CORE_REMOTE: TEST_REMOTE}, + TEST_SECTION: {Config.SECTION_REMOTE_URL: ""}, +} -TEST_AWS_REPO_BUCKET = 'dvc-test' -TEST_GCP_REPO_BUCKET = 'dvc-test' +TEST_AWS_REPO_BUCKET = "dvc-test" +TEST_GCP_REPO_BUCKET = "dvc-test" def _should_test_aws(): @@ -59,8 +69,9 @@ def _should_test_gcp(): os.unlink(creds) shutil.copyfile(TestDvc.GCP_CREDS_FILE, creds) try: - check_output(['gcloud', 'auth', 'activate-service-account', - '--key-file', creds]) + check_output( + ["gcloud", "auth", "activate-service-account", "--key-file", creds] + ) except (CalledProcessError, OSError): return False return True @@ -74,8 +85,9 @@ def _should_test_azure(): elif os.getenv("DVC_TEST_AZURE") == "false": return False - return (os.getenv("AZURE_STORAGE_CONTAINER_NAME") - and os.getenv("AZURE_STORAGE_CONNECTION_STRING")) + return os.getenv("AZURE_STORAGE_CONTAINER_NAME") and os.getenv( + "AZURE_STORAGE_CONNECTION_STRING" + ) def _should_test_ssh(): @@ -83,11 +95,11 @@ def _should_test_ssh(): return True # FIXME: enable on windows - if os.name == 'nt': + if os.name == "nt": return False try: - check_output(['ssh', '-o', 'BatchMode=yes', '127.0.0.1', 'ls']) + check_output(["ssh", "-o", "BatchMode=yes", "127.0.0.1", "ls"]) except (CalledProcessError, IOError): return False @@ -95,19 +107,17 @@ def _should_test_ssh(): def _should_test_hdfs(): - if platform.system() != 'Linux': + if platform.system() != "Linux": return False try: - check_output(['hadoop', 'version'], - shell=True, - executable=os.getenv('SHELL')) + check_output(["hadoop", "version"], shell=True, executable=os.getenv("SHELL")) except (CalledProcessError, IOError): return False - p = Popen('hadoop fs -ls hdfs://127.0.0.1/', - shell=True, - executable=os.getenv('SHELL')) + p = Popen( + "hadoop fs -ls hdfs://127.0.0.1/", shell=True, executable=os.getenv("SHELL") + ) p.communicate() if p.returncode != 0: return False @@ -124,45 +134,40 @@ def get_local_url(): def get_ssh_url(): - return 'ssh://{}@127.0.0.1:{}'.format( - getpass.getuser(), - get_local_storagepath()) + return "ssh://{}@127.0.0.1:{}".format(getpass.getuser(), get_local_storagepath()) def get_hdfs_url(): - return 'hdfs://{}@127.0.0.1{}'.format( - getpass.getuser(), - get_local_storagepath()) + return "hdfs://{}@127.0.0.1{}".format(getpass.getuser(), get_local_storagepath()) def get_aws_storagepath(): - return TEST_AWS_REPO_BUCKET + '/' + str(uuid.uuid4()) + return TEST_AWS_REPO_BUCKET + "/" + str(uuid.uuid4()) def get_aws_url(): - return 's3://' + get_aws_storagepath() + return "s3://" + get_aws_storagepath() def get_gcp_storagepath(): - return TEST_GCP_REPO_BUCKET + '/' + str(uuid.uuid4()) + return TEST_GCP_REPO_BUCKET + "/" + str(uuid.uuid4()) def get_gcp_url(): - return 'gs://' + get_gcp_storagepath() + return "gs://" + get_gcp_storagepath() def get_azure_url_compat(): container_name = os.getenv("AZURE_STORAGE_CONTAINER_NAME") connection_string = os.getenv("AZURE_STORAGE_CONNECTION_STRING") assert container_name is not None - return 'azure://ContainerName={};{}'.format(container_name, - connection_string) + return "azure://ContainerName={};{}".format(container_name, connection_string) def get_azure_url(): container_name = os.getenv("AZURE_STORAGE_CONTAINER_NAME") assert container_name is not None - return 'azure://{}/{}'.format(container_name, str(uuid.uuid4())) + return "azure://{}/{}".format(container_name, str(uuid.uuid4())) class TestDataCloud(TestDvc): @@ -173,12 +178,14 @@ class TestDataCloud(TestDvc): def test(self): config = TEST_CONFIG - clist = [('s3://mybucket/', RemoteS3), - ('gs://mybucket/', RemoteGS), - ('ssh://user@localhost:/', RemoteSSH), - ('http://localhost:8000/', RemoteHTTP), - ('azure://ContainerName=mybucket;conn_string;', RemoteAzure), - (TestDvc.mkdtemp(), RemoteLOCAL)] + clist = [ + ("s3://mybucket/", RemoteS3), + ("gs://mybucket/", RemoteGS), + ("ssh://user@localhost:/", RemoteSSH), + ("http://localhost:8000/", RemoteHTTP), + ("azure://ContainerName=mybucket;conn_string;", RemoteAzure), + (TestDvc.mkdtemp(), RemoteLOCAL), + ] for scheme, cl in clist: remote_url = scheme + str(uuid.uuid4()) @@ -194,12 +201,11 @@ class TestDataCloudBase(TestDvc): return False def _get_url(self): - return '' + return "" def _ensure_should_run(self): if not self._should_test(): - raise SkipTest('Test {} is disabled' - .format(self.__class__.__name__)) + raise SkipTest("Test {} is disabled".format(self.__class__.__name__)) def _setup_cloud(self): self._ensure_should_run() @@ -221,7 +227,7 @@ class TestDataCloudBase(TestDvc): self.assertTrue(stage is not None) cache = stage.outs[0].cache info = stage.outs[0].dumpd() - md5 = info['md5'] + md5 = info["md5"] stages = self.dvc.add(self.DATA_DIR) self.assertEqual(len(stages), 1) @@ -230,16 +236,16 @@ class TestDataCloudBase(TestDvc): cache_dir = stage_dir.outs[0].cache info_dir = stage_dir.outs[0].dumpd() - md5_dir = info_dir['md5'] + md5_dir = info_dir["md5"] with self.cloud.project.state: # Check status status = self.cloud.status([info], show_checksums=True) - expected = {md5: {'name': md5, 'status': STATUS_NEW}} + expected = {md5: {"name": md5, "status": STATUS_NEW}} self.assertEqual(status, expected) status_dir = self.cloud.status([info_dir], show_checksums=True) - expected = {md5_dir: {'name': md5_dir, 'status': STATUS_NEW}} + expected = {md5_dir: {"name": md5_dir, "status": STATUS_NEW}} self.assertEqual(status_dir, expected) # Push and check status @@ -251,40 +257,40 @@ class TestDataCloudBase(TestDvc): self.assertTrue(os.path.isfile(cache_dir)) status = self.cloud.status([info], show_checksums=True) - expected = {md5: {'name': md5, 'status': STATUS_OK}} + expected = {md5: {"name": md5, "status": STATUS_OK}} self.assertEqual(status, expected) status_dir = self.cloud.status([info_dir], show_checksums=True) - expected = {md5_dir: {'name': md5_dir, 'status': STATUS_OK}} + expected = {md5_dir: {"name": md5_dir, "status": STATUS_OK}} self.assertEqual(status_dir, expected) # Remove and check status shutil.rmtree(self.dvc.cache.local.cache_dir) status = self.cloud.status([info], show_checksums=True) - expected = {md5: {'name': md5, 'status': STATUS_DELETED}} + expected = {md5: {"name": md5, "status": STATUS_DELETED}} self.assertEqual(status, expected) status_dir = self.cloud.status([info_dir], show_checksums=True) - expected = {md5_dir: {'name': md5_dir, 'status': STATUS_DELETED}} + expected = {md5_dir: {"name": md5_dir, "status": STATUS_DELETED}} self.assertEqual(status_dir, expected) # Pull and check status self.cloud.pull([info]) self.assertTrue(os.path.exists(cache)) self.assertTrue(os.path.isfile(cache)) - with open(cache, 'r') as fd: + with open(cache, "r") as fd: self.assertEqual(fd.read(), self.FOO_CONTENTS) self.cloud.pull([info_dir]) self.assertTrue(os.path.isfile(cache_dir)) status = self.cloud.status([info], show_checksums=True) - expected = {md5: {'name': md5, 'status': STATUS_OK}} + expected = {md5: {"name": md5, "status": STATUS_OK}} self.assertEqual(status, expected) status_dir = self.cloud.status([info_dir], show_checksums=True) - expected = {md5_dir: {'name': md5_dir, 'status': STATUS_OK}} + expected = {md5_dir: {"name": md5_dir, "status": STATUS_OK}} self.assertTrue(status_dir, expected) def test(self): @@ -374,9 +380,9 @@ class TestDataCloudCLIBase(TestDvc): self.assertEqual(ret, 0) def _test_cloud(self, remote=None): - args = ['-v', '-j', '2'] + args = ["-v", "-j", "2"] if remote: - args += ['-r', remote] + args += ["-r", remote] else: args += [] @@ -393,45 +399,46 @@ class TestDataCloudCLIBase(TestDvc): cache_dir = stage_dir.outs[0].cache # FIXME check status output - self.main(['status', '-c', '--show-checksums'] + args) + self.main(["status", "-c", "--show-checksums"] + args) - self.main(['push'] + args) + self.main(["push"] + args) self.assertTrue(os.path.exists(cache)) self.assertTrue(os.path.isfile(cache)) self.assertTrue(os.path.isfile(cache_dir)) - self.main(['status', '-c', '--show-checksums'] + args) + self.main(["status", "-c", "--show-checksums"] + args) shutil.rmtree(self.dvc.cache.local.cache_dir) - self.main(['status', '-c', '--show-checksums'] + args) + self.main(["status", "-c", "--show-checksums"] + args) - self.main(['fetch'] + args) + self.main(["fetch"] + args) self.assertTrue(os.path.exists(cache)) self.assertTrue(os.path.isfile(cache)) self.assertTrue(os.path.isfile(cache_dir)) - self.main(['pull'] + args) + self.main(["pull"] + args) self.assertTrue(os.path.exists(cache)) self.assertTrue(os.path.isfile(cache)) self.assertTrue(os.path.isfile(cache_dir)) self.assertTrue(os.path.isfile(self.FOO)) self.assertTrue(os.path.isdir(self.DATA_DIR)) - with open(cache, 'r') as fd: + with open(cache, "r") as fd: self.assertEqual(fd.read(), self.FOO_CONTENTS) self.assertTrue(os.path.isfile(cache_dir)) - self.main(['status', '-c', '--show-checksums'] + args) + self.main(["status", "-c", "--show-checksums"] + args) # NOTE: check if remote gc works correctly on directories - self.main(['gc', '-c', '-f'] + args) - shutil.move(self.dvc.cache.local.cache_dir, - self.dvc.cache.local.cache_dir + '.back') + self.main(["gc", "-c", "-f"] + args) + shutil.move( + self.dvc.cache.local.cache_dir, self.dvc.cache.local.cache_dir + ".back" + ) - self.main(['fetch'] + args) + self.main(["fetch"] + args) - self.main(['pull', '-f'] + args) + self.main(["pull", "-f"] + args) self.assertTrue(os.path.exists(cache)) self.assertTrue(os.path.isfile(cache)) self.assertTrue(os.path.isfile(cache_dir)) @@ -452,8 +459,8 @@ class TestDataCloudCLIBase(TestDvc): class TestCompatRemoteLOCALCLI(TestDataCloudCLIBase): def _test(self): storagepath = get_local_storagepath() - self.main(['config', 'core.cloud', 'local']) - self.main(['config', 'local.storagepath', storagepath]) + self.main(["config", "core.cloud", "local"]) + self.main(["config", "local.storagepath", storagepath]) self._test_cloud() @@ -462,7 +469,7 @@ class TestRemoteLOCALCLI(TestDataCloudCLIBase): def _test(self): url = get_local_url() - self.main(['remote', 'add', TEST_REMOTE, url]) + self.main(["remote", "add", TEST_REMOTE, url]) self._test_cloud(TEST_REMOTE) @@ -474,7 +481,7 @@ class TestRemoteSSHCLI(TestDataCloudCLIBase): def _test(self): url = get_ssh_url() - self.main(['remote', 'add', TEST_REMOTE, url]) + self.main(["remote", "add", TEST_REMOTE, url]) self._test_cloud(TEST_REMOTE) @@ -486,7 +493,7 @@ class TestRemoteHDFSCLI(TestDataCloudCLIBase): def _test(self): url = get_hdfs_url() - self.main(['remote', 'add', TEST_REMOTE, url]) + self.main(["remote", "add", TEST_REMOTE, url]) self._test_cloud(TEST_REMOTE) @@ -497,8 +504,8 @@ class TestCompatRemoteS3CLI(TestDataCloudCLIBase): def _test(self): storagepath = get_aws_storagepath() - self.main(['config', 'core.cloud', 'aws']) - self.main(['config', 'aws.storagepath', storagepath]) + self.main(["config", "core.cloud", "aws"]) + self.main(["config", "aws.storagepath", storagepath]) self._test_cloud() @@ -510,7 +517,7 @@ class TestRemoteS3CLI(TestDataCloudCLIBase): def _test(self): url = get_aws_url() - self.main(['remote', 'add', TEST_REMOTE, url]) + self.main(["remote", "add", TEST_REMOTE, url]) self._test_cloud(TEST_REMOTE) @@ -521,8 +528,8 @@ class TestCompatRemoteGSCLI(TestDataCloudCLIBase): def _test(self): storagepath = get_gcp_storagepath() - self.main(['config', 'core.cloud', 'gcp']) - self.main(['config', 'gcp.storagepath', storagepath]) + self.main(["config", "core.cloud", "gcp"]) + self.main(["config", "gcp.storagepath", storagepath]) self._test_cloud() @@ -534,7 +541,7 @@ class TestRemoteGSCLI(TestDataCloudCLIBase): def _test(self): url = get_gcp_url() - self.main(['remote', 'add', TEST_REMOTE, url]) + self.main(["remote", "add", TEST_REMOTE, url]) self._test_cloud(TEST_REMOTE) @@ -546,7 +553,7 @@ class TestRemoteAzureCLI(TestDataCloudCLIBase): def _test(self): url = get_azure_url() - self.main(['remote', 'add', TEST_REMOTE, url]) + self.main(["remote", "add", TEST_REMOTE, url]) self._test_cloud(TEST_REMOTE) @@ -557,15 +564,15 @@ class TestDataCloudErrorCLI(TestDvc): self.assertNotEqual(ret, 0) def test_error(self): - f = 'non-existing-file' - self.main_fail(['status', '-c', f]) - self.main_fail(['push', f]) - self.main_fail(['pull', f]) - self.main_fail(['fetch', f]) + f = "non-existing-file" + self.main_fail(["status", "-c", f]) + self.main_fail(["push", f]) + self.main_fail(["pull", f]) + self.main_fail(["fetch", f]) class TestWarnOnOutdatedStage(TestDvc): - color_patch = patch.object(logger, 'colorize', new=lambda x, color='': x) + color_patch = patch.object(logger, "colorize", new=lambda x, color="": x) def main(self, args): ret = main(args) @@ -573,26 +580,28 @@ class TestWarnOnOutdatedStage(TestDvc): def _test(self): url = get_local_url() - self.main(['remote', 'add', '-d', TEST_REMOTE, url]) + self.main(["remote", "add", "-d", TEST_REMOTE, url]) self.dvc.add(self.FOO) - stage = self.dvc.run(deps=['foo'], outs=['bar'], cmd='echo bar > bar') - self.main(['push']) + stage = self.dvc.run(deps=["foo"], outs=["bar"], cmd="echo bar > bar") + self.main(["push"]) stage_file_path = stage.relpath - with open(stage_file_path, 'r') as stage_file: + with open(stage_file_path, "r") as stage_file: content = yaml.safe_load(stage_file) - del (content['outs'][0]['md5']) - with open(stage_file_path, 'w') as stage_file: + del (content["outs"][0]["md5"]) + with open(stage_file_path, "w") as stage_file: yaml.dump(content, stage_file) logger.logger.handlers[0].stream = StringIO() - self.main(['status', '-c']) - self.assertIn('Warning: Output \'bar\'(Stage: \'bar.dvc\') is ' - 'missing version info. Cache for it will not be ' - 'collected. Use dvc repro to get your pipeline up to ' - 'date.', - logger.logger.handlers[0].stream.getvalue()) + self.main(["status", "-c"]) + self.assertIn( + "Warning: Output 'bar'(Stage: 'bar.dvc') is " + "missing version info. Cache for it will not be " + "collected. Use dvc repro to get your pipeline up to " + "date.", + logger.logger.handlers[0].stream.getvalue(), + ) def test(self): self.color_patch.start() @@ -616,11 +625,7 @@ class TestRecursiveSyncOperations(TestDataCloudBase): return RemoteLOCAL def _prepare_repo(self): - self.main(['remote', - 'add', - '-d', - TEST_REMOTE, - self.cloud._cloud.cache_dir]) + self.main(["remote", "add", "-d", TEST_REMOTE, self.cloud._cloud.cache_dir]) self.create(self.DATA, self.DATA_CONTENTS) self.create(self.DATA_SUB, self.DATA_SUB_CONTENTS) @@ -638,7 +643,7 @@ class TestRecursiveSyncOperations(TestDataCloudBase): self.assertFalse(os.path.exists(self.DATA)) self.assertFalse(os.path.exists(self.DATA_SUB)) - self.main(['pull', '-R', self.DATA_DIR]) + self.main(["pull", "-R", self.DATA_DIR]) self.assertTrue(os.path.exists(self.DATA)) self.assertTrue(os.path.exists(self.DATA_SUB)) @@ -655,7 +660,7 @@ class TestRecursiveSyncOperations(TestDataCloudBase): self.assertFalse(os.path.exists(local_cache_data_path)) self.assertFalse(os.path.exists(local_cache_data_sub_path)) - self.main(['fetch', '-R', self.DATA_DIR]) + self.main(["fetch", "-R", self.DATA_DIR]) self.assertTrue(os.path.exists(local_cache_data_path)) self.assertTrue(os.path.exists(local_cache_data_sub_path)) @@ -667,7 +672,7 @@ class TestRecursiveSyncOperations(TestDataCloudBase): self.assertFalse(os.path.exists(cloud_data_path)) self.assertFalse(os.path.exists(cloud_data_sub_path)) - self.main(['push', '-R', self.DATA_DIR]) + self.main(["push", "-R", self.DATA_DIR]) self.assertTrue(os.path.exists(cloud_data_path)) self.assertTrue(os.path.exists(cloud_data_sub_path)) @@ -687,13 +692,12 @@ class TestRecursiveSyncOperations(TestDataCloudBase): class TestCheckSumRecalculation(TestDvc): - def test(self): test_collect = spy(State._collect) - with patch.object(State, '_collect', test_collect): + with patch.object(State, "_collect", test_collect): url = get_local_url() - main(['remote', 'add', '-d', TEST_REMOTE, url]) - main(['add', self.FOO]) - main(['push']) - main(['run', '-d', self.FOO, 'echo foo']) + main(["remote", "add", "-d", TEST_REMOTE, url]) + main(["add", self.FOO]) + main(["push"]) + main(["run", "-d", self.FOO, "echo foo"]) self.assertEqual(test_collect.mock.call_count, 1) diff --git a/tests/test_dependency.py b/tests/test_dependency.py index 3799025d5..bed94804d 100644 --- a/tests/test_dependency.py +++ b/tests/test_dependency.py @@ -8,8 +8,8 @@ class TestDepScheme(TestOutScheme): MAP = DEP_MAP TESTS = TestOutScheme.TESTS.copy() - TESTS['http://example.com/path/to/file'] = 'http' - TESTS['https://example.com/path/to/file'] = 'https' + TESTS["http://example.com/path/to/file"] = "http" + TESTS["https://example.com/path/to/file"] = "https" def _get(self, path): return _get(Stage(self.dvc), path, None) diff --git a/tests/test_destroy.py b/tests/test_destroy.py index cd45e43f2..07803a6b3 100644 --- a/tests/test_destroy.py +++ b/tests/test_destroy.py @@ -7,13 +7,13 @@ from tests.test_repro import TestRepro class TestDestroyNoConfirmation(TestRepro): def test(self): - ret = main(['destroy']) + ret = main(["destroy"]) self.assertNotEqual(ret, 0) class TestDestroyForce(TestRepro): def test(self): - ret = main(['destroy', '-f']) + ret = main(["destroy", "-f"]) self.assertEqual(ret, 0) self.assertFalse(os.path.exists(self.dvc.dvc_dir)) diff --git a/tests/test_gc.py b/tests/test_gc.py index bb8c961e7..7594ba816 100644 --- a/tests/test_gc.py +++ b/tests/test_gc.py @@ -16,11 +16,12 @@ class TestGC(TestDvc): self.dvc.add(self.FOO) self.dvc.add(self.DATA_DIR) - self.good_cache = [self.dvc.cache.local.get( - md5) for md5 in self.dvc.cache.local.all()] + self.good_cache = [ + self.dvc.cache.local.get(md5) for md5 in self.dvc.cache.local.all() + ] self.bad_cache = [] - for i in ['123', '234', '345']: + for i in ["123", "234", "345"]: path = os.path.join(self.dvc.cache.local.cache_dir, i[0:2], i[2:]) self.create(path, i) self.bad_cache.append(path) @@ -30,7 +31,7 @@ class TestGC(TestDvc): self._test_gc() def test_cli(self): - ret = main(['gc', '-f']) + ret = main(["gc", "-f"]) self.assertEqual(ret, 0) self._test_gc() @@ -46,47 +47,47 @@ class TestGC(TestDvc): class TestGCBranchesTags(TestDvc): def _check_cache(self, num): total = 0 - for root, dirs, files in os.walk(os.path.join('.dvc', 'cache')): + for root, dirs, files in os.walk(os.path.join(".dvc", "cache")): total += len(files) self.assertEqual(total, num) def test(self): - fname = 'file' + fname = "file" - with open(fname, 'w+') as fobj: - fobj.write('v1.0') + with open(fname, "w+") as fobj: + fobj.write("v1.0") stages = self.dvc.add(fname) self.assertEqual(len(stages), 1) - self.dvc.scm.add(['.gitignore', stages[0].relpath]) - self.dvc.scm.commit('v1.0') - self.dvc.scm.tag('v1.0') + self.dvc.scm.add([".gitignore", stages[0].relpath]) + self.dvc.scm.commit("v1.0") + self.dvc.scm.tag("v1.0") - self.dvc.scm.checkout('test', create_new=True) + self.dvc.scm.checkout("test", create_new=True) self.dvc.remove(stages[0].relpath, outs_only=True) - with open(fname, 'w+') as fobj: - fobj.write('test') + with open(fname, "w+") as fobj: + fobj.write("test") stages = self.dvc.add(fname) self.assertEqual(len(stages), 1) - self.dvc.scm.add(['.gitignore', stages[0].relpath]) - self.dvc.scm.commit('test') + self.dvc.scm.add([".gitignore", stages[0].relpath]) + self.dvc.scm.commit("test") - self.dvc.scm.checkout('master') + self.dvc.scm.checkout("master") self.dvc.remove(stages[0].relpath, outs_only=True) - with open(fname, 'w+') as fobj: - fobj.write('trash') + with open(fname, "w+") as fobj: + fobj.write("trash") stages = self.dvc.add(fname) self.assertEqual(len(stages), 1) - self.dvc.scm.add(['.gitignore', stages[0].relpath]) - self.dvc.scm.commit('trash') + self.dvc.scm.add([".gitignore", stages[0].relpath]) + self.dvc.scm.commit("trash") self.dvc.remove(stages[0].relpath, outs_only=True) - with open(fname, 'w+') as fobj: - fobj.write('master') + with open(fname, "w+") as fobj: + fobj.write("master") stages = self.dvc.add(fname) self.assertEqual(len(stages), 1) - self.dvc.scm.add(['.gitignore', stages[0].relpath]) - self.dvc.scm.commit('master') + self.dvc.scm.add([".gitignore", stages[0].relpath]) + self.dvc.scm.commit("master") self._check_cache(4) @@ -106,7 +107,7 @@ class TestGCBranchesTags(TestDvc): class TestGCMultipleProjects(TestDvc): def _check_cache(self, num): total = 0 - for root, dirs, files in os.walk(os.path.join('.dvc', 'cache')): + for root, dirs, files in os.walk(os.path.join(".dvc", "cache")): total += len(files) self.assertEqual(total, num) @@ -116,14 +117,11 @@ class TestGCMultipleProjects(TestDvc): self.additional_git = Repo.init(self.additional_path) self.additional_dvc = Project.init(self.additional_path) - cache_path = os.path.join(self._root_dir, '.dvc', 'cache') - config_path = os.path.join(self.additional_path, '.dvc', - 'config.local') + cache_path = os.path.join(self._root_dir, ".dvc", "cache") + config_path = os.path.join(self.additional_path, ".dvc", "config.local") cfg = configobj.ConfigObj() cfg.filename = config_path - cfg['cache'] = { - 'dir': cache_path - } + cfg["cache"] = {"dir": cache_path} cfg.write() self.additional_dvc = Project(self.additional_path) @@ -131,17 +129,17 @@ class TestGCMultipleProjects(TestDvc): def test(self): # ADD FILE ONLY IN MAIN PROJECT - fname = 'only_in_first' - with open(fname, 'w+') as fobj: - fobj.write('only in main project') + fname = "only_in_first" + with open(fname, "w+") as fobj: + fobj.write("only in main project") stages = self.dvc.add(fname) self.assertEqual(len(stages), 1) # ADD FILE IN MAIN PROJECT THAT IS ALSO IN SECOND PROJECT - fname = 'in_both' - with open(fname, 'w+') as fobj: - fobj.write('in both projects') + fname = "in_both" + with open(fname, "w+") as fobj: + fobj.write("in both projects") stages = self.dvc.add(fname) self.assertEqual(len(stages), 1) @@ -149,17 +147,17 @@ class TestGCMultipleProjects(TestDvc): cwd = os.getcwd() os.chdir(self.additional_path) # ADD FILE ONLY IN SECOND PROJECT - fname = os.path.join(self.additional_path, 'only_in_second') - with open(fname, 'w+') as fobj: - fobj.write('only in additional project') + fname = os.path.join(self.additional_path, "only_in_second") + with open(fname, "w+") as fobj: + fobj.write("only in additional project") stages = self.additional_dvc.add(fname) self.assertEqual(len(stages), 1) # ADD FILE IN SECOND PROJECT THAT IS ALSO IN MAIN PROJECT - fname = os.path.join(self.additional_path, 'in_both') - with open(fname, 'w+') as fobj: - fobj.write('in both projects') + fname = os.path.join(self.additional_path, "in_both") + with open(fname, "w+") as fobj: + fobj.write("in both projects") stages = self.additional_dvc.add(fname) self.assertEqual(len(stages), 1) diff --git a/tests/test_import.py b/tests/test_import.py index ae30e4150..289631402 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -14,15 +14,15 @@ from dvc.utils.compat import StringIO class TestCmdImport(TestDvc): def test(self): - ret = main(['import', self.FOO, 'import']) + ret = main(["import", self.FOO, "import"]) self.assertEqual(ret, 0) - self.assertTrue(os.path.exists('import.dvc')) + self.assertTrue(os.path.exists("import.dvc")) - ret = main(['import', 'non-existing-file', 'import']) + ret = main(["import", "non-existing-file", "import"]) self.assertNotEqual(ret, 0) def test_unsupported(self): - ret = main(['import', 'unsupported://path', 'import_unsupported']) + ret = main(["import", "unsupported://path", "import_unsupported"]) self.assertNotEqual(ret, 0) @@ -32,35 +32,37 @@ class TestDefaultOutput(TestDvc): filename = str(uuid4()) tmpfile = os.path.join(tmpdir, filename) - with open(tmpfile, 'w') as fd: - fd.write('content') + with open(tmpfile, "w") as fd: + fd.write("content") - ret = main(['import', tmpfile]) + ret = main(["import", tmpfile]) self.assertEqual(ret, 0) self.assertTrue(os.path.exists(filename)) - self.assertEqual(open(filename).read(), 'content') + self.assertEqual(open(filename).read(), "content") class TestFailedImportMessage(TestDvc): - color_patch = patch.object(logger, 'colorize', new=lambda x, color='': x) + color_patch = patch.object(logger, "colorize", new=lambda x, color="": x) def test(self): self.color_patch.start() self._test() self.color_patch.stop() - @patch('dvc.command.imp.urlparse') + @patch("dvc.command.imp.urlparse") def _test(self, imp_urlparse_patch): logger.logger.handlers[1].stream = StringIO() - page_address = 'http://somesite.com/file_name' + page_address = "http://somesite.com/file_name" def dvc_exception(*args, **kwargs): - raise DvcException('message') + raise DvcException("message") imp_urlparse_patch.side_effect = dvc_exception - main(['import', page_address]) - self.assertIn('Error: failed to import ' - 'http://somesite.com/file_name. You could also try ' - 'downloading it manually and adding it with `dvc add` ' - 'command.', - logger.logger.handlers[1].stream.getvalue()) + main(["import", page_address]) + self.assertIn( + "Error: failed to import " + "http://somesite.com/file_name. You could also try " + "downloading it manually and adding it with `dvc add` " + "command.", + logger.logger.handlers[1].stream.getvalue(), + ) diff --git a/tests/test_init.py b/tests/test_init.py index 8109deedd..7c796e1e2 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -17,7 +17,7 @@ class TestInit(TestGit): self._test_init() def test_cli(self): - ret = main(['init']) + ret = main(["init"]) self.assertEqual(ret, 0) self._test_init() @@ -25,15 +25,15 @@ class TestInit(TestGit): class TestDoubleInit(TestInit): def test(self): - ret = main(['init']) + ret = main(["init"]) self.assertEqual(ret, 0) self._test_init() - ret = main(['init']) + ret = main(["init"]) self.assertNotEqual(ret, 0) self._test_init() - ret = main(['init', '--force']) + ret = main(["init", "--force"]) self.assertEqual(ret, 0) self._test_init() @@ -44,7 +44,7 @@ class TestInitNoSCMFail(TestDir): Project.init() def test_cli(self): - ret = main(['init']) + ret = main(["init"]) self.assertNotEqual(ret, 0) @@ -59,7 +59,7 @@ class TestInitNoSCM(TestDir): self._test_init() def test_cli(self): - ret = main(['init', '--no-scm']) + ret = main(["init", "--no-scm"]) self.assertEqual(ret, 0) self._test_init() diff --git a/tests/test_install.py b/tests/test_install.py index a94458cb7..ec865b72e 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -7,20 +7,20 @@ from tests.basic_env import TestDvc class TestInstall(TestDvc): def test(self): - ret = main(['install']) + ret = main(["install"]) self.assertEqual(ret, 0) - ret = main(['install']) + ret = main(["install"]) self.assertNotEqual(ret, 0) def hook(name): - return os.path.join('.git', 'hooks', name) + return os.path.join(".git", "hooks", name) - self.assertTrue(os.path.isfile(hook('post-checkout'))) - self.assertTrue(os.path.isfile(hook('pre-commit'))) + self.assertTrue(os.path.isfile(hook("post-checkout"))) + self.assertTrue(os.path.isfile(hook("pre-commit"))) self.dvc.add(self.FOO) os.unlink(self.FOO) - self.dvc.scm.checkout('branch', create_new=True) + self.dvc.scm.checkout("branch", create_new=True) self.assertTrue(os.path.isfile(self.FOO)) diff --git a/tests/test_lock.py b/tests/test_lock.py index edf3f1967..1c5a0bda2 100644 --- a/tests/test_lock.py +++ b/tests/test_lock.py @@ -17,5 +17,5 @@ class TestLock(TestDvc): def test_cli(self): lock = Lock(self.dvc.dvc_dir) with lock: - ret = main(['add', self.FOO]) + ret = main(["add", self.FOO]) self.assertEqual(ret, 1) diff --git a/tests/test_logger.py b/tests/test_logger.py index 7d2ba5872..864f1a950 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -11,7 +11,7 @@ from dvc.utils.compat import StringIO class TestLogger(TestCase): handlers = logger.logger.handlers - color_patch = patch.object(logger, 'colorize', new=lambda x, color='': x) + color_patch = patch.object(logger, "colorize", new=lambda x, color="": x) def setUp(self): logger.logger.handlers[0].stream = StringIO() @@ -33,122 +33,124 @@ class TestLogger(TestCase): return logger.logger.handlers[1].stream.getvalue() def test_info(self): - logger.info('message') + logger.info("message") - self.assertEqual(self.stdout, 'message\n') + self.assertEqual(self.stdout, "message\n") def test_debug(self): with logger.verbose(): - logger.debug('message') + logger.debug("message") - self.assertEqual(self.stdout, 'Debug: message\n') + self.assertEqual(self.stdout, "Debug: message\n") def test_warning(self): - logger.warning('message') + logger.warning("message") - self.assertEqual(self.stdout, 'Warning: message\n') + self.assertEqual(self.stdout, "Warning: message\n") def test_error(self): - logger.error('message') + logger.error("message") output = ( - 'Error: message\n' - '\n' - 'Having any troubles? Hit us up at https://dvc.org/support,' - ' we are always happy to help!\n' + "Error: message\n" + "\n" + "Having any troubles? Hit us up at https://dvc.org/support," + " we are always happy to help!\n" ) self.assertEqual(self.stderr, output) def test_error_with_exception(self): try: - raise Exception('exception description') + raise Exception("exception description") except Exception: - logger.error('') + logger.error("") output = ( - 'Error: exception description\n' - '\n' - 'Having any troubles? Hit us up at https://dvc.org/support,' - ' we are always happy to help!\n' + "Error: exception description\n" + "\n" + "Having any troubles? Hit us up at https://dvc.org/support," + " we are always happy to help!\n" ) self.assertEqual(self.stderr, output) def test_error_with_exception_and_message(self): try: - raise Exception('exception description') + raise Exception("exception description") except Exception: - logger.error('message') + logger.error("message") output = ( - 'Error: message - exception description\n' - '\n' - 'Having any troubles? Hit us up at https://dvc.org/support,' - ' we are always happy to help!\n' + "Error: message - exception description\n" + "\n" + "Having any troubles? Hit us up at https://dvc.org/support," + " we are always happy to help!\n" ) self.assertEqual(self.stderr, output) def test_error_with_chained_exception_and_message(self): try: - raise DvcException('exception', cause=Exception('cause')) + raise DvcException("exception", cause=Exception("cause")) except Exception: - logger.error('message') + logger.error("message") output = ( - 'Error: message - exception: cause\n' - '\n' - 'Having any troubles? Hit us up at https://dvc.org/support,' - ' we are always happy to help!\n' + "Error: message - exception: cause\n" + "\n" + "Having any troubles? Hit us up at https://dvc.org/support," + " we are always happy to help!\n" ) self.assertEqual(self.stderr, output) def test_traceback(self): - stack_trace1 = 'stack_trace1\n' - stack_trace2 = 'stack_trace2\n' + stack_trace1 = "stack_trace1\n" + stack_trace2 = "stack_trace2\n" try: - exc1 = Exception('exception1') - exc2 = DvcException('exception2', cause=exc1) + exc1 = Exception("exception1") + exc2 = DvcException("exception2", cause=exc1) exc2.cause_tb = stack_trace1 - exc3 = DvcException('exception3', cause=exc2) + exc3 = DvcException("exception3", cause=exc2) exc3.cause_tb = stack_trace2 raise exc3 except Exception: stack_trace3 = traceback.format_exc() with logger.verbose(): - logger.error('message') + logger.error("message") output = ( - 'Error: message - exception3: exception2: exception1\n' - '{line}\n' - '{stack_trace1}' - '\n' - '{stack_trace2}' - '\n' - '{stack_trace3}' - '{line}\n' - '\n' - 'Having any troubles? Hit us up at https://dvc.org/support,' - ' we are always happy to help!\n' - ).format(line='-' * 60, - stack_trace1=stack_trace1, - stack_trace2=stack_trace2, - stack_trace3=stack_trace3) + "Error: message - exception3: exception2: exception1\n" + "{line}\n" + "{stack_trace1}" + "\n" + "{stack_trace2}" + "\n" + "{stack_trace3}" + "{line}\n" + "\n" + "Having any troubles? Hit us up at https://dvc.org/support," + " we are always happy to help!\n" + ).format( + line="-" * 60, + stack_trace1=stack_trace1, + stack_trace2=stack_trace2, + stack_trace3=stack_trace3, + ) self.assertEqual(self.stderr, output) def test_box(self): - logger.box('message') + logger.box("message") output = ( - '+-----------------+\n' - '| |\n' - '| message |\n' - '| |\n' - '+-----------------+\n' - '\n' + "+-----------------+\n" + "| |\n" + "| message |\n" + "| |\n" + "+-----------------+\n" + "\n" ) self.assertEqual(self.stdout, output) diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 99e22fc44..825883f43 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -10,215 +10,241 @@ from tests.basic_env import TestDvc class TestMetrics(TestDvc): def setUp(self): super(TestMetrics, self).setUp() - self.dvc.scm.commit('init') + self.dvc.scm.commit("init") - for branch in ['foo', 'bar', 'baz']: + for branch in ["foo", "bar", "baz"]: self.dvc.scm.checkout(branch, create_new=True) - with open('metric', 'w+') as fd: + with open("metric", "w+") as fd: fd.write(branch) - with open('metric_json', 'w+') as fd: - json.dump({'branch': branch}, fd) + with open("metric_json", "w+") as fd: + json.dump({"branch": branch}, fd) - with open('metric_csv', 'w+') as fd: + with open("metric_csv", "w+") as fd: fd.write(branch) - with open('metric_hcsv', 'w+') as fd: - fd.write('branch\n') + with open("metric_hcsv", "w+") as fd: + fd.write("branch\n") fd.write(branch) - with open('metric_tsv', 'w+') as fd: + with open("metric_tsv", "w+") as fd: fd.write(branch) - with open('metric_htsv', 'w+') as fd: - fd.write('branch\n') + with open("metric_htsv", "w+") as fd: + fd.write("branch\n") fd.write(branch) - self.dvc.scm.add(['metric', - 'metric_json', - 'metric_tsv', - 'metric_htsv', - 'metric_csv', - 'metric_hcsv']) - self.dvc.scm.commit('metric') + self.dvc.scm.add( + [ + "metric", + "metric_json", + "metric_tsv", + "metric_htsv", + "metric_csv", + "metric_hcsv", + ] + ) + self.dvc.scm.commit("metric") - self.dvc.scm.checkout('master') + self.dvc.scm.checkout("master") def test(self): - ret = self.dvc.metrics_show('metric', all_branches=True) + ret = self.dvc.metrics_show("metric", all_branches=True) self.assertEqual(len(ret), 3) - self.assertTrue(ret['foo']['metric'] == 'foo') - self.assertTrue(ret['bar']['metric'] == 'bar') - self.assertTrue(ret['baz']['metric'] == 'baz') + self.assertTrue(ret["foo"]["metric"] == "foo") + self.assertTrue(ret["bar"]["metric"] == "bar") + self.assertTrue(ret["baz"]["metric"] == "baz") - ret = self.dvc.metrics_show('metric_json', - typ='json', - xpath='branch', - all_branches=True) + ret = self.dvc.metrics_show( + "metric_json", typ="json", xpath="branch", all_branches=True + ) self.assertEqual(len(ret), 3) - self.assertTrue(ret['foo']['metric_json'] == ['foo']) - self.assertTrue(ret['bar']['metric_json'] == ['bar']) - self.assertTrue(ret['baz']['metric_json'] == ['baz']) + self.assertTrue(ret["foo"]["metric_json"] == ["foo"]) + self.assertTrue(ret["bar"]["metric_json"] == ["bar"]) + self.assertTrue(ret["baz"]["metric_json"] == ["baz"]) - ret = self.dvc.metrics_show('metric_tsv', - typ='tsv', - xpath='0,0', - all_branches=True) + ret = self.dvc.metrics_show( + "metric_tsv", typ="tsv", xpath="0,0", all_branches=True + ) self.assertEqual(len(ret), 3) - self.assertTrue(ret['foo']['metric_tsv'] == ['foo']) - self.assertTrue(ret['bar']['metric_tsv'] == ['bar']) - self.assertTrue(ret['baz']['metric_tsv'] == ['baz']) + self.assertTrue(ret["foo"]["metric_tsv"] == ["foo"]) + self.assertTrue(ret["bar"]["metric_tsv"] == ["bar"]) + self.assertTrue(ret["baz"]["metric_tsv"] == ["baz"]) - ret = self.dvc.metrics_show('metric_htsv', - typ='htsv', - xpath='branch,0', - all_branches=True) + ret = self.dvc.metrics_show( + "metric_htsv", typ="htsv", xpath="branch,0", all_branches=True + ) self.assertEqual(len(ret), 3) - self.assertTrue(ret['foo']['metric_htsv'] == ['foo']) - self.assertTrue(ret['bar']['metric_htsv'] == ['bar']) - self.assertTrue(ret['baz']['metric_htsv'] == ['baz']) + self.assertTrue(ret["foo"]["metric_htsv"] == ["foo"]) + self.assertTrue(ret["bar"]["metric_htsv"] == ["bar"]) + self.assertTrue(ret["baz"]["metric_htsv"] == ["baz"]) - ret = self.dvc.metrics_show('metric_csv', - typ='csv', - xpath='0,0', - all_branches=True) + ret = self.dvc.metrics_show( + "metric_csv", typ="csv", xpath="0,0", all_branches=True + ) self.assertEqual(len(ret), 3) - self.assertTrue(ret['foo']['metric_csv'] == ['foo']) - self.assertTrue(ret['bar']['metric_csv'] == ['bar']) - self.assertTrue(ret['baz']['metric_csv'] == ['baz']) + self.assertTrue(ret["foo"]["metric_csv"] == ["foo"]) + self.assertTrue(ret["bar"]["metric_csv"] == ["bar"]) + self.assertTrue(ret["baz"]["metric_csv"] == ["baz"]) - ret = self.dvc.metrics_show('metric_hcsv', - typ='hcsv', - xpath='branch,0', - all_branches=True) + ret = self.dvc.metrics_show( + "metric_hcsv", typ="hcsv", xpath="branch,0", all_branches=True + ) self.assertEqual(len(ret), 3) - self.assertTrue(ret['foo']['metric_hcsv'] == ['foo']) - self.assertTrue(ret['bar']['metric_hcsv'] == ['bar']) - self.assertTrue(ret['baz']['metric_hcsv'] == ['baz']) + self.assertTrue(ret["foo"]["metric_hcsv"] == ["foo"]) + self.assertTrue(ret["bar"]["metric_hcsv"] == ["bar"]) + self.assertTrue(ret["baz"]["metric_hcsv"] == ["baz"]) class TestMetricsReproCLI(TestDvc): def test(self): - stage = self.dvc.run(metrics_no_cache=['metrics'], - cmd='python {} {} {}'.format(self.CODE, - self.FOO, - 'metrics')) + stage = self.dvc.run( + metrics_no_cache=["metrics"], + cmd="python {} {} {}".format(self.CODE, self.FOO, "metrics"), + ) - ret = main(['repro', '-m', stage.path]) + ret = main(["repro", "-m", stage.path]) self.assertEqual(ret, 0) - ret = main(['metrics', 'remove', 'metrics']) + ret = main(["metrics", "remove", "metrics"]) self.assertEqual(ret, 0) - ret = main(['repro', '-f', '-m', stage.path]) + ret = main(["repro", "-f", "-m", stage.path]) self.assertNotEqual(ret, 0) - ret = main(['metrics', 'add', 'metrics']) + ret = main(["metrics", "add", "metrics"]) self.assertEqual(ret, 0) - ret = main(['metrics', 'modify', '-t', 'csv', '-x', '0,0', 'metrics']) + ret = main(["metrics", "modify", "-t", "csv", "-x", "0,0", "metrics"]) self.assertEqual(ret, 0) - ret = main(['repro', '-f', '-m', stage.path]) + ret = main(["repro", "-f", "-m", stage.path]) self.assertEqual(ret, 0) def test_dir(self): - os.mkdir('metrics_dir') + os.mkdir("metrics_dir") with self.assertRaises(DvcException): - self.dvc.run(metrics_no_cache=['metrics_dir']) + self.dvc.run(metrics_no_cache=["metrics_dir"]) def test_binary(self): - with open('metrics_bin', 'w+') as fd: - fd.write('\0\0\0\0\0\0\0\0') + with open("metrics_bin", "w+") as fd: + fd.write("\0\0\0\0\0\0\0\0") with self.assertRaises(DvcException): - self.dvc.run(metrics_no_cache=['metrics_bin']) + self.dvc.run(metrics_no_cache=["metrics_bin"]) class TestMetricsCLICompat(TestMetrics): def test(self): # FIXME check output - ret = main(['metrics', 'show', '--all-branches', 'metric', '-v']) + ret = main(["metrics", "show", "--all-branches", "metric", "-v"]) self.assertEqual(ret, 0) - ret = main(['metrics', 'show', '--all-branches', 'metric_json', - '--json-path', 'branch']) + ret = main( + [ + "metrics", + "show", + "--all-branches", + "metric_json", + "--json-path", + "branch", + ] + ) self.assertEqual(ret, 0) - ret = main(['metrics', 'show', '--all-branches', 'metric_tsv', - '--tsv-path', '0,0']) + ret = main( + ["metrics", "show", "--all-branches", "metric_tsv", "--tsv-path", "0,0"] + ) self.assertEqual(ret, 0) - ret = main(['metrics', 'show', '--all-branches', 'metric_htsv', - '--htsv-path', 'branch,0']) + ret = main( + [ + "metrics", + "show", + "--all-branches", + "metric_htsv", + "--htsv-path", + "branch,0", + ] + ) self.assertEqual(ret, 0) - ret = main(['metrics', 'show', '--all-branches', 'metric_csv', - '--csv-path', '0,0']) + ret = main( + ["metrics", "show", "--all-branches", "metric_csv", "--csv-path", "0,0"] + ) self.assertEqual(ret, 0) - ret = main(['metrics', 'show', '--all-branches', 'metric_hcsv', - '--hcsv-path', 'branch,0']) + ret = main( + [ + "metrics", + "show", + "--all-branches", + "metric_hcsv", + "--hcsv-path", + "branch,0", + ] + ) self.assertEqual(ret, 0) class TestMetricsCLI(TestMetrics): def test(self): # FIXME check output - ret = main(['metrics', 'show', '-a', 'metric', '-v']) + ret = main(["metrics", "show", "-a", "metric", "-v"]) self.assertEqual(ret, 0) - ret = main(['metrics', 'show', '-a', 'metric_json', - '-t', 'json', '-x', 'branch']) + ret = main( + ["metrics", "show", "-a", "metric_json", "-t", "json", "-x", "branch"] + ) self.assertEqual(ret, 0) - ret = main(['metrics', 'show', '-a', 'metric_tsv', - '-t', 'tsv', '-x', '0,0']) + ret = main(["metrics", "show", "-a", "metric_tsv", "-t", "tsv", "-x", "0,0"]) self.assertEqual(ret, 0) - ret = main(['metrics', 'show', '-a', 'metric_htsv', - '-t', 'htsv', '-x', 'branch,0']) + ret = main( + ["metrics", "show", "-a", "metric_htsv", "-t", "htsv", "-x", "branch,0"] + ) self.assertEqual(ret, 0) - ret = main(['metrics', 'show', '-a', 'metric_csv', - '-t', 'csv', '-x', '0,0']) + ret = main(["metrics", "show", "-a", "metric_csv", "-t", "csv", "-x", "0,0"]) self.assertEqual(ret, 0) - ret = main(['metrics', 'show', '-a', 'metric_hcsv', - '-t', 'hcsv', '-x', 'branch,0']) + ret = main( + ["metrics", "show", "-a", "metric_hcsv", "-t", "hcsv", "-x", "branch,0"] + ) self.assertEqual(ret, 0) def test_dir(self): - os.mkdir('metrics_dir') + os.mkdir("metrics_dir") with self.assertRaises(DvcException): - self.dvc.run(outs_no_cache=['metrics_dir']) - self.dvc.metrics_add('metrics_dir') + self.dvc.run(outs_no_cache=["metrics_dir"]) + self.dvc.metrics_add("metrics_dir") def test_binary(self): - with open('metrics_bin', 'w+') as fd: - fd.write('\0\0\0\0\0\0\0\0') + with open("metrics_bin", "w+") as fd: + fd.write("\0\0\0\0\0\0\0\0") with self.assertRaises(DvcException): - self.dvc.run(outs_no_cache=['metrics_bin']) - self.dvc.metrics_add('metrics_bin') + self.dvc.run(outs_no_cache=["metrics_bin"]) + self.dvc.metrics_add("metrics_bin") def test_non_existing(self): - ret = main(['metrics', 'add', 'non-existing']) + ret = main(["metrics", "add", "non-existing"]) self.assertNotEqual(ret, 0) - ret = main(['metrics', 'modify', 'non-existing']) + ret = main(["metrics", "modify", "non-existing"]) self.assertNotEqual(ret, 0) - ret = main(['metrics', 'remove', 'non-existing']) + ret = main(["metrics", "remove", "non-existing"]) self.assertNotEqual(ret, 0) class TestNoMetrics(TestDvc): def test(self): - ret = main(['metrics', 'show']) + ret = main(["metrics", "show"]) self.assertNotEqual(ret, 0) @@ -227,34 +253,38 @@ class TestCachedMetrics(TestDvc): self.dvc.scm.checkout(branch) self.dvc.checkout(force=True) - with open('metrics.json', 'w+') as fd: - json.dump({'metrics': branch}, fd) + with open("metrics.json", "w+") as fd: + json.dump({"metrics": branch}, fd) - stages = self.dvc.add('metrics.json') + stages = self.dvc.add("metrics.json") self.assertEqual(len(stages), 1) stage = stages[0] self.assertNotEqual(stage, None) - self.dvc.scm.add(['.gitignore', 'metrics.json.dvc']) + self.dvc.scm.add([".gitignore", "metrics.json.dvc"]) self.dvc.scm.commit(branch) def test(self): - self.dvc.scm.commit('init') + self.dvc.scm.commit("init") - self.dvc.scm.branch('one') - self.dvc.scm.branch('two') + self.dvc.scm.branch("one") + self.dvc.scm.branch("two") - self._do_write('master') - self._do_write('one') - self._do_write('two') + self._do_write("master") + self._do_write("one") + self._do_write("two") - self.dvc = Project('.') + self.dvc = Project(".") - res = self.dvc.metrics_show('metrics.json', - all_branches=True, - typ='json', - xpath='metrics') + res = self.dvc.metrics_show( + "metrics.json", all_branches=True, typ="json", xpath="metrics" + ) - self.assertEqual(res, {"master": {"metrics.json": ["master"]}, - "one": {"metrics.json": ["one"]}, - "two": {"metrics.json": ["two"]}}) + self.assertEqual( + res, + { + "master": {"metrics.json": ["master"]}, + "one": {"metrics.json": ["one"]}, + "two": {"metrics.json": ["two"]}, + }, + ) diff --git a/tests/test_move.py b/tests/test_move.py index 8b651cd2d..fb8c75c2e 100644 --- a/tests/test_move.py +++ b/tests/test_move.py @@ -11,7 +11,7 @@ from tests.utils import load_stage_file class TestMove(TestDvc): def test(self): - dst = self.FOO + '1' + dst = self.FOO + "1" self.dvc.add(self.FOO) self.dvc.move(self.FOO, dst) @@ -22,12 +22,12 @@ class TestMove(TestDvc): class TestMoveNonExistentFile(TestDvc): def test(self): with self.assertRaises(DvcException): - self.dvc.move('non_existent_file', 'dst') + self.dvc.move("non_existent_file", "dst") class TestMoveDirectory(TestDvc): def test(self): - dst = 'dst' + dst = "dst" stages = self.dvc.add(self.DATA_DIR) self.assertEqual(len(stages), 1) self.assertTrue(stages[0] is not None) @@ -42,10 +42,10 @@ class TestCmdMove(TestDvc): self.assertEqual(len(stages), 1) self.assertTrue(stages[0] is not None) - ret = main(['move', self.FOO, self.FOO + '1']) + ret = main(["move", self.FOO, self.FOO + "1"]) self.assertEqual(ret, 0) - ret = main(['move', 'non-existing-file', 'dst']) + ret = main(["move", "non-existing-file", "dst"]) self.assertNotEqual(ret, 0) @@ -55,29 +55,29 @@ class TestMoveNotDataSource(TestRepro): self.dvc = Project(self._root_dir) with self.assertRaises(MoveNotDataSourceError): - self.dvc.move(self.file1, 'dst') + self.dvc.move(self.file1, "dst") - ret = main(['move', self.file1, 'dst']) + ret = main(["move", self.file1, "dst"]) self.assertNotEqual(ret, 0) class TestMoveFileWithExtension(TestDvc): def test(self): - with open(os.path.join(self.dvc.root_dir, 'file.csv'), 'w') as fd: - fd.write('1,2,3\n') + with open(os.path.join(self.dvc.root_dir, "file.csv"), "w") as fd: + fd.write("1,2,3\n") - self.dvc.add('file.csv') + self.dvc.add("file.csv") - self.assertTrue(os.path.exists('file.csv')) - self.assertTrue(os.path.exists('file.csv.dvc')) + self.assertTrue(os.path.exists("file.csv")) + self.assertTrue(os.path.exists("file.csv.dvc")) - ret = main(['move', 'file.csv', 'other_name.csv']) + ret = main(["move", "file.csv", "other_name.csv"]) self.assertEqual(ret, 0) - self.assertFalse(os.path.exists('file.csv')) - self.assertFalse(os.path.exists('file.csv.dvc')) - self.assertTrue(os.path.exists('other_name.csv')) - self.assertTrue(os.path.exists('other_name.csv.dvc')) + self.assertFalse(os.path.exists("file.csv")) + self.assertFalse(os.path.exists("file.csv.dvc")) + self.assertTrue(os.path.exists("other_name.csv")) + self.assertTrue(os.path.exists("other_name.csv.dvc")) class TestMoveFileToDirectory(TestDvc): @@ -101,14 +101,14 @@ class TestMoveFileToDirectory(TestDvc): class TestMoveFileToDirectoryWithoutSpecifiedTargetName(TestDvc): def test(self): foo_stage_file_path = self.FOO + Stage.STAGE_FILE_SUFFIX - ret = main(['add', self.FOO]) + ret = main(["add", self.FOO]) self.assertEqual(ret, 0) self.assertTrue(os.path.exists(foo_stage_file_path)) target_foo_path = os.path.join(self.DATA_DIR, self.FOO) target_foo_stage_file_path = target_foo_path + Stage.STAGE_FILE_SUFFIX - ret = main(['move', self.FOO, self.DATA_DIR]) + ret = main(["move", self.FOO, self.DATA_DIR]) self.assertEqual(ret, 0) self.assertFalse(os.path.exists(self.FOO)) @@ -118,41 +118,39 @@ class TestMoveFileToDirectoryWithoutSpecifiedTargetName(TestDvc): self.assertTrue(os.path.exists(target_foo_stage_file_path)) new_stage = load_stage_file(target_foo_stage_file_path) - self.assertEqual(self.FOO, new_stage['outs'][0]['path']) + self.assertEqual(self.FOO, new_stage["outs"][0]["path"]) class TestMoveDirectoryShouldNotOverwriteExisting(TestDvc): def test(self): - new_dir_name = 'data_dir2' + new_dir_name = "data_dir2" os.mkdir(new_dir_name) - ret = main(['add', self.DATA_DIR]) + ret = main(["add", self.DATA_DIR]) self.assertEqual(ret, 0) - ret = main(['move', self.DATA_DIR, new_dir_name]) + ret = main(["move", self.DATA_DIR, new_dir_name]) self.assertEqual(ret, 0) self.assertFalse(os.path.exists(self.DATA_DIR)) - self.assertFalse( - os.path.exists(self.DATA_DIR + Stage.STAGE_FILE_SUFFIX)) + self.assertFalse(os.path.exists(self.DATA_DIR + Stage.STAGE_FILE_SUFFIX)) self.assertTrue(os.path.exists(new_dir_name)) self.assertTrue(os.path.isfile(new_dir_name + Stage.STAGE_FILE_SUFFIX)) - self.assertEqual(os.listdir(new_dir_name), - [self.DATA_DIR]) + self.assertEqual(os.listdir(new_dir_name), [self.DATA_DIR]) class TestMoveFileBetweenDirectories(TestDvc): def test(self): data_stage_file = self.DATA + Stage.STAGE_FILE_SUFFIX - ret = main(['add', self.DATA]) + ret = main(["add", self.DATA]) self.assertEqual(ret, 0) self.assertTrue(os.path.exists(data_stage_file)) - new_data_dir = 'data_dir2' + new_data_dir = "data_dir2" os.makedirs(new_data_dir) - ret = main(['move', self.DATA, new_data_dir]) + ret = main(["move", self.DATA, new_data_dir]) self.assertEqual(ret, 0) new_data_path = os.path.join(new_data_dir, os.path.basename(self.DATA)) @@ -165,5 +163,4 @@ class TestMoveFileBetweenDirectories(TestDvc): self.assertTrue(os.path.exists(new_data_stage_file)) new_stage_file = load_stage_file(new_data_stage_file) - self.assertEqual(os.path.basename(self.DATA), - new_stage_file['outs'][0]['path']) + self.assertEqual(os.path.basename(self.DATA), new_stage_file["outs"][0]["path"]) diff --git a/tests/test_output.py b/tests/test_output.py index e2bb7c9cc..9ecb9d0db 100644 --- a/tests/test_output.py +++ b/tests/test_output.py @@ -8,18 +8,18 @@ class TestOutScheme(TestDvc): MAP = OUTS_MAP TESTS = { - 's3://bucket/path': 's3', - 'gs://bucket/path': 'gs', - 'ssh://example.com:/dir/path': 'ssh', - 'hdfs://example.com/dir/path': 'hdfs', - 'path/to/file': 'local', - 'path\\to\\file': 'local', - 'file': 'local', - './file': 'local', - '.\\file': 'local', - '../file': 'local', - '..\\file': 'local', - 'unknown://path': 'local', + "s3://bucket/path": "s3", + "gs://bucket/path": "gs", + "ssh://example.com:/dir/path": "ssh", + "hdfs://example.com/dir/path": "hdfs", + "path/to/file": "local", + "path\\to\\file": "local", + "file": "local", + "./file": "local", + ".\\file": "local", + "../file": "local", + "..\\file": "local", + "unknown://path": "local", } def _get(self, path): diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 7350f7fce..a95182b95 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -8,184 +8,184 @@ import os class TestPipelineShowSingle(TestDvc): def setUp(self): super(TestPipelineShowSingle, self).setUp() - self.stage = 'foo.dvc' - self.dotFile = 'graph.dot' - ret = main(['add', self.FOO]) + self.stage = "foo.dvc" + self.dotFile = "graph.dot" + ret = main(["add", self.FOO]) self.assertEqual(ret, 0) def test(self): - ret = main(['pipeline', 'show', self.stage]) + ret = main(["pipeline", "show", self.stage]) self.assertEqual(ret, 0) def test_commands(self): - ret = main(['pipeline', 'show', self.stage, '--commands']) + ret = main(["pipeline", "show", self.stage, "--commands"]) self.assertEqual(ret, 0) def test_outs(self): - ret = main(['pipeline', 'show', self.stage, '--outs']) + ret = main(["pipeline", "show", self.stage, "--outs"]) self.assertEqual(ret, 0) def test_ascii(self): - ret = main(['pipeline', 'show', '--ascii', self.stage]) + ret = main(["pipeline", "show", "--ascii", self.stage]) self.assertEqual(ret, 0) def test_dot(self): - ret = main(['pipeline', 'show', '--dot', self.dotFile, self.stage]) + ret = main(["pipeline", "show", "--dot", self.dotFile, self.stage]) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.dotFile)) def test_ascii_commands(self): - ret = main(['pipeline', 'show', '--ascii', self.stage, '--commands']) + ret = main(["pipeline", "show", "--ascii", self.stage, "--commands"]) self.assertEqual(ret, 0) def test_ascii_outs(self): - ret = main(['pipeline', 'show', '--ascii', self.stage, '--outs']) + ret = main(["pipeline", "show", "--ascii", self.stage, "--outs"]) self.assertEqual(ret, 0) def test_dot_commands(self): - ret = main(['pipeline', 'show', '--dot', self.dotFile, self.stage, - '--commands']) + ret = main( + ["pipeline", "show", "--dot", self.dotFile, self.stage, "--commands"] + ) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.dotFile)) def test_dot_outs(self): - ret = main(['pipeline', 'show', '--dot', self.dotFile, self.stage, - '--outs']) + ret = main(["pipeline", "show", "--dot", self.dotFile, self.stage, "--outs"]) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.dotFile)) def test_not_dvc_file(self): - ret = main(['pipeline', 'show', self.FOO]) + ret = main(["pipeline", "show", self.FOO]) self.assertNotEqual(ret, 0) def test_non_existing(self): - ret = main(['pipeline', 'show', 'non-existing']) + ret = main(["pipeline", "show", "non-existing"]) self.assertNotEqual(ret, 0) class TestPipelineShow(TestRepro): def setUp(self): super(TestPipelineShow, self).setUp() - self.dotFile = 'graph.dot' + self.dotFile = "graph.dot" def test(self): - ret = main(['pipeline', 'show', self.file1_stage]) + ret = main(["pipeline", "show", self.file1_stage]) self.assertEqual(ret, 0) def test_commands(self): - ret = main(['pipeline', 'show', self.file1_stage, '--commands']) + ret = main(["pipeline", "show", self.file1_stage, "--commands"]) self.assertEqual(ret, 0) def test_outs(self): - ret = main(['pipeline', 'show', self.file1_stage, '--outs']) + ret = main(["pipeline", "show", self.file1_stage, "--outs"]) self.assertEqual(ret, 0) def test_ascii(self): - ret = main(['pipeline', 'show', '--ascii', self.file1_stage]) + ret = main(["pipeline", "show", "--ascii", self.file1_stage]) self.assertEqual(ret, 0) def test_dot(self): - ret = main(['pipeline', 'show', '--dot', self.dotFile, - self.file1_stage]) + ret = main(["pipeline", "show", "--dot", self.dotFile, self.file1_stage]) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.dotFile)) def test_ascii_commands(self): - ret = main(['pipeline', 'show', '--ascii', self.file1_stage, - '--commands']) + ret = main(["pipeline", "show", "--ascii", self.file1_stage, "--commands"]) self.assertEqual(ret, 0) def test_ascii_outs(self): - ret = main(['pipeline', 'show', '--ascii', self.file1_stage, '--outs']) + ret = main(["pipeline", "show", "--ascii", self.file1_stage, "--outs"]) self.assertEqual(ret, 0) def test_dot_commands(self): - ret = main(['pipeline', 'show', '--dot', self.dotFile, - self.file1_stage, '--commands']) + ret = main( + ["pipeline", "show", "--dot", self.dotFile, self.file1_stage, "--commands"] + ) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.dotFile)) def test_dot_outs(self): - ret = main(['pipeline', 'show', '--dot', self.dotFile, - self.file1_stage, '--outs']) + ret = main( + ["pipeline", "show", "--dot", self.dotFile, self.file1_stage, "--outs"] + ) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.dotFile)) def test_not_dvc_file(self): - ret = main(['pipeline', 'show', self.file1]) + ret = main(["pipeline", "show", self.file1]) self.assertNotEqual(ret, 0) def test_non_existing(self): - ret = main(['pipeline', 'show', 'non-existing']) + ret = main(["pipeline", "show", "non-existing"]) self.assertNotEqual(ret, 0) class TestPipelineShowDeep(TestReproChangedDeepData): def setUp(self): super(TestPipelineShowDeep, self).setUp() - self.dotFile = 'graph.dot' + self.dotFile = "graph.dot" def test(self): - ret = main(['pipeline', 'show', self.file1_stage]) + ret = main(["pipeline", "show", self.file1_stage]) self.assertEqual(ret, 0) def test_commands(self): - ret = main(['pipeline', 'show', self.file1_stage, '--commands']) + ret = main(["pipeline", "show", self.file1_stage, "--commands"]) self.assertEqual(ret, 0) def test_outs(self): - ret = main(['pipeline', 'show', self.file1_stage, '--outs']) + ret = main(["pipeline", "show", self.file1_stage, "--outs"]) self.assertEqual(ret, 0) def test_ascii(self): - ret = main(['pipeline', 'show', '--ascii', self.file1_stage]) + ret = main(["pipeline", "show", "--ascii", self.file1_stage]) self.assertEqual(ret, 0) def test_dot(self): - ret = main(['pipeline', 'show', '--dot', self.dotFile, - self.file1_stage]) + ret = main(["pipeline", "show", "--dot", self.dotFile, self.file1_stage]) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.dotFile)) def test_ascii_commands(self): - ret = main(['pipeline', 'show', '--ascii', self.file1_stage, - '--commands']) + ret = main(["pipeline", "show", "--ascii", self.file1_stage, "--commands"]) self.assertEqual(ret, 0) def test_ascii_outs(self): - ret = main(['pipeline', 'show', '--ascii', self.file1_stage, '--outs']) + ret = main(["pipeline", "show", "--ascii", self.file1_stage, "--outs"]) self.assertEqual(ret, 0) def test_dot_commands(self): - ret = main(['pipeline', 'show', '--dot', self.dotFile, - self.file1_stage, '--commands']) + ret = main( + ["pipeline", "show", "--dot", self.dotFile, self.file1_stage, "--commands"] + ) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.dotFile)) def test_dot_outs(self): - ret = main(['pipeline', 'show', '--dot', self.dotFile, - self.file1_stage, '--outs']) + ret = main( + ["pipeline", "show", "--dot", self.dotFile, self.file1_stage, "--outs"] + ) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(self.dotFile)) def test_not_dvc_file(self): - ret = main(['pipeline', 'show', self.file1]) + ret = main(["pipeline", "show", self.file1]) self.assertNotEqual(ret, 0) def test_non_existing(self): - ret = main(['pipeline', 'show', 'non-existing']) + ret = main(["pipeline", "show", "non-existing"]) self.assertNotEqual(ret, 0) class TestPipelineListEmpty(TestDvc): def test(self): - ret = main(['pipeline', 'list']) + ret = main(["pipeline", "list"]) self.assertEqual(ret, 0) class TestPipelineListSingle(TestPipelineShowDeep): def test(self): - ret = main(['pipeline', 'list']) + ret = main(["pipeline", "list"]) self.assertEqual(ret, 0) @@ -195,9 +195,9 @@ class TestProjectPipeline(TestDvc): self.assertEqual(len(pipelines), 0) def one_pipeline(self): - self.dvc.add('foo') - self.dvc.run(deps=['foo'], outs=['bar'], cmd='') - self.dvc.run(deps=['bar'], outs=['baz'], cmd='echo baz > baz') + self.dvc.add("foo") + self.dvc.run(deps=["foo"], outs=["bar"], cmd="") + self.dvc.run(deps=["bar"], outs=["baz"], cmd="echo baz > baz") pipelines = self.dvc.pipelines() self.assertEqual(len(pipelines), 1) @@ -205,11 +205,11 @@ class TestProjectPipeline(TestDvc): self.assertEqual(pipelines[0].edges, 2) def two_pipelines(self): - self.dvc.add('foo') - self.dvc.run(deps=['foo'], outs=['bar'], cmd='') - self.dvc.run(deps=['bar'], outs=['baz'], cmd='echo baz > baz') + self.dvc.add("foo") + self.dvc.run(deps=["foo"], outs=["bar"], cmd="") + self.dvc.run(deps=["bar"], outs=["baz"], cmd="echo baz > baz") - self.dvc.add('code.py') + self.dvc.add("code.py") pipelines = self.dvc.pipelines() @@ -220,8 +220,8 @@ class TestProjectPipeline(TestDvc): self.assertEqual(pipelines[1].edges, 0) def locked_stage(self): - self.dvc.add('foo') - self.dvc.lock_stage('foo.dvc') + self.dvc.add("foo") + self.dvc.lock_stage("foo.dvc") pipelines = self.dvc.pipelines() self.assertEqual(len(pipelines), 0) diff --git a/tests/test_remote.py b/tests/test_remote.py index 2b7e148ce..dea01f72f 100644 --- a/tests/test_remote.py +++ b/tests/test_remote.py @@ -10,30 +10,30 @@ from tests.basic_env import TestDvc class TestRemote(TestDvc): def test(self): - remotes = ['a', 'b', 'c'] + remotes = ["a", "b", "c"] - self.assertEqual(main(['remote', 'list']), 0) - self.assertNotEqual(main(['remote', 'remove', remotes[0]]), 0) + self.assertEqual(main(["remote", "list"]), 0) + self.assertNotEqual(main(["remote", "remove", remotes[0]]), 0) for r in remotes: - self.assertEqual(main(['remote', 'add', '--default', r, - 's3://bucket/name']), 0) + self.assertEqual( + main(["remote", "add", "--default", r, "s3://bucket/name"]), 0 + ) - self.assertEqual(main(['remote', 'list']), 0) + self.assertEqual(main(["remote", "list"]), 0) - self.assertEqual(main(['remote', 'remove', remotes[0]]), 0) - self.assertEqual(main(['remote', 'modify', remotes[0], - 'option', 'value']), 0) + self.assertEqual(main(["remote", "remove", remotes[0]]), 0) + self.assertEqual(main(["remote", "modify", remotes[0], "option", "value"]), 0) - self.assertEqual(main(['remote', 'list']), 0) + self.assertEqual(main(["remote", "list"]), 0) def test_failed_write(self): class A(object): system = False glob = False local = False - name = 'myremote' - url = 's3://remote' + name = "myremote" + url = "s3://remote" unset = False args = A() @@ -44,40 +44,42 @@ class TestRemote(TestDvc): self.assertNotEqual(ret, 0) def test_relative_path(self): - dname = os.path.join('..', 'path', 'to', 'dir') - ret = main(['remote', 'add', 'mylocal', dname]) + dname = os.path.join("..", "path", "to", "dir") + ret = main(["remote", "add", "mylocal", dname]) self.assertEqual(ret, 0) # NOTE: we are in the project's root and config is in .dvc/, so # dir path written to config should be just one level above. - rel = os.path.join('..', dname) + rel = os.path.join("..", dname) config = configobj.ConfigObj(self.dvc.config.config_file) - self.assertEqual(config['remote "mylocal"']['url'], rel) + self.assertEqual(config['remote "mylocal"']["url"], rel) class TestRemoteRemoveDefault(TestDvc): def test(self): - remote = 'mys3' - self.assertEqual(main(['remote', 'add', '--default', remote, - 's3://bucket/name']), 0) - self.assertEqual(main(['remote', 'modify', remote, 'profile', - 'default']), 0) - self.assertEqual(main(['config', '--local', 'core.remote', remote]), 0) + remote = "mys3" + self.assertEqual( + main(["remote", "add", "--default", remote, "s3://bucket/name"]), 0 + ) + self.assertEqual(main(["remote", "modify", remote, "profile", "default"]), 0) + self.assertEqual(main(["config", "--local", "core.remote", remote]), 0) - config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, - Config.CONFIG)) - local_config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, - Config.CONFIG_LOCAL)) - self.assertEqual(config[Config.SECTION_CORE] - [Config.SECTION_CORE_REMOTE], remote) - self.assertEqual(local_config[Config.SECTION_CORE] - [Config.SECTION_CORE_REMOTE], remote) + config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, Config.CONFIG)) + local_config = configobj.ConfigObj( + os.path.join(self.dvc.dvc_dir, Config.CONFIG_LOCAL) + ) + self.assertEqual( + config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE], remote + ) + self.assertEqual( + local_config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE], remote + ) - self.assertEqual(main(['remote', 'remove', remote]), 0) - config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, - Config.CONFIG)) - local_config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, - Config.CONFIG_LOCAL)) + self.assertEqual(main(["remote", "remove", remote]), 0) + config = configobj.ConfigObj(os.path.join(self.dvc.dvc_dir, Config.CONFIG)) + local_config = configobj.ConfigObj( + os.path.join(self.dvc.dvc_dir, Config.CONFIG_LOCAL) + ) section = Config.SECTION_REMOTE_FMT.format(remote) self.assertTrue(section not in config.keys()) @@ -92,31 +94,31 @@ class TestRemoteRemoveDefault(TestDvc): class TestRemoteRemove(TestDvc): def test(self): - ret = main(['config', 'core.jobs', '1']) + ret = main(["config", "core.jobs", "1"]) self.assertEqual(ret, 0) - remote = 'mys3' - ret = main(['remote', 'add', remote, 's3://bucket/name']) + remote = "mys3" + ret = main(["remote", "add", remote, "s3://bucket/name"]) self.assertEqual(ret, 0) - ret = main(['remote', 'remove', remote]) + ret = main(["remote", "remove", remote]) self.assertEqual(ret, 0) class TestRemoteDefault(TestDvc): def test(self): - remote = 'mys3' - ret = main(['remote', 'add', 'mys3', 's3://bucket/path']) + remote = "mys3" + ret = main(["remote", "add", "mys3", "s3://bucket/path"]) self.assertEqual(ret, 0) - ret = main(['remote', 'default', 'mys3']) + ret = main(["remote", "default", "mys3"]) self.assertEqual(ret, 0) config_file = os.path.join(self.dvc.dvc_dir, Config.CONFIG) config = configobj.ConfigObj(config_file) default = config[Config.SECTION_CORE][Config.SECTION_CORE_REMOTE] self.assertEqual(default, remote) - ret = main(['remote', 'default', '--unset']) + ret = main(["remote", "default", "--unset"]) self.assertEqual(ret, 0) config = configobj.ConfigObj(config_file) core = config.get(Config.SECTION_CORE) diff --git a/tests/test_remove.py b/tests/test_remove.py index 7e2b7bf15..d0511e1e6 100644 --- a/tests/test_remove.py +++ b/tests/test_remove.py @@ -30,7 +30,7 @@ class TestRemove(TestDvc): class TestRemoveNonExistentFile(TestDvc): def test(self): with self.assertRaises(StageFileDoesNotExistError): - self.dvc.remove('non_existent_dvc_file') + self.dvc.remove("non_existent_dvc_file") class TestRemoveDirectory(TestDvc): @@ -51,28 +51,26 @@ class TestCmdRemove(TestDvc): self.assertEqual(len(stages), 1) stage = stages[0] self.assertTrue(stage is not None) - ret = main(['remove', - stage.path]) + ret = main(["remove", stage.path]) self.assertEqual(ret, 0) - ret = main(['remove', - 'non-existing-dvc-file']) + ret = main(["remove", "non-existing-dvc-file"]) self.assertNotEqual(ret, 0) class TestRemovePurge(TestDvc): def test(self): dvcfile = self.dvc.add(self.FOO)[0].path - ret = main(['remove', '--purge', '--force', dvcfile]) + ret = main(["remove", "--purge", "--force", dvcfile]) self.assertEqual(ret, 0) self.assertFalse(os.path.exists(self.FOO)) self.assertFalse(os.path.exists(dvcfile)) - @patch('dvc.prompt.confirm', return_value=False) + @patch("dvc.prompt.confirm", return_value=False) def test_force(self, mock_prompt): dvcfile = self.dvc.add(self.FOO)[0].path - ret = main(['remove', '--purge', dvcfile]) + ret = main(["remove", "--purge", dvcfile]) mock_prompt.assert_called() self.assertEqual(ret, 1) diff --git a/tests/test_repro.py b/tests/test_repro.py index ff6327116..c512d91de 100644 --- a/tests/test_repro.py +++ b/tests/test_repro.py @@ -38,45 +38,41 @@ class TestRepro(TestDvc): self.foo_stage = stages[0] self.assertTrue(self.foo_stage is not None) - self.file1 = 'file1' - self.file1_stage = self.file1 + '.dvc' - self.dvc.run(fname=self.file1_stage, - outs=[self.file1], - deps=[self.FOO, self.CODE], - cmd='python {} {} {}'.format(self.CODE, - self.FOO, - self.file1)) + self.file1 = "file1" + self.file1_stage = self.file1 + ".dvc" + self.dvc.run( + fname=self.file1_stage, + outs=[self.file1], + deps=[self.FOO, self.CODE], + cmd="python {} {} {}".format(self.CODE, self.FOO, self.file1), + ) class TestReproFail(TestRepro): def test(self): os.unlink(self.CODE) - ret = main(['repro', self.file1_stage]) + ret = main(["repro", self.file1_stage]) self.assertNotEqual(ret, 0) class TestReproCyclicGraph(TestDvc): def test(self): - self.dvc.run(deps=[self.FOO], - outs=['bar.txt'], - cmd='echo bar > bar.txt') + self.dvc.run(deps=[self.FOO], outs=["bar.txt"], cmd="echo bar > bar.txt") - self.dvc.run(deps=['bar.txt'], - outs=['baz.txt'], - cmd='echo baz > baz.txt') + self.dvc.run(deps=["bar.txt"], outs=["baz.txt"], cmd="echo baz > baz.txt") - with open('cycle.dvc', 'w') as fd: + with open("cycle.dvc", "w") as fd: stage_dump = { - 'cmd': 'echo baz > foo', - 'deps': [{'path': 'baz.txt'}], - 'outs': [{'path': self.FOO}], + "cmd": "echo baz > foo", + "deps": [{"path": "baz.txt"}], + "outs": [{"path": self.FOO}], } yaml.safe_dump(stage_dump, fd, default_flow_style=False) with self.assertRaises(CyclicGraphError): - self.dvc.reproduce('cycle.dvc') + self.dvc.reproduce("cycle.dvc") class TestReproWorkingDirectoryAsOutput(TestDvc): @@ -88,6 +84,7 @@ class TestReproWorkingDirectoryAsOutput(TestDvc): | dir | dir-1 | False | | . | something | False | """ + def test(self): # File structure: # . @@ -96,20 +93,20 @@ class TestReproWorkingDirectoryAsOutput(TestDvc): # |__ dir2 # |__ something.dvc (stage.cwd == ./dir2) - os.mkdir(os.path.join(self.dvc.root_dir, 'dir1')) + os.mkdir(os.path.join(self.dvc.root_dir, "dir1")) self.dvc.run( - cwd='dir1', - outs=['../dir2'], - cmd='mkdir {path}'.format(path=os.path.join('..', 'dir2')) + cwd="dir1", + outs=["../dir2"], + cmd="mkdir {path}".format(path=os.path.join("..", "dir2")), ) - faulty_stage_path = os.path.join('dir2', 'something.dvc') + faulty_stage_path = os.path.join("dir2", "something.dvc") - with open(faulty_stage_path, 'w') as fd: + with open(faulty_stage_path, "w") as fd: stage_dump = { - 'cmd': 'echo something > something', - 'outs': [{'path': 'something'}], + "cmd": "echo something > something", + "outs": [{"path": "something"}], } yaml.safe_dump(stage_dump, fd, default_flow_style=False) @@ -119,6 +116,7 @@ class TestReproWorkingDirectoryAsOutput(TestDvc): def test_nested(self): from dvc.stage import Stage + # # . # |-- a @@ -127,39 +125,39 @@ class TestReproWorkingDirectoryAsOutput(TestDvc): # | |__ error.dvc (stage.cwd == 'a/nested/dir') # |__ b # |__ nested.dvc (stage.out == 'a/nested') - dir1 = 'b' - dir2 = 'a' + dir1 = "b" + dir2 = "a" os.mkdir(dir1) os.mkdir(dir2) - nested_dir = os.path.join(dir2, 'nested') + nested_dir = os.path.join(dir2, "nested") out_dir = os.path.relpath(nested_dir, dir1) nested_stage = self.dvc.run( - cwd=dir1, # b - outs=[out_dir], # ../a/nested - cmd='mkdir {path}'.format(path=out_dir) + cwd=dir1, # b + outs=[out_dir], # ../a/nested + cmd="mkdir {path}".format(path=out_dir), ) - os.mkdir(os.path.join(nested_dir, 'dir')) + os.mkdir(os.path.join(nested_dir, "dir")) - error_stage_path = os.path.join(nested_dir, 'dir', 'error.dvc') + error_stage_path = os.path.join(nested_dir, "dir", "error.dvc") - with open(error_stage_path, 'w') as fd: + with open(error_stage_path, "w") as fd: stage_dump = { - 'cmd': 'echo something > something', - 'outs': [{'path': 'something'}], + "cmd": "echo something > something", + "outs": [{"path": "something"}], } yaml.safe_dump(stage_dump, fd, default_flow_style=False) # NOTE: os.walk() walks in a sorted order and we need dir2 subdirs to # be processed before dir1 to load error.dvc first. - with patch.object(Project, 'stages') as mock_stages: + with patch.object(Project, "stages") as mock_stages: mock_stages.return_value = [ nested_stage, - Stage.load(self.dvc, error_stage_path) + Stage.load(self.dvc, error_stage_path), ] with self.assertRaises(WorkingDirectoryAsOutputError): @@ -175,24 +173,21 @@ class TestReproWorkingDirectoryAsOutput(TestDvc): # |-- a # |__ a.dvc (stage.cwd == something-1) - self.dvc.run(outs=['something'], cmd='mkdir something') + self.dvc.run(outs=["something"], cmd="mkdir something") - os.mkdir('something-1') + os.mkdir("something-1") - stage = os.path.join('something-1', 'a.dvc') + stage = os.path.join("something-1", "a.dvc") - with open(stage, 'w') as fd: - stage_dump = { - 'cmd': 'echo a > a', - 'outs': [{'path': 'a'}], - } + with open(stage, "w") as fd: + stage_dump = {"cmd": "echo a > a", "outs": [{"path": "a"}]} yaml.safe_dump(stage_dump, fd, default_flow_style=False) try: self.dvc.reproduce(stage) except WorkingDirectoryAsOutputError: - self.fail('should not raise WorkingDirectoryAsOutputError') + self.fail("should not raise WorkingDirectoryAsOutputError") class TestReproDepUnderDir(TestDvc): @@ -202,14 +197,14 @@ class TestReproDepUnderDir(TestDvc): self.dir_stage = stages[0] self.assertTrue(self.dir_stage is not None) - self.file1 = 'file1' - self.file1_stage = self.file1 + '.dvc' - self.dvc.run(fname=self.file1_stage, - outs=[self.file1], - deps=[self.DATA, self.CODE], - cmd='python {} {} {}'.format(self.CODE, - self.DATA, - self.file1)) + self.file1 = "file1" + self.file1_stage = self.file1 + ".dvc" + self.dvc.run( + fname=self.file1_stage, + outs=[self.file1], + deps=[self.DATA, self.CODE], + cmd="python {} {} {}".format(self.CODE, self.DATA, self.file1), + ) self.assertTrue(filecmp.cmp(self.file1, self.DATA, shallow=False)) @@ -231,18 +226,17 @@ class TestReproDepDirWithOutputsUnderIt(TestDvc): self.assertEqual(len(stages), 1) self.assertTrue(stages[0] is not None) - stage = self.dvc.run(fname='Dvcfile', - deps=[self.DATA, self.DATA_SUB]) + stage = self.dvc.run(fname="Dvcfile", deps=[self.DATA, self.DATA_SUB]) self.assertTrue(stage is not None) - file1 = 'file1' - file1_stage = file1 + '.dvc' - stage = self.dvc.run(fname=file1_stage, - deps=[self.DATA_DIR], - outs=[file1], - cmd='python {} {} {}'.format(self.CODE, - self.DATA, - file1)) + file1 = "file1" + file1_stage = file1 + ".dvc" + stage = self.dvc.run( + fname=file1_stage, + deps=[self.DATA_DIR], + outs=[file1], + cmd="python {} {} {}".format(self.CODE, self.DATA, file1), + ) self.assertTrue(stage is not None) os.unlink(self.DATA) @@ -254,16 +248,16 @@ class TestReproDepDirWithOutputsUnderIt(TestDvc): class TestReproNoDeps(TestRepro): def test(self): - out = 'out' - code_file = 'out.py' - stage_file = 'out.dvc' - code = 'import uuid\nwith open("{}", "w+") as fd:\n' \ - '\tfd.write(str(uuid.uuid4()))\n'.format(out) - with open(code_file, 'w+') as fd: + out = "out" + code_file = "out.py" + stage_file = "out.dvc" + code = ( + 'import uuid\nwith open("{}", "w+") as fd:\n' + "\tfd.write(str(uuid.uuid4()))\n".format(out) + ) + with open(code_file, "w+") as fd: fd.write(code) - self.dvc.run(fname=stage_file, - outs=[out], - cmd='python {}'.format(code_file)) + self.dvc.run(fname=stage_file, outs=[out], cmd="python {}".format(code_file)) stages = self.dvc.reproduce(stage_file) self.assertEqual(len(stages), 1) @@ -287,8 +281,7 @@ class TestReproChangedCode(TestRepro): def swap_code(self): os.unlink(self.CODE) new_contents = self.CODE_CONTENTS - new_contents += "\nshutil.copyfile('{}', " \ - "sys.argv[2])\n".format(self.BAR) + new_contents += "\nshutil.copyfile('{}', " "sys.argv[2])\n".format(self.BAR) self.create(self.CODE, new_contents) @@ -315,14 +308,14 @@ class TestReproDry(TestReproChangedData): self.assertTrue(len(stages), 2) self.assertFalse(filecmp.cmp(self.file1, self.BAR, shallow=False)) - ret = main(['repro', '--dry', self.file1_stage]) + ret = main(["repro", "--dry", self.file1_stage]) self.assertEqual(ret, 0) self.assertFalse(filecmp.cmp(self.file1, self.BAR, shallow=False)) class TestReproUpToDate(TestRepro): def test(self): - ret = main(['repro', self.file1_stage]) + ret = main(["repro", self.file1_stage]) self.assertEqual(ret, 0) @@ -330,30 +323,36 @@ class TestReproDryNoExec(TestDvc): def test(self): deps = [] for d in range(3): - idir = 'idir{}'.format(d) - odir = 'odir{}'.format(d) + idir = "idir{}".format(d) + odir = "odir{}".format(d) - deps.append('-d') + deps.append("-d") deps.append(odir) os.mkdir(idir) - f = os.path.join(idir, 'file') - with open(f, 'w+') as fobj: + f = os.path.join(idir, "file") + with open(f, "w+") as fobj: fobj.write(str(d)) - ret = main(['run', - '--no-exec', - '-d', idir, - '-o', odir, - "python -c 'import shutil; " - "shutil.copytree(\"{}\", \"{}\")'".format(idir, odir)]) + ret = main( + [ + "run", + "--no-exec", + "-d", + idir, + "-o", + odir, + "python -c 'import shutil; " + 'shutil.copytree("{}", "{}")\''.format(idir, odir), + ] + ) self.assertEqual(ret, 0) - ret = main(['run', '--no-exec', '-f', 'Dvcfile'] + deps) + ret = main(["run", "--no-exec", "-f", "Dvcfile"] + deps) self.assertEqual(ret, 0) - ret = main(['repro', '--dry']) + ret = main(["repro", "--dry"]) self.assertEqual(ret, 0) @@ -361,14 +360,14 @@ class TestReproChangedDeepData(TestReproChangedData): def setUp(self): super(TestReproChangedDeepData, self).setUp() - self.file2 = 'file2' - self.file2_stage = self.file2 + '.dvc' - self.dvc.run(fname=self.file2_stage, - outs=[self.file2], - deps=[self.file1, self.CODE], - cmd='python {} {} {}'.format(self.CODE, - self.file1, - self.file2)) + self.file2 = "file2" + self.file2_stage = self.file2 + ".dvc" + self.dvc.run( + fname=self.file2_stage, + outs=[self.file2], + deps=[self.file1, self.CODE], + cmd="python {} {} {}".format(self.CODE, self.file1, self.file2), + ) def test(self): self.swap_foo_with_bar() @@ -387,32 +386,32 @@ class TestReproIgnoreBuildCache(TestDvc): foo_stage = stages[0] self.assertTrue(foo_stage is not None) - code1 = 'code1.py' + code1 = "code1.py" shutil.copyfile(self.CODE, code1) - file1 = 'file1' - file1_stage = file1 + '.dvc' - stage = self.dvc.run(fname=file1_stage, - outs=[file1], - deps=[self.FOO, code1], - cmd='python {} {} {}'.format(code1, - self.FOO, - file1)) + file1 = "file1" + file1_stage = file1 + ".dvc" + stage = self.dvc.run( + fname=file1_stage, + outs=[file1], + deps=[self.FOO, code1], + cmd="python {} {} {}".format(code1, self.FOO, file1), + ) self.assertTrue(stage is not None) - code2 = 'code2.py' + code2 = "code2.py" shutil.copyfile(self.CODE, code2) - file2 = 'file2' - file2_stage = file2 + '.dvc' - stage = self.dvc.run(fname=file2_stage, - outs=[file2], - deps=[file1, code2], - cmd='python {} {} {}'.format(code2, - file1, - file2)) + file2 = "file2" + file2_stage = file2 + ".dvc" + stage = self.dvc.run( + fname=file2_stage, + outs=[file2], + deps=[file1, code2], + cmd="python {} {} {}".format(code2, file1, file2), + ) self.assertTrue(stage is not None) - with open(code1, 'a') as fobj: - fobj.write('\n\n') + with open(code1, "a") as fobj: + fobj.write("\n\n") stages = self.dvc.reproduce(file2_stage) self.assertEqual(len(stages), 1) @@ -423,13 +422,11 @@ class TestReproIgnoreBuildCache(TestDvc): class TestReproPipeline(TestReproChangedDeepData): def test(self): - stages = self.dvc.reproduce(self.file1_stage, - force=True, - pipeline=True) + stages = self.dvc.reproduce(self.file1_stage, force=True, pipeline=True) self.assertEqual(len(stages), 3) def test_cli(self): - ret = main(['repro', '--pipeline', '-f', self.file1_stage]) + ret = main(["repro", "--pipeline", "-f", self.file1_stage]) self.assertEqual(ret, 0) @@ -447,23 +444,23 @@ class TestReproPipelines(TestDvc): self.bar_stage = stages[0] self.assertTrue(self.bar_stage is not None) - self.file1 = 'file1' - self.file1_stage = self.file1 + '.dvc' - self.dvc.run(fname=self.file1_stage, - outs=[self.file1], - deps=[self.FOO, self.CODE], - cmd='python {} {} {}'.format(self.CODE, - self.FOO, - self.file1)) + self.file1 = "file1" + self.file1_stage = self.file1 + ".dvc" + self.dvc.run( + fname=self.file1_stage, + outs=[self.file1], + deps=[self.FOO, self.CODE], + cmd="python {} {} {}".format(self.CODE, self.FOO, self.file1), + ) - self.file2 = 'file2' - self.file2_stage = self.file2 + '.dvc' - self.dvc.run(fname=self.file2_stage, - outs=[self.file2], - deps=[self.BAR, self.CODE], - cmd='python {} {} {}'.format(self.CODE, - self.BAR, - self.file2)) + self.file2 = "file2" + self.file2_stage = self.file2 + ".dvc" + self.dvc.run( + fname=self.file2_stage, + outs=[self.file2], + deps=[self.BAR, self.CODE], + cmd="python {} {} {}".format(self.CODE, self.BAR, self.file2), + ) def test(self): stages = self.dvc.reproduce(all_pipelines=True, force=True) @@ -475,29 +472,29 @@ class TestReproPipelines(TestDvc): self.assertTrue(self.file2_stage in names) def test_cli(self): - ret = main(['repro', '-f', '-P']) + ret = main(["repro", "-f", "-P"]) self.assertEqual(ret, 0) class TestReproLocked(TestReproChangedData): def test(self): - file2 = 'file2' - file2_stage = file2 + '.dvc' - self.dvc.run(fname=file2_stage, - outs=[file2], - deps=[self.file1, self.CODE], - cmd='python {} {} {}'.format(self.CODE, - self.file1, - file2)) + file2 = "file2" + file2_stage = file2 + ".dvc" + self.dvc.run( + fname=file2_stage, + outs=[file2], + deps=[self.file1, self.CODE], + cmd="python {} {} {}".format(self.CODE, self.file1, file2), + ) self.swap_foo_with_bar() - ret = main(['lock', file2_stage]) + ret = main(["lock", file2_stage]) self.assertEqual(ret, 0) stages = self.dvc.reproduce(file2_stage) self.assertEqual(len(stages), 0) - ret = main(['unlock', file2_stage]) + ret = main(["unlock", file2_stage]) self.assertEqual(ret, 0) stages = self.dvc.reproduce(file2_stage) self.assertTrue(filecmp.cmp(self.file1, self.BAR, shallow=False)) @@ -506,23 +503,23 @@ class TestReproLocked(TestReproChangedData): def test_non_existing(self): with self.assertRaises(StageFileDoesNotExistError): - self.dvc.lock_stage('non-existing-stage') + self.dvc.lock_stage("non-existing-stage") - ret = main(['lock', 'non-existing-stage']) + ret = main(["lock", "non-existing-stage"]) self.assertNotEqual(ret, 0) class TestReproLockedCallback(TestDvc): def test(self): - file1 = 'file1' - file1_stage = file1 + '.dvc' + file1 = "file1" + file1_stage = file1 + ".dvc" # NOTE: purposefully not specifying dependencies # to create a callbacs stage. - stage = self.dvc.run(fname=file1_stage, - outs=[file1], - cmd='python {} {} {}'.format(self.CODE, - self.FOO, - file1)) + stage = self.dvc.run( + fname=file1_stage, + outs=[file1], + cmd="python {} {} {}".format(self.CODE, self.FOO, file1), + ) self.assertTrue(stage is not None) self.assertEqual(stage.relpath, file1_stage) @@ -561,12 +558,14 @@ class TestReproMetricsAddUnchanged(TestDvc): self.assertEqual(len(stages), 1) self.assertTrue(stages[0] is not None) - file1 = 'file1' - file1_stage = file1 + '.dvc' - self.dvc.run(fname=file1_stage, - outs_no_cache=[file1], - deps=[self.FOO, self.CODE], - cmd='python {} {} {}'.format(self.CODE, self.FOO, file1)) + file1 = "file1" + file1_stage = file1 + ".dvc" + self.dvc.run( + fname=file1_stage, + outs_no_cache=[file1], + deps=[self.FOO, self.CODE], + cmd="python {} {} {}".format(self.CODE, self.FOO, file1), + ) stages = self.dvc.reproduce(file1_stage) self.assertEqual(len(stages), 0) @@ -611,22 +610,26 @@ class TestReproDataSource(TestReproChangedData): class TestReproChangedDir(TestDvc): def test(self): - file_name = 'file' + file_name = "file" shutil.copyfile(self.FOO, file_name) - stage_name = 'dir.dvc' - dir_name = 'dir' - dir_code = 'dir.py' - code = "import os; import shutil; os.mkdir(\"{}\"); " \ - "shutil.copyfile(\"{}\", os.path.join(\"{}\", \"{}\"))" + stage_name = "dir.dvc" + dir_name = "dir" + dir_code = "dir.py" + code = ( + 'import os; import shutil; os.mkdir("{}"); ' + 'shutil.copyfile("{}", os.path.join("{}", "{}"))' + ) - with open(dir_code, 'w+') as fd: + with open(dir_code, "w+") as fd: fd.write(code.format(dir_name, file_name, dir_name, file_name)) - self.dvc.run(fname=stage_name, - outs=[dir_name], - deps=[file_name, dir_code], - cmd="python {}".format(dir_code)) + self.dvc.run( + fname=stage_name, + outs=[dir_name], + deps=[file_name, dir_code], + cmd="python {}".format(dir_code), + ) stages = self.dvc.reproduce(stage_name) self.assertEqual(len(stages), 0) @@ -640,25 +643,27 @@ class TestReproChangedDir(TestDvc): class TestReproChangedDirData(TestDvc): def test(self): - dir_name = 'dir' - dir_code = 'dir_code.py' + dir_name = "dir" + dir_code = "dir_code.py" - with open(dir_code, 'w+') as fd: - fd.write("import os; import sys; import shutil; " - "shutil.copytree(sys.argv[1], sys.argv[2])") + with open(dir_code, "w+") as fd: + fd.write( + "import os; import sys; import shutil; " + "shutil.copytree(sys.argv[1], sys.argv[2])" + ) - stage = self.dvc.run(outs=[dir_name], - deps=[self.DATA_DIR, dir_code], - cmd="python {} {} {}".format(dir_code, - self.DATA_DIR, - dir_name)) + stage = self.dvc.run( + outs=[dir_name], + deps=[self.DATA_DIR, dir_code], + cmd="python {} {} {}".format(dir_code, self.DATA_DIR, dir_name), + ) self.assertTrue(stage is not None) stages = self.dvc.reproduce(stage.path) self.assertEqual(len(stages), 0) - with open(self.DATA_SUB, 'a') as fd: - fd.write('add') + with open(self.DATA_SUB, "a") as fd: + fd.write("add") stages = self.dvc.reproduce(stage.path) self.assertEqual(len(stages), 1) @@ -671,7 +676,7 @@ class TestReproChangedDirData(TestDvc): self.assertTrue(stages[0] is not None) # Check that dvc registers mtime change for the directory. - System.hardlink(self.DATA_SUB, self.DATA_SUB + '.lnk') + System.hardlink(self.DATA_SUB, self.DATA_SUB + ".lnk") stages = self.dvc.reproduce(stage.path) self.assertEqual(len(stages), 1) self.assertTrue(stages[0] is not None) @@ -679,13 +684,13 @@ class TestReproChangedDirData(TestDvc): class TestReproMissingMd5InStageFile(TestRepro): def test(self): - with open(self.file1_stage, 'r') as fd: + with open(self.file1_stage, "r") as fd: d = yaml.load(fd) - del(d[Stage.PARAM_OUTS][0][RemoteLOCAL.PARAM_CHECKSUM]) - del(d[Stage.PARAM_DEPS][0][RemoteLOCAL.PARAM_CHECKSUM]) + del (d[Stage.PARAM_OUTS][0][RemoteLOCAL.PARAM_CHECKSUM]) + del (d[Stage.PARAM_DEPS][0][RemoteLOCAL.PARAM_CHECKSUM]) - with open(self.file1_stage, 'w') as fd: + with open(self.file1_stage, "w") as fd: yaml.dump(d, fd) stages = self.dvc.reproduce(self.file1_stage) @@ -696,21 +701,19 @@ class TestCmdRepro(TestReproChangedData): def test(self): self.swap_foo_with_bar() - ret = main(['status']) + ret = main(["status"]) self.assertEqual(ret, 0) - ret = main(['repro', - self.file1_stage]) + ret = main(["repro", self.file1_stage]) self.assertEqual(ret, 0) - ret = main(['repro', - 'non-existing-file']) + ret = main(["repro", "non-existing-file"]) self.assertNotEqual(ret, 0) class TestCmdReproChdir(TestDvc): def test(self): - dname = 'dir' + dname = "dir" os.mkdir(dname) foo = os.path.join(dname, self.FOO) bar = os.path.join(dname, self.BAR) @@ -718,12 +721,20 @@ class TestCmdReproChdir(TestDvc): shutil.copyfile(self.FOO, foo) shutil.copyfile(self.CODE, code) - ret = main(['run', - '-f', 'Dvcfile', - '-c', dname, - '-d', self.FOO, - '-o', self.BAR, - 'python {} {} {}'.format(self.CODE, self.FOO, self.BAR)]) + ret = main( + [ + "run", + "-f", + "Dvcfile", + "-c", + dname, + "-d", + self.FOO, + "-o", + self.BAR, + "python {} {} {}".format(self.CODE, self.FOO, self.BAR), + ] + ) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(foo)) self.assertTrue(os.path.isfile(bar)) @@ -731,8 +742,7 @@ class TestCmdReproChdir(TestDvc): os.unlink(bar) - ret = main(['repro', - '-c', dname]) + ret = main(["repro", "-c", dname]) self.assertEqual(ret, 0) self.assertTrue(os.path.isfile(foo)) self.assertTrue(os.path.isfile(bar)) @@ -753,29 +763,26 @@ class TestReproExternalBase(TestDvc): @property def scheme_sep(self): - return '://' + return "://" @property def sep(self): - return '/' + return "/" def check_already_cached(self, stage): stage.outs[0].remove() - patch_download = patch.object(stage.deps[0], - 'download', - wraps=stage.deps[0].download) + patch_download = patch.object( + stage.deps[0], "download", wraps=stage.deps[0].download + ) - patch_checkout = patch.object(stage.outs[0], - 'checkout', - wraps=stage.outs[0].checkout) + patch_checkout = patch.object( + stage.outs[0], "checkout", wraps=stage.outs[0].checkout + ) - patch_run = patch.object(stage, '_run', wraps=stage._run) + patch_run = patch.object(stage, "_run", wraps=stage._run) - with self.dvc.state, \ - patch_download as mock_download, \ - patch_checkout as mock_checkout, \ - patch_run as mock_run: + with self.dvc.state, patch_download as mock_download, patch_checkout as mock_checkout, patch_run as mock_run: stage.run() @@ -784,81 +791,74 @@ class TestReproExternalBase(TestDvc): mock_checkout.assert_called_once() def corrupted_cache(self): - os.unlink('bar.dvc') + os.unlink("bar.dvc") - stage = self.dvc.run(deps=[self.FOO], - outs=[self.BAR], - cmd='echo bar > bar') + stage = self.dvc.run(deps=[self.FOO], outs=[self.BAR], cmd="echo bar > bar") - with open(self.BAR, 'w') as fd: - fd.write('corrupting the cache') + with open(self.BAR, "w") as fd: + fd.write("corrupting the cache") - patch_checkout = patch.object(stage.outs[0], - 'checkout', - wraps=stage.outs[0].checkout) + patch_checkout = patch.object( + stage.outs[0], "checkout", wraps=stage.outs[0].checkout + ) - patch_run = patch.object(stage, '_run', wraps=stage._run) + patch_run = patch.object(stage, "_run", wraps=stage._run) - with self.dvc.state, \ - patch_checkout as mock_checkout, \ - patch_run as mock_run: + with self.dvc.state, patch_checkout as mock_checkout, patch_run as mock_run: stage.run() mock_run.assert_called_once() mock_checkout.assert_not_called() - @patch('dvc.prompt.confirm', return_value=True) + @patch("dvc.prompt.confirm", return_value=True) def test(self, mock_prompt): if not self.should_test(): return - cache = (self.scheme + self.scheme_sep + self.bucket + self.sep - + str(uuid.uuid4())) + cache = ( + self.scheme + self.scheme_sep + self.bucket + self.sep + str(uuid.uuid4()) + ) - ret = main(['config', 'cache.' + self.cache_scheme, 'myrepo']) + ret = main(["config", "cache." + self.cache_scheme, "myrepo"]) self.assertEqual(ret, 0) - ret = main(['remote', 'add', 'myrepo', cache]) + ret = main(["remote", "add", "myrepo", cache]) self.assertEqual(ret, 0) - remote_name = 'myremote' + remote_name = "myremote" remote_key = str(uuid.uuid4()) - remote = (self.scheme + self.scheme_sep + self.bucket + self.sep - + remote_key) + remote = self.scheme + self.scheme_sep + self.bucket + self.sep + remote_key - ret = main(['remote', 'add', remote_name, remote]) + ret = main(["remote", "add", remote_name, remote]) self.assertEqual(ret, 0) - self.dvc = Project('.') + self.dvc = Project(".") foo_key = remote_key + self.sep + self.FOO bar_key = remote_key + self.sep + self.BAR - foo_path = (self.scheme + self.scheme_sep + self.bucket + self.sep - + foo_key) - bar_path = (self.scheme + self.scheme_sep + self.bucket + self.sep - + bar_key) + foo_path = self.scheme + self.scheme_sep + self.bucket + self.sep + foo_key + bar_path = self.scheme + self.scheme_sep + self.bucket + self.sep + bar_key # Using both plain and remote notation - out_foo_path = 'remote://' + remote_name + '/' + self.FOO + out_foo_path = "remote://" + remote_name + "/" + self.FOO out_bar_path = bar_path self.write(self.bucket, foo_key, self.FOO_CONTENTS) - import_stage = self.dvc.imp(out_foo_path, 'import') + import_stage = self.dvc.imp(out_foo_path, "import") - self.assertTrue(os.path.exists('import')) - self.assertTrue(filecmp.cmp('import', self.FOO, shallow=False)) + self.assertTrue(os.path.exists("import")) + self.assertTrue(filecmp.cmp("import", self.FOO, shallow=False)) self.assertEqual(self.dvc.status(import_stage.path), {}) self.check_already_cached(import_stage) - import_remote_stage = self.dvc.imp(out_foo_path, - out_foo_path + '_imported') + import_remote_stage = self.dvc.imp(out_foo_path, out_foo_path + "_imported") self.assertEqual(self.dvc.status(import_remote_stage.path), {}) - cmd_stage = self.dvc.run(outs=[out_bar_path], - deps=[out_foo_path], - cmd=self.cmd(foo_path, bar_path)) + cmd_stage = self.dvc.run( + outs=[out_bar_path], deps=[out_foo_path], cmd=self.cmd(foo_path, bar_path) + ) self.assertEqual(self.dvc.status(cmd_stage.path), {}) self.assertEqual(self.dvc.status(), {}) @@ -870,8 +870,8 @@ class TestReproExternalBase(TestDvc): stages = self.dvc.reproduce(import_stage.path) self.assertEqual(len(stages), 1) - self.assertTrue(os.path.exists('import')) - self.assertTrue(filecmp.cmp('import', self.BAR, shallow=False)) + self.assertTrue(os.path.exists("import")) + self.assertTrue(filecmp.cmp("import", self.BAR, shallow=False)) self.assertEqual(self.dvc.status(import_stage.path), {}) stages = self.dvc.reproduce(import_remote_stage.path) @@ -901,17 +901,17 @@ class TestReproExternalS3(TestReproExternalBase): @property def scheme(self): - return 's3' + return "s3" @property def bucket(self): return TEST_AWS_REPO_BUCKET def cmd(self, i, o): - return 'aws s3 cp {} {}'.format(i, o) + return "aws s3 cp {} {}".format(i, o) def write(self, bucket, key, body): - s3 = boto3.resource('s3') + s3 = boto3.resource("s3") s3.Bucket(bucket).put_object(Key=key, Body=body) @@ -921,14 +921,14 @@ class TestReproExternalGS(TestReproExternalBase): @property def scheme(self): - return 'gs' + return "gs" @property def bucket(self): return TEST_GCP_REPO_BUCKET def cmd(self, i, o): - return 'gsutil cp {} {}'.format(i, o) + return "gsutil cp {} {}".format(i, o) def write(self, bucket, key, body): client = gc.Client() @@ -942,46 +942,52 @@ class TestReproExternalHDFS(TestReproExternalBase): @property def scheme(self): - return 'hdfs' + return "hdfs" @property def bucket(self): - return '{}@127.0.0.1'.format(getpass.getuser()) + return "{}@127.0.0.1".format(getpass.getuser()) def cmd(self, i, o): - return 'hadoop fs -cp {} {}'.format(i, o) + return "hadoop fs -cp {} {}".format(i, o) def write(self, bucket, key, body): - url = self.scheme + '://' + bucket + '/' + key - p = Popen('hadoop fs -rm -f {}'.format(url), - shell=True, - executable=os.getenv('SHELL'), - stdin=PIPE, - stdout=PIPE, - stderr=PIPE) + url = self.scheme + "://" + bucket + "/" + key + p = Popen( + "hadoop fs -rm -f {}".format(url), + shell=True, + executable=os.getenv("SHELL"), + stdin=PIPE, + stdout=PIPE, + stderr=PIPE, + ) p.communicate() - p = Popen('hadoop fs -mkdir -p {}'.format(posixpath.dirname(url)), - shell=True, - executable=os.getenv('SHELL'), - stdin=PIPE, - stdout=PIPE, - stderr=PIPE) + p = Popen( + "hadoop fs -mkdir -p {}".format(posixpath.dirname(url)), + shell=True, + executable=os.getenv("SHELL"), + stdin=PIPE, + stdout=PIPE, + stderr=PIPE, + ) out, err = p.communicate() if p.returncode != 0: print(out) print(err) self.assertEqual(p.returncode, 0) - with open('tmp', 'w+') as fd: + with open("tmp", "w+") as fd: fd.write(body) - p = Popen('hadoop fs -copyFromLocal {} {}'.format('tmp', url), - shell=True, - executable=os.getenv('SHELL'), - stdin=PIPE, - stdout=PIPE, - stderr=PIPE) + p = Popen( + "hadoop fs -copyFromLocal {} {}".format("tmp", url), + shell=True, + executable=os.getenv("SHELL"), + stdin=PIPE, + stdout=PIPE, + stderr=PIPE, + ) out, err = p.communicate() if p.returncode != 0: print(out) @@ -997,50 +1003,54 @@ class TestReproExternalSSH(TestReproExternalBase): @property def scheme(self): - return 'ssh' + return "ssh" @property def bucket(self): if not self._dir: self._dir = TestDvc.mkdtemp() - return '{}@127.0.0.1:{}'.format(getpass.getuser(), self._dir) + return "{}@127.0.0.1:{}".format(getpass.getuser(), self._dir) def cmd(self, i, o): - i = i.strip('ssh://') - o = o.strip('ssh://') - return 'scp {} {}'.format(i, o) + i = i.strip("ssh://") + o = o.strip("ssh://") + return "scp {} {}".format(i, o) def write(self, bucket, key, body): - dest = '{}@127.0.0.1'.format(getpass.getuser()) + dest = "{}@127.0.0.1".format(getpass.getuser()) path = posixpath.join(self._dir, key) - p = Popen('ssh {} rm {}'.format(dest, path), - shell=True, - executable=os.getenv('SHELL'), - stdin=PIPE, - stdout=PIPE, - stderr=PIPE) + p = Popen( + "ssh {} rm {}".format(dest, path), + shell=True, + executable=os.getenv("SHELL"), + stdin=PIPE, + stdout=PIPE, + stderr=PIPE, + ) p.communicate() - p = Popen('ssh {} "mkdir -p $(dirname {})"'.format(dest, path), - shell=True, - executable=os.getenv('SHELL'), - stdin=PIPE, - stdout=PIPE, - stderr=PIPE) + p = Popen( + 'ssh {} "mkdir -p $(dirname {})"'.format(dest, path), + shell=True, + executable=os.getenv("SHELL"), + stdin=PIPE, + stdout=PIPE, + stderr=PIPE, + ) out, err = p.communicate() if p.returncode != 0: print(out) print(err) self.assertEqual(p.returncode, 0) - p = Popen('echo "{}" | ssh {} "tr -d \'\n\' > {}"'.format(body, - dest, - path), - shell=True, - executable=os.getenv('SHELL'), - stdin=PIPE, - stdout=PIPE, - stderr=PIPE) + p = Popen( + 'echo "{}" | ssh {} "tr -d \'\n\' > {}"'.format(body, dest, path), + shell=True, + executable=os.getenv("SHELL"), + stdin=PIPE, + stdout=PIPE, + stderr=PIPE, + ) out, err = p.communicate() if p.returncode != 0: print(out) @@ -1058,15 +1068,15 @@ class TestReproExternalLOCAL(TestReproExternalBase): @property def cache_scheme(self): - return 'local' + return "local" @property def scheme(self): - return '' + return "" @property def scheme_sep(self): - return '' + return "" @property def sep(self): @@ -1077,9 +1087,9 @@ class TestReproExternalLOCAL(TestReproExternalBase): return self.tmpdir def cmd(self, i, o): - if os.name == 'nt': - return 'copy {} {}'.format(i, o) - return 'cp {} {}'.format(i, o) + if os.name == "nt": + return "copy {} {}".format(i, o) + return "cp {} {}".format(i, o) def write(self, bucket, key, body): path = os.path.join(bucket, key) @@ -1088,7 +1098,7 @@ class TestReproExternalLOCAL(TestReproExternalBase): if not os.path.exists(dname): os.makedirs(dname) - with open(path, 'w+') as fd: + with open(path, "w+") as fd: fd.write(body) @@ -1097,11 +1107,11 @@ class TestReproExternalHTTP(TestReproExternalBase): @property def remote(self): - return 'http://localhost:8000/' + return "http://localhost:8000/" @property def local_cache(self): - return os.path.join(self.dvc.dvc_dir, 'cache') + return os.path.join(self.dvc.dvc_dir, "cache") @property def external_cache_id(self): @@ -1115,27 +1125,27 @@ class TestReproExternalHTTP(TestReproExternalBase): return urljoin(self.remote, self.external_cache_id) def test(self): - ret1 = main(['remote', 'add', 'mycache', self.external_cache]) - ret2 = main(['remote', 'add', 'myremote', self.remote]) + ret1 = main(["remote", "add", "mycache", self.external_cache]) + ret2 = main(["remote", "add", "myremote", self.remote]) self.assertEqual(ret1, 0) self.assertEqual(ret2, 0) - self.dvc = Project('.') + self.dvc = Project(".") # Import with StaticFileServer(): import_url = urljoin(self.remote, self.FOO) - import_output = 'imported_file' + import_output = "imported_file" import_stage = self.dvc.imp(import_url, import_output) self.assertTrue(os.path.exists(import_output)) self.assertTrue(filecmp.cmp(import_output, self.FOO, shallow=False)) - self.dvc.remove('imported_file.dvc') + self.dvc.remove("imported_file.dvc") - with StaticFileServer(handler='Content-MD5'): + with StaticFileServer(handler="Content-MD5"): import_url = urljoin(self.remote, self.FOO) - import_output = 'imported_file' + import_output = "imported_file" import_stage = self.dvc.imp(import_url, import_output) self.assertTrue(os.path.exists(import_output)) @@ -1144,15 +1154,15 @@ class TestReproExternalHTTP(TestReproExternalBase): # Run --deps with StaticFileServer(): run_dependency = urljoin(self.remote, self.BAR) - run_output = 'remote_file' + run_output = "remote_file" cmd = 'open("{}", "w+")'.format(run_output) - with open('create-output.py', 'w') as fd: + with open("create-output.py", "w") as fd: fd.write(cmd) - run_stage = self.dvc.run(deps=[run_dependency], - outs=[run_output], - cmd='python create-output.py') + run_stage = self.dvc.run( + deps=[run_dependency], outs=[run_output], cmd="python create-output.py" + ) self.assertTrue(run_stage is not None) self.assertTrue(os.path.exists(run_output)) @@ -1165,62 +1175,66 @@ class TestReproExternalHTTP(TestReproExternalBase): self.assertFalse(os.path.exists(self.local_cache)) with StaticFileServer(): - self.dvc.pull(import_stage.path, remote='mycache') + self.dvc.pull(import_stage.path, remote="mycache") self.assertTrue(os.path.exists(import_output)) class TestReproShell(TestDvc): def test(self): - if os.name == 'nt': + if os.name == "nt": return - fname = 'shell.txt' - stage = fname + '.dvc' + fname = "shell.txt" + stage = fname + ".dvc" - self.dvc.run(fname=stage, - outs=[fname], - cmd='echo $SHELL > {}'.format(fname)) + self.dvc.run(fname=stage, outs=[fname], cmd="echo $SHELL > {}".format(fname)) - with open(fname, 'r') as fd: - self.assertEqual(os.getenv('SHELL'), fd.read().strip()) + with open(fname, "r") as fd: + self.assertEqual(os.getenv("SHELL"), fd.read().strip()) os.unlink(fname) self.dvc.reproduce(stage) - with open(fname, 'r') as fd: - self.assertEqual(os.getenv('SHELL'), fd.read().strip()) + with open(fname, "r") as fd: + self.assertEqual(os.getenv("SHELL"), fd.read().strip()) class TestReproNoSCM(TestRepro): def test(self): shutil.rmtree(self.dvc.scm.dir) - ret = main(['repro', self.file1_stage]) + ret = main(["repro", self.file1_stage]) self.assertEqual(ret, 0) class TestReproAllPipelines(TestDvc): def test(self): - self.dvc.run(fname='start.dvc', - outs=['start.txt'], - cmd='echo start > start.txt') + self.dvc.run( + fname="start.dvc", outs=["start.txt"], cmd="echo start > start.txt" + ) - self.dvc.run(fname='middle.dvc', - deps=['start.txt'], - outs=['middle.txt'], - cmd='echo middle > middle.txt') + self.dvc.run( + fname="middle.dvc", + deps=["start.txt"], + outs=["middle.txt"], + cmd="echo middle > middle.txt", + ) - self.dvc.run(fname='final.dvc', - deps=['middle.txt'], - outs=['final.txt'], - cmd='echo final > final.txt') + self.dvc.run( + fname="final.dvc", + deps=["middle.txt"], + outs=["final.txt"], + cmd="echo final > final.txt", + ) - self.dvc.run(fname='disconnected.dvc', - outs=['disconnected.txt'], - cmd='echo other > disconnected.txt') + self.dvc.run( + fname="disconnected.dvc", + outs=["disconnected.txt"], + cmd="echo other > disconnected.txt", + ) - with patch.object(Stage, 'reproduce') as mock_reproduce: - ret = main(['repro', '--all-pipelines']) + with patch.object(Stage, "reproduce") as mock_reproduce: + ret = main(["repro", "--all-pipelines"]) self.assertEqual(ret, 0) self.assertEqual(mock_reproduce.call_count, 4) diff --git a/tests/test_run.py b/tests/test_run.py index 150398714..94d407c08 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -10,33 +10,37 @@ from dvc.system import System from dvc.stage import Stage from dvc.stage import StageFileBadNameError, MissingDep from dvc.stage import StageBadCwdError, StageFileAlreadyExistsError -from dvc.exceptions import (OutputDuplicationError, - CircularDependencyError, - CyclicGraphError, - ArgumentDuplicationError, - WorkingDirectoryAsOutputError) +from dvc.exceptions import ( + OutputDuplicationError, + CircularDependencyError, + CyclicGraphError, + ArgumentDuplicationError, + WorkingDirectoryAsOutputError, +) from tests.basic_env import TestDvc class TestRun(TestDvc): def test(self): - cmd = 'python {} {} {}'.format(self.CODE, self.FOO, 'out') + cmd = "python {} {} {}".format(self.CODE, self.FOO, "out") deps = [self.FOO, self.CODE] - outs = [os.path.join(self.dvc.root_dir, 'out')] + outs = [os.path.join(self.dvc.root_dir, "out")] outs_no_cache = [] - fname = 'out.dvc' + fname = "out.dvc" cwd = os.curdir self.dvc.add(self.FOO) - stage = self.dvc.run(cmd=cmd, - deps=deps, - outs=outs, - outs_no_cache=outs_no_cache, - fname=fname, - cwd=cwd) + stage = self.dvc.run( + cmd=cmd, + deps=deps, + outs=outs, + outs_no_cache=outs_no_cache, + fname=fname, + cwd=cwd, + ) - self.assertTrue(filecmp.cmp(self.FOO, 'out', shallow=False)) + self.assertTrue(filecmp.cmp(self.FOO, "out", shallow=False)) self.assertTrue(os.path.isfile(stage.path)) self.assertEqual(stage.cmd, cmd) self.assertEqual(len(stage.deps), len(deps)) @@ -46,301 +50,338 @@ class TestRun(TestDvc): self.assertTrue(stage.path, fname) with self.assertRaises(OutputDuplicationError): - stage = self.dvc.run(cmd=cmd, - deps=deps, - outs=outs, - outs_no_cache=outs_no_cache, - fname='duplicate' + fname, - cwd=cwd) + stage = self.dvc.run( + cmd=cmd, + deps=deps, + outs=outs, + outs_no_cache=outs_no_cache, + fname="duplicate" + fname, + cwd=cwd, + ) class TestRunEmpty(TestDvc): def test(self): - self.dvc.run(cmd='', - deps=[], - outs=[], - outs_no_cache=[], - fname='empty.dvc', - cwd=os.curdir) + self.dvc.run( + cmd="", deps=[], outs=[], outs_no_cache=[], fname="empty.dvc", cwd=os.curdir + ) class TestRunMissingDep(TestDvc): def test(self): with self.assertRaises(MissingDep): - self.dvc.run(cmd='', - deps=['non-existing-dep'], - outs=[], - outs_no_cache=[], - fname='empty.dvc', - cwd=os.curdir) + self.dvc.run( + cmd="", + deps=["non-existing-dep"], + outs=[], + outs_no_cache=[], + fname="empty.dvc", + cwd=os.curdir, + ) class TestRunBadStageFilename(TestDvc): def test(self): with self.assertRaises(StageFileBadNameError): - self.dvc.run(cmd='', - deps=[], - outs=[], - outs_no_cache=[], - fname='empty', - cwd=os.curdir) + self.dvc.run( + cmd="", deps=[], outs=[], outs_no_cache=[], fname="empty", cwd=os.curdir + ) with self.assertRaises(StageFileBadNameError): - self.dvc.run(cmd='', - deps=[], - outs=[], - outs_no_cache=[], - fname=os.path.join(self.DATA_DIR, 'empty.dvc'), - cwd=os.curdir) + self.dvc.run( + cmd="", + deps=[], + outs=[], + outs_no_cache=[], + fname=os.path.join(self.DATA_DIR, "empty.dvc"), + cwd=os.curdir, + ) class TestRunNoExec(TestDvc): def test(self): - self.dvc.run(cmd='python {} {} {}'.format(self.CODE, self.FOO, 'out'), - no_exec=True) - self.assertFalse(os.path.exists('out')) + self.dvc.run( + cmd="python {} {} {}".format(self.CODE, self.FOO, "out"), no_exec=True + ) + self.assertFalse(os.path.exists("out")) class TestRunCircularDependency(TestDvc): def test(self): with self.assertRaises(CircularDependencyError): - self.dvc.run(cmd='', - deps=[self.FOO], - outs=[self.FOO], - fname='circular-dependency.dvc') + self.dvc.run( + cmd="", + deps=[self.FOO], + outs=[self.FOO], + fname="circular-dependency.dvc", + ) def test_outs_no_cache(self): with self.assertRaises(CircularDependencyError): - self.dvc.run(cmd='', - deps=[self.FOO], - outs_no_cache=[self.FOO], - fname='circular-dependency.dvc') + self.dvc.run( + cmd="", + deps=[self.FOO], + outs_no_cache=[self.FOO], + fname="circular-dependency.dvc", + ) def test_non_normalized_paths(self): with self.assertRaises(CircularDependencyError): - self.dvc.run(cmd='', - deps=['./foo'], - outs=['foo'], - fname='circular-dependency.dvc') + self.dvc.run( + cmd="", deps=["./foo"], outs=["foo"], fname="circular-dependency.dvc" + ) def test_graph(self): - self.dvc.run(deps=[self.FOO], - outs=['bar.txt'], - cmd='echo bar > bar.txt') + self.dvc.run(deps=[self.FOO], outs=["bar.txt"], cmd="echo bar > bar.txt") - self.dvc.run(deps=['bar.txt'], - outs=['baz.txt'], - cmd='echo baz > baz.txt') + self.dvc.run(deps=["bar.txt"], outs=["baz.txt"], cmd="echo baz > baz.txt") with self.assertRaises(CyclicGraphError): - self.dvc.run(deps=['baz.txt'], - outs=[self.FOO], - cmd='echo baz > foo') + self.dvc.run(deps=["baz.txt"], outs=[self.FOO], cmd="echo baz > foo") class TestRunDuplicatedArguments(TestDvc): def test(self): with self.assertRaises(ArgumentDuplicationError): - self.dvc.run(cmd='', - deps=[], - outs=[self.FOO, self.FOO], - fname='circular-dependency.dvc') + self.dvc.run( + cmd="", + deps=[], + outs=[self.FOO, self.FOO], + fname="circular-dependency.dvc", + ) def test_outs_no_cache(self): with self.assertRaises(ArgumentDuplicationError): - self.dvc.run(cmd='', - outs=[self.FOO], - outs_no_cache=[self.FOO], - fname='circular-dependency.dvc') + self.dvc.run( + cmd="", + outs=[self.FOO], + outs_no_cache=[self.FOO], + fname="circular-dependency.dvc", + ) def test_non_normalized_paths(self): with self.assertRaises(ArgumentDuplicationError): - self.dvc.run(cmd='', - deps=[], - outs=['foo', './foo'], - fname='circular-dependency.dvc') + self.dvc.run( + cmd="", deps=[], outs=["foo", "./foo"], fname="circular-dependency.dvc" + ) class TestRunWorkingDirectoryAsOutput(TestDvc): def test(self): - self.dvc.run(cmd='', - deps=[], - outs=[self.DATA_DIR]) + self.dvc.run(cmd="", deps=[], outs=[self.DATA_DIR]) with self.assertRaises(WorkingDirectoryAsOutputError): - self.dvc.run(cmd='', - cwd=self.DATA_DIR, - outs=[self.FOO], - fname='inside-cwd.dvc') + self.dvc.run( + cmd="", cwd=self.DATA_DIR, outs=[self.FOO], fname="inside-cwd.dvc" + ) class TestRunBadCwd(TestDvc): def test(self): with self.assertRaises(StageBadCwdError): - self.dvc.run(cmd='', - cwd=self.mkdtemp()) + self.dvc.run(cmd="", cwd=self.mkdtemp()) class TestCmdRun(TestDvc): def test_run(self): - ret = main(['run', - '-d', self.FOO, - '-d', self.CODE, - '-o', 'out', - '-f', 'out.dvc', - 'python', self.CODE, self.FOO, 'out']) + ret = main( + [ + "run", + "-d", + self.FOO, + "-d", + self.CODE, + "-o", + "out", + "-f", + "out.dvc", + "python", + self.CODE, + self.FOO, + "out", + ] + ) - stage = Stage.load(self.dvc, fname='out.dvc') + stage = Stage.load(self.dvc, fname="out.dvc") self.assertEqual(ret, 0) - self.assertTrue(os.path.isfile('out')) - self.assertTrue(os.path.isfile('out.dvc')) - self.assertTrue(filecmp.cmp(self.FOO, 'out', shallow=False)) - self.assertEqual(stage.cmd, 'python code.py foo out') + self.assertTrue(os.path.isfile("out")) + self.assertTrue(os.path.isfile("out.dvc")) + self.assertTrue(filecmp.cmp(self.FOO, "out", shallow=False)) + self.assertEqual(stage.cmd, "python code.py foo out") def test_run_args_from_cli(self): - ret = main(['run', 'echo', 'foo']) - stage = Stage.load(self.dvc, fname='Dvcfile') + ret = main(["run", "echo", "foo"]) + stage = Stage.load(self.dvc, fname="Dvcfile") self.assertEqual(ret, 0) - self.assertEqual(stage.cmd, 'echo foo') + self.assertEqual(stage.cmd, "echo foo") def test_run_bad_command(self): - ret = main(['run', 'non-existing-command']) + ret = main(["run", "non-existing-command"]) self.assertNotEqual(ret, 0) def test_run_args_with_spaces(self): - ret = main(['run', 'echo', 'foo bar']) - stage = Stage.load(self.dvc, fname='Dvcfile') + ret = main(["run", "echo", "foo bar"]) + stage = Stage.load(self.dvc, fname="Dvcfile") self.assertEqual(ret, 0) self.assertEqual(stage.cmd, 'echo "foo bar"') - @mock.patch.object(subprocess, 'Popen', side_effect=KeyboardInterrupt) + @mock.patch.object(subprocess, "Popen", side_effect=KeyboardInterrupt) def test_keyboard_interrupt(self, mock_popen): - ret = main(['run', 'mycmd']) + ret = main(["run", "mycmd"]) self.assertEqual(ret, 252) class TestRunRemoveOuts(TestDvc): def test(self): - with open(self.CODE, 'w+') as fobj: + with open(self.CODE, "w+") as fobj: fobj.write("import sys\n") fobj.write("import os\n") fobj.write("if os.path.exists(sys.argv[1]):\n") fobj.write(" sys.exit(1)\n") fobj.write("open(sys.argv[1], 'w+').close()\n") - ret = main(['run', - '--remove-outs', - '-d', self.CODE, - '-o', self.FOO, - 'python', self.CODE, self.FOO]) + ret = main( + [ + "run", + "--remove-outs", + "-d", + self.CODE, + "-o", + self.FOO, + "python", + self.CODE, + self.FOO, + ] + ) self.assertEqual(ret, 0) class TestRunUnprotectOutsCopy(TestDvc): def test(self): - with open(self.CODE, 'w+') as fobj: + with open(self.CODE, "w+") as fobj: fobj.write("import sys\n") fobj.write("with open(sys.argv[1], 'a+') as fobj:\n") fobj.write(" fobj.write('foo')\n") - ret = main(['config', 'cache.protected', 'true']) + ret = main(["config", "cache.protected", "true"]) self.assertEqual(ret, 0) - ret = main(['config', 'cache.type', 'copy']) + ret = main(["config", "cache.type", "copy"]) self.assertEqual(ret, 0) - ret = main(['run', - '-d', self.CODE, - '-o', self.FOO, - 'python', self.CODE, self.FOO]) + ret = main( + ["run", "-d", self.CODE, "-o", self.FOO, "python", self.CODE, self.FOO] + ) self.assertEqual(ret, 0) self.assertFalse(os.access(self.FOO, os.W_OK)) - self.assertEqual(open(self.FOO, 'r').read(), 'foofoo') + self.assertEqual(open(self.FOO, "r").read(), "foofoo") - ret = main(['run', - '--overwrite-dvcfile', - '--ignore-build-cache', - '-d', self.CODE, - '-o', self.FOO, - 'python', self.CODE, self.FOO]) + ret = main( + [ + "run", + "--overwrite-dvcfile", + "--ignore-build-cache", + "-d", + self.CODE, + "-o", + self.FOO, + "python", + self.CODE, + self.FOO, + ] + ) self.assertEqual(ret, 0) self.assertFalse(os.access(self.FOO, os.W_OK)) - self.assertEqual(open(self.FOO, 'r').read(), 'foofoofoo') + self.assertEqual(open(self.FOO, "r").read(), "foofoofoo") class TestRunUnprotectOutsSymlink(TestDvc): def test(self): - with open(self.CODE, 'w+') as fobj: + with open(self.CODE, "w+") as fobj: fobj.write("import sys\n") fobj.write("import os\n") fobj.write("assert os.path.exists(sys.argv[1])\n") fobj.write("with open(sys.argv[1], 'a+') as fobj:\n") fobj.write(" fobj.write('foo')\n") - ret = main(['config', 'cache.protected', 'true']) + ret = main(["config", "cache.protected", "true"]) self.assertEqual(ret, 0) - ret = main(['config', 'cache.type', 'symlink']) + ret = main(["config", "cache.type", "symlink"]) self.assertEqual(ret, 0) self.assertEqual(ret, 0) - ret = main(['run', - '-d', self.CODE, - '-o', self.FOO, - 'python', self.CODE, self.FOO]) + ret = main( + ["run", "-d", self.CODE, "-o", self.FOO, "python", self.CODE, self.FOO] + ) self.assertEqual(ret, 0) self.assertFalse(os.access(self.FOO, os.W_OK)) self.assertTrue(System.is_symlink(self.FOO)) - self.assertEqual(open(self.FOO, 'r').read(), 'foofoo') + self.assertEqual(open(self.FOO, "r").read(), "foofoo") - ret = main(['run', - '--overwrite-dvcfile', - '--ignore-build-cache', - '-d', self.CODE, - '-o', self.FOO, - 'python', self.CODE, self.FOO]) + ret = main( + [ + "run", + "--overwrite-dvcfile", + "--ignore-build-cache", + "-d", + self.CODE, + "-o", + self.FOO, + "python", + self.CODE, + self.FOO, + ] + ) self.assertEqual(ret, 0) self.assertFalse(os.access(self.FOO, os.W_OK)) self.assertTrue(System.is_symlink(self.FOO)) - self.assertEqual(open(self.FOO, 'r').read(), 'foofoofoo') + self.assertEqual(open(self.FOO, "r").read(), "foofoofoo") class TestRunUnprotectOutsHardlink(TestDvc): def test(self): - with open(self.CODE, 'w+') as fobj: + with open(self.CODE, "w+") as fobj: fobj.write("import sys\n") fobj.write("import os\n") fobj.write("assert os.path.exists(sys.argv[1])\n") fobj.write("with open(sys.argv[1], 'a+') as fobj:\n") fobj.write(" fobj.write('foo')\n") - ret = main(['config', 'cache.protected', 'true']) + ret = main(["config", "cache.protected", "true"]) self.assertEqual(ret, 0) - ret = main(['config', 'cache.type', 'hardlink']) + ret = main(["config", "cache.type", "hardlink"]) self.assertEqual(ret, 0) self.assertEqual(ret, 0) - ret = main(['run', - '-d', self.CODE, - '-o', self.FOO, - 'python', self.CODE, self.FOO]) + ret = main( + ["run", "-d", self.CODE, "-o", self.FOO, "python", self.CODE, self.FOO] + ) self.assertEqual(ret, 0) self.assertFalse(os.access(self.FOO, os.W_OK)) self.assertTrue(System.is_hardlink(self.FOO)) - self.assertEqual(open(self.FOO, 'r').read(), 'foofoo') + self.assertEqual(open(self.FOO, "r").read(), "foofoo") - ret = main(['run', - '--overwrite-dvcfile', - '--ignore-build-cache', - '-d', self.CODE, - '-o', self.FOO, - 'python', self.CODE, self.FOO]) + ret = main( + [ + "run", + "--overwrite-dvcfile", + "--ignore-build-cache", + "-d", + self.CODE, + "-o", + self.FOO, + "python", + self.CODE, + self.FOO, + ] + ) self.assertEqual(ret, 0) self.assertFalse(os.access(self.FOO, os.W_OK)) self.assertTrue(System.is_hardlink(self.FOO)) - self.assertEqual(open(self.FOO, 'r').read(), 'foofoofoo') + self.assertEqual(open(self.FOO, "r").read(), "foofoofoo") class TestCmdRunOverwrite(TestDvc): @@ -352,64 +393,94 @@ class TestCmdRunOverwrite(TestDvc): # below don't change). import time - ret = main(['run', - '-d', self.FOO, - '-d', self.CODE, - '-o', 'out', - '-f', 'out.dvc', - 'python', self.CODE, self.FOO, 'out']) + ret = main( + [ + "run", + "-d", + self.FOO, + "-d", + self.CODE, + "-o", + "out", + "-f", + "out.dvc", + "python", + self.CODE, + self.FOO, + "out", + ] + ) self.assertEqual(ret, 0) - stage_mtime = os.path.getmtime('out.dvc') + stage_mtime = os.path.getmtime("out.dvc") time.sleep(1) - ret = main(['run', - '-d', self.FOO, - '-d', self.CODE, - '-o', 'out', - '-f', 'out.dvc', - 'python', self.CODE, self.FOO, 'out']) + ret = main( + [ + "run", + "-d", + self.FOO, + "-d", + self.CODE, + "-o", + "out", + "-f", + "out.dvc", + "python", + self.CODE, + self.FOO, + "out", + ] + ) self.assertEqual(ret, 0) # NOTE: check that dvcfile was NOT overwritten - self.assertEqual(stage_mtime, os.path.getmtime('out.dvc')) - stage_mtime = os.path.getmtime('out.dvc') + self.assertEqual(stage_mtime, os.path.getmtime("out.dvc")) + stage_mtime = os.path.getmtime("out.dvc") time.sleep(1) - ret = main(['run', - '-d', self.FOO, - '-d', self.CODE, - '--overwrite-dvcfile', - '--ignore-build-cache', - '-o', 'out', - '-f', 'out.dvc', - 'python', self.CODE, self.FOO, 'out']) + ret = main( + [ + "run", + "-d", + self.FOO, + "-d", + self.CODE, + "--overwrite-dvcfile", + "--ignore-build-cache", + "-o", + "out", + "-f", + "out.dvc", + "python", + self.CODE, + self.FOO, + "out", + ] + ) self.assertEqual(ret, 0) # NOTE: check that dvcfile was overwritten - self.assertNotEqual(stage_mtime, os.path.getmtime('out.dvc')) - stage_mtime = os.path.getmtime('out.dvc') + self.assertNotEqual(stage_mtime, os.path.getmtime("out.dvc")) + stage_mtime = os.path.getmtime("out.dvc") time.sleep(1) - ret = main(['run', - '--overwrite-dvcfile', - '-f', 'out.dvc', - '-d', self.BAR]) + ret = main(["run", "--overwrite-dvcfile", "-f", "out.dvc", "-d", self.BAR]) self.assertEqual(ret, 0) # NOTE: check that dvcfile was overwritten - self.assertNotEqual(stage_mtime, os.path.getmtime('out.dvc')) + self.assertNotEqual(stage_mtime, os.path.getmtime("out.dvc")) class TestRunDeterministicBase(TestDvc): def setUp(self): super(TestRunDeterministicBase, self).setUp() - self.out_file = 'out' - self.stage_file = self.out_file + '.dvc' - self.cmd = 'python {} {} {}'.format(self.CODE, self.FOO, self.out_file) + self.out_file = "out" + self.stage_file = self.out_file + ".dvc" + self.cmd = "python {} {} {}".format(self.CODE, self.FOO, self.out_file) self.deps = [self.FOO, self.CODE] self.outs = [self.out_file] self.overwrite = False @@ -418,12 +489,14 @@ class TestRunDeterministicBase(TestDvc): self._run() def _run(self): - self.stage = self.dvc.run(cmd=self.cmd, - fname=self.stage_file, - overwrite=self.overwrite, - ignore_build_cache=self.ignore_build_cache, - deps=self.deps, - outs=self.outs) + self.stage = self.dvc.run( + cmd=self.cmd, + fname=self.stage_file, + overwrite=self.overwrite, + ignore_build_cache=self.ignore_build_cache, + deps=self.deps, + outs=self.outs, + ) class TestRunDeterministic(TestRunDeterministicBase): @@ -485,6 +558,6 @@ class TestRunDeterministicChangedOut(TestRunDeterministicBase): class TestRunDeterministicChangedCmd(TestRunDeterministicBase): def test(self): - self.cmd += ' arg' + self.cmd += " arg" with self.assertRaises(StageFileAlreadyExistsError): self._run() diff --git a/tests/test_scm.py b/tests/test_scm.py index fd24fbf11..1b92991b0 100644 --- a/tests/test_scm.py +++ b/tests/test_scm.py @@ -22,9 +22,9 @@ class TestSCMGit(TestGit): def test_commit(self): G = Git(self._root_dir) - G.add(['foo']) - G.commit('add') - self.assertTrue('foo' in self.git.git.ls_files()) + G.add(["foo"]) + G.commit("add") + self.assertTrue("foo" in self.git.git.ls_files()) class TestSCMGitSubmodule(TestGitSubmodule): @@ -36,17 +36,16 @@ class TestSCMGitSubmodule(TestGitSubmodule): def test_commit_in_submodule(self): G = Git(self._root_dir) - G.add(['foo']) - G.commit('add') - self.assertTrue('foo' in self.git.git.ls_files()) + G.add(["foo"]) + G.commit("add") + self.assertTrue("foo" in self.git.git.ls_files()) class TestIgnore(TestGit): def _count_gitignore(self): - with open(Git.GITIGNORE, 'r') as fd: + with open(Git.GITIGNORE, "r") as fd: lines = fd.readlines() - return len(list(filter(lambda x: x.strip() == '/' + self.FOO, - lines))) + return len(list(filter(lambda x: x.strip() == "/" + self.FOO, lines))) def test(self): git = Git(self._root_dir) diff --git a/tests/test_stage.py b/tests/test_stage.py index ba227c5d0..e44595c8d 100644 --- a/tests/test_stage.py +++ b/tests/test_stage.py @@ -25,7 +25,7 @@ class TestSchemaCmd(TestSchema): Stage.validate(d) def test_cmd_str(self): - d = {Stage.PARAM_CMD: 'cmd'} + d = {Stage.PARAM_CMD: "cmd"} Stage.validate(d) @@ -52,11 +52,11 @@ class TestSchemaDepsOuts(TestSchema): Stage.validate(d) def test_list(self): - lst = [{OutputLOCAL.PARAM_PATH: 'foo', - RemoteLOCAL.PARAM_CHECKSUM: '123'}, - {OutputLOCAL.PARAM_PATH: 'bar', - RemoteLOCAL.PARAM_CHECKSUM: None}, - {OutputLOCAL.PARAM_PATH: 'baz'}] + lst = [ + {OutputLOCAL.PARAM_PATH: "foo", RemoteLOCAL.PARAM_CHECKSUM: "123"}, + {OutputLOCAL.PARAM_PATH: "bar", RemoteLOCAL.PARAM_CHECKSUM: None}, + {OutputLOCAL.PARAM_PATH: "baz"}, + ] d = {Stage.PARAM_DEPS: lst} Stage.validate(d) @@ -69,26 +69,27 @@ class TestSchemaDepsOuts(TestSchema): class TestReload(TestDvc): def test(self): import yaml + stages = self.dvc.add(self.FOO) self.assertEqual(len(stages), 1) stage = stages[0] self.assertTrue(stage is not None) - with open(stage.relpath, 'r') as fobj: + with open(stage.relpath, "r") as fobj: d = yaml.safe_load(fobj) # NOTE: checking that reloaded stage didn't change its checksum - md5 = '11111111111111111111111111111111' + md5 = "11111111111111111111111111111111" d[stage.PARAM_MD5] = md5 - with open(stage.relpath, 'w') as fobj: + with open(stage.relpath, "w") as fobj: yaml.safe_dump(d, fobj, default_flow_style=False) stage = Stage.load(self.dvc, stage.relpath) self.assertTrue(stage is not None) stage.dump() - with open(stage.relpath, 'r') as fobj: + with open(stage.relpath, "r") as fobj: d = yaml.safe_load(fobj) self.assertEqual(d[stage.PARAM_MD5], md5) diff --git a/tests/test_state.py b/tests/test_state.py index 713009198..3d642e69d 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -21,8 +21,8 @@ class TestState(TestDvc): self.assertEqual(entry_md5, md5) os.unlink(path) - with open(path, 'a') as fd: - fd.write('1') + with open(path, "a") as fd: + fd.write("1") md5 = file_md5(path)[0] entry_md5 = state.update(path) @@ -33,14 +33,14 @@ class TestStateOverflow(TestDvc): def test(self): # NOTE: trying to add more entries than state can handle, # to see if it will clean up and vacuum successfully - ret = main(['config', 'state.row_limit', '10']) + ret = main(["config", "state.row_limit", "10"]) self.assertEqual(ret, 0) - dname = 'dir' + dname = "dir" os.mkdir(dname) for i in range(20): - with open(os.path.join(dname, str(i)), 'w+') as fobj: + with open(os.path.join(dname, str(i)), "w+") as fobj: fobj.write(str(i)) - ret = main(['add', 'dir']) + ret = main(["add", "dir"]) self.assertEqual(ret, 0) diff --git a/tests/test_status.py b/tests/test_status.py index 4999ce6e2..b5668169d 100644 --- a/tests/test_status.py +++ b/tests/test_status.py @@ -9,11 +9,11 @@ class TestStatus(TestDvc): def test_quiet(self): self.dvc.add(self.FOO) - ret = main(['status', '--quiet']) + ret = main(["status", "--quiet"]) self.assertEqual(ret, 0) os.remove(self.FOO) os.rename(self.BAR, self.FOO) - ret = main(['status', '--quiet']) + ret = main(["status", "--quiet"]) self.assertEqual(ret, 1) diff --git a/tests/test_system.py b/tests/test_system.py index ef47177a2..45b79cebd 100644 --- a/tests/test_system.py +++ b/tests/test_system.py @@ -6,11 +6,11 @@ from tests.basic_env import TestDir class TestSystem(TestDir): def test_getdirinfo(self): - if os.name != 'nt': + if os.name != "nt": return # simulate someone holding an exclussive access - locked = 'locked' + locked = "locked" fd = os.open(locked, os.O_WRONLY | os.O_CREAT | os.O_EXCL) # should not raise WinError diff --git a/tests/test_unprotect.py b/tests/test_unprotect.py index d51990a60..fdaa0ebd6 100644 --- a/tests/test_unprotect.py +++ b/tests/test_unprotect.py @@ -8,23 +8,24 @@ from tests.basic_env import TestDvc class TestUnprotect(TestDvc): def test(self): - ret = main(['config', 'cache.protected', 'true']) + ret = main(["config", "cache.protected", "true"]) self.assertEqual(ret, 0) - ret = main(['add', self.FOO]) + ret = main(["add", self.FOO]) self.assertEqual(ret, 0) - self.assertNotEqual(os.stat(self.FOO).st_mode & (stat.S_IREAD | - stat.S_IRGRP | - stat.S_IROTH), 0) - self.assertEqual(os.stat(self.FOO).st_mode & (stat.S_IWRITE | - stat.S_IWGRP | - stat.S_IWOTH), 0) + self.assertNotEqual( + os.stat(self.FOO).st_mode & (stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH), 0 + ) + self.assertEqual( + os.stat(self.FOO).st_mode & (stat.S_IWRITE | stat.S_IWGRP | stat.S_IWOTH), 0 + ) - ret = main(['unprotect', self.FOO]) + ret = main(["unprotect", self.FOO]) self.assertEqual(ret, 0) - self.assertNotEqual(os.stat(self.FOO).st_mode & (stat.S_IREAD | - stat.S_IRGRP | - stat.S_IROTH | - stat.S_IWRITE), 0) + self.assertNotEqual( + os.stat(self.FOO).st_mode + & (stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH | stat.S_IWRITE), + 0, + ) diff --git a/tests/test_updater.py b/tests/test_updater.py index d3e372ada..b2d2adfab 100644 --- a/tests/test_updater.py +++ b/tests/test_updater.py @@ -6,17 +6,17 @@ from tests.basic_env import TestDvc class TestUpdater(TestDvc): def test(self): # NOTE: only test on travis CRON to avoid generating too much logs - travis = os.getenv('TRAVIS') == 'true' + travis = os.getenv("TRAVIS") == "true" if not travis: return - cron = os.getenv('TRAVIS_EVENT_TYPE') == 'cron' + cron = os.getenv("TRAVIS_EVENT_TYPE") == "cron" if not cron: return env = os.environ.copy() - if os.getenv('CI'): - del os.environ['CI'] + if os.getenv("CI"): + del os.environ["CI"] self.dvc.updater.check() self.dvc.updater.check() diff --git a/tests/test_utils.py b/tests/test_utils.py index 929d19da5..a613e44b7 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -8,12 +8,12 @@ from tests.basic_env import TestDvc class TestUtils(TestDvc): def test_copyfile(self): - src = 'file1' - dest = 'file2' - dest_dir = 'testdir' + src = "file1" + dest = "file2" + dest_dir = "testdir" - with open(src, 'w+') as f: - f.write('file1contents') + with open(src, "w+") as f: + f.write("file1contents") os.mkdir(dest_dir) @@ -21,34 +21,38 @@ class TestUtils(TestDvc): self.assertTrue(filecmp.cmp(src, dest, shallow=False)) utils.copyfile(src, dest_dir) - self.assertTrue(filecmp.cmp(src, - '{}/{}'.format(dest_dir, src), - shallow=False)) + self.assertTrue(filecmp.cmp(src, "{}/{}".format(dest_dir, src), shallow=False)) shutil.rmtree(dest_dir) os.remove(src) os.remove(dest) def test_file_md5_crlf(self): - with open('cr', 'wb+') as fd: - fd.write(b'a\nb\nc') - with open('crlf', 'wb+') as fd: - fd.write(b'a\r\nb\r\nc') + with open("cr", "wb+") as fd: + fd.write(b"a\nb\nc") + with open("crlf", "wb+") as fd: + fd.write(b"a\r\nb\r\nc") - self.assertEqual(utils.file_md5('cr')[0], utils.file_md5('crlf')[0]) + self.assertEqual(utils.file_md5("cr")[0], utils.file_md5("crlf")[0]) def test_dict_md5(self): - d = {'cmd': 'python code.py foo file1', - 'locked': 'true', - 'outs': [{'path': 'file1', - 'metric': {'type': 'raw'}, - 'cache': False, - 'md5': 'acbd18db4cc2f85cedef654fccc4a4d8'}], - 'deps': [{'path': 'foo', - 'md5': 'acbd18db4cc2f85cedef654fccc4a4d8'}, - {'path': 'code.py', - 'md5': 'd05447644b89960913c7eee5fd776adb'}]} + d = { + "cmd": "python code.py foo file1", + "locked": "true", + "outs": [ + { + "path": "file1", + "metric": {"type": "raw"}, + "cache": False, + "md5": "acbd18db4cc2f85cedef654fccc4a4d8", + } + ], + "deps": [ + {"path": "foo", "md5": "acbd18db4cc2f85cedef654fccc4a4d8"}, + {"path": "code.py", "md5": "d05447644b89960913c7eee5fd776adb"}, + ], + } - md5 = '8b263fa05ede6c3145c164829be694b4' + md5 = "8b263fa05ede6c3145c164829be694b4" - self.assertEqual(md5, utils.dict_md5(d, exclude=['metric', 'locked'])) + self.assertEqual(md5, utils.dict_md5(d, exclude=["metric", "locked"])) diff --git a/tests/unit/daemon.py b/tests/unit/daemon.py index 3ab101978..43c9de862 100644 --- a/tests/unit/daemon.py +++ b/tests/unit/daemon.py @@ -7,12 +7,12 @@ import dvc.daemon as daemon class TestDaemon(TestCase): - @mock.patch('dvc.daemon._spawn_posix') - @mock.patch('dvc.daemon._spawn_windows') + @mock.patch("dvc.daemon._spawn_posix") + @mock.patch("dvc.daemon._spawn_windows") def test(self, mock_windows, mock_posix): - daemon.daemon(['updater']) + daemon.daemon(["updater"]) - if os.name == 'nt': + if os.name == "nt": mock_posix.assert_not_called() mock_windows.assert_called() args, kwargs = mock_windows.call_args @@ -22,10 +22,10 @@ class TestDaemon(TestCase): args, kwargs = mock_posix.call_args env = args[1] - self.assertTrue('PYTHONPATH' in env.keys()) + self.assertTrue("PYTHONPATH" in env.keys()) file_path = os.path.abspath(inspect.stack()[0][1]) file_dir = os.path.dirname(file_path) test_dir = os.path.dirname(file_dir) dvc_dir = os.path.dirname(test_dir) - self.assertEqual(env['PYTHONPATH'], dvc_dir) + self.assertEqual(env["PYTHONPATH"], dvc_dir) diff --git a/tests/unit/remote/azure.py b/tests/unit/remote/azure.py index 7f00a84d6..e089b8f86 100644 --- a/tests/unit/remote/azure.py +++ b/tests/unit/remote/azure.py @@ -4,31 +4,29 @@ from dvc.remote.azure import RemoteAzure class TestRemoteAzure(TestCase): - container_name = 'container-name' + container_name = "container-name" connection_string = ( - 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;' - 'AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsu' - 'Fq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;' - 'BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;' + "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;" + "AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsu" + "Fq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;" + "BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" ) def test_init_compat(self): - url = ( - 'azure://ContainerName={container_name};{connection_string}' - .format(container_name=self.container_name, - connection_string=self.connection_string) + url = "azure://ContainerName={container_name};{connection_string}".format( + container_name=self.container_name, connection_string=self.connection_string ) - config = {'url': url} + config = {"url": url} remote = RemoteAzure(None, config) self.assertEqual(remote.url, url) - self.assertEqual(remote.prefix, '') + self.assertEqual(remote.prefix, "") self.assertEqual(remote.bucket, self.container_name) self.assertEqual(remote.connection_string, self.connection_string) def test_init(self): - prefix = 'some/prefix' - url = 'azure://{}/{}'.format(self.container_name, prefix) - config = {'url': url, 'connection_string': self.connection_string} + prefix = "some/prefix" + url = "azure://{}/{}".format(self.container_name, prefix) + config = {"url": url, "connection_string": self.connection_string} remote = RemoteAzure(None, config) self.assertEqual(remote.url, url) self.assertEqual(remote.prefix, prefix) diff --git a/tests/unit/remote/base.py b/tests/unit/remote/base.py index 68bc343b8..ecf434818 100644 --- a/tests/unit/remote/base.py +++ b/tests/unit/remote/base.py @@ -9,7 +9,7 @@ class TestMissingDeps(TestCase): def test(self): remotes = REMOTES + [RemoteLOCAL, RemoteBase] for remote_class in remotes: - REQUIRES = {'foo': None, 'bar': None, 'mock': mock} + REQUIRES = {"foo": None, "bar": None, "mock": mock} with mock.patch.object(remote_class, "REQUIRES", REQUIRES): with self.assertRaises(RemoteMissingDepsError): remote_class(None, {}) diff --git a/tests/unit/remote/gs.py b/tests/unit/remote/gs.py index c159a08ea..20f503952 100644 --- a/tests/unit/remote/gs.py +++ b/tests/unit/remote/gs.py @@ -5,11 +5,11 @@ from dvc.remote.gs import RemoteGS class TestRemoteGS(TestCase): - BUCKET = 'bucket' - PREFIX = 'prefix' - URL = 'gs://{}/{}'.format(BUCKET, PREFIX) - PROJECT = 'PROJECT' - CONFIG = {'projectname': PROJECT, 'url': URL} + BUCKET = "bucket" + PREFIX = "prefix" + URL = "gs://{}/{}".format(BUCKET, PREFIX) + PROJECT = "PROJECT" + CONFIG = {"projectname": PROJECT, "url": URL} def test_init(self): remote = RemoteGS(None, self.CONFIG) @@ -18,7 +18,7 @@ class TestRemoteGS(TestCase): self.assertEqual(remote.bucket, self.BUCKET) self.assertEqual(remote.projectname, self.PROJECT) - @mock.patch('google.cloud.storage.Client') + @mock.patch("google.cloud.storage.Client") def test_gs(self, mock_client): remote = RemoteGS(None, self.CONFIG) remote.gs() diff --git a/tests/unit/remote/ssh.py b/tests/unit/remote/ssh.py index 13aafea48..ad768806f 100644 --- a/tests/unit/remote/ssh.py +++ b/tests/unit/remote/ssh.py @@ -7,31 +7,31 @@ from dvc.remote.ssh import RemoteSSH class TestRemoteSSH(TestCase): def test_user(self): - url = 'ssh://127.0.0.1:/path/to/dir' - config = {'url': url} + url = "ssh://127.0.0.1:/path/to/dir" + config = {"url": url} remote = RemoteSSH(None, config) self.assertEqual(remote.user, getpass.getuser()) - user = 'test1' - url = 'ssh://{}@127.0.0.1:/path/to/dir'.format(user) - config = {'url': url} + user = "test1" + url = "ssh://{}@127.0.0.1:/path/to/dir".format(user) + config = {"url": url} remote = RemoteSSH(None, config) self.assertEqual(remote.user, user) - user = 'test2' - config['user'] = user + user = "test2" + config["user"] = user remote = RemoteSSH(None, config) self.assertEqual(remote.user, user) def test_url(self): - user = 'test' - host = '123.45.67.89' + user = "test" + host = "123.45.67.89" port = 1234 - path = '/path/to/dir' + path = "/path/to/dir" # URL ssh://[user@]host.xz[:port]/path - url = 'ssh://{}@{}:{}{}'.format(user, host, port, path) - config = {'url': url} + url = "ssh://{}@{}:{}{}".format(user, host, port, path) + config = {"url": url} remote = RemoteSSH(None, config) @@ -41,8 +41,8 @@ class TestRemoteSSH(TestCase): self.assertEqual(remote.prefix, path) # SCP-like URL ssh://[user@]host.xz:path - url = 'ssh://{}@{}:{}'.format(user, host, path) - config = {'url': url} + url = "ssh://{}@{}:{}".format(user, host, path) + config = {"url": url} remote = RemoteSSH(None, config) @@ -52,18 +52,18 @@ class TestRemoteSSH(TestCase): self.assertEqual(remote.prefix, path) def test_port(self): - url = 'ssh://127.0.0.1/path/to/dir' - config = {'url': url} + url = "ssh://127.0.0.1/path/to/dir" + config = {"url": url} remote = RemoteSSH(None, config) self.assertEqual(remote.port, remote.DEFAULT_PORT) port = 1234 - url = 'ssh://127.0.0.1:{}/path/to/dir'.format(port) - config = {'url': url} + url = "ssh://127.0.0.1:{}/path/to/dir".format(port) + config = {"url": url} remote = RemoteSSH(None, config) self.assertEqual(remote.port, port) port = 4321 - config['port'] = port + config["port"] = port remote = RemoteSSH(None, config) self.assertEqual(remote.port, port) diff --git a/tests/unit/stage.py b/tests/unit/stage.py index 2934643bc..92862fcd1 100644 --- a/tests/unit/stage.py +++ b/tests/unit/stage.py @@ -7,13 +7,9 @@ from dvc.stage import Stage class TestStageChecksum(TestCase): def test(self): stage = Stage(None) - outs = [{'path': 'a', 'md5': '123456789'}] - deps = [{'path': 'b', 'md5': '987654321'}] - d = {'md5': '123456', - 'cmd': 'mycmd', - 'outs': outs, - 'deps': deps} + outs = [{"path": "a", "md5": "123456789"}] + deps = [{"path": "b", "md5": "987654321"}] + d = {"md5": "123456", "cmd": "mycmd", "outs": outs, "deps": deps} - with mock.patch.object(stage, 'dumpd', return_value=d): - self.assertEqual(stage._compute_md5(), - 'e9521a22111493406ea64a88cda63e0b') + with mock.patch.object(stage, "dumpd", return_value=d): + self.assertEqual(stage._compute_md5(), "e9521a22111493406ea64a88cda63e0b") diff --git a/tests/unit/state.py b/tests/unit/state.py index e967da717..ab585ea69 100644 --- a/tests/unit/state.py +++ b/tests/unit/state.py @@ -12,10 +12,12 @@ class TestMtimeAndSize(TestDvc): dir_time, dir_size = State._mtime_and_size(self.DATA_DIR) actual_file_size = os.path.getsize(self.DATA) - actual_dir_size = (os.path.getsize(self.DATA_DIR) - + os.path.getsize(self.DATA) - + os.path.getsize(self.DATA_SUB_DIR) - + os.path.getsize(self.DATA_SUB)) + actual_dir_size = ( + os.path.getsize(self.DATA_DIR) + + os.path.getsize(self.DATA) + + os.path.getsize(self.DATA_SUB_DIR) + + os.path.getsize(self.DATA_SUB) + ) self.assertIs(type(file_time), str) self.assertIs(type(file_size), str) diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index cbb6bf558..d4e2f91e1 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -14,5 +14,5 @@ def spy(method_to_decorate): def load_stage_file(path): - with open(path, 'r') as fobj: + with open(path, "r") as fobj: return yaml.safe_load(fobj) diff --git a/tests/utils/httpd.py b/tests/utils/httpd.py index be5c44cf0..4d11d6913 100644 --- a/tests/utils/httpd.py +++ b/tests/utils/httpd.py @@ -10,9 +10,9 @@ class ETagHandler(SimpleHTTPRequestHandler): file = self.translate_path(self.path) if not os.path.isdir(file) and os.path.exists(file): - with open(file, 'r') as fd: - etag = hashlib.md5(fd.read().encode('utf8')).hexdigest() - self.send_header('ETag', etag) + with open(file, "r") as fd: + etag = hashlib.md5(fd.read().encode("utf8")).hexdigest() + self.send_header("ETag", etag) SimpleHTTPRequestHandler.end_headers(self) @@ -22,9 +22,9 @@ class ContentMD5Handler(SimpleHTTPRequestHandler): file = self.translate_path(self.path) if not os.path.isdir(file) and os.path.exists(file): - with open(file, 'r') as fd: - md5 = hashlib.md5(fd.read().encode('utf8')).hexdigest() - self.send_header('Content-MD5', md5) + with open(file, "r") as fd: + md5 = hashlib.md5(fd.read().encode("utf8")).hexdigest() + self.send_header("Content-MD5", md5) SimpleHTTPRequestHandler.end_headers(self) @@ -32,11 +32,11 @@ class ContentMD5Handler(SimpleHTTPRequestHandler): class StaticFileServer: __server_lock = threading.Lock() - def __init__(self, handler='etag'): - handler_class = ETagHandler if handler == 'etag' else ContentMD5Handler + def __init__(self, handler="etag"): + handler_class = ETagHandler if handler == "etag" else ContentMD5Handler self.__server_lock.acquire() - self.httpd = HTTPServer(('localhost', 8000), handler_class) + self.httpd = HTTPServer(("localhost", 8000), handler_class) def __enter__(self): self.server_thread = threading.Thread(target=self.httpd.serve_forever)