From a86470ccbc693d6f0a4f8066cbd1357e5191c4ff Mon Sep 17 00:00:00 2001 From: skshetry <18718008+skshetry@users.noreply.github.com> Date: Sat, 4 Feb 2023 09:13:07 +0545 Subject: [PATCH] update black and change line-length to 88 (#8953) --- .flake8 | 2 +- .pre-commit-config.yaml | 2 +- dvc/_debug.py | 24 ++--- dvc/api/data.py | 8 +- dvc/cli/__init__.py | 25 +++--- dvc/cli/actions.py | 8 +- dvc/cli/utils.py | 4 +- dvc/commands/add.py | 5 +- dvc/commands/cache.py | 10 ++- dvc/commands/check_ignore.py | 20 ++--- dvc/commands/checkout.py | 8 +- dvc/commands/commit.py | 4 +- dvc/commands/config.py | 12 +-- dvc/commands/dag.py | 10 +-- dvc/commands/data.py | 29 +++---- dvc/commands/data_sync.py | 25 +++--- dvc/commands/diff.py | 23 ++--- dvc/commands/experiments/__init__.py | 9 +- dvc/commands/experiments/apply.py | 10 +-- dvc/commands/experiments/branch.py | 6 +- dvc/commands/experiments/diff.py | 6 +- dvc/commands/experiments/exec_run.py | 5 +- dvc/commands/experiments/gc.py | 9 +- dvc/commands/experiments/init.py | 45 ++++++---- dvc/commands/experiments/pull.py | 12 +-- dvc/commands/experiments/push.py | 21 ++--- dvc/commands/experiments/remove.py | 1 - dvc/commands/experiments/run.py | 8 +- dvc/commands/experiments/save.py | 1 - dvc/commands/experiments/show.py | 29 ++----- dvc/commands/gc.py | 17 ++-- dvc/commands/get.py | 10 +-- dvc/commands/imp.py | 6 +- dvc/commands/imp_url.py | 43 ++++----- dvc/commands/init.py | 8 +- dvc/commands/install.py | 6 +- dvc/commands/machine.py | 37 ++------ dvc/commands/move.py | 8 +- dvc/commands/params.py | 2 +- dvc/commands/plots.py | 45 +++------- dvc/commands/queue/kill.py | 3 +- dvc/commands/queue/logs.py | 4 +- dvc/commands/queue/remove.py | 2 - dvc/commands/queue/start.py | 14 +-- dvc/commands/queue/status.py | 8 +- dvc/commands/queue/stop.py | 1 - dvc/commands/remote.py | 16 +--- dvc/commands/remove.py | 3 +- dvc/commands/repro.py | 31 +++---- dvc/commands/run.py | 10 +-- dvc/commands/stage.py | 32 ++++--- dvc/commands/status.py | 4 +- dvc/commands/update.py | 6 +- dvc/commands/version.py | 4 +- dvc/compare.py | 35 ++------ dvc/config.py | 7 +- dvc/config_schema.py | 19 +--- dvc/daemon.py | 4 +- dvc/dagascii.py | 4 +- dvc/data_cloud.py | 6 +- dvc/dependency/__init__.py | 4 +- dvc/dependency/base.py | 8 +- dvc/dependency/param.py | 4 +- dvc/dependency/repo.py | 29 ++----- dvc/dvcfile.py | 33 ++----- dvc/exceptions.py | 29 ++----- dvc/external_repo.py | 12 +-- dvc/fs/__init__.py | 4 +- dvc/fs/callbacks.py | 4 +- dvc/fs/dvc.py | 27 ++---- dvc/ignore.py | 36 +++----- dvc/info.py | 6 +- dvc/lock.py | 14 +-- dvc/logger.py | 23 ++--- dvc/machine/__init__.py | 4 +- dvc/machine/backend/base.py | 4 +- dvc/machine/backend/terraform.py | 8 +- dvc/output.py | 51 +++-------- dvc/parsing/__init__.py | 31 +++---- dvc/parsing/context.py | 29 ++----- dvc/parsing/interpolate.py | 9 +- dvc/parsing/versions.py | 4 +- dvc/proc/exceptions.py | 4 +- dvc/proc/manager.py | 4 +- dvc/progress.py | 14 +-- dvc/prompt.py | 4 +- dvc/render/convert.py | 8 +- dvc/render/converter/vega.py | 14 ++- dvc/render/match.py | 13 +-- dvc/repo/__init__.py | 34 ++------ dvc/repo/add.py | 11 +-- dvc/repo/brancher.py | 12 +-- dvc/repo/checkout.py | 15 +--- dvc/repo/commit.py | 7 +- dvc/repo/data.py | 15 +--- dvc/repo/diff.py | 26 ++---- dvc/repo/experiments/__init__.py | 19 ++-- dvc/repo/experiments/apply.py | 13 ++- dvc/repo/experiments/branch.py | 12 +-- dvc/repo/experiments/diff.py | 4 +- dvc/repo/experiments/exceptions.py | 20 ++--- dvc/repo/experiments/executor/base.py | 50 +++++------ dvc/repo/experiments/executor/local.py | 1 - dvc/repo/experiments/executor/ssh.py | 24 ++--- dvc/repo/experiments/init.py | 20 ++--- dvc/repo/experiments/pull.py | 11 +-- dvc/repo/experiments/push.py | 8 +- dvc/repo/experiments/queue/base.py | 63 +++++--------- dvc/repo/experiments/queue/celery.py | 36 ++------ dvc/repo/experiments/queue/remove.py | 4 +- dvc/repo/experiments/queue/tasks.py | 4 +- dvc/repo/experiments/queue/tempdir.py | 20 ++--- dvc/repo/experiments/queue/workspace.py | 22 ++--- dvc/repo/experiments/refs.py | 1 - dvc/repo/experiments/save.py | 14 +-- dvc/repo/experiments/show.py | 9 +- dvc/repo/experiments/utils.py | 12 +-- dvc/repo/fetch.py | 4 +- dvc/repo/gc.py | 5 +- dvc/repo/get.py | 4 +- dvc/repo/imp.py | 4 +- dvc/repo/imp_url.py | 8 +- dvc/repo/imports.py | 4 +- dvc/repo/index.py | 23 ++--- dvc/repo/metrics/diff.py | 4 +- dvc/repo/metrics/show.py | 17 ++-- dvc/repo/move.py | 4 +- dvc/repo/params/diff.py | 4 +- dvc/repo/params/show.py | 26 +----- dvc/repo/plots/__init__.py | 22 ++--- dvc/repo/plots/diff.py | 4 +- dvc/repo/push.py | 5 +- dvc/repo/reproduce.py | 13 ++- dvc/repo/run.py | 7 +- dvc/repo/scm_context.py | 15 +--- dvc/repo/stage.py | 31 ++----- dvc/repo/status.py | 22 ++--- dvc/repo/update.py | 20 ++--- dvc/repo/worktree.py | 30 ++----- dvc/rwlock.py | 16 ++-- dvc/schema.py | 9 +- dvc/scm.py | 21 ++--- dvc/stage/__init__.py | 52 +++-------- dvc/stage/cache.py | 18 +--- dvc/stage/exceptions.py | 9 +- dvc/stage/loader.py | 20 ++--- dvc/stage/monitor.py | 12 +-- dvc/stage/run.py | 10 +-- dvc/stage/serialize.py | 7 +- dvc/stage/utils.py | 32 ++----- dvc/testing/api_tests.py | 12 +-- dvc/testing/fixtures.py | 24 ++--- dvc/testing/path_info.py | 8 +- dvc/testing/remote_tests.py | 16 +--- dvc/testing/tmp_dir.py | 15 +--- dvc/testing/workspace_tests.py | 60 +++---------- dvc/ui/__init__.py | 27 ++---- dvc/ui/pager.py | 3 +- dvc/ui/prompt.py | 20 ++--- dvc/ui/table.py | 8 +- dvc/updater.py | 15 +--- dvc/utils/__init__.py | 16 +--- dvc/utils/cli_parse.py | 1 - dvc/utils/collections.py | 25 ++---- dvc/utils/diff.py | 6 +- dvc/utils/fs.py | 4 +- dvc/utils/humanize.py | 12 +-- dvc/utils/hydra.py | 3 +- dvc/utils/plots.py | 4 +- dvc/utils/serialize/__init__.py | 8 +- dvc/utils/serialize/_common.py | 12 +-- dvc/utils/serialize/_py.py | 17 +--- dvc/utils/strictyaml.py | 12 +-- dvc/utils/table.py | 8 +- dvc/utils/threadpool.py | 6 +- pyproject.toml | 4 +- scripts/build.py | 12 +-- scripts/fpm/build.py | 4 +- scripts/fpm/notarize.py | 4 +- scripts/pyinstaller/hooks/hook-dvc.py | 4 +- scripts/pyinstaller/hooks/hook-dvc.tree.gs.py | 4 +- .../hooks/hook-dvc.utils.flatten.py | 4 +- .../hook-google_compute_engine.logger.py | 4 +- scripts/pyinstaller/hooks/hook-pydrive2.py | 4 +- tests/conftest.py | 5 +- tests/func/api/test_data.py | 40 +++------ tests/func/experiments/test_checkpoints.py | 29 ++----- tests/func/experiments/test_experiments.py | 41 +++------ tests/func/experiments/test_init.py | 36 +++----- tests/func/experiments/test_remote.py | 52 +++-------- tests/func/experiments/test_remove.py | 32 +++---- tests/func/experiments/test_save.py | 5 +- tests/func/experiments/test_set_params.py | 4 +- tests/func/experiments/test_show.py | 80 ++++++----------- tests/func/experiments/test_utils.py | 4 +- tests/func/machine/test_machine_config.py | 20 ++--- tests/func/machine/test_machine_status.py | 13 ++- tests/func/metrics/test_diff.py | 32 ++----- tests/func/metrics/test_show.py | 72 ++++----------- tests/func/params/test_diff.py | 20 ++--- tests/func/params/test_show.py | 57 +++--------- tests/func/parsing/test_errors.py | 77 ++++++++-------- tests/func/parsing/test_foreach.py | 16 +--- tests/func/parsing/test_interpolated_entry.py | 32 +++---- tests/func/parsing/test_resolver.py | 24 +++-- tests/func/plots/test_diff.py | 6 +- tests/func/plots/test_modify.py | 8 +- tests/func/plots/test_show.py | 27 ++---- tests/func/test_add.py | 33 ++----- tests/func/test_analytics.py | 4 +- tests/func/test_check_ignore.py | 12 +-- tests/func/test_checkout.py | 24 ++--- tests/func/test_commit.py | 12 +-- tests/func/test_config.py | 39 +++------ tests/func/test_data_cloud.py | 39 +++------ tests/func/test_data_status.py | 24 ++--- tests/func/test_diff.py | 16 +--- tests/func/test_dvcfile.py | 19 ++-- tests/func/test_external_repo.py | 4 +- tests/func/test_gc.py | 36 +++----- tests/func/test_get.py | 20 ++--- tests/func/test_ignore.py | 20 ++--- tests/func/test_import.py | 48 +++------- tests/func/test_import_url.py | 28 ++---- tests/func/test_init.py | 7 +- tests/func/test_install.py | 16 ++-- tests/func/test_lock.py | 4 +- tests/func/test_lockfile.py | 16 +--- tests/func/test_ls.py | 13 +-- tests/func/test_merge_driver.py | 15 +--- tests/func/test_odb.py | 4 +- tests/func/test_remote.py | 47 +++------- tests/func/test_remove.py | 5 +- tests/func/test_repo_index.py | 16 +--- tests/func/test_repro.py | 63 ++++---------- tests/func/test_repro_multistage.py | 29 ++----- tests/func/test_run_cache.py | 6 +- tests/func/test_run_multistage.py | 20 ++--- tests/func/test_run_single_stage.py | 12 +-- tests/func/test_stage.py | 20 +---- tests/func/test_stage_load.py | 87 ++++++------------- tests/func/test_state.py | 4 +- tests/func/test_status.py | 34 ++------ tests/func/test_unprotect.py | 4 +- tests/func/test_update.py | 42 ++------- tests/func/test_used_objs.py | 8 +- tests/func/utils/test_strict_yaml.py | 13 +-- tests/integration/plots/test_plots.py | 86 +++++------------- .../integration/plots/test_repo_plots_api.py | 26 ++---- tests/unit/cli/test_main.py | 10 +-- tests/unit/command/ls/test_ls.py | 24 ++--- tests/unit/command/ls/test_ls_colors.py | 12 +-- tests/unit/command/test_add.py | 8 +- tests/unit/command/test_checkout.py | 4 +- tests/unit/command/test_config.py | 4 +- tests/unit/command/test_dag.py | 32 ++----- tests/unit/command/test_data_status.py | 4 +- tests/unit/command/test_diff.py | 22 +++-- tests/unit/command/test_experiments.py | 50 +++-------- tests/unit/command/test_get.py | 8 +- tests/unit/command/test_imp_url.py | 8 +- tests/unit/command/test_machine.py | 17 +--- tests/unit/command/test_metrics.py | 8 +- tests/unit/command/test_params.py | 4 +- tests/unit/command/test_plots.py | 47 +++------- tests/unit/command/test_queue.py | 1 - tests/unit/command/test_repro.py | 4 +- tests/unit/data/db/test_local.py | 20 ++--- tests/unit/dependency/test_params.py | 51 +++++------ tests/unit/fs/test_azure.py | 18 ++-- tests/unit/fs/test_data.py | 12 +-- tests/unit/fs/test_dvc.py | 56 +++--------- tests/unit/fs/test_s3.py | 3 +- tests/unit/fs/test_tree.py | 8 +- tests/unit/output/test_annotations.py | 4 +- tests/unit/output/test_load.py | 4 +- tests/unit/proc/test_manager.py | 5 +- tests/unit/proc/test_process.py | 4 +- tests/unit/remote/test_webdav.py | 12 +-- tests/unit/remote/test_webhdfs.py | 4 +- tests/unit/render/test_convert.py | 12 +-- tests/unit/render/test_match.py | 14 +-- tests/unit/repo/experiments/conftest.py | 10 +-- .../repo/experiments/queue/test_celery.py | 13 +-- .../repo/experiments/queue/test_remove.py | 34 +++----- .../repo/experiments/test_executor_status.py | 18 +--- tests/unit/repo/experiments/test_remove.py | 3 +- tests/unit/repo/experiments/test_utils.py | 1 - tests/unit/repo/plots/test_diff.py | 4 +- tests/unit/repo/test_repo.py | 4 +- tests/unit/repo/test_reproduce.py | 16 +--- tests/unit/repo/test_scm_context.py | 6 +- tests/unit/stage/test_cache.py | 6 +- tests/unit/stage/test_loader_pipeline_file.py | 16 +--- .../stage/test_serialize_pipeline_file.py | 21 ++--- .../stage/test_serialize_pipeline_lock.py | 42 +++------ tests/unit/stage/test_stage.py | 4 +- tests/unit/stage/test_utils.py | 4 +- tests/unit/test_analytics.py | 4 +- tests/unit/test_api.py | 4 +- tests/unit/test_compare.py | 74 ++++------------ tests/unit/test_dvcfile.py | 13 +-- tests/unit/test_external_repo.py | 11 +-- tests/unit/test_hashinfo.py | 4 +- tests/unit/test_ignore.py | 8 +- tests/unit/test_info.py | 12 +-- tests/unit/test_interpolate.py | 4 +- tests/unit/test_lockfile.py | 4 +- tests/unit/test_tabular_data.py | 8 +- tests/unit/test_updater.py | 22 ++--- tests/unit/ui/test_pager.py | 27 ++---- tests/unit/ui/test_prompt.py | 11 +-- tests/unit/ui/test_table.py | 45 ++++++---- tests/unit/utils/test_fs.py | 8 +- tests/unit/utils/test_humanize.py | 9 +- tests/unit/utils/test_utils.py | 18 +--- tests/utils/plots.py | 4 +- 317 files changed, 1520 insertions(+), 3738 deletions(-) diff --git a/.flake8 b/.flake8 index 879ffb212..fd87e09b7 100644 --- a/.flake8 +++ b/.flake8 @@ -11,7 +11,7 @@ ignore= P1 # use `!r` instead of surrounding by quotes B028 -max_line_length = 79 +max_line_length = 88 max-complexity = 10 select = B,C,E,F,W,T4,B902,T,P,TYP show_source = true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f690e4521..253e27166 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,7 +35,7 @@ repos: args: - --py38-plus - repo: https://github.com/psf/black - rev: 22.12.0 + rev: 23.1.0 hooks: - id: black - repo: https://github.com/PyCQA/isort diff --git a/dvc/_debug.py b/dvc/_debug.py index 2cf877d76..1c4001b2d 100644 --- a/dvc/_debug.py +++ b/dvc/_debug.py @@ -16,9 +16,7 @@ def viztracer_profile( try: import viztracer # pylint: disable=import-error except ImportError: - print( # noqa: T201 - "Failed to run profiler, viztracer is not installed" - ) + print("Failed to run profiler, viztracer is not installed") # noqa: T201 yield return @@ -80,9 +78,7 @@ def instrument(html_output=False): try: from pyinstrument import Profiler # pylint: disable=import-error except ImportError: - print( # noqa: T201 - "Failed to run profiler, pyinstrument is not installed" - ) + print("Failed to run profiler, pyinstrument is not installed") # noqa: T201 yield return @@ -194,9 +190,7 @@ def debugtools(args: Optional["Namespace"] = None, **kwargs): path_func = _get_path_func("viztracer", "json") depth = kw.get("viztracer_depth") or -1 log_async = kw.get("viztracer_async") or False - prof = viztracer_profile( - path=path_func, depth=depth, log_async=log_async - ) + prof = viztracer_profile(path=path_func, depth=depth, log_async=log_async) stack.enter_context(prof) yield @@ -204,12 +198,8 @@ def debugtools(args: Optional["Namespace"] = None, **kwargs): def add_debugging_flags(parser): from argparse import SUPPRESS - parser.add_argument( - "--cprofile", action="store_true", default=False, help=SUPPRESS - ) - parser.add_argument( - "--yappi", action="store_true", default=False, help=SUPPRESS - ) + parser.add_argument("--cprofile", action="store_true", default=False, help=SUPPRESS) + parser.add_argument("--yappi", action="store_true", default=False, help=SUPPRESS) parser.add_argument( "--yappi-separate-threads", action="store_true", @@ -224,9 +214,7 @@ def add_debugging_flags(parser): "--viztracer-async", action="store_true", default=False, help=SUPPRESS ) parser.add_argument("--cprofile-dump", help=SUPPRESS) - parser.add_argument( - "--pdb", action="store_true", default=False, help=SUPPRESS - ) + parser.add_argument("--pdb", action="store_true", default=False, help=SUPPRESS) parser.add_argument( "--instrument", action="store_true", default=False, help=SUPPRESS ) diff --git a/dvc/api/data.py b/dvc/api/data.py index 69b1423ce..423941b21 100644 --- a/dvc/api/data.py +++ b/dvc/api/data.py @@ -34,9 +34,7 @@ def get_url(path, repo=None, rev=None, remote=None): class _OpenContextManager(GCM): - def __init__( - self, func, args, kwds - ): # pylint: disable=super-init-not-called + def __init__(self, func, args, kwds): # pylint: disable=super-init-not-called self.gen = func(*args, **kwds) self.func, self.args, self.kwds = ( # type: ignore[assignment] func, @@ -45,9 +43,7 @@ class _OpenContextManager(GCM): ) def __getattr__(self, name): - raise AttributeError( - "dvc.api.open() should be used in a with statement." - ) + raise AttributeError("dvc.api.open() should be used in a with statement.") def open( # noqa, pylint: disable=redefined-builtin diff --git a/dvc/cli/__init__.py b/dvc/cli/__init__.py index 531b10c92..9b14059bb 100644 --- a/dvc/cli/__init__.py +++ b/dvc/cli/__init__.py @@ -56,8 +56,10 @@ def _log_exceptions(exc: Exception) -> Optional[int]: if exc.errno == errno.EMFILE: logger.exception( - "too many open files, please visit " - "%s to see how to handle this problem", + ( + "too many open files, please visit " + "%s to see how to handle this problem" + ), error_link("many-files"), extra={"tb_only": True}, ) @@ -80,7 +82,7 @@ def _log_exceptions(exc: Exception) -> Optional[int]: if cmd: link = format_link("https://dvc.org/doc/install") hint = ( - f"To install dvc with those dependencies, run:\n" + "To install dvc with those dependencies, run:\n" "\n" f"\t{cmd}\n" "\n" @@ -91,8 +93,7 @@ def _log_exceptions(exc: Exception) -> Optional[int]: hint = f"\nPlease report this bug to {link}. Thank you!" logger.exception( - "URL '%s' is supported but requires these missing " - "dependencies: %s. %s", + "URL '%s' is supported but requires these missing dependencies: %s. %s", exc.url, exc.missing_deps, hint, @@ -118,9 +119,11 @@ def _log_exceptions(exc: Exception) -> Optional[int]: directory = relpath(exc.directory) logger.exception( - "Could not open pickled '%s' cache.\n" - "Remove the '%s' directory and then retry this command." - "\nSee %s for more information.", + ( + "Could not open pickled '%s' cache.\n" + "Remove the '%s' directory and then retry this command." + "\nSee %s for more information." + ), exc.type, directory, error_link("pickle"), @@ -181,11 +184,7 @@ def main(argv=None): # noqa: C901 set_loggers_level(level) if level and level <= logging.DEBUG: - from platform import ( - platform, - python_implementation, - python_version, - ) + from platform import platform, python_implementation, python_version from dvc import __version__ from dvc.utils.pkg import PKG diff --git a/dvc/cli/actions.py b/dvc/cli/actions.py index 299974d39..5a05e7912 100644 --- a/dvc/cli/actions.py +++ b/dvc/cli/actions.py @@ -2,9 +2,7 @@ from argparse import _AppendAction class CommaSeparatedArgs(_AppendAction): # pylint: disable=protected-access - def __call__( - self, parser, namespace, values, option_string=None # noqa: ARG002 - ): + def __call__(self, parser, namespace, values, option_string=None): # noqa: ARG002 from funcy import ldistinct items = getattr(namespace, self.dest) or [] @@ -13,9 +11,7 @@ class CommaSeparatedArgs(_AppendAction): # pylint: disable=protected-access class KeyValueArgs(_AppendAction): - def __call__( - self, parser, namespace, values, option_string=None # noqa: ARG002 - ): + def __call__(self, parser, namespace, values, option_string=None): # noqa: ARG002 items = getattr(namespace, self.dest) or {} for value in filter(bool, values): key, _, value = value.partition("=") diff --git a/dvc/cli/utils.py b/dvc/cli/utils.py index 537d6d11e..15d9671da 100644 --- a/dvc/cli/utils.py +++ b/dvc/cli/utils.py @@ -28,7 +28,5 @@ def hide_subparsers_from_help(subparsers): # Need to set `add_help=False`, but avoid setting `help` # (not even to `argparse.SUPPPRESS`). # NOTE: The argument is the parent subparser, not the subcommand parser. - cmds = [ - cmd for cmd, parser in subparsers.choices.items() if parser.add_help - ] + cmds = [cmd for cmd, parser in subparsers.choices.items() if parser.add_help] subparsers.metavar = "{{{}}}".format(",".join(cmds)) diff --git a/dvc/commands/add.py b/dvc/commands/add.py index 5b7c6549a..d5dd0d004 100644 --- a/dvc/commands/add.py +++ b/dvc/commands/add.py @@ -41,10 +41,7 @@ def _add_annotating_args(parser: argparse.ArgumentParser) -> None: class CmdAdd(CmdBase): def run(self): - from dvc.exceptions import ( - DvcException, - RecursiveAddingWhileUsingFilename, - ) + from dvc.exceptions import DvcException, RecursiveAddingWhileUsingFilename try: if len(self.args.targets) > 1 and self.args.file: diff --git a/dvc/commands/cache.py b/dvc/commands/cache.py index 569b09212..cab800535 100644 --- a/dvc/commands/cache.py +++ b/dvc/commands/cache.py @@ -68,10 +68,12 @@ def add_parser(subparsers, parent_parser): ) cache_dir_parser.add_argument( "value", - help="Path to cache directory. Relative paths are resolved relative " - "to the current directory and saved to config relative to the " - "config file location. If no path is provided, it returns the " - "current cache directory.", + help=( + "Path to cache directory. Relative paths are resolved relative " + "to the current directory and saved to config relative to the " + "config file location. If no path is provided, it returns the " + "current cache directory." + ), nargs="?", ).complete = completion.DIR cache_dir_parser.set_defaults(func=CmdCacheDir) diff --git a/dvc/commands/check_ignore.py b/dvc/commands/check_ignore.py index bc21e4ef0..69a33291d 100644 --- a/dvc/commands/check_ignore.py +++ b/dvc/commands/check_ignore.py @@ -62,9 +62,7 @@ class CmdCheckIgnore(CmdBase): raise DvcException("cannot have both `targets` and `--stdin`") if self.args.non_matching and not self.args.details: - raise DvcException( - "`--non-matching` is only valid with `--details`" - ) + raise DvcException("`--non-matching` is only valid with `--details`") if self.args.all and not self.args.details: raise DvcException("`--all` is only valid with `--details`") @@ -80,9 +78,7 @@ class CmdCheckIgnore(CmdBase): def add_parser(subparsers, parent_parser): - ADD_HELP = ( - "Check whether files or directories are excluded due to `.dvcignore`." - ) + ADD_HELP = "Check whether files or directories are excluded due to `.dvcignore`." parser = subparsers.add_parser( "check-ignore", @@ -103,16 +99,20 @@ def add_parser(subparsers, parent_parser): "--all", action="store_true", default=False, - help="Include the target paths which don't match any pattern " - "in the `--details` list.", + help=( + "Include the target paths which don't match any pattern " + "in the `--details` list." + ), ) parser.add_argument( "-n", "--non-matching", action="store_true", default=False, - help="Include the target paths which don't match any pattern " - "in the `--details` list.", + help=( + "Include the target paths which don't match any pattern " + "in the `--details` list." + ), ) parser.add_argument( "--stdin", diff --git a/dvc/commands/checkout.py b/dvc/commands/checkout.py index 7d323de78..228b2d508 100644 --- a/dvc/commands/checkout.py +++ b/dvc/commands/checkout.py @@ -22,9 +22,7 @@ def log_changes(stats): continue for entry in entries: - ui.write( - f"[{color}]{state[0].upper()}", entry, styled=True, sep="\t" - ) + ui.write(f"[{color}]{state[0].upper()}", entry, styled=True, sep="\t") class CmdCheckout(CmdBase): @@ -46,9 +44,7 @@ class CmdCheckout(CmdBase): if self.args.summary: default_message = "No changes." - msg = get_summary( - sorted(stats.items(), key=operator.itemgetter(0)) - ) + msg = get_summary(sorted(stats.items(), key=operator.itemgetter(0))) ui.write(msg or default_message) else: log_changes(stats) diff --git a/dvc/commands/commit.py b/dvc/commands/commit.py index ba155a29b..7285dc144 100644 --- a/dvc/commands/commit.py +++ b/dvc/commands/commit.py @@ -24,9 +24,7 @@ class CmdCommit(CmdBase): force=self.args.force, ) except DvcException: - logger.exception( - "failed to commit%s", (" " + target) if target else "" - ) + logger.exception("failed to commit%s", (" " + target) if target else "") return 1 return 0 diff --git a/dvc/commands/config.py b/dvc/commands/config.py index ddad68b06..e1f710a2b 100644 --- a/dvc/commands/config.py +++ b/dvc/commands/config.py @@ -17,9 +17,7 @@ def _name_type(value): match = re.match(NAME_REGEX, value) if not match: raise argparse.ArgumentTypeError( - "name argument should look like " - "remote.name.option or " - "section.option" + "name argument should look like remote.name.option or section.option" ) return ( bool(match.group("remote")), @@ -70,9 +68,7 @@ class CmdConfig(CmdBaseNoRepo): for level in levels: conf = self.config.read(level) - prefix = self._config_file_prefix( - self.args.show_origin, self.config, level - ) + prefix = self._config_file_prefix(self.args.show_origin, self.config, level) configs = list(self._format_config(conf, prefix)) if configs: ui.write("\n".join(configs)) @@ -131,9 +127,7 @@ class CmdConfig(CmdBaseNoRepo): raise ConfigError(f"{name} '{section}' doesn't exist") if opt and opt not in conf[section]: - raise ConfigError( - f"option '{opt}' doesn't exist in {name} '{section}'" - ) + raise ConfigError(f"option '{opt}' doesn't exist in {name} '{section}'") def _get_appropriate_levels(self, levels): if levels: diff --git a/dvc/commands/dag.py b/dvc/commands/dag.py index e30974c32..6bf7b8722 100644 --- a/dvc/commands/dag.py +++ b/dvc/commands/dag.py @@ -86,9 +86,7 @@ def _collect_targets(repo, target, outs): targets.extend([str(out) for out in stage.outs]) continue - for out in outs_trie.itervalues( # noqa: B301 - prefix=repo.fs.path.parts(path) - ): + for out in outs_trie.itervalues(prefix=repo.fs.path.parts(path)): # noqa: B301 targets.extend(str(out)) return targets @@ -204,7 +202,9 @@ def add_parser(subparsers, parent_parser): dag_parser.add_argument( "target", nargs="?", - help="Stage name or output to show pipeline for. " - "Finds all stages in the workspace by default.", + help=( + "Stage name or output to show pipeline for. " + "Finds all stages in the workspace by default." + ), ) dag_parser.set_defaults(func=CmdDAG) diff --git a/dvc/commands/data.py b/dvc/commands/data.py index eca85b575..d77e6f507 100644 --- a/dvc/commands/data.py +++ b/dvc/commands/data.py @@ -8,11 +8,7 @@ from funcy import chunks, compact, log_durations from dvc.cli import completion from dvc.cli.actions import CommaSeparatedArgs from dvc.cli.command import CmdBase -from dvc.cli.utils import ( - append_doc_link, - fix_subparsers, - hide_subparsers_from_help, -) +from dvc.cli.utils import append_doc_link, fix_subparsers, hide_subparsers_from_help from dvc.ui import ui from dvc.utils import colorize @@ -39,20 +35,19 @@ class CmdDataStatus(CmdBase): } HINTS = { "not_in_cache": ('use "dvc fetch ..." to download files',), - "committed": ( - "git commit the corresponding dvc files to update the repo", - ), + "committed": ("git commit the corresponding dvc files to update the repo",), "uncommitted": ( 'use "dvc commit ..." to track changes', 'use "dvc checkout ..." to discard changes', ), "untracked": ( - 'use "git add ..." or ' - '"dvc add ..." to commit to git or to dvc', + ( + 'use "git add ..." or ' + '"dvc add ..." to commit to git or to dvc' + ), ), "git_dirty": ( - "there are {}changes not tracked by dvc, " - 'use "git status" to see', + 'there are {}changes not tracked by dvc, use "git status" to see', ), } @@ -96,8 +91,7 @@ class CmdDataStatus(CmdBase): if isinstance(stage_status, dict): items = [ - ": ".join([state, file]) - for file, state in stage_status.items() + ": ".join([state, file]) for file, state in stage_status.items() ] else: items = stage_status @@ -180,9 +174,7 @@ class CmdDataLs(CmdBase): filter_labels = set(self.args.labels) filter_types = set(self.args.type) - d = ls( - self.repo, targets=self.args.targets, recursive=self.args.recursive - ) + d = ls(self.repo, targets=self.args.targets, recursive=self.args.recursive) self._show_table( d, filter_labels=filter_labels, @@ -205,8 +197,7 @@ def add_parser(subparsers, parent_parser): fix_subparsers(data_subparsers) DATA_STATUS_HELP = ( - "Show changes between the last git commit, " - "the dvcfiles and the workspace." + "Show changes between the last git commit, the dvcfiles and the workspace." ) data_status_parser = data_subparsers.add_parser( "status", diff --git a/dvc/commands/data_sync.py b/dvc/commands/data_sync.py index 3258a0b60..ce84d05a8 100644 --- a/dvc/commands/data_sync.py +++ b/dvc/commands/data_sync.py @@ -262,9 +262,7 @@ def add_parser(subparsers, _parent_parser): push_parser.set_defaults(func=CmdDataPush) # Fetch - FETCH_HELP = ( - "Download files or directories from remote storage to the cache." - ) + FETCH_HELP = "Download files or directories from remote storage to the cache." fetch_parser = subparsers.add_parser( "fetch", @@ -320,9 +318,7 @@ def add_parser(subparsers, _parent_parser): fetch_parser.set_defaults(func=CmdDataFetch) # Status - STATUS_HELP = ( - "Show changed stages, compare local cache and a remote storage." - ) + STATUS_HELP = "Show changed stages, compare local cache and a remote storage." status_parser = subparsers.add_parser( "status", @@ -360,24 +356,29 @@ def add_parser(subparsers, _parent_parser): "--all-branches", action="store_true", default=False, - help="Show status of a local cache compared to a remote repository " - "for all branches.", + help=( + "Show status of a local cache compared to a remote repository " + "for all branches." + ), ) status_parser.add_argument( "-T", "--all-tags", action="store_true", default=False, - help="Show status of a local cache compared to a remote repository " - "for all tags.", + help=( + "Show status of a local cache compared to a remote repository for all tags." + ), ) status_parser.add_argument( "-A", "--all-commits", action="store_true", default=False, - help="Show status of a local cache compared to a remote repository " - "for all commits.", + help=( + "Show status of a local cache compared to a remote repository " + "for all commits." + ), ) status_parser.add_argument( "-d", diff --git a/dvc/commands/diff.py b/dvc/commands/diff.py index 1b2da7732..9ab14983f 100644 --- a/dvc/commands/diff.py +++ b/dvc/commands/diff.py @@ -112,9 +112,7 @@ class CmdDiff(CmdBase): return states_summary = ", ".join( - f"{summary[state]} {state}" - for state in states - if summary[state] > 0 + f"{summary[state]} {state}" for state in states if summary[state] > 0 ) ui.write("files summary:", states_summary) @@ -122,9 +120,7 @@ class CmdDiff(CmdBase): from dvc.exceptions import DvcException try: - diff = self.repo.diff( - self.args.a_rev, self.args.b_rev, self.args.targets - ) + diff = self.repo.diff(self.args.a_rev, self.args.b_rev, self.args.targets) show_hash = self.args.show_hash hide_missing = self.args.b_rev or self.args.hide_missing if hide_missing: @@ -133,9 +129,11 @@ class CmdDiff(CmdBase): for key, entries in diff.items(): entries = sorted( entries, - key=lambda entry: entry["path"]["old"] - if isinstance(entry["path"], dict) - else entry["path"], + key=lambda entry: ( + entry["path"]["old"] + if isinstance(entry["path"], dict) + else entry["path"] + ), ) if not show_hash: for entry in entries: @@ -170,10 +168,7 @@ def add_parser(subparsers, parent_parser): diff_parser.add_argument( "--targets", nargs="*", - help=( - "Specific DVC-tracked files to compare. " - "Accepts one or more file paths." - ), + help="Specific DVC-tracked files to compare. Accepts one or more file paths.", metavar="", ).complete = completion.FILE diff_parser.add_argument( @@ -184,7 +179,7 @@ def add_parser(subparsers, parent_parser): ) diff_parser.add_argument( "b_rev", - help=("New Git commit to compare (defaults to the current workspace)"), + help="New Git commit to compare (defaults to the current workspace)", nargs="?", ) diff_parser.add_argument( diff --git a/dvc/commands/experiments/__init__.py b/dvc/commands/experiments/__init__.py index adbe6b923..7546b0ae7 100644 --- a/dvc/commands/experiments/__init__.py +++ b/dvc/commands/experiments/__init__.py @@ -1,10 +1,6 @@ import argparse -from dvc.cli.utils import ( - append_doc_link, - fix_subparsers, - hide_subparsers_from_help, -) +from dvc.cli.utils import append_doc_link, fix_subparsers, hide_subparsers_from_help from dvc.commands.experiments import ( apply, branch, @@ -54,8 +50,7 @@ def add_parser(subparsers, parent_parser): experiments_subparsers = experiments_parser.add_subparsers( dest="cmd", - help="Use `dvc experiments CMD --help` to display " - "command-specific help.", + help="Use `dvc experiments CMD --help` to display command-specific help.", ) fix_subparsers(experiments_subparsers) diff --git a/dvc/commands/experiments/apply.py b/dvc/commands/experiments/apply.py index 12e652ae5..642a3fa22 100644 --- a/dvc/commands/experiments/apply.py +++ b/dvc/commands/experiments/apply.py @@ -10,19 +10,13 @@ logger = logging.getLogger(__name__) class CmdExperimentsApply(CmdBase): def run(self): - - self.repo.experiments.apply( - self.args.experiment, force=self.args.force - ) + self.repo.experiments.apply(self.args.experiment, force=self.args.force) return 0 def add_parser(experiments_subparsers, parent_parser): - - EXPERIMENTS_APPLY_HELP = ( - "Apply the changes from an experiment to your workspace." - ) + EXPERIMENTS_APPLY_HELP = "Apply the changes from an experiment to your workspace." experiments_apply_parser = experiments_subparsers.add_parser( "apply", parents=[parent_parser], diff --git a/dvc/commands/experiments/branch.py b/dvc/commands/experiments/branch.py index 022a4d891..6d8137513 100644 --- a/dvc/commands/experiments/branch.py +++ b/dvc/commands/experiments/branch.py @@ -9,14 +9,12 @@ logger = logging.getLogger(__name__) class CmdExperimentsBranch(CmdBase): def run(self): - self.repo.experiments.branch(self.args.experiment, self.args.branch) return 0 def add_parser(experiments_subparsers, parent_parser): - EXPERIMENTS_BRANCH_HELP = "Promote an experiment to a Git branch." experiments_branch_parser = experiments_subparsers.add_parser( "branch", @@ -28,7 +26,5 @@ def add_parser(experiments_subparsers, parent_parser): experiments_branch_parser.add_argument( "experiment", help="Experiment to be promoted." ) - experiments_branch_parser.add_argument( - "branch", help="Name for the new Git branch" - ) + experiments_branch_parser.add_argument("branch", help="Name for the new Git branch") experiments_branch_parser.set_defaults(func=CmdExperimentsBranch) diff --git a/dvc/commands/experiments/diff.py b/dvc/commands/experiments/diff.py index 4956d34d1..de3fc5e50 100644 --- a/dvc/commands/experiments/diff.py +++ b/dvc/commands/experiments/diff.py @@ -13,7 +13,6 @@ logger = logging.getLogger(__name__) class CmdExperimentsDiff(CmdBase): def run(self): - try: diff = self.repo.experiments.diff( a_rev=self.args.a_rev, @@ -53,10 +52,7 @@ class CmdExperimentsDiff(CmdBase): def add_parser(experiments_subparsers, parent_parser): - - EXPERIMENTS_DIFF_HELP = ( - "Show changes between experiments in the DVC repository." - ) + EXPERIMENTS_DIFF_HELP = "Show changes between experiments in the DVC repository." experiments_diff_parser = experiments_subparsers.add_parser( "diff", diff --git a/dvc/commands/experiments/exec_run.py b/dvc/commands/experiments/exec_run.py index d10a79bde..a327f1fbe 100644 --- a/dvc/commands/experiments/exec_run.py +++ b/dvc/commands/experiments/exec_run.py @@ -9,10 +9,7 @@ class CmdExecutorRun(CmdBaseNoRepo): """Run an experiment executor.""" def run(self): - from dvc.repo.experiments.executor.base import ( - BaseExecutor, - ExecutorInfo, - ) + from dvc.repo.experiments.executor.base import BaseExecutor, ExecutorInfo from dvc.utils.serialize import load_json info = ExecutorInfo.from_dict(load_json(self.args.infofile)) diff --git a/dvc/commands/experiments/gc.py b/dvc/commands/experiments/gc.py index 1f01ccf4d..d0cf8ec5c 100644 --- a/dvc/commands/experiments/gc.py +++ b/dvc/commands/experiments/gc.py @@ -41,10 +41,7 @@ class CmdExperimentsGC(CmdBase): msg += " and all git tags" if self.args.commit_date: - msg += ( - " and all git commits before date " - f"{self.args.commit_date}" - ) + msg += f" and all git commits before date {self.args.commit_date}" msg += " of the current repo." if self.args.queued: @@ -79,11 +76,9 @@ class CmdExperimentsGC(CmdBase): def add_parser(experiments_subparsers, parent_parser): - EXPERIMENTS_GC_HELP = "Garbage collect unneeded experiments." EXPERIMENTS_GC_DESCRIPTION = ( - "Removes all experiments which are not derived from the specified" - "Git revisions." + "Removes all experiments which are not derived from the specifiedGit revisions." ) experiments_gc_parser = experiments_subparsers.add_parser( "gc", diff --git a/dvc/commands/experiments/init.py b/dvc/commands/experiments/init.py index 68feaaf26..1d2791328 100644 --- a/dvc/commands/experiments/init.py +++ b/dvc/commands/experiments/init.py @@ -92,9 +92,7 @@ class CmdExperimentsInit(CmdBase): if new_out_dirs: out_dirs_paths = humanize.join(map(path_fmt, new_out_dirs)) - ui.write( - f"Creating output directories: {out_dirs_paths}", styled=True - ) + ui.write(f"Creating output directories: {out_dirs_paths}", styled=True) ui.write( f"Creating [b]{self.args.name}[/b] stage in [green]dvc.yaml[/]", @@ -120,7 +118,6 @@ class CmdExperimentsInit(CmdBase): def add_parser(experiments_subparsers, parent_parser): - EXPERIMENTS_INIT_HELP = "Quickly setup any project to use experiments." experiments_init_parser = experiments_subparsers.add_parser( "init", @@ -167,35 +164,47 @@ def add_parser(experiments_subparsers, parent_parser): ) experiments_init_parser.add_argument( "--code", - help="Path to the source file or directory " - "which your experiments depend" - f" (default: {CmdExperimentsInit.CODE})", + help=( + "Path to the source file or directory " + "which your experiments depend" + f" (default: {CmdExperimentsInit.CODE})" + ), ) experiments_init_parser.add_argument( "--data", - help="Path to the data file or directory " - "which your experiments depend" - f" (default: {CmdExperimentsInit.DATA})", + help=( + "Path to the data file or directory " + "which your experiments depend" + f" (default: {CmdExperimentsInit.DATA})" + ), ) experiments_init_parser.add_argument( "--models", - help="Path to the model file or directory for your experiments" - f" (default: {CmdExperimentsInit.MODELS})", + help=( + "Path to the model file or directory for your experiments" + f" (default: {CmdExperimentsInit.MODELS})" + ), ) experiments_init_parser.add_argument( "--params", - help="Path to the parameters file for your experiments" - f" (default: {CmdExperimentsInit.DEFAULT_PARAMS})", + help=( + "Path to the parameters file for your experiments" + f" (default: {CmdExperimentsInit.DEFAULT_PARAMS})" + ), ) experiments_init_parser.add_argument( "--metrics", - help="Path to the metrics file for your experiments" - f" (default: {CmdExperimentsInit.DEFAULT_METRICS})", + help=( + "Path to the metrics file for your experiments" + f" (default: {CmdExperimentsInit.DEFAULT_METRICS})" + ), ) experiments_init_parser.add_argument( "--plots", - help="Path to the plots file or directory for your experiments" - f" (default: {CmdExperimentsInit.PLOTS})", + help=( + "Path to the plots file or directory for your experiments" + f" (default: {CmdExperimentsInit.PLOTS})" + ), ) experiments_init_parser.add_argument( "--live", diff --git a/dvc/commands/experiments/pull.py b/dvc/commands/experiments/pull.py index b13484c02..98acc40ec 100644 --- a/dvc/commands/experiments/pull.py +++ b/dvc/commands/experiments/pull.py @@ -11,9 +11,7 @@ logger = logging.getLogger(__name__) class CmdExperimentsPull(CmdBase): def raise_error_if_all_disabled(self): - if not any( - [self.args.experiment, self.args.all_commits, self.args.rev] - ): + if not any([self.args.experiment, self.args.all_commits, self.args.rev]): raise InvalidArgumentError( "Either provide an `experiment` argument, or use the " "`--rev` or `--all-commits` flag." @@ -74,10 +72,7 @@ def add_parser(experiments_subparsers, parent_parser): "--no-cache", action="store_false", dest="pull_cache", - help=( - "Do not pull cached outputs for this experiment from DVC remote " - "storage." - ), + help="Do not pull cached outputs for this experiment from DVC remote storage.", ) experiments_pull_parser.add_argument( "-r", @@ -92,8 +87,7 @@ def add_parser(experiments_subparsers, parent_parser): type=int, metavar="", help=( - "Number of jobs to run simultaneously when pulling from DVC " - "remote storage." + "Number of jobs to run simultaneously when pulling from DVC remote storage." ), ) experiments_pull_parser.add_argument( diff --git a/dvc/commands/experiments/push.py b/dvc/commands/experiments/push.py index 73c61bd57..9e0ade18b 100644 --- a/dvc/commands/experiments/push.py +++ b/dvc/commands/experiments/push.py @@ -12,16 +12,13 @@ logger = logging.getLogger(__name__) class CmdExperimentsPush(CmdBase): def raise_error_if_all_disabled(self): - if not any( - [self.args.experiment, self.args.all_commits, self.args.rev] - ): + if not any([self.args.experiment, self.args.all_commits, self.args.rev]): raise InvalidArgumentError( "Either provide an `experiment` argument, or use the " "`--rev` or `--all-commits` flag." ) def run(self): - self.raise_error_if_all_disabled() pushed_exps = self.repo.experiments.push( @@ -47,8 +44,10 @@ class CmdExperimentsPush(CmdBase): if not self.args.push_cache: ui.write( "To push cached outputs", - "for this experiment to DVC remote storage," - "re-run this command without '--no-cache'.", + ( + "for this experiment to DVC remote storage," + "re-run this command without '--no-cache'." + ), ) return 0 @@ -76,10 +75,7 @@ def add_parser(experiments_subparsers, parent_parser): "--no-cache", action="store_false", dest="push_cache", - help=( - "Do not push cached outputs for this experiment to DVC remote " - "storage." - ), + help="Do not push cached outputs for this experiment to DVC remote storage.", ) experiments_push_parser.add_argument( "-r", @@ -93,10 +89,7 @@ def add_parser(experiments_subparsers, parent_parser): "--jobs", type=int, metavar="", - help=( - "Number of jobs to run simultaneously when pushing to DVC remote " - "storage." - ), + help="Number of jobs to run simultaneously when pushing to DVC remote storage.", ) experiments_push_parser.add_argument( "--run-cache", diff --git a/dvc/commands/experiments/remove.py b/dvc/commands/experiments/remove.py index 9814f1400..ce827f4e8 100644 --- a/dvc/commands/experiments/remove.py +++ b/dvc/commands/experiments/remove.py @@ -24,7 +24,6 @@ class CmdExperimentsRemove(CmdBase): ) def run(self): - self.check_arguments() removed_list = self.repo.experiments.remove( diff --git a/dvc/commands/experiments/run.py b/dvc/commands/experiments/run.py index 2e9eac741..bbc595487 100644 --- a/dvc/commands/experiments/run.py +++ b/dvc/commands/experiments/run.py @@ -17,13 +17,10 @@ class CmdExperimentsRun(CmdRepro): if self.args.checkpoint_resume: if self.args.reset: - raise InvalidArgumentError( - "--reset and --rev are mutually exclusive." - ) + raise InvalidArgumentError("--reset and --rev are mutually exclusive.") if not (self.args.queue or self.args.tmp_dir): raise InvalidArgumentError( - "--rev can only be used in conjunction with " - "--queue or --temp." + "--rev can only be used in conjunction with --queue or --temp." ) if self.args.reset: @@ -51,7 +48,6 @@ class CmdExperimentsRun(CmdRepro): def add_parser(experiments_subparsers, parent_parser): - EXPERIMENTS_RUN_HELP = "Run or resume an experiment." experiments_run_parser = experiments_subparsers.add_parser( "run", diff --git a/dvc/commands/experiments/save.py b/dvc/commands/experiments/save.py index e459a111c..d0c3863f8 100644 --- a/dvc/commands/experiments/save.py +++ b/dvc/commands/experiments/save.py @@ -11,7 +11,6 @@ logger = logging.getLogger(__name__) class CmdExperimentsSave(CmdBase): def run(self): - try: ref = self.repo.experiments.save( name=self.args.name, diff --git a/dvc/commands/experiments/show.py b/dvc/commands/experiments/show.py index b97aacbc9..43e72d01a 100644 --- a/dvc/commands/experiments/show.py +++ b/dvc/commands/experiments/show.py @@ -84,9 +84,7 @@ def _collect_rows( # noqa: C901 sort_by, metric_names, param_names ) reverse = sort_order == "desc" - experiments = _sort_exp( - experiments, sort_path, sort_name, sort_type, reverse - ) + experiments = _sort_exp(experiments, sort_path, sort_name, sort_type, reverse) new_checkpoint = True for i, (rev, results) in enumerate(experiments.items()): @@ -116,9 +114,7 @@ def _collect_rows( # noqa: C901 typ = "baseline" elif tip: if tip == parent_tip: - typ = ( - "checkpoint_tip" if new_checkpoint else "checkpoint_commit" - ) + typ = "checkpoint_tip" if new_checkpoint else "checkpoint_commit" elif parent_rev == base_rev: typ = "checkpoint_base" else: @@ -135,9 +131,7 @@ def _collect_rows( # noqa: C901 row_dict["Experiment"] = exp.get("name", "") row_dict["rev"] = name_rev row_dict["typ"] = typ - row_dict["Created"] = format_time( - exp.get("timestamp"), fill_value, iso - ) + row_dict["Created"] = format_time(exp.get("timestamp"), fill_value, iso) row_dict["parent"] = parent row_dict["State"] = state row_dict["Executor"] = exp.get("executor", fill_value) @@ -371,9 +365,7 @@ def show_experiments( # noqa: C901 if not csv: merge_headers = ["Experiment", "rev", "typ", "parent"] - td.column("Experiment")[:] = map( - prepare_exp_id, td.as_dict(merge_headers) - ) + td.column("Experiment")[:] = map(prepare_exp_id, td.as_dict(merge_headers)) td.drop(*merge_headers[1:]) headers = { @@ -410,9 +402,7 @@ def show_experiments( # noqa: C901 cols_to_drop = set() if drop is not None: - cols_to_drop = { - col for col in td.keys() if re.match(drop, col) # noqa: SIM118 - } + cols_to_drop = {col for col in td.keys() if re.match(drop, col)} # noqa: SIM118 if pcp: cols_to_drop.add("Created") td.drop(*cols_to_drop) @@ -491,9 +481,7 @@ class CmdExperimentsShow(CmdBase): ui.write_json(all_experiments, default=_format_json) else: precision = ( - self.args.precision or None - if self.args.csv - else DEFAULT_PRECISION + self.args.precision or None if self.args.csv else DEFAULT_PRECISION ) fill_value = "" if self.args.csv else FILL_VALUE iso = self.args.csv @@ -582,10 +570,7 @@ def add_parser(experiments_subparsers, parent_parser): ) experiments_show_parser.add_argument( "--sort-order", - help=( - "Sort order to use with --sort-by." - " Defaults to ascending ('asc')." - ), + help="Sort order to use with --sort-by. Defaults to ascending ('asc').", choices=("asc", "desc"), default="asc", ) diff --git a/dvc/commands/gc.py b/dvc/commands/gc.py index 588b36016..da4c70e2b 100644 --- a/dvc/commands/gc.py +++ b/dvc/commands/gc.py @@ -40,14 +40,9 @@ class CmdGC(CmdBase): elif self.args.all_tags: msg += " and all git tags" if self.args.commit_date: - msg += ( - " and all git commits before date " - f"{self.args.commit_date}" - ) + msg += f" and all git commits before date {self.args.commit_date}" if self.args.rev: - msg += ( - f" and last {self.args.num} commits from {self.args.rev}" - ) + msg += f" and last {self.args.num} commits from {self.args.rev}" if self.args.all_experiments: msg += " and all experiments" @@ -198,9 +193,11 @@ def add_parser(subparsers, parent_parser): dest="repos", type=str, nargs="*", - help="Keep data files required by these projects " - "in addition to the current one. " - "Useful if you share a single cache across repos.", + help=( + "Keep data files required by these projects " + "in addition to the current one. " + "Useful if you share a single cache across repos." + ), metavar="", ) gc_parser.set_defaults(func=CmdGC) diff --git a/dvc/commands/get.py b/dvc/commands/get.py index 7b59d965f..908596997 100644 --- a/dvc/commands/get.py +++ b/dvc/commands/get.py @@ -16,9 +16,7 @@ class CmdGet(CmdBaseNoRepo): from dvc.ui import ui try: - url = get_url( - self.args.path, repo=self.args.url, rev=self.args.rev - ) + url = get_url(self.args.path, repo=self.args.url, rev=self.args.rev) ui.write(url, force=True) except DvcException: logger.exception("failed to show URL") @@ -85,8 +83,10 @@ def add_parser(subparsers, parent_parser): get_parser.add_argument( "--show-url", action="store_true", - help="Print the storage location (URL) the target data would be " - "downloaded from, and exit.", + help=( + "Print the storage location (URL) the target data would be " + "downloaded from, and exit." + ), ) get_parser.add_argument( "-j", diff --git a/dvc/commands/imp.py b/dvc/commands/imp.py index 862729ecf..6c71e7e8c 100644 --- a/dvc/commands/imp.py +++ b/dvc/commands/imp.py @@ -90,8 +90,10 @@ def add_parser(subparsers, parent_parser): "--no-download", action="store_true", default=False, - help="Create .dvc file including target data hash value(s)" - " but do not actually download the file(s).", + help=( + "Create .dvc file including target data hash value(s)" + " but do not actually download the file(s)." + ), ) import_parser.add_argument( "-j", diff --git a/dvc/commands/imp_url.py b/dvc/commands/imp_url.py index 8e5d570ef..30b72f9db 100644 --- a/dvc/commands/imp_url.py +++ b/dvc/commands/imp_url.py @@ -29,8 +29,10 @@ class CmdImportUrl(CmdBase): ) except DvcException: logger.exception( - "failed to import %s. You could also try downloading " - "it manually, and adding it with `dvc add`.", + ( + "failed to import %s. You could also try downloading " + "it manually, and adding it with `dvc add`." + ), self.args.url, ) return 1 @@ -40,9 +42,7 @@ class CmdImportUrl(CmdBase): def add_parser(subparsers, parent_parser): from .add import _add_annotating_args - IMPORT_HELP = ( - "Download or copy file from URL and take it under DVC control." - ) + IMPORT_HELP = "Download or copy file from URL and take it under DVC control." import_parser = subparsers.add_parser( "import-url", @@ -53,16 +53,18 @@ def add_parser(subparsers, parent_parser): ) import_parser.add_argument( "url", - help="Location of the data to download. Supported URLs:\n" - "/absolute/path/to/file/or/dir\n" - "relative/path/to/file/or/dir\n" - "C:\\\\path\\to\\file\\or\\dir\n" - "https://example.com/path/to/file\n" - "s3://bucket/key/path\n" - "gs://bucket/path/to/file/or/dir\n" - "hdfs://example.com/path/to/file\n" - "ssh://example.com/absolute/path/to/file/or/dir\n" - "remote://remote_name/path/to/file/or/dir (see `dvc remote`)", + help=( + "Location of the data to download. Supported URLs:\n" + "/absolute/path/to/file/or/dir\n" + "relative/path/to/file/or/dir\n" + "C:\\\\path\\to\\file\\or\\dir\n" + "https://example.com/path/to/file\n" + "s3://bucket/key/path\n" + "gs://bucket/path/to/file/or/dir\n" + "hdfs://example.com/path/to/file\n" + "ssh://example.com/absolute/path/to/file/or/dir\n" + "remote://remote_name/path/to/file/or/dir (see `dvc remote`)" + ), ) import_parser.add_argument( "out", nargs="?", help="Destination path to put files to." @@ -95,8 +97,10 @@ def add_parser(subparsers, parent_parser): "--no-download", action="store_true", default=False, - help="Create .dvc file including target data hash value(s)" - " but do not actually download the file(s).", + help=( + "Create .dvc file including target data hash value(s)" + " but do not actually download the file(s)." + ), ) import_parser.add_argument( "-j", @@ -112,10 +116,7 @@ def add_parser(subparsers, parent_parser): "--version-aware", action="store_true", default=False, - help=( - "Import using cloud versioning. Implied if the URL contains a " - "version ID." - ), + help="Import using cloud versioning. Implied if the URL contains a version ID.", ) _add_annotating_args(import_parser) import_parser.set_defaults(func=CmdImportUrl) diff --git a/dvc/commands/init.py b/dvc/commands/init.py index 3d0c774bb..ca4491939 100644 --- a/dvc/commands/init.py +++ b/dvc/commands/init.py @@ -75,8 +75,9 @@ def add_parser(subparsers, parent_parser): "--no-scm", action="store_true", default=False, - help="Initiate DVC in directory that is " - "not tracked by any SCM tool (e.g. Git).", + help=( + "Initiate DVC in directory that is not tracked by any SCM tool (e.g. Git)." + ), ) init_parser.add_argument( "-f", @@ -84,8 +85,7 @@ def add_parser(subparsers, parent_parser): action="store_true", default=False, help=( - "Overwrite existing '.dvc/' directory. " - "This operation removes local cache." + "Overwrite existing '.dvc/' directory. This operation removes local cache." ), ) init_parser.add_argument( diff --git a/dvc/commands/install.py b/dvc/commands/install.py index 6860b6af4..96029d9e8 100644 --- a/dvc/commands/install.py +++ b/dvc/commands/install.py @@ -31,7 +31,9 @@ def add_parser(subparsers, parent_parser): "--use-pre-commit-tool", action="store_true", default=False, - help="Install DVC hooks using pre-commit " - "(https://pre-commit.com) if it is installed.", + help=( + "Install DVC hooks using pre-commit " + "(https://pre-commit.com) if it is installed." + ), ) install_parser.set_defaults(func=CmdInstall) diff --git a/dvc/commands/machine.py b/dvc/commands/machine.py index a208e73f3..c32d0da20 100644 --- a/dvc/commands/machine.py +++ b/dvc/commands/machine.py @@ -18,7 +18,6 @@ class MachineDisabledError(ConfigError): class CmdMachineConfig(CmdConfig): def __init__(self, args): - super().__init__(args) if not self.config["feature"].get("machine", False): raise MachineDisabledError @@ -163,14 +162,11 @@ class CmdMachineRename(CmdBase): all_config = self.config.load_config_to_level(None) if self.args.new in all_config.get("machine", {}): raise ConfigError( - "Rename failed. Machine '{}' already exists.".format( - self.args.new - ) + f"Rename failed. Machine '{self.args.new}' already exists." ) ui.write(f"Rename machine '{self.args.name}' to '{self.args.new}'.") def run(self): - self._check_before_rename() with self.config.edit(self.args.level) as conf: @@ -209,9 +205,7 @@ class CmdMachineDefault(CmdMachineConfig): if self.args.unset: conf["core"].pop("machine", None) else: - merged_conf = self.config.load_config_to_level( - self.args.level - ) + merged_conf = self.config.load_config_to_level(self.args.level) if ( self.args.name in conf["machine"] or self.args.name in merged_conf["machine"] @@ -219,8 +213,7 @@ class CmdMachineDefault(CmdMachineConfig): conf["core"]["machine"] = self.args.name else: raise ConfigError( - "default machine must be present in machine " - "list." + "default machine must be present in machine list." ) return 0 @@ -251,7 +244,6 @@ class CmdMachineStatus(CmdBase): all_status: List[Dict], td: TabularData, ): - if not all_status: row = [ name, @@ -343,8 +335,7 @@ def add_parser(subparsers, parent_parser): "cloud", help="Machine cloud. See full list of supported clouds at {}".format( format_link( - "https://github.com/iterative/" - "terraform-provider-iterative#machine" + "https://github.com/iterative/terraform-provider-iterative#machine" ) ), ) @@ -372,9 +363,7 @@ def add_parser(subparsers, parent_parser): help=machine_DEFAULT_HELP, formatter_class=argparse.RawDescriptionHelpFormatter, ) - machine_default_parser.add_argument( - "name", nargs="?", help="Name of the machine" - ) + machine_default_parser.add_argument("name", nargs="?", help="Name of the machine") machine_default_parser.add_argument( "-u", "--unset", @@ -414,9 +403,7 @@ def add_parser(subparsers, parent_parser): formatter_class=argparse.RawDescriptionHelpFormatter, ) machine_modify_parser.add_argument("name", help="Name of the machine") - machine_modify_parser.add_argument( - "option", help="Name of the option to modify." - ) + machine_modify_parser.add_argument("option", help="Name of the option to modify.") machine_modify_parser.add_argument( "value", nargs="?", help="(optional) Value of the option." ) @@ -449,9 +436,7 @@ def add_parser(subparsers, parent_parser): help=machine_REMOVE_HELP, formatter_class=argparse.RawDescriptionHelpFormatter, ) - machine_remove_parser.add_argument( - "name", help="Name of the machine to remove." - ) + machine_remove_parser.add_argument("name", help="Name of the machine to remove.") machine_remove_parser.set_defaults(func=CmdMachineRemove) machine_CREATE_HELP = "Create and start a machine instance." @@ -462,14 +447,10 @@ def add_parser(subparsers, parent_parser): help=machine_CREATE_HELP, formatter_class=argparse.RawDescriptionHelpFormatter, ) - machine_create_parser.add_argument( - "name", help="Name of the machine to create." - ) + machine_create_parser.add_argument("name", help="Name of the machine to create.") machine_create_parser.set_defaults(func=CmdMachineCreate) - machine_STATUS_HELP = ( - "List the status of running instances for one/all machines." - ) + machine_STATUS_HELP = "List the status of running instances for one/all machines." machine_status_parser = machine_subparsers.add_parser( "status", parents=[parent_parser], diff --git a/dvc/commands/move.py b/dvc/commands/move.py index a79492998..8d3f950d1 100644 --- a/dvc/commands/move.py +++ b/dvc/commands/move.py @@ -14,9 +14,7 @@ class CmdMove(CmdBase): try: self.repo.move(self.args.src, self.args.dst) except DvcException: - msg = "failed to move '{}' -> '{}'".format( - self.args.src, self.args.dst - ) + msg = f"failed to move '{self.args.src}' -> '{self.args.dst}'" logger.exception(msg) return 1 return 0 @@ -40,7 +38,5 @@ def add_parser(subparsers, parent_parser): move_parser.add_argument( "src", help="Source path to a data file or directory." ).complete = completion.FILE - move_parser.add_argument( - "dst", help="Destination path." - ).complete = completion.FILE + move_parser.add_argument("dst", help="Destination path.").complete = completion.FILE move_parser.set_defaults(func=CmdMove) diff --git a/dvc/commands/params.py b/dvc/commands/params.py index 8f82368b8..dfdf68218 100644 --- a/dvc/commands/params.py +++ b/dvc/commands/params.py @@ -79,7 +79,7 @@ def add_parser(subparsers, parent_parser): params_diff_parser.add_argument( "b_rev", nargs="?", - help=("New Git commit to compare (defaults to the current workspace)"), + help="New Git commit to compare (defaults to the current workspace)", ) params_diff_parser.add_argument( "--targets", diff --git a/dvc/commands/plots.py b/dvc/commands/plots.py index 71e3e5f0f..75a36c315 100644 --- a/dvc/commands/plots.py +++ b/dvc/commands/plots.py @@ -18,9 +18,7 @@ logger = logging.getLogger(__name__) def _show_json(renderers, split=False): from dvc.render.convert import to_json - result = { - renderer.name: to_json(renderer, split) for renderer in renderers - } + result = {renderer.name: to_json(renderer, split) for renderer in renderers} ui.write_json(result) @@ -90,10 +88,8 @@ class CmdPlots(CmdBase): ) if html_template_path and not os.path.isabs(html_template_path): assert self.repo.dvc_dir - html_template_path = os.path.join( - self.repo.dvc_dir, html_template_path - ) - return html_template_path + html_template_path = os.path.join(self.repo.dvc_dir, html_template_path) + return html_template_path # noqa: RET504 def run(self) -> int: # noqa: C901 from pathlib import Path @@ -106,14 +102,11 @@ class CmdPlots(CmdBase): logger.error("please specify a target for `--show-vega`") return 1 if len(self.args.targets) > 1: - logger.error( - "you can only specify one target for `--show-vega`" - ) + logger.error("you can only specify one target for `--show-vega`") return 1 if self.args.json: logger.error( - "'--show-vega' and '--json' are mutually exclusive " - "options." + "'--show-vega' and '--json' are mutually exclusive options." ) return 1 @@ -126,17 +119,14 @@ class CmdPlots(CmdBase): if not plots_data: ui.error_write( - "No plots were loaded, " - "visualization file will not be created." + "No plots were loaded, visualization file will not be created." ) out: str = self.args.out or self.repo.config.get("plots", {}).get( "out_dir", "dvc_plots" ) - renderers_out = ( - out if self.args.json else os.path.join(out, "static") - ) + renderers_out = out if self.args.json else os.path.join(out, "static") renderers = match_defs_renderers( data=plots_data, @@ -249,8 +239,7 @@ def add_parser(subparsers, parent_parser): fix_subparsers(plots_subparsers) SHOW_HELP = ( - "Generate plots from target files or from `plots`" - " definitions in `dvc.yaml`." + "Generate plots from target files or from `plots` definitions in `dvc.yaml`." ) plots_show_parser = plots_subparsers.add_parser( "show", @@ -273,8 +262,7 @@ def add_parser(subparsers, parent_parser): plots_show_parser.set_defaults(func=CmdPlotsShow) PLOTS_DIFF_HELP = ( - "Show multiple versions of a plot by overlaying them " - "in a single image." + "Show multiple versions of a plot by overlaying them in a single image." ) plots_diff_parser = plots_subparsers.add_parser( "diff", @@ -332,9 +320,7 @@ def add_parser(subparsers, parent_parser): ) plots_modify_parser.set_defaults(func=CmdPlotsModify) - TEMPLATES_HELP = ( - "List built-in plots templates or show JSON specification for one." - ) + TEMPLATES_HELP = "List built-in plots templates or show JSON specification for one." plots_templates_parser = plots_subparsers.add_parser( "templates", parents=[parent_parser], @@ -360,11 +346,8 @@ def _add_props_arguments(parser): "--template", nargs="?", default=None, - help=( - "Special JSON or HTML schema file to inject with the data. " - "See {}".format( - format_link("https://man.dvc.org/plots#plot-templates") - ) + help="Special JSON or HTML schema file to inject with the data. See {}".format( + format_link("https://man.dvc.org/plots#plot-templates") ), metavar="", ).complete = completion.FILE @@ -381,9 +364,7 @@ def _add_props_arguments(parser): default=None, # Use default None to distinguish when it's not used help="Provided CSV or TSV datafile does not have a header.", ) - parser.add_argument( - "--title", default=None, metavar="", help="Plot title." - ) + parser.add_argument("--title", default=None, metavar="", help="Plot title.") parser.add_argument( "--x-label", default=None, help="X axis label", metavar="" ) diff --git a/dvc/commands/queue/kill.py b/dvc/commands/queue/kill.py index c2c9fc454..11483259d 100644 --- a/dvc/commands/queue/kill.py +++ b/dvc/commands/queue/kill.py @@ -20,8 +20,7 @@ class CmdQueueKill(CmdBase): def add_parser(queue_subparsers, parent_parser): QUEUE_KILL_HELP = ( - "Gracefully interrupt running experiment queue tasks " - "(equivalent to Ctrl-C)" + "Gracefully interrupt running experiment queue tasks (equivalent to Ctrl-C)" ) queue_kill_parser = queue_subparsers.add_parser( "kill", diff --git a/dvc/commands/queue/logs.py b/dvc/commands/queue/logs.py index e970f217a..e445dc56b 100644 --- a/dvc/commands/queue/logs.py +++ b/dvc/commands/queue/logs.py @@ -34,9 +34,7 @@ def add_parser(queue_subparsers, parent_parser): queue_logs_parser.add_argument( "-e", "--encoding", - help=( - "Text encoding for log output. Defaults to system locale encoding." - ), + help="Text encoding for log output. Defaults to system locale encoding.", metavar="", ) queue_logs_parser.add_argument( diff --git a/dvc/commands/queue/remove.py b/dvc/commands/queue/remove.py index 6699b5fd2..2957ea613 100644 --- a/dvc/commands/queue/remove.py +++ b/dvc/commands/queue/remove.py @@ -28,7 +28,6 @@ class CmdQueueRemove(CmdBase): ) def run(self): - self.check_arguments() if self.args.all: @@ -57,7 +56,6 @@ class CmdQueueRemove(CmdBase): def add_parser(queue_subparsers, parent_parser): - QUEUE_REMOVE_HELP = "Remove queued and completed tasks from the queue." queue_remove_parser = queue_subparsers.add_parser( "remove", diff --git a/dvc/commands/queue/start.py b/dvc/commands/queue/start.py index 2bb5d917c..d8e5e245c 100644 --- a/dvc/commands/queue/start.py +++ b/dvc/commands/queue/start.py @@ -12,20 +12,15 @@ class CmdQueueStart(CmdBase): """Start exp queue workers.""" def run(self): - started = self.repo.experiments.celery_queue.start_workers( - self.args.jobs - ) + started = self.repo.experiments.celery_queue.start_workers(self.args.jobs) suffix = "s" if started > 1 else "" - ui.write( - f"Started '{started}' new experiments task queue worker{suffix}." - ) + ui.write(f"Started '{started}' new experiments task queue worker{suffix}.") return 0 def add_parser(queue_subparsers, parent_parser): - QUEUE_START_HELP = "Start the experiments task queue worker." queue_start_parser = queue_subparsers.add_parser( "start", @@ -39,10 +34,7 @@ def add_parser(queue_subparsers, parent_parser): "--jobs", type=int, default=1, - help=( - "Maximum number of concurrent queue workers to start. " - "Defaults to 1." - ), + help="Maximum number of concurrent queue workers to start. Defaults to 1.", metavar="", ) queue_start_parser.set_defaults(func=CmdQueueStart) diff --git a/dvc/commands/queue/status.py b/dvc/commands/queue/status.py index 0ffcdea82..d235d986a 100644 --- a/dvc/commands/queue/status.py +++ b/dvc/commands/queue/status.py @@ -39,9 +39,7 @@ class CmdQueueStatus(CmdBase): ui.write() worker_status = self.repo.experiments.celery_queue.worker_status() - active_count = len( - [name for name, task in worker_status.items() if task] - ) + active_count = len([name for name, task in worker_status.items() if task]) idle_count = len(worker_status) - active_count ui.write(f"Worker status: {active_count} active, {idle_count} idle") @@ -50,9 +48,7 @@ class CmdQueueStatus(CmdBase): def add_parser(queue_subparsers, parent_parser): - QUEUE_STATUS_HELP = ( - "Show the status of experiments queue tasks and workers." - ) + QUEUE_STATUS_HELP = "Show the status of experiments queue tasks and workers." queue_status_parser = queue_subparsers.add_parser( "status", parents=[parent_parser], diff --git a/dvc/commands/queue/stop.py b/dvc/commands/queue/stop.py index 04f474311..d45e1d0ab 100644 --- a/dvc/commands/queue/stop.py +++ b/dvc/commands/queue/stop.py @@ -26,7 +26,6 @@ class CmdQueueStop(CmdBase): def add_parser(queue_subparsers, parent_parser): - QUEUE_STOP_HELP = "Stop all experiments task queue workers." queue_stop_parser = queue_subparsers.add_parser( "stop", diff --git a/dvc/commands/remote.py b/dvc/commands/remote.py index ebafa6beb..600117f09 100644 --- a/dvc/commands/remote.py +++ b/dvc/commands/remote.py @@ -95,9 +95,7 @@ class CmdRemoteDefault(CmdRemote): if self.args.unset: conf["core"].pop("remote", None) else: - merged_conf = self.config.load_config_to_level( - self.args.level - ) + merged_conf = self.config.load_config_to_level(self.args.level) if ( self.args.name in conf["remote"] or self.args.name in merged_conf["remote"] @@ -208,9 +206,7 @@ def add_parser(subparsers, parent_parser): help=REMOTE_DEFAULT_HELP, formatter_class=argparse.RawDescriptionHelpFormatter, ) - remote_default_parser.add_argument( - "name", nargs="?", help="Name of the remote" - ) + remote_default_parser.add_argument("name", nargs="?", help="Name of the remote") remote_default_parser.add_argument( "-u", "--unset", @@ -229,9 +225,7 @@ def add_parser(subparsers, parent_parser): formatter_class=argparse.RawDescriptionHelpFormatter, ) remote_modify_parser.add_argument("name", help="Name of the remote") - remote_modify_parser.add_argument( - "option", help="Name of the option to modify." - ) + remote_modify_parser.add_argument("option", help="Name of the option to modify.") remote_modify_parser.add_argument( "value", nargs="?", help="(optional) Value of the option." ) @@ -262,9 +256,7 @@ def add_parser(subparsers, parent_parser): help=REMOTE_REMOVE_HELP, formatter_class=argparse.RawDescriptionHelpFormatter, ) - remote_remove_parser.add_argument( - "name", help="Name of the remote to remove." - ) + remote_remove_parser.add_argument("name", help="Name of the remote to remove.") remote_remove_parser.set_defaults(func=CmdRemoteRemove) REMOTE_RENAME_HELP = "Rename a DVC remote" remote_rename_parser = remote_subparsers.add_parser( diff --git a/dvc/commands/remove.py b/dvc/commands/remove.py index 0da9c1765..424d37ec1 100644 --- a/dvc/commands/remove.py +++ b/dvc/commands/remove.py @@ -22,8 +22,7 @@ class CmdRemove(CmdBase): def add_parser(subparsers, parent_parser): REMOVE_HELP = ( - "Remove stages from dvc.yaml and/or" - " stop tracking files or directories." + "Remove stages from dvc.yaml and/or stop tracking files or directories." ) remove_parser = subparsers.add_parser( "remove", diff --git a/dvc/commands/repro.py b/dvc/commands/repro.py index 305dbe5c5..fdb0659e2 100644 --- a/dvc/commands/repro.py +++ b/dvc/commands/repro.py @@ -10,9 +10,7 @@ class CmdRepro(CmdBase): def run(self): from dvc.ui import ui - stages = self.repo.reproduce( - **self._common_kwargs, **self._repro_kwargs - ) + stages = self.repo.reproduce(**self._common_kwargs, **self._repro_kwargs) if len(stages) == 0: ui.write(CmdDataStatus.UP_TO_DATE_MSG) else: @@ -55,16 +53,14 @@ def add_arguments(repro_parser): repro_parser.add_argument( "targets", nargs="*", - help=( - """\ + help="""\ Stages to reproduce. 'dvc.yaml' by default. The targets can be path to a dvc.yaml file or `.dvc` file, or a stage name from dvc.yaml file from current working directory. To run a stage from dvc.yaml from other directories, the target must be a path followed by colon `:` and then the stage name name. -""" - ), +""", ).complete = completion.DVCFILES_AND_STAGE repro_parser.add_argument( "-f", @@ -85,16 +81,14 @@ and then the stage name name. "--single-item", action="store_true", default=False, - help="Reproduce only single data item without recursive dependencies " - "check.", + help="Reproduce only single data item without recursive dependencies check.", ) repro_parser.add_argument( "-p", "--pipeline", action="store_true", default=False, - help="Reproduce the whole pipeline that the specified targets " - "belong to.", + help="Reproduce the whole pipeline that the specified targets belong to.", ) repro_parser.add_argument( "-P", @@ -127,8 +121,10 @@ and then the stage name name. "--force-downstream", action="store_true", default=False, - help="Reproduce all descendants of a changed stage even if their " - "direct dependencies didn't change.", + help=( + "Reproduce all descendants of a changed stage even if their " + "direct dependencies didn't change." + ), ) repro_parser.add_argument( "--pull", @@ -143,15 +139,14 @@ and then the stage name name. "--dry", action="store_true", default=False, - help="Only print the commands that would be executed without " - "actually executing.", + help=( + "Only print the commands that would be executed without actually executing." + ), ) def add_parser(subparsers, parent_parser): - REPRO_HELP = ( - "Reproduce complete or partial pipelines by executing their stages." - ) + REPRO_HELP = "Reproduce complete or partial pipelines by executing their stages." repro_parser = subparsers.add_parser( "repro", parents=[parent_parser], diff --git a/dvc/commands/run.py b/dvc/commands/run.py index bc17e4ad5..a5e74f11f 100644 --- a/dvc/commands/run.py +++ b/dvc/commands/run.py @@ -39,7 +39,7 @@ class CmdRun(CmdBase): { "cmd": parse_cmd(self.args.command), "fname": kwargs.pop("file"), - "no_exec": (self.args.no_exec or bool(self.args.checkpoints)), + "no_exec": self.args.no_exec or bool(self.args.checkpoints), "run_cache": not kwargs.pop("no_run_cache"), } ) @@ -55,9 +55,7 @@ class CmdRun(CmdBase): def add_parser(subparsers, parent_parser): from dvc.commands.stage import _add_common_args - RUN_HELP = ( - "Generate a dvc.yaml file from a command and execute the command." - ) + RUN_HELP = "Generate a dvc.yaml file from a command and execute the command." run_parser = subparsers.add_parser( "run", parents=[parent_parser], @@ -66,9 +64,7 @@ def add_parser(subparsers, parent_parser): formatter_class=argparse.RawDescriptionHelpFormatter, ) run_parser.add_argument("-n", "--name", help="Stage name.") - run_parser.add_argument( - "--file", metavar="", help=argparse.SUPPRESS - ) + run_parser.add_argument("--file", metavar="", help=argparse.SUPPRESS) run_parser.add_argument( "--single-stage", action="store_true", diff --git a/dvc/commands/stage.py b/dvc/commands/stage.py index 85936d4fb..f3c11aec3 100644 --- a/dvc/commands/stage.py +++ b/dvc/commands/stage.py @@ -44,9 +44,7 @@ def generate_description(stage: "Stage") -> str: return "; ".join(desc) -def prepare_description( - stage: "Stage", max_length: int = MAX_TEXT_LENGTH -) -> str: +def prepare_description(stage: "Stage", max_length: int = MAX_TEXT_LENGTH) -> str: desc = stage.short_description() or generate_description(stage) return truncate_text(desc, max_length) @@ -57,9 +55,9 @@ def prepare_stages_data( max_length: int = MAX_TEXT_LENGTH, ) -> Dict[str, str]: return { - stage.addressing: prepare_description(stage, max_length=max_length) - if description - else "" + stage.addressing: ( + prepare_description(stage, max_length=max_length) if description else "" + ) for stage in stages } @@ -75,9 +73,7 @@ class CmdStageList(CmdBase): # removing duplicates while maintaining order collected = chain.from_iterable( - self.repo.stage.collect( - target=target, recursive=self.args.recursive - ) + self.repo.stage.collect(target=target, recursive=self.args.recursive) for target in self.args.targets ) return dict.fromkeys(collected).keys() @@ -173,8 +169,7 @@ def _add_common_args(parser): "--outs-no-cache", action="append", default=[], - help="Declare output file or directory " - "(do not put into DVC cache).", + help="Declare output file or directory (do not put into DVC cache).", metavar="", ).complete = completion.FILE parser.add_argument( @@ -182,8 +177,10 @@ def _add_common_args(parser): "--checkpoints", action="append", default=[], - help="Declare checkpoint output file or directory for 'dvc exp run'. " - "Not compatible with 'dvc repro'.", + help=( + "Declare checkpoint output file or directory for 'dvc exp run'. " + "Not compatible with 'dvc repro'." + ), metavar="", ).complete = completion.FILE parser.add_argument( @@ -196,16 +193,17 @@ def _add_common_args(parser): "--outs-persist", action="append", default=[], - help="Declare output file or directory that will not be " - "removed upon repro.", + help="Declare output file or directory that will not be removed upon repro.", metavar="", ) parser.add_argument( "--outs-persist-no-cache", action="append", default=[], - help="Declare output file or directory that will not be " - "removed upon repro (do not put into DVC cache).", + help=( + "Declare output file or directory that will not be " + "removed upon repro (do not put into DVC cache)." + ), metavar="", ) parser.add_argument( diff --git a/dvc/commands/status.py b/dvc/commands/status.py index 94e261d22..fe8076148 100644 --- a/dvc/commands/status.py +++ b/dvc/commands/status.py @@ -81,9 +81,7 @@ class CmdDataStatus(CmdDataBase): if not self.repo.index.stages: ui.write(self.EMPTY_PROJECT_MSG) elif self.args.cloud or self.args.remote: - remote = self.args.remote or self.repo.config["core"].get( - "remote" - ) + remote = self.args.remote or self.repo.config["core"].get("remote") ui.write(self.IN_SYNC_MSG.format(remote=remote)) else: ui.write(self.UP_TO_DATE_MSG) diff --git a/dvc/commands/update.py b/dvc/commands/update.py index ce5446d41..6b724a116 100644 --- a/dvc/commands/update.py +++ b/dvc/commands/update.py @@ -61,8 +61,10 @@ def add_parser(subparsers, parent_parser): "--no-download", action="store_true", default=False, - help="Update .dvc file git revision/hash value(s)" - " but do not download the file(s).", + help=( + "Update .dvc file git revision/hash value(s)" + " but do not download the file(s)." + ), ) update_parser.add_argument( "--to-remote", diff --git a/dvc/commands/version.py b/dvc/commands/version.py index e2f7a19e0..2eec11cb4 100644 --- a/dvc/commands/version.py +++ b/dvc/commands/version.py @@ -21,9 +21,7 @@ class CmdVersion(CmdBaseNoRepo): def add_parser(subparsers, parent_parser): - VERSION_HELP = ( - "Display the DVC version and system/environment information." - ) + VERSION_HELP = "Display the DVC version and system/environment information." version_parser = subparsers.add_parser( "version", parents=[parent_parser], diff --git a/dvc/compare.py b/dvc/compare.py index 7199cced2..1e0f7b2eb 100644 --- a/dvc/compare.py +++ b/dvc/compare.py @@ -64,9 +64,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]): def keys(self) -> List[str]: return self._keys - def _iter_col_row( - self, row: Sequence["CellT"] - ) -> Iterator[Tuple["CellT", Column]]: + def _iter_col_row(self, row: Sequence["CellT"]) -> Iterator[Tuple["CellT", Column]]: for val, col in zip_longest(row, self.columns): if col is None: break @@ -103,9 +101,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]): ... @overload - def __setitem__( - self, item: slice, value: Iterable[Sequence["CellT"]] - ) -> None: + def __setitem__(self, item: slice, value: Iterable[Sequence["CellT"]]) -> None: ... def __setitem__(self, item, value) -> None: @@ -113,8 +109,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]): if isinstance(item, slice): n = len(self.columns) normalized_rows = ( - chain(val, repeat(self._fill_value, n - len(val))) - for val in value + chain(val, repeat(self._fill_value, n - len(val))) for val in value ) # we need to transpose those rows into columnar format # as we work in terms of column-based arrays @@ -212,9 +207,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]): self, cols: Optional[Iterable[str]] = None ) -> Iterable[Dict[str, "CellT"]]: keys = self.keys() if cols is None else set(cols) - return [ - {k: self._columns[k][i] for k in keys} for i in range(len(self)) - ] + return [{k: self._columns[k][i] for k in keys} for i in range(len(self))] def dropna( # noqa: C901 self, @@ -224,13 +217,10 @@ class TabularData(MutableSequence[Sequence["CellT"]]): ): if axis not in ["rows", "cols"]: raise ValueError( - f"Invalid 'axis' value {axis}." - "Choose one of ['rows', 'cols']" + f"Invalid 'axis' value {axis}.Choose one of ['rows', 'cols']" ) if how not in ["any", "all"]: - raise ValueError( - f"Invalid 'how' value {how}. Choose one of ['any', 'all']" - ) + raise ValueError(f"Invalid 'how' value {how}. Choose one of ['any', 'all']") match_line: Set = set() match_any = True @@ -258,11 +248,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]): if axis == "rows": for name in self.keys(): # noqa: SIM118 self._columns[name] = Column( - [ - x - for n, x in enumerate(self._columns[name]) - if n not in to_drop - ] + [x for n, x in enumerate(self._columns[name]) if n not in to_drop] ) else: self.drop(*to_drop) @@ -275,8 +261,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]): ): if axis not in ["rows", "cols"]: raise ValueError( - f"Invalid 'axis' value {axis}." - "Choose one of ['rows', 'cols']" + f"Invalid 'axis' value {axis}.Choose one of ['rows', 'cols']" ) if axis == "cols": @@ -362,9 +347,7 @@ def diff_table( for item, change in sorted(diff_in_file.items()): old_value = with_value(change.get("old"), fill_value) new_value = with_value(change.get("new"), fill_value) - diff_value = with_value( - change.get("diff", on_empty_diff), fill_value - ) + diff_value = with_value(change.get("diff", on_empty_diff), fill_value) td.append( [ fname, diff --git a/dvc/config.py b/dvc/config.py index 007d1b6aa..112caef7e 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -153,9 +153,7 @@ class Config(dict): with open(config_file, "w+", encoding="utf-8"): return Config(dvc_dir) - def load( - self, validate: bool = True, config: Optional["DictStrAny"] = None - ): + def load(self, validate: bool = True, config: Optional["DictStrAny"] = None): """Loads config from all the config files. Raises: @@ -376,6 +374,5 @@ def merge(into, update): def _lower_keys(data): return { - k.lower(): _lower_keys(v) if isinstance(v, dict) else v - for k, v in data.items() + k.lower(): _lower_keys(v) if isinstance(v, dict) else v for k, v in data.items() } diff --git a/dvc/config_schema.py b/dvc/config_schema.py index a5c077d3c..614025d9f 100644 --- a/dvc/config_schema.py +++ b/dvc/config_schema.py @@ -2,16 +2,7 @@ import os from urllib.parse import urlparse from funcy import walk_values -from voluptuous import ( - All, - Any, - Coerce, - Invalid, - Lower, - Optional, - Range, - Schema, -) +from voluptuous import All, Any, Coerce, Invalid, Lower, Optional, Range, Schema Bool = All( Lower, @@ -34,9 +25,7 @@ def supported_cache_type(types): unsupported = set(types) - {"reflink", "hardlink", "symlink", "copy"} if unsupported: - raise Invalid( - "Unsupported cache type(s): {}".format(", ".join(unsupported)) - ) + raise Invalid("Unsupported cache type(s): {}".format(", ".join(unsupported))) return types @@ -258,9 +247,7 @@ SCHEMA = { "machine": { str: { "cloud": All(Lower, Choices("aws", "azure")), - "region": All( - Lower, Choices("us-west", "us-east", "eu-west", "eu-north") - ), + "region": All(Lower, Choices("us-west", "us-east", "eu-west", "eu-north")), "image": str, "spot": Bool, "spot_price": Coerce(float), diff --git a/dvc/daemon.py b/dvc/daemon.py index 1311bb9dc..6266d7a8b 100644 --- a/dvc/daemon.py +++ b/dvc/daemon.py @@ -40,9 +40,7 @@ def _spawn_windows(cmd, env): startupinfo = STARTUPINFO() startupinfo.dwFlags |= STARTF_USESHOWWINDOW - _popen( - cmd, env=env, creationflags=creationflags, startupinfo=startupinfo - ) + _popen(cmd, env=env, creationflags=creationflags, startupinfo=startupinfo) def _spawn_posix(cmd, env): diff --git a/dvc/dagascii.py b/dvc/dagascii.py index c4c25b8a2..5be1c2250 100644 --- a/dvc/dagascii.py +++ b/dvc/dagascii.py @@ -272,8 +272,6 @@ def draw(vertices, edges): vertex.view.h, ) - canvas.text( - int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data - ) + canvas.text(int(round(x - minx)) + 1, int(round(y - miny)) + 1, vertex.data) return canvas.draw() diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index d89bce521..5abdd860d 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -111,8 +111,10 @@ class DataCloud: for hash_info in status.missing ) logger.warning( - "Some of the cache files do not exist neither locally " - "nor on remote. Missing cache files:\n%s", + ( + "Some of the cache files do not exist neither locally " + "nor on remote. Missing cache files:\n%s" + ), missing_desc, ) diff --git a/dvc/dependency/__init__.py b/dvc/dependency/__init__.py index a1bd0eaa4..de4b9ef8b 100644 --- a/dvc/dependency/__init__.py +++ b/dvc/dependency/__init__.py @@ -90,6 +90,4 @@ def _merge_params(s_list): def loads_params(stage, s_list): d = _merge_params(s_list) - return [ - ParamsDependency(stage, path, params) for path, params in d.items() - ] + return [ParamsDependency(stage, path, params) for path, params in d.items()] diff --git a/dvc/dependency/base.py b/dvc/dependency/base.py index 9588be3e1..2aa90a549 100644 --- a/dvc/dependency/base.py +++ b/dvc/dependency/base.py @@ -47,9 +47,7 @@ class Dependency(Output): self.def_path = self.fs.path.version_path( self.def_path, self.meta.version_id ) - self.fs_path = self.fs.path.version_path( - self.fs_path, self.meta.version_id - ) + self.fs_path = self.fs.path.version_path(self.fs_path, self.meta.version_id) def download(self, to, jobs=None): fs_download(self.fs, self.fs_path, to, jobs=jobs) @@ -57,9 +55,7 @@ class Dependency(Output): def save(self): super().save() if self.fs.version_aware: - self.fs_path = self.fs.path.version_path( - self.fs_path, self.meta.version_id - ) + self.fs_path = self.fs.path.version_path(self.fs_path, self.meta.version_id) def dumpd(self, **kwargs): if self.fs.version_aware: diff --git a/dvc/dependency/param.py b/dvc/dependency/param.py index 99e3603af..a83ad9448 100644 --- a/dvc/dependency/param.py +++ b/dvc/dependency/param.py @@ -154,9 +154,7 @@ class ParamsDependency(Dependency): try: return load_path(self.fs_path, self.repo.fs) except ParseError as exc: - raise BadParamFileError( - f"Unable to read parameters from '{self}'" - ) from exc + raise BadParamFileError(f"Unable to read parameters from '{self}'") from exc def get_hash(self): info = self.read_params() diff --git a/dvc/dependency/repo.py b/dvc/dependency/repo.py index b9ab815b2..31f49a493 100644 --- a/dvc/dependency/repo.py +++ b/dvc/dependency/repo.py @@ -35,9 +35,7 @@ class RepoDependency(Dependency): } } - def __init__( - self, def_repo: Dict[str, str], stage: "Stage", *args, **kwargs - ): + def __init__(self, def_repo: Dict[str, str], stage: "Stage", *args, **kwargs): self.def_repo = def_repo self._objs: Dict[str, "HashFile"] = {} self._meta: Dict[str, "Meta"] = {} @@ -100,17 +98,13 @@ class RepoDependency(Dependency): # immutable, hence its impossible for checksum to change. return False - def get_used_objs( - self, **kwargs - ) -> Dict[Optional["ObjectDB"], Set["HashInfo"]]: + def get_used_objs(self, **kwargs) -> Dict[Optional["ObjectDB"], Set["HashInfo"]]: used, _, _ = self._get_used_and_obj(**kwargs) return used def _get_used_and_obj( self, obj_only: bool = False, **kwargs - ) -> Tuple[ - Dict[Optional["ObjectDB"], Set["HashInfo"]], "Meta", "HashFile" - ]: + ) -> Tuple[Dict[Optional["ObjectDB"], Set["HashInfo"]], "Meta", "HashFile"]: from dvc.config import NoRemoteError from dvc.exceptions import NoOutputOrStageError, PathMissingError from dvc.utils import as_posix @@ -163,9 +157,7 @@ class RepoDependency(Dependency): used_obj_ids[object_store].update(oid for _, _, oid in obj) return used_obj_ids, meta, obj - def _check_circular_import( - self, odb: "ObjectDB", obj_ids: Set["HashInfo"] - ) -> None: + def _check_circular_import(self, odb: "ObjectDB", obj_ids: Set["HashInfo"]) -> None: from dvc.exceptions import CircularImportError from dvc.fs.dvc import DVCFileSystem from dvc_data.hashfile.db.reference import ReferenceHashFileDB @@ -187,10 +179,7 @@ class RepoDependency(Dependency): for obj in iter_objs(): if not isinstance(obj.fs, DVCFileSystem): continue - if ( - obj.fs.repo_url in checked_urls - or obj.fs.repo.root_dir in checked_urls - ): + if obj.fs.repo_url in checked_urls or obj.fs.repo.root_dir in checked_urls: continue self_url = self.repo.url or self.repo.root_dir if ( @@ -221,9 +210,7 @@ class RepoDependency(Dependency): ) return meta - def _make_repo( - self, locked: bool = True, **kwargs - ) -> "ContextManager[Repo]": + def _make_repo(self, locked: bool = True, **kwargs) -> "ContextManager[Repo]": from dvc.external_repo import external_repo d = self.def_repo @@ -232,6 +219,4 @@ class RepoDependency(Dependency): def _get_rev(self, locked: bool = True): d = self.def_repo - return (d.get(self.PARAM_REV_LOCK) if locked else None) or d.get( - self.PARAM_REV - ) + return (d.get(self.PARAM_REV_LOCK) if locked else None) or d.get(self.PARAM_REV) diff --git a/dvc/dvcfile.py b/dvc/dvcfile.py index da177020b..4eb78bf21 100644 --- a/dvc/dvcfile.py +++ b/dvc/dvcfile.py @@ -63,9 +63,7 @@ def is_valid_filename(path): def is_dvc_file(path): - return os.path.isfile(path) and ( - is_valid_filename(path) or is_lock_file(path) - ) + return os.path.isfile(path) and (is_valid_filename(path) or is_lock_file(path)) def is_lock_file(path): @@ -77,9 +75,7 @@ def is_git_ignored(repo, path): from dvc.scm import NoSCMError try: - return isinstance(repo.fs, LocalFileSystem) and repo.scm.is_ignored( - path - ) + return isinstance(repo.fs, LocalFileSystem) and repo.scm.is_ignored(path) except NoSCMError: return False @@ -152,9 +148,7 @@ class FileMixin: # 4. when the file is git ignored if not self.exists(): dvc_ignored = self.repo.dvcignore.is_ignored_file(self.path) - raise StageFileDoesNotExistError( - self.path, dvc_ignored=dvc_ignored - ) + raise StageFileDoesNotExistError(self.path, dvc_ignored=dvc_ignored) self._verify_filename() if not self.repo.fs.isfile(self.path): @@ -271,9 +265,7 @@ class ProjectFile(FileMixin): @staticmethod def _check_if_parametrized(stage, action: str = "dump") -> None: if stage.raw_data.parametrized: - raise ParametrizedDumpError( - f"cannot {action} a parametrized {stage}" - ) + raise ParametrizedDumpError(f"cannot {action} a parametrized {stage}") def _dump_pipeline_file(self, stage): self._check_if_parametrized(stage) @@ -286,9 +278,7 @@ class ProjectFile(FileMixin): data["stages"] = data.get("stages", {}) existing_entry = stage.name in data["stages"] action = "Modifying" if existing_entry else "Adding" - logger.info( - "%s stage '%s' in '%s'", action, stage.name, self.relpath - ) + logger.info("%s stage '%s' in '%s'", action, stage.name, self.relpath) if existing_entry: orig_stage_data = data["stages"][stage.name] @@ -300,9 +290,7 @@ class ProjectFile(FileMixin): @property def stage(self): - raise DvcException( - "ProjectFile has multiple stages. Please specify it's name." - ) + raise DvcException("ProjectFile has multiple stages. Please specify it's name.") @cached_property def contents(self) -> Dict[str, Any]: @@ -366,10 +354,7 @@ class ProjectFile(FileMixin): def get_lockfile_schema(d): - from dvc.schema import ( - COMPILED_LOCKFILE_V1_SCHEMA, - COMPILED_LOCKFILE_V2_SCHEMA, - ) + from dvc.schema import COMPILED_LOCKFILE_V1_SCHEMA, COMPILED_LOCKFILE_V2_SCHEMA schema = { LOCKFILE_VERSION.V1: COMPILED_LOCKFILE_V1_SCHEMA, @@ -422,9 +407,7 @@ class Lockfile(FileMixin): with modify_yaml(self.path, fs=self.repo.fs) as data: version = LOCKFILE_VERSION.from_dict(data) if version == LOCKFILE_VERSION.V1: - logger.info( - "Migrating lock file '%s' from v1 to v2", self.relpath - ) + logger.info("Migrating lock file '%s' from v1 to v2", self.relpath) migrate_lock_v1_to_v2(data, self.latest_version_info) else: if not data: diff --git a/dvc/exceptions.py b/dvc/exceptions.py index dc9ac5df3..5d6975f39 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -150,8 +150,9 @@ class CyclicGraphError(DvcException): class ConfirmRemoveError(DvcException): def __init__(self, path): super().__init__( - "unable to remove '{}' without a confirmation. Use " - "`-f` to force.".format(path) + "unable to remove '{}' without a confirmation. Use `-f` to force.".format( + path + ) ) @@ -177,9 +178,7 @@ class BadMetricError(DvcException): class RecursiveAddingWhileUsingFilename(DvcException): def __init__(self): - super().__init__( - "cannot use `fname` with multiple targets or `-R|--recursive`" - ) + super().__init__("cannot use `fname` with multiple targets or `-R|--recursive`") class OverlappingOutputPathsError(DvcException): @@ -206,9 +205,7 @@ class FileMissingError(DvcException): def __init__(self, path, hint=None): self.path = path hint = "" if hint is None else f". {hint}" - super().__init__( - f"Can't find '{path}' neither locally nor on remote{hint}" - ) + super().__init__(f"Can't find '{path}' neither locally nor on remote{hint}") class FileTransferError(DvcException): @@ -250,9 +247,7 @@ class CollectCacheError(DvcException): class NoRemoteInExternalRepoError(DvcException): def __init__(self, url): - super().__init__( - f"No DVC remote is specified in target repository '{url}'." - ) + super().__init__(f"No DVC remote is specified in target repository '{url}'.") class NoOutputInExternalRepoError(DvcException): @@ -277,8 +272,7 @@ class PathMissingError(DvcException): " neither as a DVC output nor as a Git-tracked file." ) default_msg_dvc_only = ( - "The path '{}' does not exist in the target repository '{}'" - " as an DVC output." + "The path '{}' does not exist in the target repository '{}' as an DVC output." ) def __init__(self, path, repo, dvc_only=False): @@ -294,7 +288,6 @@ class URLMissingError(DvcException): class RemoteCacheRequiredError(DvcException): def __init__(self, scheme, fs_path): - super().__init__( ( "Current operation was unsuccessful because '{}' requires " @@ -319,8 +312,7 @@ class NoOutputOrStageError(DvcException): def __init__(self, target, file): super().__init__( - f"'{target}' " - f"does not exist as an output or a stage name in '{file}'" + f"'{target}' does not exist as an output or a stage name in '{file}'" ) @@ -329,11 +321,8 @@ class MergeError(DvcException): class CacheLinkError(DvcException): - SUPPORT_LINK = "See {} for more information.".format( - format_link( - "https://dvc.org/doc/user-guide/troubleshooting#cache-types" - ) + format_link("https://dvc.org/doc/user-guide/troubleshooting#cache-types") ) def __init__(self, fs_paths): diff --git a/dvc/external_repo.py b/dvc/external_repo.py index 2c5f7821c..8954df2b3 100644 --- a/dvc/external_repo.py +++ b/dvc/external_repo.py @@ -88,9 +88,7 @@ def external_repo( raise NoRemoteInExternalRepoError(url) from exc except OutputNotFoundError as exc: if exc.repo is repo: - raise NoOutputInExternalRepoError( - exc.output, repo.root_dir, url - ) from exc + raise NoOutputInExternalRepoError(exc.output, repo.root_dir, url) from exc raise except FileMissingError as exc: raise PathMissingError(exc.path, url) from exc @@ -234,9 +232,7 @@ def _clone_default_branch(url, rev, for_write=False): # noqa: C901 os.path.join(clone_path, Git.GIT_DIR, "shallow") ) if shallow: - logger.debug( - "erepo: using shallow clone for branch '%s'", rev - ) + logger.debug("erepo: using shallow clone for branch '%s'", rev) except CloneError: git_dir = os.path.join(clone_path, ".git") if os.path.exists(git_dir): @@ -292,8 +288,6 @@ def _remove(path): try: os_retry(remove)(path) except PermissionError: - logger.warning( - "Failed to remove '%s'", relpath(path), exc_info=True - ) + logger.warning("Failed to remove '%s'", relpath(path), exc_info=True) else: remove(path) diff --git a/dvc/fs/__init__.py b/dvc/fs/__init__.py index 06b1a45ca..b59d41088 100644 --- a/dvc/fs/__init__.py +++ b/dvc/fs/__init__.py @@ -72,9 +72,7 @@ def get_fs_config(config, **kwargs): except KeyError: from dvc.config import RemoteNotFoundError - raise RemoteNotFoundError( # noqa: B904 - f"remote '{name}' doesn't exist" - ) + raise RemoteNotFoundError(f"remote '{name}' doesn't exist") # noqa: B904 else: remote_conf = kwargs return _resolve_remote_refs(config, remote_conf) diff --git a/dvc/fs/callbacks.py b/dvc/fs/callbacks.py index 25555b7bf..024aa9f0c 100644 --- a/dvc/fs/callbacks.py +++ b/dvc/fs/callbacks.py @@ -58,9 +58,7 @@ class RichCallback(Callback): @cached_property def task(self) -> "TaskID": - return self.progress.add_task( - **self._task_kwargs # type: ignore[arg-type] - ) + return self.progress.add_task(**self._task_kwargs) # type: ignore[arg-type] def __enter__(self): return self diff --git a/dvc/fs/dvc.py b/dvc/fs/dvc.py index 6928a425a..ad9936b93 100644 --- a/dvc/fs/dvc.py +++ b/dvc/fs/dvc.py @@ -6,16 +6,7 @@ import os import posixpath import threading from contextlib import suppress -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - Optional, - Tuple, - Type, - Union, -) +from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple, Type, Union from fsspec.spec import AbstractFileSystem from funcy import wrap_with @@ -126,9 +117,7 @@ class _DVCFileSystem(AbstractFileSystem): # pylint:disable=abstract-method super().__init__() if repo is None: - repo = self._make_repo( - url=url, rev=rev, subrepos=subrepos, **repo_kwargs - ) + repo = self._make_repo(url=url, rev=rev, subrepos=subrepos, **repo_kwargs) assert repo is not None # pylint: disable=protected-access repo_factory = repo._fs_conf["repo_factory"] @@ -144,9 +133,7 @@ class _DVCFileSystem(AbstractFileSystem): # pylint:disable=abstract-method relparts = () assert repo is not None if repo.fs.path.isin(repo.fs.path.getcwd(), repo.root_dir): - relparts = repo.fs.path.relparts( - repo.fs.path.getcwd(), repo.root_dir - ) + relparts = repo.fs.path.relparts(repo.fs.path.getcwd(), repo.root_dir) return self.root_marker + self.sep.join(relparts) self.path = Path(self.sep, getcwd=_getcwd) @@ -222,9 +209,7 @@ class _DVCFileSystem(AbstractFileSystem): # pylint:disable=abstract-method scm=self.repo.scm, repo_factory=self.repo_factory, ) - self._datafss[key] = DataFileSystem( - index=repo.index.data["repo"] - ) + self._datafss[key] = DataFileSystem(index=repo.index.data["repo"]) self._subrepos_trie[key] = repo def _is_dvc_repo(self, dir_path): @@ -331,9 +316,7 @@ class _DVCFileSystem(AbstractFileSystem): # pylint:disable=abstract-method ignore_subrepos = kwargs.get("ignore_subrepos", True) return self._info(key, path, ignore_subrepos=ignore_subrepos) - def _info( # noqa: C901 - self, key, path, ignore_subrepos=True, check_ignored=True - ): + def _info(self, key, path, ignore_subrepos=True, check_ignored=True): # noqa: C901 repo, dvc_fs, subkey = self._get_subrepo_info(key) dvc_info = None diff --git a/dvc/ignore.py b/dvc/ignore.py index e2b63dae6..08b08d418 100644 --- a/dvc/ignore.py +++ b/dvc/ignore.py @@ -30,9 +30,7 @@ class DvcIgnorePatterns(DvcIgnore): from pathspec.patterns.gitwildmatch import _DIR_MARK if pattern_list and isinstance(pattern_list[0], str): - pattern_list = [ - PatternInfo(pattern, "") for pattern in pattern_list - ] + pattern_list = [PatternInfo(pattern, "") for pattern in pattern_list] self.sep = sep self.pattern_list = pattern_list @@ -40,20 +38,14 @@ class DvcIgnorePatterns(DvcIgnore): self.regex_pattern_list = [] for count, pattern in enumerate(pattern_list): - pattern, group = GitWildMatchPattern.pattern_to_regex( - pattern.patterns - ) + pattern, group = GitWildMatchPattern.pattern_to_regex(pattern.patterns) if pattern: - pattern = pattern.replace( - f"<{_DIR_MARK}>", f"<{_DIR_MARK}{count}>" - ) + pattern = pattern.replace(f"<{_DIR_MARK}>", f"<{_DIR_MARK}{count}>") self.regex_pattern_list.append((pattern, group)) self.ignore_spec = [ (ignore, re.compile("|".join(item[0] for item in group))) - for ignore, group in groupby( - self.regex_pattern_list, lambda x: x[1] - ) + for ignore, group in groupby(self.regex_pattern_list, lambda x: x[1]) if ignore is not None ] @@ -64,9 +56,7 @@ class DvcIgnorePatterns(DvcIgnore): with fs.open(path, encoding="utf-8") as fobj: path_spec_lines = [ PatternInfo(line, f"{name}:{line_no + 1}:{line}") - for line_no, line in enumerate( - map(str.strip, fobj.readlines()) - ) + for line_no, line in enumerate(map(str.strip, fobj.readlines())) if line and not (line.strip().startswith("#")) ] @@ -158,9 +148,7 @@ class DvcIgnorePatterns(DvcIgnore): return bool(self.pattern_list) -CheckIgnoreResult = namedtuple( - "CheckIgnoreResult", ["file", "match", "patterns"] -) +CheckIgnoreResult = namedtuple("CheckIgnoreResult", ["file", "match", "patterns"]) def _no_match(path): @@ -249,9 +237,7 @@ class DvcIgnoreFilter: dnames = [] for dname in dnames: - self._update_sub_repo( - self.fs.path.join(dirname, dname), ignore_trie - ) + self._update_sub_repo(self.fs.path.join(dirname, dname), ignore_trie) def _update_sub_repo(self, path, ignore_trie: Trie): from dvc.repo import Repo @@ -409,9 +395,7 @@ class DvcIgnoreFilter: # https://github.com/iterative/dvc/issues/5046 full_target = self.fs.path.abspath(target) if not self._outside_repo(full_target): - dirname, basename = self.fs.path.split( - self.fs.path.normpath(full_target) - ) + dirname, basename = self.fs.path.split(self.fs.path.normpath(full_target)) pattern = self._get_trie_pattern(dirname) if pattern: matches = pattern.matches( @@ -433,9 +417,9 @@ class DvcIgnoreFilter: return self.is_ignored_file(path, ignore_subrepos) if fs.isdir(path): return self.is_ignored_dir(path, ignore_subrepos) - return self.is_ignored_file( + return self.is_ignored_file(path, ignore_subrepos) or self.is_ignored_dir( path, ignore_subrepos - ) or self.is_ignored_dir(path, ignore_subrepos) + ) def init(path): diff --git a/dvc/info.py b/dvc/info.py index f70d5c520..2dfb1ec2b 100644 --- a/dvc/info.py +++ b/dvc/info.py @@ -30,8 +30,7 @@ def get_dvc_info(): info = [ dvc_version, "-" * len(dvc_version), - f"Platform: Python {platform.python_version()} on " - f"{platform.platform()}", + f"Platform: Python {platform.python_version()} on {platform.platform()}", f"Subprojects:{_get_subprojects()}", f"Supports:{_get_supported_remotes()}", ] @@ -125,8 +124,7 @@ def _get_supported_remotes(): dependencies = [] for requirement in fs_cls.REQUIRES: dependencies.append( - f"{requirement} = " - f"{importlib_metadata.version(requirement)}" + f"{requirement} = {importlib_metadata.version(requirement)}" ) remote_info = scheme diff --git a/dvc/lock.py b/dvc/lock.py index 050c0d01a..de8929c34 100644 --- a/dvc/lock.py +++ b/dvc/lock.py @@ -20,9 +20,7 @@ FAILED_TO_LOCK_MESSAGE = ( "Unable to acquire lock. Most likely another DVC process is running or " "was terminated abruptly. Check the page {} for other possible reasons " "and to learn how to resolve this." -).format( - format_link("https://dvc.org/doc/user-guide/troubleshooting#lock-issue") -) +).format(format_link("https://dvc.org/doc/user-guide/troubleshooting#lock-issue")) class LockError(DvcException): @@ -61,9 +59,7 @@ class LockBase(ABC): class LockNoop(LockBase): - def __init__( - self, *args, **kwargs - ): # pylint: disable=super-init-not-called + def __init__(self, *args, **kwargs): # pylint: disable=super-init-not-called self._lock = False def lock(self): @@ -107,10 +103,8 @@ class Lock(LockBase): with Tqdm( bar_format="{desc}", disable=not self._friendly, - desc=( - "If DVC froze, see `hardlink_lock` in {}".format( - format_link("https://man.dvc.org/config#core") - ) + desc="If DVC froze, see `hardlink_lock` in {}".format( + format_link("https://man.dvc.org/config#core") ), ): self._lock = zc.lockfile.LockFile(self._lockfile) diff --git a/dvc/logger.py b/dvc/logger.py index ecf525a68..21c943653 100644 --- a/dvc/logger.py +++ b/dvc/logger.py @@ -159,9 +159,7 @@ class LoggerHandler(logging.StreamHandler): pass # noqa: S110 msg = self.format(record) - Tqdm.write( - msg, file=self.stream, end=getattr(self, "terminator", "\n") - ) + Tqdm.write(msg, file=self.stream, end=getattr(self, "terminator", "\n")) self.flush() except (BrokenPipeError, RecursionError): raise @@ -171,9 +169,7 @@ class LoggerHandler(logging.StreamHandler): def _is_verbose(): return ( - logging.NOTSET - < logging.getLogger("dvc").getEffectiveLevel() - <= logging.DEBUG + logging.NOTSET < logging.getLogger("dvc").getEffectiveLevel() <= logging.DEBUG ) @@ -186,16 +182,11 @@ def _iter_causes(exc): def _stack_trace(exc_info): import traceback - return ( - "\n" - "{red}{line}{nc}\n" - "{trace}" - "{red}{line}{nc}".format( - red=colorama.Fore.RED, - line="-" * 60, - trace="".join(traceback.format_exception(*exc_info)), - nc=colorama.Fore.RESET, - ) + return "\n{red}{line}{nc}\n{trace}{red}{line}{nc}".format( + red=colorama.Fore.RED, + line="-" * 60, + trace="".join(traceback.format_exception(*exc_info)), + nc=colorama.Fore.RESET, ) diff --git a/dvc/machine/__init__.py b/dvc/machine/__init__.py index 8c575402d..4e2a1d305 100644 --- a/dvc/machine/__init__.py +++ b/dvc/machine/__init__.py @@ -67,9 +67,7 @@ class MachineBackends(Mapping): initialized = self.initialized.get(key) if not initialized: backend = self.backends[key] - initialized = backend( - os.path.join(self.tmp_dir, key), **self.kwargs - ) + initialized = backend(os.path.join(self.tmp_dir, key), **self.kwargs) self.initialized[key] = initialized return initialized diff --git a/dvc/machine/backend/base.py b/dvc/machine/backend/base.py index ba2bc366e..f72425cc6 100644 --- a/dvc/machine/backend/base.py +++ b/dvc/machine/backend/base.py @@ -21,9 +21,7 @@ class BaseMachineBackend(ABC): """Stop and destroy all instances of the specified machine.""" @abstractmethod - def instances( - self, name: Optional[str] = None, **config - ) -> Iterator[dict]: + def instances(self, name: Optional[str] = None, **config) -> Iterator[dict]: """Iterate over status of all instances of the specified machine.""" def close(self): # noqa: B027 diff --git a/dvc/machine/backend/terraform.py b/dvc/machine/backend/terraform.py index 953986fcd..9d00946b9 100644 --- a/dvc/machine/backend/terraform.py +++ b/dvc/machine/backend/terraform.py @@ -56,12 +56,8 @@ class TerraformBackend(BaseMachineBackend): create = partialmethod(_tpi_func, "create") # type: ignore[assignment] destroy = partialmethod(_tpi_func, "destroy") # type: ignore[assignment] - instances = partialmethod( - _tpi_func, "instances" - ) # type: ignore[assignment] - run_shell = partialmethod( - _tpi_func, "run_shell" - ) # type: ignore[assignment] + instances = partialmethod(_tpi_func, "instances") # type: ignore[assignment] + run_shell = partialmethod(_tpi_func, "run_shell") # type: ignore[assignment] def get_executor_kwargs(self, name: str, **config) -> dict: with self.make_tpi(name) as tpi: diff --git a/dvc/output.py b/dvc/output.py index 4248b2723..31471a92b 100644 --- a/dvc/output.py +++ b/dvc/output.py @@ -151,9 +151,7 @@ def _merge_data(s_list): d[key].update({}) continue if not isinstance(key, dict): - raise ValueError( # noqa: TRY004 - f"'{type(key).__name__}' not supported." - ) + raise ValueError(f"'{type(key).__name__}' not supported.") # noqa: TRY004 for k, flags in key.items(): if not isinstance(flags, dict): @@ -336,9 +334,7 @@ class Output: if fs_config is not None: fs_kwargs.update(**fs_config) - fs_cls, fs_config, fs_path = get_cloud_fs( - self.repo, url=path, **fs_kwargs - ) + fs_cls, fs_config, fs_path = get_cloud_fs(self.repo, url=path, **fs_kwargs) self.fs = fs_cls(**fs_config) if ( @@ -500,9 +496,7 @@ class Output: @property def cache_path(self): - return self.odb.fs.unstrip_protocol( - self.odb.oid_to_path(self.hash_info.value) - ) + return self.odb.fs.unstrip_protocol(self.odb.oid_to_path(self.hash_info.value)) def get_hash(self): _, hash_info = self._get_hash_meta() @@ -691,9 +685,7 @@ class Output: dry_run=True, ) if not self.IS_DEPENDENCY: - logger.debug( - "Output '%s' doesn't use cache. Skipping saving.", self - ) + logger.debug("Output '%s' doesn't use cache. Skipping saving.", self) self.hash_info = self.obj.hash_info self.files = None @@ -724,9 +716,7 @@ class Output: if self.use_cache: granular = ( - self.is_dir_checksum - and filter_info - and filter_info != self.fs_path + self.is_dir_checksum and filter_info and filter_info != self.fs_path ) if granular: obj = self._commit_granular_dir(filter_info) @@ -757,9 +747,7 @@ class Output: self.set_exec() def _commit_granular_dir(self, filter_info) -> Optional["HashFile"]: - prefix = self.fs.path.parts( - self.fs.path.relpath(filter_info, self.fs_path) - ) + prefix = self.fs.path.parts(self.fs.path.relpath(filter_info, self.fs_path)) staging, _, save_obj = build( self.odb, self.fs_path, @@ -794,9 +782,7 @@ class Output: ret.update(split_file_meta_from_cloud(meta_d)) if self.is_in_repo: - path = self.fs.path.as_posix( - relpath(self.fs_path, self.stage.wdir) - ) + path = self.fs.path.as_posix(relpath(self.fs_path, self.stage.wdir)) else: path = self.def_path @@ -837,16 +823,13 @@ class Output: if obj: obj = cast("Tree", obj) ret[self.PARAM_FILES] = [ - split_file_meta_from_cloud(f) - for f in obj.as_list(with_meta=True) + split_file_meta_from_cloud(f) for f in obj.as_list(with_meta=True) ] return ret def verify_metric(self): if self.fs.protocol != "local": - raise DvcException( - f"verify metric is not supported for {self.protocol}" - ) + raise DvcException(f"verify metric is not supported for {self.protocol}") if not self.metric: return @@ -902,9 +885,7 @@ class Output: ) -> Optional[Tuple[bool, Optional[bool]]]: if not self.use_cache: if progress_callback != DEFAULT_CALLBACK: - progress_callback.relative_update( - self.get_files_number(filter_info) - ) + progress_callback.relative_update(self.get_files_number(filter_info)) return None obj = self.get_obj(filter_info=filter_info) @@ -1071,9 +1052,7 @@ class Output: obj = self.get_obj() if filter_info and filter_info != self.fs_path: assert obj - prefix = self.fs.path.parts( - self.fs.path.relpath(filter_info, self.fs_path) - ) + prefix = self.fs.path.parts(self.fs.path.relpath(filter_info, self.fs_path)) obj = cast(Tree, obj) return obj.filter(prefix) return cast(Tree, obj) @@ -1181,14 +1160,10 @@ class Output: other.pop(opt, None) if my != other: - raise MergeError( - "unable to auto-merge outputs with different options" - ) + raise MergeError("unable to auto-merge outputs with different options") if not out.is_dir_checksum: - raise MergeError( - "unable to auto-merge outputs that are not directories" - ) + raise MergeError("unable to auto-merge outputs that are not directories") def merge(self, ancestor, other, allowed=None): from dvc_data.hashfile.tree import MergeError as TreeMergeError diff --git a/dvc/parsing/__init__.py b/dvc/parsing/__init__.py index b0dbf4e47..1c1e03b92 100644 --- a/dvc/parsing/__init__.py +++ b/dvc/parsing/__init__.py @@ -72,9 +72,7 @@ def _format_preamble(msg: str, path: str, spacing: str = " ") -> str: def format_and_raise(exc: Exception, msg: str, path: str) -> "NoReturn": spacing = ( - "\n" - if isinstance(exc, (ParseError, MergeError, VarsAlreadyLoaded)) - else " " + "\n" if isinstance(exc, (ParseError, MergeError, VarsAlreadyLoaded)) else " " ) message = _format_preamble(msg, path, spacing) + str(exc) @@ -116,8 +114,7 @@ def check_interpolations(data: "DictStrAny", where: str, path: str): def func(s: "DictStrAny") -> None: if is_interpolated_string(s): raise ResolveError( - _format_preamble(f"'{where}'", path) - + "interpolating is not allowed" + _format_preamble(f"'{where}'", path) + "interpolating is not allowed" ) return recurse(func)(data) @@ -188,9 +185,7 @@ class DataResolver: def resolve(self): """Used for testing purposes, otherwise use resolve_one().""" data = join(map(self.resolve_one, self.get_keys())) - logger.trace( # type: ignore[attr-defined] - "Resolved dvc.yaml:\n%s", data - ) + logger.trace("Resolved dvc.yaml:\n%s", data) # type: ignore[attr-defined] return {STAGES_KWD: data} def has_key(self, key: str): @@ -203,9 +198,9 @@ class DataResolver: return False if key: - return isinstance( - definition, ForeachDefinition - ) and definition.has_member(key) + return isinstance(definition, ForeachDefinition) and definition.has_member( + key + ) return not isinstance(definition, ForeachDefinition) @collecting @@ -309,9 +304,7 @@ class EntryDefinition: value, skip_interpolation_checks=skip_checks, key=key ) except (ParseError, KeyNotInContext) as exc: - format_and_raise( - exc, f"'{self.where}.{self.name}.{key}'", self.relpath - ) + format_and_raise(exc, f"'{self.where}.{self.name}.{key}'", self.relpath) class IterationPair(NamedTuple): @@ -354,9 +347,7 @@ class ForeachDefinition: try: iterable = self.context.resolve(self.foreach_data, unwrap=False) except (ContextError, ParseError) as exc: - format_and_raise( - exc, f"'{self.where}.{self.name}.foreach'", self.relpath - ) + format_and_raise(exc, f"'{self.where}.{self.name}.foreach'", self.relpath) # foreach data can be a resolved dictionary/list. self._check_is_map_or_seq(iterable) @@ -381,8 +372,10 @@ class ForeachDefinition: if warn_for: linking_verb = "is" if len(warn_for) == 1 else "are" logger.warning( - "%s %s already specified, " - "will be overwritten for stages generated from '%s'", + ( + "%s %s already specified, " + "will be overwritten for stages generated from '%s'" + ), " and ".join(warn_for), linking_verb, self.name, diff --git a/dvc/parsing/context.py b/dvc/parsing/context.py index 92387c041..0cb73b650 100644 --- a/dvc/parsing/context.py +++ b/dvc/parsing/context.py @@ -48,9 +48,7 @@ class MergeError(ContextError): self.key = key to_node = into[key] if not isinstance(to_node, Node) or not isinstance(new, Node): - super().__init__( - f"cannot merge '{key}' as it already exists in {into}" - ) + super().__init__(f"cannot merge '{key}' as it already exists in {into}") return assert isinstance(to_node, Node) @@ -138,9 +136,7 @@ class Node: @dataclass class Value(Node): _value: Any - meta: Meta = field( - compare=False, default_factory=_default_meta, repr=False - ) + meta: Meta = field(compare=False, default_factory=_default_meta, repr=False) def __repr__(self): return repr(self._value) @@ -182,10 +178,7 @@ class Container(Node, ABC): # noqa: B024 assert meta container = CtxDict if isinstance(value, dict) else CtxList return container(value, meta=meta) - msg = ( - "Unsupported value of type " - f"'{type(value).__name__}' in '{meta}'" - ) + msg = f"Unsupported value of type '{type(value).__name__}' in '{meta}'" raise TypeError(msg) def __repr__(self): @@ -219,9 +212,7 @@ class Container(Node, ABC): # noqa: B024 try: d = self[index] except LookupError as exc: - raise ValueError( - f"Could not find '{index}' in {self.data}" - ) from exc + raise ValueError(f"Could not find '{index}' in {self.data}") from exc if not rems: return d @@ -380,9 +371,7 @@ class Context(CtxDict): data = {key: data[key] for key in select_keys} except KeyError as exc: key, *_ = exc.args - raise ParamsLoadError( - f"could not find '{key}' in '{path}'" - ) from exc + raise ParamsLoadError(f"could not find '{key}' in '{path}'") from exc meta = Meta(source=path, local=False) ctx = cls(data, meta=meta) @@ -480,9 +469,7 @@ class Context(CtxDict): But for now, just `item` and `key`, this should do. """ # using dict to make the error messages ordered - new = dict.fromkeys( - [key for key in keys if key not in self._reserved_keys] - ) + new = dict.fromkeys([key for key in keys if key not in self._reserved_keys]) self._reserved_keys.update(new) try: yield @@ -542,9 +529,7 @@ class Context(CtxDict): if is_exact_string(src, matches): # replace "${enabled}", if `enabled` is a boolean, with it's actual # value rather than it's string counterparts. - expr = get_expression( - matches[0], skip_checks=skip_interpolation_checks - ) + expr = get_expression(matches[0], skip_checks=skip_interpolation_checks) value = self.select(expr, unwrap=unwrap) validate_value(value, key) return value diff --git a/dvc/parsing/interpolate.py b/dvc/parsing/interpolate.py index d31931e1b..413c10cbb 100644 --- a/dvc/parsing/interpolate.py +++ b/dvc/parsing/interpolate.py @@ -99,7 +99,6 @@ def _(obj: dict): # noqa: C901 result = "" for k, v in flatten(obj).items(): - if isinstance(v, bool): if v: result += f"--{k} " @@ -115,9 +114,7 @@ def _(obj: dict): # noqa: C901 if isinstance(i, str): i = escape_str(i) elif isinstance(i, Iterable): - raise ParseError( - f"Cannot interpolate nested iterable in '{k}'" - ) + raise ParseError(f"Cannot interpolate nested iterable in '{k}'") if config.get("list", "nargs") == "append": result += f"--{k} {i} " @@ -206,9 +203,7 @@ def validate_value(value, key): if not_primitive and not_foreach: if isinstance(value, dict) and key == "cmd": return True - raise ParseError( - f"Cannot interpolate data of type '{type(value).__name__}'" - ) + raise ParseError(f"Cannot interpolate data of type '{type(value).__name__}'") def str_interpolate( diff --git a/dvc/parsing/versions.py b/dvc/parsing/versions.py index 80fcd18b6..2055a4d09 100644 --- a/dvc/parsing/versions.py +++ b/dvc/parsing/versions.py @@ -9,9 +9,7 @@ META_KWD = "meta" def lockfile_version_schema(value): expected = [LOCKFILE_VERSION.V2.value] # pylint: disable=no-member - msg = "invalid schema version {}, expected one of {}".format( - value, expected - ) + msg = f"invalid schema version {value}, expected one of {expected}" return validators.Any(*expected, msg=msg)(value) diff --git a/dvc/proc/exceptions.py b/dvc/proc/exceptions.py index 5db4ee8b4..bdf593760 100644 --- a/dvc/proc/exceptions.py +++ b/dvc/proc/exceptions.py @@ -13,9 +13,7 @@ class ProcessNotFoundError(DvcException): class TimeoutExpired(DvcException): def __init__(self, cmd, timeout): - super().__init__( - f"'{cmd}' did not complete before timeout '{timeout}'" - ) + super().__init__(f"'{cmd}' did not complete before timeout '{timeout}'") self.cmd = cmd self.timeout = timeout diff --git a/dvc/proc/manager.py b/dvc/proc/manager.py index bc52fb563..4fc9d1a89 100644 --- a/dvc/proc/manager.py +++ b/dvc/proc/manager.py @@ -91,9 +91,7 @@ class ProcessManager: raise UnsupportedSignalError(sig) def handle_closed_process(): - logging.warning( - "Process %s had already aborted unexpectedly.", name - ) + logging.warning("Process %s had already aborted unexpectedly.", name) process_info.returncode = -1 self[name] = process_info diff --git a/dvc/progress.py b/dvc/progress.py index 082c875bf..e1831df8d 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -28,9 +28,7 @@ class Tqdm(tqdm): "{postfix[info]}{n_fmt}/{total_fmt}" " [{elapsed}<{remaining}, {rate_fmt:>11}]" ) - BAR_FMT_NOTOTAL = ( - "{desc}{bar:b}|{postfix[info]}{n_fmt} [{elapsed}, {rate_fmt:>11}]" - ) + BAR_FMT_NOTOTAL = "{desc}{bar:b}|{postfix[info]}{n_fmt} [{elapsed}, {rate_fmt:>11}]" BYTES_DEFAULTS = { "unit": "B", "unit_scale": True, @@ -75,11 +73,7 @@ class Tqdm(tqdm): if not disable: disable = logger.getEffectiveLevel() > level # auto-disable based on TTY - if ( - not disable - and not env2bool(DVC_IGNORE_ISATTY) - and hasattr(file, "isatty") - ): + if not disable and not env2bool(DVC_IGNORE_ISATTY) and hasattr(file, "isatty"): disable = not file.isatty() super().__init__( iterable=iterable, @@ -95,9 +89,7 @@ class Tqdm(tqdm): if bar_format is None: if self.__len__(): self.bar_format = ( - self.BAR_FMT_DEFAULT_NESTED - if self.pos - else self.BAR_FMT_DEFAULT + self.BAR_FMT_DEFAULT_NESTED if self.pos else self.BAR_FMT_DEFAULT ) else: self.bar_format = self.BAR_FMT_NOTOTAL diff --git a/dvc/prompt.py b/dvc/prompt.py index d6664d3ef..8c358b504 100644 --- a/dvc/prompt.py +++ b/dvc/prompt.py @@ -24,9 +24,7 @@ def ask(prompt: str, limited_to: Optional[Collection[str]] = None): if answer in limited_to: return answer - logger.info( - "Your response must be one of: %s. Please try again.", limited_to - ) + logger.info("Your response must be one of: %s. Please try again.", limited_to) def confirm(statement: str) -> bool: diff --git a/dvc/render/convert.py b/dvc/render/convert.py index 492459af7..dec94b5cd 100644 --- a/dvc/render/convert.py +++ b/dvc/render/convert.py @@ -2,13 +2,7 @@ import json from collections import defaultdict from typing import Dict, List, Union -from dvc.render import ( - REVISION_FIELD, - REVISIONS_KEY, - SRC_FIELD, - TYPE_KEY, - VERSION_FIELD, -) +from dvc.render import REVISION_FIELD, REVISIONS_KEY, SRC_FIELD, TYPE_KEY, VERSION_FIELD from dvc.render.converter.image import ImageConverter from dvc.render.converter.vega import VegaConverter diff --git a/dvc/render/converter/vega.py b/dvc/render/converter/vega.py index bb203cc5b..705e0ecae 100644 --- a/dvc/render/converter/vega.py +++ b/dvc/render/converter/vega.py @@ -81,9 +81,9 @@ def _is_datapoints(lst: List[Dict]): to unexpected behavior """ - return all(isinstance(item, dict) for item in lst) and set( - first(lst).keys() - ) == {key for keys in lst for key in keys} + return all(isinstance(item, dict) for item in lst) and set(first(lst).keys()) == { + key for keys in lst for key in keys + } def get_datapoints(file_content: Dict): @@ -195,7 +195,6 @@ class VegaConverter(Converter): return "x" def flat_datapoints(self, revision): # noqa: C901 - file2datapoints, properties = self.convert() props_update = {} @@ -218,7 +217,7 @@ class VegaConverter(Converter): num_ys = len(ys) if num_xs > 1 and num_xs != num_ys: raise DvcException( - f"Cannot have different number of x and y data sources. Found " + "Cannot have different number of x and y data sources. Found " f"{num_xs} x and {num_ys} y data sources." ) @@ -316,9 +315,7 @@ def _update_from_field( source_field = field if len(source_datapoints) != len(target_datapoints): - raise IndexError( - "Source and target datapoints must have the same length" - ) + raise IndexError("Source and target datapoints must have the same length") for index, datapoint in enumerate(target_datapoints): source_datapoint = source_datapoints[index] @@ -332,6 +329,5 @@ def _update_from_index(datapoints: List[Dict], new_field: str): def _update_all(datapoints: List[Dict], update_dict: Dict): - for datapoint in datapoints: datapoint.update(update_dict) diff --git a/dvc/render/match.py b/dvc/render/match.py index 788254138..ad3d3b2ba 100644 --- a/dvc/render/match.py +++ b/dvc/render/match.py @@ -62,7 +62,6 @@ def match_defs_renderers( out=None, templates_dir: Optional["StrPath"] = None, ): - from dvc_render import ImageRenderer, VegaRenderer plots_data = PlotsData(data) @@ -80,9 +79,7 @@ def match_defs_renderers( for rev, inner_id, plot_definition in group: plot_sources = infer_data_sources(inner_id, plot_definition) - definitions_data = plots_data.get_definition_data( - plot_sources, rev - ) + definitions_data = plots_data.get_definition_data(plot_sources, rev) if ImageRenderer.matches(inner_id, None): renderer_cls = ImageRenderer @@ -91,9 +88,7 @@ def match_defs_renderers( renderer_cls = VegaRenderer renderer_id = plot_id - converter = _get_converter( - renderer_cls, inner_id, props, definitions_data - ) + converter = _get_converter(renderer_cls, inner_id, props, definitions_data) dps, rev_props = converter.flat_datapoints(rev) if not final_props and rev_props: @@ -103,7 +98,5 @@ def match_defs_renderers( if "title" not in final_props: final_props["title"] = renderer_id if renderer_cls is not None: - renderers.append( - renderer_cls(plot_datapoints, renderer_id, **final_props) - ) + renderers.append(renderer_cls(plot_datapoints, renderer_id, **final_props)) return renderers diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 8290f49fe..c70ce83c0 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -3,16 +3,7 @@ import os from collections import defaultdict from contextlib import contextmanager from functools import wraps -from typing import ( - TYPE_CHECKING, - Callable, - Iterable, - List, - Optional, - Tuple, - Union, - cast, -) +from typing import TYPE_CHECKING, Callable, Iterable, List, Optional, Tuple, Union, cast from dvc.exceptions import FileMissingError from dvc.exceptions import IsADirectoryError as DvcIsADirectoryError @@ -148,9 +139,7 @@ class Repo: import hashlib - root_dir_hash = hashlib.sha224( - self.root_dir.encode("utf-8") - ).hexdigest() + root_dir_hash = hashlib.sha224(self.root_dir.encode("utf-8")).hexdigest() db_dir = self.fs.path.join( base_db_dir, @@ -230,9 +219,7 @@ class Repo: self.lock = make_lock( self.fs.path.join(self.tmp_dir, "lock"), tmp_dir=self.tmp_dir, - hardlink_lock=self.config["core"].get( - "hardlink_lock", False - ), + hardlink_lock=self.config["core"].get("hardlink_lock", False), friendly=True, ) state_db_dir = self._get_database_dir("state") @@ -338,8 +325,7 @@ class Repo: from dvc.machine import MachineManager if self.tmp_dir and ( - self.config["feature"].get("machine", False) - or env2bool("DVC_TEST") + self.config["feature"].get("machine", False) or env2bool("DVC_TEST") ): return MachineManager(self) return None @@ -393,14 +379,10 @@ class Repo: return os.path.join(root_dir, cls.DVC_DIR) @staticmethod - def init( - root_dir=os.curdir, no_scm=False, force=False, subdir=False - ) -> "Repo": + def init(root_dir=os.curdir, no_scm=False, force=False, subdir=False) -> "Repo": from dvc.repo.init import init - return init( - root_dir=root_dir, no_scm=no_scm, force=force, subdir=subdir - ) + return init(root_dir=root_dir, no_scm=no_scm, force=force, subdir=subdir) def unprotect(self, target): return self.odb.repo.unprotect(target) @@ -530,9 +512,7 @@ class Repo: def dvcfs(self) -> "DVCFileSystem": from dvc.fs.dvc import DVCFileSystem - return DVCFileSystem( - repo=self, subrepos=self.subrepos, **self._fs_conf - ) + return DVCFileSystem(repo=self, subrepos=self.subrepos, **self._fs_conf) @cached_property def index_db_dir(self): diff --git a/dvc/repo/add.py b/dvc/repo/add.py index d662c4197..e9b88db65 100644 --- a/dvc/repo/add.py +++ b/dvc/repo/add.py @@ -15,12 +15,7 @@ from dvc.exceptions import ( ) from dvc.repo.scm_context import scm_context from dvc.ui import ui -from dvc.utils import ( - LARGE_DIR_SIZE, - glob_targets, - resolve_output, - resolve_paths, -) +from dvc.utils import LARGE_DIR_SIZE, glob_targets, resolve_output, resolve_paths from dvc.utils.collections import ensure_list, validate from . import locked @@ -108,9 +103,7 @@ def translate_graph_error(stages: Stages) -> Iterator[None]: def progress_iter(stages: Stages) -> Iterator["Stage"]: total = len(stages) desc = "Adding..." - with ui.progress( - stages, total=total, desc=desc, unit="file", leave=True - ) as pbar: + with ui.progress(stages, total=total, desc=desc, unit="file", leave=True) as pbar: if total == 1: pbar.bar_format = desc pbar.refresh() diff --git a/dvc/repo/brancher.py b/dvc/repo/brancher.py index 6182309bb..5df2967a4 100644 --- a/dvc/repo/brancher.py +++ b/dvc/repo/brancher.py @@ -49,15 +49,11 @@ def brancher( # noqa: E302 repo_root_parts = () if self.fs.path.isin(self.root_dir, self.scm.root_dir): - repo_root_parts = self.fs.path.relparts( - self.root_dir, self.scm.root_dir - ) + repo_root_parts = self.fs.path.relparts(self.root_dir, self.scm.root_dir) cwd_parts = () if self.fs.path.isin(self.fs.path.getcwd(), self.scm.root_dir): - cwd_parts = self.fs.path.relparts( - self.fs.path.getcwd(), self.scm.root_dir - ) + cwd_parts = self.fs.path.relparts(self.fs.path.getcwd(), self.scm.root_dir) saved_fs = self.fs saved_root = self.root_dir @@ -91,9 +87,7 @@ def brancher( # noqa: E302 self.root_dir = self.fs.path.join("/", *repo_root_parts) if cwd_parts: - cwd = self.fs.path.join( # type: ignore[unreachable] - "/", *cwd_parts - ) + cwd = self.fs.path.join("/", *cwd_parts) # type: ignore[unreachable] self.fs.path.chdir(cwd) # ignore revs that don't contain repo root diff --git a/dvc/repo/checkout.py b/dvc/repo/checkout.py index 8e5db918e..51c3dc5f7 100644 --- a/dvc/repo/checkout.py +++ b/dvc/repo/checkout.py @@ -2,11 +2,7 @@ import logging import os from typing import TYPE_CHECKING, Dict, List, Set -from dvc.exceptions import ( - CheckoutError, - CheckoutErrorSuggestGit, - NoOutputOrStageError, -) +from dvc.exceptions import CheckoutError, CheckoutErrorSuggestGit, NoOutputOrStageError from dvc.utils import relpath from . import locked @@ -35,18 +31,13 @@ def _remove_unused_links(repo): def get_all_files_numbers(pairs): - return sum( - stage.get_all_files_number(filter_info) for stage, filter_info in pairs - ) + return sum(stage.get_all_files_number(filter_info) for stage, filter_info in pairs) def _collect_pairs( self: "Repo", targets, with_deps: bool, recursive: bool ) -> Set["StageInfo"]: - from dvc.stage.exceptions import ( - StageFileBadNameError, - StageFileDoesNotExistError, - ) + from dvc.stage.exceptions import StageFileBadNameError, StageFileDoesNotExistError pairs: Set["StageInfo"] = set() for target in targets: diff --git a/dvc/repo/commit.py b/dvc/repo/commit.py index 4747eac3e..6f70134ef 100644 --- a/dvc/repo/commit.py +++ b/dvc/repo/commit.py @@ -30,8 +30,7 @@ def prompt_to_commit(stage, changes, force=False): if not (force or prompt.confirm(_prepare_message(stage, changes))): raise StageCommitError( - "unable to commit changed {}. Use `-f|--force` to " - "force.".format(stage) + f"unable to commit changed {stage}. Use `-f|--force` to force." ) @@ -61,8 +60,6 @@ def commit( if any(changes): prompt_to_commit(stage, changes, force=force) stage.save(allow_missing=allow_missing) - stage.commit( - filter_info=stage_info.filter_info, allow_missing=allow_missing - ) + stage.commit(filter_info=stage_info.filter_info, allow_missing=allow_missing) stage.dump(update_pipeline=False) return [s.stage for s in stages_info] diff --git a/dvc/repo/data.py b/dvc/repo/data.py index 2fceb2cb7..fb84f6b5f 100644 --- a/dvc/repo/data.py +++ b/dvc/repo/data.py @@ -112,9 +112,7 @@ class GitInfo(TypedDict, total=False): is_dirty: bool -def _git_info( - scm: Union["Git", "NoSCM"], untracked_files: str = "all" -) -> GitInfo: +def _git_info(scm: Union["Git", "NoSCM"], untracked_files: str = "all") -> GitInfo: from scmrepo.exceptions import SCMError from dvc.scm import NoSCM @@ -145,9 +143,7 @@ def _git_info( def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> Dict[str, List[str]]: from .index import build_data_index - workspace = build_data_index( - repo.index, repo.root_dir, repo.fs, compute_hash=True - ) + workspace = build_data_index(repo.index, repo.root_dir, repo.fs, compute_hash=True) return _diff( repo.index.data["repo"], @@ -180,9 +176,7 @@ class Status(TypedDict): git: GitInfo -def _transform_git_paths_to_dvc( - repo: "Repo", files: Iterable[str] -) -> List[str]: +def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> List[str]: """Transform files rel. to Git root to DVC root, and drop outside files.""" rel = repo.fs.path.relpath(repo.root_dir, repo.scm.root_dir).rstrip("/") @@ -237,8 +231,7 @@ def ls( ) -> Iterator[Dict[str, Any]]: targets = targets or [None] pairs = chain.from_iterable( - repo.stage.collect_granular(target, recursive=recursive) - for target in targets + repo.stage.collect_granular(target, recursive=recursive) for target in targets ) for stage, filter_info in pairs: for out in stage.filter_outs(filter_info): diff --git a/dvc/repo/diff.py b/dvc/repo/diff.py index 917112455..59fca9f1a 100644 --- a/dvc/repo/diff.py +++ b/dvc/repo/diff.py @@ -63,17 +63,13 @@ def diff( targets_paths = None if targets: # convert targets to paths, and capture any missing targets - targets_paths, missing_targets[rev] = _targets_to_paths( - dvcfs, targets - ) + targets_paths, missing_targets[rev] = _targets_to_paths(dvcfs, targets) results[rev] = _paths_checksums(self, targets_paths) if targets: # check for overlapping missing targets between a_rev and b_rev - for target in set(missing_targets[a_rev]) & set( - missing_targets[b_rev] - ): + for target in set(missing_targets[a_rev]) & set(missing_targets[b_rev]): raise PathMissingError(target, self) old = results[a_rev] @@ -112,9 +108,7 @@ def diff( if old[path] != new[path] ], "renamed": renamed, - "not in cache": [ - {"path": path, "hash": old[path]} for path in missing - ], + "not in cache": [{"path": path, "hash": old[path]} for path in missing], } return ret if any(ret.values()) else {} @@ -162,8 +156,7 @@ def _output_paths( # noqa: C901 continue yield_output = targets is None or any( - output.fs.path.isin_or_eq(output.fs_path, target) - for target in targets + output.fs.path.isin_or_eq(output.fs_path, target) for target in targets ) obj: Optional["HashFile"] @@ -192,15 +185,12 @@ def _output_paths( # noqa: C901 or ( isinstance(targets, list) and any( - output.fs.path.isin(target, output.fs_path) - for target in targets + output.fs.path.isin(target, output.fs_path) for target in targets ) ) ): obj = cast("Tree", obj) - yield from _dir_output_paths( - output.fs, output.fs_path, obj, targets - ) + yield from _dir_output_paths(output.fs, output.fs_path, obj, targets) def _dir_output_paths( @@ -277,8 +267,6 @@ def _calculate_renamed( continue old_path = old_paths.pop(index) - renamed.append( - {"path": {"old": old_path, "new": path}, "hash": path_hash} - ) + renamed.append({"path": {"old": old_path, "new": path}, "hash": path_hash}) return renamed diff --git a/dvc/repo/experiments/__init__.py b/dvc/repo/experiments/__init__.py index eb8fb198a..c90447cb2 100644 --- a/dvc/repo/experiments/__init__.py +++ b/dvc/repo/experiments/__init__.py @@ -279,20 +279,15 @@ class Experiments: branch: Optional[str] = None try: allow_multiple = bool(kwargs.get("params", None)) - branch = self.get_branch_by_rev( - resume_rev, allow_multiple=allow_multiple - ) + branch = self.get_branch_by_rev(resume_rev, allow_multiple=allow_multiple) if not branch: raise DvcException( - "Could not find checkpoint experiment " - f"'{resume_rev[:7]}'" + f"Could not find checkpoint experiment '{resume_rev[:7]}'" ) baseline_rev = self._get_baseline(branch) except MultipleBranchError as exc: baselines = { - info.baseline_sha - for info in exc.ref_infos - if info.baseline_sha + info.baseline_sha for info in exc.ref_infos if info.baseline_sha } if len(baselines) == 1: baseline_rev = baselines.pop() @@ -341,9 +336,7 @@ class Experiments: self.scm.remove_ref(EXEC_APPLY) @unlocked_repo - def _reproduce_queue( - self, queue: "BaseStashQueue", **kwargs - ) -> Dict[str, str]: + def _reproduce_queue(self, queue: "BaseStashQueue", **kwargs) -> Dict[str, str]: """Reproduce queued experiments. Arguments: @@ -416,9 +409,7 @@ class Experiments: """ result: Dict[str, Optional[str]] = {} exclude = f"{EXEC_NAMESPACE}/*" - ref_dict = self.scm.describe( - revs, base=EXPS_NAMESPACE, exclude=exclude - ) + ref_dict = self.scm.describe(revs, base=EXPS_NAMESPACE, exclude=exclude) for rev in revs: name: Optional[str] = None ref = ref_dict[rev] diff --git a/dvc/repo/experiments/apply.py b/dvc/repo/experiments/apply.py index 3823ab44c..843bfeb3e 100644 --- a/dvc/repo/experiments/apply.py +++ b/dvc/repo/experiments/apply.py @@ -8,11 +8,7 @@ from dvc.repo.scm_context import scm_context from dvc.scm import Git from dvc.utils.fs import remove -from .exceptions import ( - ApplyConflictError, - BaselineMismatchError, - InvalidExpRevError, -) +from .exceptions import ApplyConflictError, BaselineMismatchError, InvalidExpRevError from .executor.base import BaseExecutor from .refs import EXEC_APPLY @@ -43,7 +39,9 @@ def apply(repo: "Repo", rev: str, force: bool = True, **kwargs): # noqa: C901 ( exp_ref_info, queue_entry, - ) = exps.celery_queue.get_ref_and_entry_by_names(rev)[rev] + ) = exps.celery_queue.get_ref_and_entry_by_names( + rev + )[rev] baseline_sha = repo.scm.get_rev() if exp_ref_info: if baseline_sha != exp_ref_info.baseline_sha: @@ -79,8 +77,7 @@ def apply(repo: "Repo", rev: str, force: bool = True, **kwargs): # noqa: C901 repo.scm.set_ref(EXEC_APPLY, exp_rev) logger.info( - "Changes for experiment '%s' have been applied to your current " - "workspace.", + "Changes for experiment '%s' have been applied to your current workspace.", rev, ) diff --git a/dvc/repo/experiments/branch.py b/dvc/repo/experiments/branch.py index 3d47e5946..10b946e41 100644 --- a/dvc/repo/experiments/branch.py +++ b/dvc/repo/experiments/branch.py @@ -33,9 +33,11 @@ def branch(repo, exp_rev, branch_name, *args, **kwargs): break if not ref_info: msg = [ - f"Ambiguous experiment name '{exp_rev}' can refer to " - "multiple experiments. To create a branch use a full " - "experiment ref:", + ( + f"Ambiguous experiment name '{exp_rev}' can refer to " + "multiple experiments. To create a branch use a full " + "experiment ref:" + ), "", ] msg.extend([str(info) for info in ref_infos]) @@ -46,9 +48,7 @@ def branch(repo, exp_rev, branch_name, *args, **kwargs): branch_ref = f"refs/heads/{branch_name}" if repo.scm.get_ref(branch_ref): - raise InvalidArgumentError( - f"Git branch '{branch_name}' already exists." - ) + raise InvalidArgumentError(f"Git branch '{branch_name}' already exists.") target = repo.scm.get_ref(str(ref_info)) repo.scm.set_ref( diff --git a/dvc/repo/experiments/diff.py b/dvc/repo/experiments/diff.py index f8a9fbe91..0cda9bf38 100644 --- a/dvc/repo/experiments/diff.py +++ b/dvc/repo/experiments/diff.py @@ -23,9 +23,7 @@ def diff(repo, *args, a_rev=None, b_rev=None, param_deps=False, **kwargs): rev = resolve_rev(repo.scm, b_rev) new = collect_experiment_commit(repo, rev, param_deps=param_deps) else: - new = collect_experiment_commit( - repo, "workspace", param_deps=param_deps - ) + new = collect_experiment_commit(repo, "workspace", param_deps=param_deps) with_unchanged = kwargs.pop("all", False) diff --git a/dvc/repo/experiments/exceptions.py b/dvc/repo/experiments/exceptions.py index a609d2adb..383bb603a 100644 --- a/dvc/repo/experiments/exceptions.py +++ b/dvc/repo/experiments/exceptions.py @@ -40,7 +40,7 @@ class CheckpointExistsError(DvcException): "\tdvc exp run -f ...\n\n" "To resume the existing experiment, run:\n\n" f"\tdvc exp apply {name}\n" - f"\tdvc exp run\n" + "\tdvc exp run\n" ) super().__init__(msg) self.name = name @@ -54,16 +54,13 @@ class InvalidExpRefError(DvcException): class InvalidExpRevError(InvalidArgumentError): def __init__(self, rev): - super().__init__( - f"'{rev}' does not appear to be an experiment commit." - ) + super().__init__(f"'{rev}' does not appear to be an experiment commit.") class MultipleBranchError(DvcException): def __init__(self, rev, ref_infos): super().__init__( - f"Ambiguous commit '{rev[:7]}' belongs to multiple experiment " - "branches." + f"Ambiguous commit '{rev[:7]}' belongs to multiple experiment branches." ) self.rev = rev self.ref_infos = ref_infos @@ -110,17 +107,12 @@ class UnresolvedExpNamesError(InvalidArgumentError): unresolved_names = "; ".join(unresolved_list) if not git_remote: if len(unresolved_list) > 1: - super().__init__( - f"'{unresolved_names}' are not valid {self.NAME}s" - ) + super().__init__(f"'{unresolved_names}' are not valid {self.NAME}s") else: - super().__init__( - f"'{unresolved_names}' is not a valid {self.NAME}" - ) + super().__init__(f"'{unresolved_names}' is not a valid {self.NAME}") else: super().__init__( - f"Experiment '{unresolved_names}' does not exist " - f"in '{git_remote}'" + f"Experiment '{unresolved_names}' does not exist in '{git_remote}'" ) diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py index 6c0864638..3fb27e712 100644 --- a/dvc/repo/experiments/executor/base.py +++ b/dvc/repo/experiments/executor/base.py @@ -25,10 +25,7 @@ from scmrepo.exceptions import SCMError from dvc.env import DVC_EXP_AUTO_PUSH, DVC_EXP_GIT_REMOTE from dvc.exceptions import DvcException -from dvc.repo.experiments.exceptions import ( - CheckpointExistsError, - ExperimentExistsError, -) +from dvc.repo.experiments.exceptions import CheckpointExistsError, ExperimentExistsError from dvc.repo.experiments.refs import ( EXEC_BASELINE, EXEC_BRANCH, @@ -214,11 +211,7 @@ class BaseExecutor(ABC): if info.result_hash: result: Optional["ExecutorResult"] = ExecutorResult( info.result_hash, - ( - ExpRefInfo.from_ref(info.result_ref) - if info.result_ref - else None - ), + (ExpRefInfo.from_ref(info.result_ref) if info.result_ref else None), info.result_force, ) else: @@ -286,7 +279,6 @@ class BaseExecutor(ABC): open_func = fs.open fs.makedirs(dpath) else: - open_func = open os.makedirs(dpath, exist_ok=True) @@ -336,9 +328,7 @@ class BaseExecutor(ABC): if on_diverged: return on_diverged(orig_ref, has_checkpoint) - self._raise_ref_conflict( - dest_scm, orig_ref, new_rev, has_checkpoint - ) + self._raise_ref_conflict(dest_scm, orig_ref, new_rev, has_checkpoint) logger.debug("Reproduced existing experiment '%s'", orig_ref) return False @@ -372,10 +362,12 @@ class BaseExecutor(ABC): if git_remote == dvc.root_dir: logger.warning( - "'%s' points to the current Git repo, experiment " - "Git refs will not be pushed. But DVC cache and run cache " - "will automatically be pushed to the default DVC remote " - "(if any) on each experiment commit.", + ( + "'%s' points to the current Git repo, experiment " + "Git refs will not be pushed. But DVC cache and run cache " + "will automatically be pushed to the default DVC remote " + "(if any) on each experiment commit." + ), git_remote, ) try: @@ -529,17 +521,17 @@ class BaseExecutor(ABC): if auto_push: cls._auto_push(dvc, dvc.scm, git_remote) ref: Optional[str] = dvc.scm.get_ref(EXEC_BRANCH, follow=False) - exp_ref: Optional["ExpRefInfo"] = ( - ExpRefInfo.from_ref(ref) if ref else None - ) + exp_ref: Optional["ExpRefInfo"] = ExpRefInfo.from_ref(ref) if ref else None if cls.WARN_UNTRACKED: untracked = dvc.scm.untracked_files() if untracked: logger.warning( - "The following untracked files were present in " - "the experiment directory after reproduction but " - "will not be included in experiment commits:\n" - "\t%s", + ( + "The following untracked files were present in " + "the experiment directory after reproduction but " + "will not be included in experiment commits:\n" + "\t%s" + ), ", ".join(untracked), ) return ref, exp_ref, repro_force @@ -622,8 +614,10 @@ class BaseExecutor(ABC): ) except BaseException as exc: # noqa: BLE001, pylint: disable=W0703 logger.warning( - "Something went wrong while auto pushing experiment " - "to the remote '%s': %s", + ( + "Something went wrong while auto pushing experiment " + "to the remote '%s': %s" + ), git_remote, exc, ) @@ -639,9 +633,7 @@ class BaseExecutor(ABC): stages: Iterable["PipelineStage"], ): exp_hash = cls.hash_exp(list(stages) + list(unchanged)) - exp_rev = cls.commit( - scm, exp_hash, exp_name=name, force=force, checkpoint=True - ) + exp_rev = cls.commit(scm, exp_hash, exp_name=name, force=force, checkpoint=True) if env2bool(DVC_EXP_AUTO_PUSH): git_remote = os.getenv(DVC_EXP_GIT_REMOTE) diff --git a/dvc/repo/experiments/executor/local.py b/dvc/repo/experiments/executor/local.py index 7b2386089..bb463ca81 100644 --- a/dvc/repo/experiments/executor/local.py +++ b/dvc/repo/experiments/executor/local.py @@ -27,7 +27,6 @@ from dvc.utils.objects import cached_property from .base import BaseExecutor, TaskStatus if TYPE_CHECKING: - from dvc.repo import Repo from dvc.repo.experiments.refs import ExpRefInfo from dvc.repo.experiments.stash import ExpStashEntry diff --git a/dvc/repo/experiments/executor/ssh.py b/dvc/repo/experiments/executor/ssh.py index 100e6b6f4..7e097aead 100644 --- a/dvc/repo/experiments/executor/ssh.py +++ b/dvc/repo/experiments/executor/ssh.py @@ -137,9 +137,7 @@ class SSHExecutor(BaseExecutor): fs.makedirs(self.root_dir) self._ssh_cmd(fs, "git init .") self._ssh_cmd(fs, "git config user.name dvc-exp") - self._ssh_cmd( - fs, "git config user.email dvc-exp@noreply.localhost" - ) + self._ssh_cmd(fs, "git config user.email dvc-exp@noreply.localhost") result = self._ssh_cmd(fs, "pwd") path = result.stdout.strip() @@ -163,12 +161,8 @@ class SSHExecutor(BaseExecutor): push_refspec(scm, self.git_url, [(branch, branch)], **kwargs) self._ssh_cmd(fs, f"git symbolic-ref {EXEC_BRANCH} {branch}") else: - self._ssh_cmd( - fs, f"git symbolic-ref -d {EXEC_BRANCH}", check=False - ) - self._ssh_cmd( - fs, f"git update-ref -d {EXEC_CHECKPOINT}", check=False - ) + self._ssh_cmd(fs, f"git symbolic-ref -d {EXEC_BRANCH}", check=False) + self._ssh_cmd(fs, f"git update-ref -d {EXEC_CHECKPOINT}", check=False) # checkout EXEC_HEAD and apply EXEC_MERGE on top of it without # committing @@ -266,13 +260,9 @@ class SSHExecutor(BaseExecutor): with _sshfs(fs_factory) as fs: while not fs.exists("/var/log/dvc-machine-init.log"): - logger.info( - "Waiting for dvc-machine startup script to complete..." - ) + logger.info("Waiting for dvc-machine startup script to complete...") time.sleep(5) - logger.info( - "Reproducing experiment on '%s'", fs.fs_args.get("host") - ) + logger.info("Reproducing experiment on '%s'", fs.fs_args.get("host")) with TemporaryFile(mode="w+", encoding="utf-8") as fobj: json.dump(info.asdict(), fobj) fobj.seek(0) @@ -280,9 +270,7 @@ class SSHExecutor(BaseExecutor): cmd = ["source ~/.profile"] script_path = cls._setup_script_path(info.dvc_dir) if fs.exists(posixpath.join(info.root_dir, script_path)): - cmd.extend( - [f"pushd {info.root_dir}", f"source {script_path}", "popd"] - ) + cmd.extend([f"pushd {info.root_dir}", f"source {script_path}", "popd"]) exec_cmd = f"dvc exp exec-run --infofile {infofile}" if log_level is not None: if log_level <= logging.TRACE: # type: ignore[attr-defined] diff --git a/dvc/repo/experiments/init.py b/dvc/repo/experiments/init.py index d8c078b5b..a776b73e2 100644 --- a/dvc/repo/experiments/init.py +++ b/dvc/repo/experiments/init.py @@ -41,9 +41,7 @@ PROMPTS = { def _prompts( keys: Iterable[str], defaults: Optional[Dict[str, str]] = None, - validator: Optional[ - Callable[[str, str], Union[str, Tuple[str, str]]] - ] = None, + validator: Optional[Callable[[str, str], Union[str, Tuple[str, str]]]] = None, allow_omission: bool = True, stream: Optional[TextIO] = None, ) -> Dict[str, Optional[str]]: @@ -78,9 +76,7 @@ def _disable_logging(highest_level=logging.CRITICAL): def init_interactive( defaults: Dict[str, str], provided: Dict[str, str], - validator: Optional[ - Callable[[str, str], Union[str, Tuple[str, str]]] - ] = None, + validator: Optional[Callable[[str, str], Union[str, Tuple[str, str]]]] = None, stream: Optional[TextIO] = None, ) -> Dict[str, str]: command_prompts = lremove(provided.keys(), ["cmd"]) @@ -123,14 +119,10 @@ def _check_stage_exists( from dvc.stage.exceptions import DuplicateStageName hint = "Use '--force' to overwrite." - raise DuplicateStageName( - f"Stage '{name}' already exists in 'dvc.yaml'. {hint}" - ) + raise DuplicateStageName(f"Stage '{name}' already exists in 'dvc.yaml'. {hint}") -def validate_prompts( - repo: "Repo", key: str, value: str -) -> Union[Any, Tuple[Any, str]]: +def validate_prompts(repo: "Repo", key: str, value: str) -> Union[Any, Tuple[Any, str]]: from dvc.ui.prompt import InvalidResponse msg_format = "[yellow]'{0}' does not exist, the {1} will be created.[/]" @@ -248,9 +240,7 @@ def init( try: ParamsDependency(None, params, repo=repo).validate_filepath() except ParamsIsADirectoryError as exc: - raise DvcException( # noqa: B904 - f"{exc}." - ) # swallow cause for display + raise DvcException(f"{exc}.") # noqa: B904 # swallow cause for display except MissingParamsFile: pass diff --git a/dvc/repo/experiments/pull.py b/dvc/repo/experiments/pull.py index 42de96f1e..ca76eea18 100644 --- a/dvc/repo/experiments/pull.py +++ b/dvc/repo/experiments/pull.py @@ -67,8 +67,7 @@ def pull( # noqa: C901 if pull_cache: pull_cache_ref = ( - pull_result[SyncStatus.UP_TO_DATE] - + pull_result[SyncStatus.SUCCESS] + pull_result[SyncStatus.UP_TO_DATE] + pull_result[SyncStatus.SUCCESS] ) _pull_cache(repo, pull_cache_ref, **kwargs) @@ -95,9 +94,7 @@ def _pull( def group_result(refspec): return results[str(refspec)] - pull_result: Mapping[SyncStatus, List["ExpRefInfo"]] = group_by( - group_result, refs - ) + pull_result: Mapping[SyncStatus, List["ExpRefInfo"]] = group_by(group_result, refs) return pull_result @@ -114,6 +111,4 @@ def _pull_cache( refs = [refs] revs = list(exp_commits(repo.scm, refs)) logger.debug("dvc fetch experiment '%s'", refs) - repo.fetch( - jobs=jobs, remote=dvc_remote, run_cache=run_cache, revs=revs, odb=odb - ) + repo.fetch(jobs=jobs, remote=dvc_remote, run_cache=run_cache, revs=revs, odb=odb) diff --git a/dvc/repo/experiments/push.py b/dvc/repo/experiments/push.py index 62eaf3c72..36ec8ddc1 100644 --- a/dvc/repo/experiments/push.py +++ b/dvc/repo/experiments/push.py @@ -29,7 +29,6 @@ def push( # noqa: C901 push_cache: bool = False, **kwargs, ) -> Iterable[str]: - exp_ref_set: Set["ExpRefInfo"] = set() if all_commits: exp_ref_set.update(exp_refs(repo.scm)) @@ -67,8 +66,7 @@ def push( # noqa: C901 ) if push_cache: push_cache_ref = ( - push_result[SyncStatus.UP_TO_DATE] - + push_result[SyncStatus.SUCCESS] + push_result[SyncStatus.UP_TO_DATE] + push_result[SyncStatus.SUCCESS] ) _push_cache(repo, push_cache_ref, **kwargs) return [ref.name for ref in push_result[SyncStatus.SUCCESS]] @@ -101,9 +99,7 @@ def _push( def group_result(refspec): return results[str(refspec)] - pull_result: Mapping[SyncStatus, List["ExpRefInfo"]] = group_by( - group_result, refs - ) + pull_result: Mapping[SyncStatus, List["ExpRefInfo"]] = group_by(group_result, refs) return pull_result diff --git a/dvc/repo/experiments/queue/base.py b/dvc/repo/experiments/queue/base.py index 9f84b8642..3aef018e2 100644 --- a/dvc/repo/experiments/queue/base.py +++ b/dvc/repo/experiments/queue/base.py @@ -23,10 +23,7 @@ from dvc.dependency import ParamsDependency from dvc.env import DVC_EXP_BASELINE_REV, DVC_EXP_NAME, DVCLIVE_RESUME from dvc.exceptions import DvcException from dvc.lock import LockError -from dvc.repo.experiments.exceptions import ( - CheckpointExistsError, - ExperimentExistsError, -) +from dvc.repo.experiments.exceptions import CheckpointExistsError, ExperimentExistsError from dvc.repo.experiments.executor.base import BaseExecutor from dvc.repo.experiments.executor.local import WorkspaceExecutor from dvc.repo.experiments.refs import ExpRefInfo @@ -101,9 +98,7 @@ class BaseStashQueue(ABC): Maps queued experiments to (Git) stash reflog entries. """ - def __init__( - self, repo: "Repo", ref: str, failed_ref: Optional[str] = None - ): + def __init__(self, repo: "Repo", ref: str, failed_ref: Optional[str] = None): """Construct a queue. Arguments: @@ -172,9 +167,7 @@ class BaseStashQueue(ABC): name_to_remove: List[str] = [] entry_to_remove: List[ExpStashEntry] = [] - queue_entries = self.match_queue_entry_by_name( - revs, self.iter_queued() - ) + queue_entries = self.match_queue_entry_by_name(revs, self.iter_queued()) for name, entry in queue_entries.items(): if entry: entry_to_remove.append(self.stash.stash_revs[entry.stash_rev]) @@ -366,19 +359,16 @@ class BaseStashQueue(ABC): branch_name = f"{resume_rev[:7]}" if self.scm.is_dirty(untracked_files=False): ui.write( - "Modified checkpoint experiment based on " - f"'{branch_name}' will be created", + ( + "Modified checkpoint experiment based on " + f"'{branch_name}' will be created" + ), ) branch = None - elif ( - not branch - or self.scm.get_ref(branch) != resume_rev - ): + elif not branch or self.scm.get_ref(branch) != resume_rev: err_msg = [ - ( - "Nothing to do for unchanged checkpoint " - f"'{resume_rev[:7]}'. " - ) + "Nothing to do for unchanged checkpoint " + f"'{resume_rev[:7]}'. " ] if branch: err_msg.append( @@ -394,9 +384,7 @@ class BaseStashQueue(ABC): ) ] if len(names) > 3: - names[3:] = [ - f"... ({len(names) - 3} more)" - ] + names[3:] = [f"... ({len(names) - 3} more)"] err_msg.append( "To resume an experiment containing this " "checkpoint, apply one of these heads:\n" @@ -410,9 +398,11 @@ class BaseStashQueue(ABC): ) if name: logger.warning( - "Ignoring option '--name %s' for resumed " - "experiment. Existing experiment name will" - "be preserved instead.", + ( + "Ignoring option '--name %s' for resumed " + "experiment. Existing experiment name will" + "be preserved instead." + ), name, ) @@ -449,9 +439,7 @@ class BaseStashQueue(ABC): # NOTE: this set_ref + reset() is equivalent to # `git reset orig_head` (our SCM reset() only operates # on HEAD rather than any arbitrary commit) - self.scm.set_ref( - "HEAD", orig_head, message="dvc: restore HEAD" - ) + self.scm.set_ref("HEAD", orig_head, message="dvc: restore HEAD") self.scm.reset() # Revert any of our changes before prior unstashing self.scm.reset(hard=True) @@ -503,9 +491,7 @@ class BaseStashQueue(ABC): def _pack_args(self, *args, **kwargs) -> None: import pickle - if os.path.exists(self.args_file) and self.scm.is_tracked( - self.args_file - ): + if os.path.exists(self.args_file) and self.scm.is_tracked(self.args_file): logger.warning( ( "Temporary DVC file '.dvc/tmp/%s' exists and was " @@ -524,9 +510,7 @@ class BaseStashQueue(ABC): extra = int(data.get("extra", 0)) + 1 else: extra = None - BaseExecutor.pack_repro_args( - self.args_file, *args, extra=extra, **kwargs - ) + BaseExecutor.pack_repro_args(self.args_file, *args, extra=extra, **kwargs) self.scm.add(self.args_file) @staticmethod @@ -605,9 +589,7 @@ class BaseStashQueue(ABC): ) -> BaseExecutor: stash_entry = cls.get_stash_entry(exp, queue_entry) - executor = executor_cls.from_stash_entry( - exp.repo, stash_entry, **kwargs - ) + executor = executor_cls.from_stash_entry(exp.repo, stash_entry, **kwargs) stash_rev = queue_entry.stash_rev infofile = exp.celery_queue.get_infofile_path(stash_rev) @@ -689,10 +671,7 @@ class BaseStashQueue(ABC): for entry in concat(*entries): if isinstance(entry, QueueDoneResult): queue_entry: QueueEntry = entry.entry - if ( - entry.result is not None - and entry.result.ref_info is not None - ): + if entry.result is not None and entry.result.ref_info is not None: name: Optional[str] = entry.result.ref_info.name else: name = queue_entry.name diff --git a/dvc/repo/experiments/queue/celery.py b/dvc/repo/experiments/queue/celery.py index e03fa01ae..d46662d2c 100644 --- a/dvc/repo/experiments/queue/celery.py +++ b/dvc/repo/experiments/queue/celery.py @@ -30,12 +30,7 @@ from dvc.repo.experiments.utils import EXEC_TMP_DIR, get_exp_rwlock from dvc.ui import ui from dvc.utils.objects import cached_property -from .base import ( - BaseStashQueue, - ExpRefAndQueueEntry, - QueueDoneResult, - QueueEntry, -) +from .base import BaseStashQueue, ExpRefAndQueueEntry, QueueDoneResult, QueueEntry from .exceptions import CannotKillTasksError from .tasks import run_exp from .utils import fetch_running_exp_from_temp_dir @@ -166,9 +161,7 @@ class LocalCeleryQueue(BaseStashQueue): started = 0 for num in range(1, 1 + count): - wdir_hash = hashlib.sha256(self.wdir.encode("utf-8")).hexdigest()[ - :6 - ] + wdir_hash = hashlib.sha256(self.wdir.encode("utf-8")).hexdigest()[:6] node_name = f"dvc-exp-{wdir_hash}-{num}@localhost" if node_name in active_worker: logger.debug("Exp queue worker %s already exist", node_name) @@ -212,7 +205,6 @@ class LocalCeleryQueue(BaseStashQueue): yield _MessageEntry(msg, QueueEntry.from_dict(entry_dict)) def _iter_active_tasks(self) -> Generator[_TaskEntry, None, None]: - for msg, entry in self._iter_processed(): task_id = msg.headers["id"] result: AsyncResult = AsyncResult(task_id) @@ -221,7 +213,6 @@ class LocalCeleryQueue(BaseStashQueue): yield _TaskEntry(result, entry) def _iter_done_tasks(self) -> Generator[_TaskEntry, None, None]: - for msg, entry in self._iter_processed(): task_id = msg.headers["id"] result: AsyncResult = AsyncResult(task_id) @@ -273,9 +264,7 @@ class LocalCeleryQueue(BaseStashQueue): task_id = msg.headers["id"] result: AsyncResult = AsyncResult(task_id) if not result.ready(): - logger.debug( - "Waiting for exp task '%s' to complete", result.id - ) + logger.debug("Waiting for exp task '%s' to complete", result.id) try: result.get(timeout=timeout) except _CeleryTimeout as exc: @@ -289,7 +278,6 @@ class LocalCeleryQueue(BaseStashQueue): def get_result( self, entry: QueueEntry, timeout: Optional[float] = None ) -> Optional["ExecutorResult"]: - try: return self._get_done_result(entry, timeout) except FileNotFoundError: @@ -353,9 +341,7 @@ class LocalCeleryQueue(BaseStashQueue): if remained_revs: raise CannotKillTasksError(remained_revs) - def _kill_entries( - self, entries: Dict[QueueEntry, str], force: bool - ) -> None: + def _kill_entries(self, entries: Dict[QueueEntry, str], force: bool) -> None: logger.debug( "Found active tasks: '%s' to kill", list(entries.values()), @@ -368,9 +354,9 @@ class LocalCeleryQueue(BaseStashQueue): self._mark_inactive_tasks_failure(inactive_entries) def kill(self, revs: Collection[str], force: bool = False) -> None: - name_dict: Dict[ - str, Optional[QueueEntry] - ] = self.match_queue_entry_by_name(set(revs), self.iter_active()) + name_dict: Dict[str, Optional[QueueEntry]] = self.match_queue_entry_by_name( + set(revs), self.iter_active() + ) missing_revs: List[str] = [] to_kill: Dict[QueueEntry, str] = {} @@ -413,9 +399,7 @@ class LocalCeleryQueue(BaseStashQueue): {rev}, self.iter_active(), self.iter_done() ).get(rev) if queue_entry is None: - if self.match_queue_entry_by_name({rev}, self.iter_queued()).get( - rev - ): + if self.match_queue_entry_by_name({rev}, self.iter_queued()).get(rev): raise DvcException( f"Experiment '{rev}' is in queue but has not been started" ) @@ -467,9 +451,7 @@ class LocalCeleryQueue(BaseStashQueue): result: Dict[str, Dict] = {} for entry in self.iter_active(): result.update( - fetch_running_exp_from_temp_dir( - self, entry.stash_rev, fetch_refs - ) + fetch_running_exp_from_temp_dir(self, entry.stash_rev, fetch_refs) ) return result diff --git a/dvc/repo/experiments/queue/remove.py b/dvc/repo/experiments/queue/remove.py index 6f961d257..ef857280c 100644 --- a/dvc/repo/experiments/queue/remove.py +++ b/dvc/repo/experiments/queue/remove.py @@ -49,9 +49,7 @@ def remove_tasks( # noqa: C901 for ( msg, queue_entry, - ) in ( - celery_queue._iter_processed() # pylint: disable=protected-access - ): + ) in celery_queue._iter_processed(): # pylint: disable=protected-access if queue_entry not in done_entry_set: continue task_id = msg.headers["id"] diff --git a/dvc/repo/experiments/queue/tasks.py b/dvc/repo/experiments/queue/tasks.py index 9f8af5675..08e07ef8b 100644 --- a/dvc/repo/experiments/queue/tasks.py +++ b/dvc/repo/experiments/queue/tasks.py @@ -68,9 +68,7 @@ def collect_exp( exec_result = executor_info.result try: if exec_result is not None: - BaseStashQueue.collect_executor( - repo.experiments, executor, exec_result - ) + BaseStashQueue.collect_executor(repo.experiments, executor, exec_result) else: logger.debug("Experiment failed (Exec result was None)") celery_queue.stash_failed(entry) diff --git a/dvc/repo/experiments/queue/tempdir.py b/dvc/repo/experiments/queue/tempdir.py index 15b23881e..27bcaab61 100644 --- a/dvc/repo/experiments/queue/tempdir.py +++ b/dvc/repo/experiments/queue/tempdir.py @@ -109,29 +109,21 @@ class TempDirQueue(WorkspaceQueue): log_errors=True, ) if not exec_result.exp_hash: - raise DvcException( - f"Failed to reproduce experiment '{rev[:7]}'" - ) + raise DvcException(f"Failed to reproduce experiment '{rev[:7]}'") if exec_result.ref_info: results[rev].update( - self.collect_executor( - self.repo.experiments, executor, exec_result - ) + self.collect_executor(self.repo.experiments, executor, exec_result) ) except CheckpointKilledError: results[rev].update( - self.collect_executor( - self.repo.experiments, executor, exec_result - ) + self.collect_executor(self.repo.experiments, executor, exec_result) ) return results except DvcException: raise except Exception as exc: # noqa: BLE001 - raise DvcException( - f"Failed to reproduce experiment '{rev[:7]}'" - ) from exc + raise DvcException(f"Failed to reproduce experiment '{rev[:7]}'") from exc finally: executor.cleanup(infofile) return results @@ -148,8 +140,6 @@ class TempDirQueue(WorkspaceQueue): result: Dict[str, Dict] = {} for entry in self.iter_active(): result.update( - fetch_running_exp_from_temp_dir( - self, entry.stash_rev, fetch_refs - ) + fetch_running_exp_from_temp_dir(self, entry.stash_rev, fetch_refs) ) return result diff --git a/dvc/repo/experiments/queue/workspace.py b/dvc/repo/experiments/queue/workspace.py index ba4afb5ac..4cf7afe57 100644 --- a/dvc/repo/experiments/queue/workspace.py +++ b/dvc/repo/experiments/queue/workspace.py @@ -118,14 +118,10 @@ class WorkspaceQueue(BaseStashQueue): log_errors=not isinstance(executor, WorkspaceExecutor), ) if not exec_result.exp_hash: - raise DvcException( - f"Failed to reproduce experiment '{rev[:7]}'" - ) + raise DvcException(f"Failed to reproduce experiment '{rev[:7]}'") if exec_result.ref_info: results[rev].update( - self.collect_executor( - self.repo.experiments, executor, exec_result - ) + self.collect_executor(self.repo.experiments, executor, exec_result) ) except CheckpointKilledError: # Checkpoint errors have already been logged @@ -133,9 +129,7 @@ class WorkspaceQueue(BaseStashQueue): except DvcException: raise except Exception as exc: # noqa: BLE001 - raise DvcException( - f"Failed to reproduce experiment '{rev[:7]}'" - ) from exc + raise DvcException(f"Failed to reproduce experiment '{rev[:7]}'") from exc finally: post_live_metrics( "done", @@ -214,8 +208,7 @@ class WorkspaceQueue(BaseStashQueue): return True cmd = lock_info["cmd"] logger.warning( - "Process '%s' with (Pid %s), in RWLock-file '%s'" - " had been killed.", + "Process '%s' with (Pid %s), in RWLock-file '%s' had been killed.", cmd, pid, relpath(path), @@ -224,16 +217,13 @@ class WorkspaceQueue(BaseStashQueue): return False except json.JSONDecodeError: logger.warning( - "Unable to read RWLock-file '%s'. JSON structure is" - " corrupted", + "Unable to read RWLock-file '%s'. JSON structure is corrupted", relpath(path), ) except Invalid: logger.warning("RWLock-file '%s' format error.", relpath(path)) if autocorrect: - logger.warning( - "Delete corrupted RWLock-file '%s'", relpath(path) - ) + logger.warning("Delete corrupted RWLock-file '%s'", relpath(path)) remove(path) return False diff --git a/dvc/repo/experiments/refs.py b/dvc/repo/experiments/refs.py index cca3f0e78..87ef230bc 100644 --- a/dvc/repo/experiments/refs.py +++ b/dvc/repo/experiments/refs.py @@ -21,7 +21,6 @@ STASHES = {WORKSPACE_STASH, CELERY_STASH} class ExpRefInfo: - namespace = EXPS_NAMESPACE def __init__(self, baseline_sha: str, name: Optional[str] = None): diff --git a/dvc/repo/experiments/save.py b/dvc/repo/experiments/save.py index c89443989..6a2aa084d 100644 --- a/dvc/repo/experiments/save.py +++ b/dvc/repo/experiments/save.py @@ -59,15 +59,15 @@ def save( _, _, untracked = repo.scm.status() if include_untracked: - untracked = [ - file for file in untracked if file not in include_untracked - ] + untracked = [file for file in untracked if file not in include_untracked] if untracked: logger.warning( - "The following untracked files were present in " - "the workspace before saving but " - "will not be included in the experiment commit:\n" - "\t%s", + ( + "The following untracked files were present in " + "the workspace before saving but " + "will not be included in the experiment commit:\n" + "\t%s" + ), ", ".join(untracked), ) diff --git a/dvc/repo/experiments/show.py b/dvc/repo/experiments/show.py index 286e10b8d..83b9f3624 100644 --- a/dvc/repo/experiments/show.py +++ b/dvc/repo/experiments/show.py @@ -75,9 +75,7 @@ def collect_experiment_commit( commit = repo.scm.resolve_commit(rev) result["timestamp"] = datetime.fromtimestamp(commit.commit_time) - params = _gather_params( - repo, targets=None, deps=param_deps, onerror=onerror - ) + params = _gather_params(repo, targets=None, deps=param_deps, onerror=onerror) if params: result["params"] = params @@ -223,8 +221,7 @@ def get_branch_names( ) -> Dict[str, Optional[str]]: names: Dict[str, Optional[str]] = {} for base in [ - f"refs/exps/{baseline[:2]}/{baseline[2:]}/" - for baseline in baseline_set + f"refs/exps/{baseline[:2]}/{baseline[2:]}/" for baseline in baseline_set ] + ["refs/heads/", "refs/tags/"]: for ref in scm.iter_refs(base=base): if ref: @@ -242,7 +239,6 @@ def update_names( # noqa: C901 branch_names: Dict[str, Optional[str]], result: Dict[str, Dict[str, Any]], ): - rev_set = set() for baseline in result: for rev in result[baseline]: @@ -401,7 +397,6 @@ def show( onerror: Optional[Callable] = None, fetch_running: bool = True, ): - if repo.scm.no_commits: return {} diff --git a/dvc/repo/experiments/utils.py b/dvc/repo/experiments/utils.py index ec040b8a5..a48aa3d01 100644 --- a/dvc/repo/experiments/utils.py +++ b/dvc/repo/experiments/utils.py @@ -112,9 +112,7 @@ def exp_refs_by_baseline( return result -def iter_remote_refs( - scm: "Git", url: str, base: Optional[str] = None, **kwargs -): +def iter_remote_refs(scm: "Git", url: str, base: Optional[str] = None, **kwargs): from scmrepo.exceptions import AuthError, InvalidRemote from dvc.scm import GitAuthError, InvalidRemoteSCMRepo @@ -160,14 +158,10 @@ def push_refspec( results = scm.push_refspecs( url, refspecs, force=force, on_diverged=on_diverged, **kwargs ) - diverged = [ - ref for ref in results if results[ref] == SyncStatus.DIVERGED - ] + diverged = [ref for ref in results if results[ref] == SyncStatus.DIVERGED] if diverged: - raise SCMError( - f"local ref '{diverged}' diverged from remote '{url}'" - ) + raise SCMError(f"local ref '{diverged}' diverged from remote '{url}'") except AuthError as exc: raise GitAuthError(str(exc)) # noqa: B904 diff --git a/dvc/repo/fetch.py b/dvc/repo/fetch.py index 50b749ef6..f425cd743 100644 --- a/dvc/repo/fetch.py +++ b/dvc/repo/fetch.py @@ -194,7 +194,5 @@ def _fetch_worktree( all_tags=all_tags, all_commits=all_commits, ): - downloaded += fetch_worktree( - repo, remote, targets=targets, jobs=jobs, **kwargs - ) + downloaded += fetch_worktree(repo, remote, targets=targets, jobs=jobs, **kwargs) return downloaded diff --git a/dvc/repo/gc.py b/dvc/repo/gc.py index 845f178bd..ea74950bd 100644 --- a/dvc/repo/gc.py +++ b/dvc/repo/gc.py @@ -19,9 +19,7 @@ def _validate_args(**kwargs): "needs to be set." ) if kwargs.get("num") and not kwargs.get("rev"): - raise InvalidArgumentError( - "`--num` can only be used alongside `--rev`" - ) + raise InvalidArgumentError("`--num` can only be used alongside `--rev`") @locked @@ -42,7 +40,6 @@ def gc( rev: Optional[str] = None, num: Optional[int] = None, ): - # require `workspace` to be true to come into effect. # assume `workspace` to be enabled if any of `all_tags`, `all_commits`, # `all_experiments` or `all_branches` are enabled. diff --git a/dvc/repo/get.py b/dvc/repo/get.py index a3758818b..6e5498094 100644 --- a/dvc/repo/get.py +++ b/dvc/repo/get.py @@ -16,8 +16,7 @@ logger = logging.getLogger(__name__) class GetDVCFileError(DvcException): def __init__(self): super().__init__( - "the given path is a DVC file, you must specify a data file " - "or a directory" + "the given path is a DVC file, you must specify a data file or a directory" ) @@ -59,7 +58,6 @@ def get(url, path, out=None, rev=None, jobs=None): fs: Union[DataFileSystem, "DVCFileSystem"] if os.path.isabs(path): - fs = DataFileSystem(index=repo.index.data["local"]) fs_path = fs.from_os_path(path) else: diff --git a/dvc/repo/imp.py b/dvc/repo/imp.py index 5fc572543..14454c1aa 100644 --- a/dvc/repo/imp.py +++ b/dvc/repo/imp.py @@ -3,6 +3,4 @@ def imp(self, url, path, out=None, fname=None, rev=None, **kwargs): if rev is not None: erepo["rev"] = rev - return self.imp_url( - path, out=out, fname=fname, erepo=erepo, frozen=True, **kwargs - ) + return self.imp_url(path, out=out, fname=fname, erepo=erepo, frozen=True, **kwargs) diff --git a/dvc/repo/imp_url.py b/dvc/repo/imp_url.py index aec70cbf5..fc6445cac 100644 --- a/dvc/repo/imp_url.py +++ b/dvc/repo/imp_url.py @@ -34,9 +34,7 @@ def imp_url( # noqa: C901 version_aware: bool = False, ): out = resolve_output(url, out) - path, wdir, out = resolve_paths( - self, out, always_local=to_remote and not out - ) + path, wdir, out = resolve_paths(self, out, always_local=to_remote and not out) if to_remote and (no_exec or no_download): raise InvalidArgumentError( @@ -44,9 +42,7 @@ def imp_url( # noqa: C901 ) if not to_remote and remote: - raise InvalidArgumentError( - "--remote can't be used without --to-remote" - ) + raise InvalidArgumentError("--remote can't be used without --to-remote") # NOTE: when user is importing something from within their own repository if ( diff --git a/dvc/repo/imports.py b/dvc/repo/imports.py index 838e1259b..18e6f4a9c 100644 --- a/dvc/repo/imports.py +++ b/dvc/repo/imports.py @@ -49,9 +49,7 @@ def unfetched_view( return unfetched, changed_deps -def partial_view( - index: "Index", targets: "TargetType", **kwargs -) -> "IndexView": +def partial_view(index: "Index", targets: "TargetType", **kwargs) -> "IndexView": return index.targets_view( targets, stage_filter=lambda s: s.is_partial_import, diff --git a/dvc/repo/index.py b/dvc/repo/index.py index 116c7d771..da8f73c7c 100644 --- a/dvc/repo/index.py +++ b/dvc/repo/index.py @@ -306,9 +306,7 @@ class Index: data_index.odb_map[key] = out.odb try: - data_index.remote_map[key] = self.repo.cloud.get_remote_odb( - out.remote - ) + data_index.remote_map[key] = self.repo.cloud.get_remote_odb(out.remote) except NoRemoteError: pass @@ -347,9 +345,7 @@ class Index: collected = [] for target in targets: try: - collected.extend( - self.repo.stage.collect_granular(target, **kwargs) - ) + collected.extend(self.repo.stage.collect_granular(target, **kwargs)) except DvcException as exc: onerror(target, exc) self._collected_targets[targets_hash] = collected @@ -369,9 +365,7 @@ class Index: from collections import defaultdict used: "ObjectContainer" = defaultdict(set) - pairs = self.collect_targets( - targets, recursive=recursive, with_deps=with_deps - ) + pairs = self.collect_targets(targets, recursive=recursive, with_deps=with_deps) for stage, filter_info in pairs: for odb, objs in stage.get_used_objs( remote=remote, @@ -467,9 +461,7 @@ class IndexView: key = key + out.fs.path.relparts(filter_info, out.fs_path) if out.meta.isdir: prefixes[workspace].recursive.add(key) - prefixes[workspace].explicit.update( - key[:i] for i in range(len(key), 0, -1) - ) + prefixes[workspace].explicit.update(key[:i] for i in range(len(key), 0, -1)) return prefixes @cached_property @@ -494,8 +486,7 @@ class IndexView: try: prefixes = self._data_prefixes[workspace] return key in prefixes.explicit or any( - key[: len(prefix)] == prefix - for prefix in prefixes.recursive + key[: len(prefix)] == prefix for prefix in prefixes.recursive ) except KeyError: return False @@ -503,9 +494,7 @@ class IndexView: data: Dict[str, Union["DataIndex", "DataIndexView"]] = {} for workspace, data_index in self._index.data.items(): if self.stages: - data[workspace] = view( - data_index, partial(key_filter, workspace) - ) + data[workspace] = view(data_index, partial(key_filter, workspace)) else: data[workspace] = DataIndex() return data diff --git a/dvc/repo/metrics/diff.py b/dvc/repo/metrics/diff.py index abede5c8a..7719f7957 100644 --- a/dvc/repo/metrics/diff.py +++ b/dvc/repo/metrics/diff.py @@ -18,6 +18,4 @@ def diff(repo, *args, a_rev=None, b_rev=None, **kwargs): old = metrics.get(a_rev, {}).get("data", {}) new = metrics.get(b_rev, {}).get("data", {}) - return _diff( - format_dict(old), format_dict(new), with_unchanged=with_unchanged - ) + return _diff(format_dict(old), format_dict(new), with_unchanged=with_unchanged) diff --git a/dvc/repo/metrics/show.py b/dvc/repo/metrics/show.py index 0ce1c3420..cfa1e2550 100644 --- a/dvc/repo/metrics/show.py +++ b/dvc/repo/metrics/show.py @@ -8,12 +8,7 @@ from dvc.fs.dvc import DVCFileSystem from dvc.repo import locked from dvc.repo.collect import collect from dvc.scm import NoSCMError -from dvc.utils import ( - as_posix, - error_handler, - errored_revisions, - onerror_collect, -) +from dvc.utils import as_posix, error_handler, errored_revisions, onerror_collect from dvc.utils.collections import ensure_list from dvc.utils.serialize import load_path @@ -38,9 +33,7 @@ def _to_fs_paths(metrics: List["Output"]) -> List["str"]: def _collect_top_level_metrics(repo): top_metrics = repo.index._metrics # pylint: disable=protected-access for dvcfile, metrics in top_metrics.items(): - wdir = repo.fs.path.relpath( - repo.fs.path.parent(dvcfile), repo.root_dir - ) + wdir = repo.fs.path.relpath(repo.fs.path.parent(dvcfile), repo.root_dir) for file in metrics: path = repo.fs.path.join(wdir, as_posix(file)) yield repo.fs.path.normpath(path) @@ -67,8 +60,10 @@ def _extract_metrics(metrics, path, rev): ret[key] = m else: logger.debug( - "Could not parse '%s' metric from '%s' at '%s' " - "due to its unsupported type: '%s'", + ( + "Could not parse '%s' metric from '%s' at '%s' " + "due to its unsupported type: '%s'" + ), key, path, rev, diff --git a/dvc/repo/move.py b/dvc/repo/move.py index 78ca603a2..5806fbc08 100644 --- a/dvc/repo/move.py +++ b/dvc/repo/move.py @@ -56,9 +56,7 @@ def move(self, from_path, to_path): os.path.dirname(to_path), os.path.basename(to_path) + DVC_FILE_SUFFIX, ) - new_wdir = os.path.abspath( - os.path.join(os.curdir, os.path.dirname(to_path)) - ) + new_wdir = os.path.abspath(os.path.join(os.curdir, os.path.dirname(to_path))) to_path = os.path.relpath(to_path, new_wdir) new_stage = self.stage.create( single_stage=True, diff --git a/dvc/repo/params/diff.py b/dvc/repo/params/diff.py index 79c68b1e4..2ddb6bd3f 100644 --- a/dvc/repo/params/diff.py +++ b/dvc/repo/params/diff.py @@ -18,6 +18,4 @@ def diff(repo, *args, a_rev=None, b_rev=None, **kwargs): old = params.get(a_rev, {}).get("data", {}) new = params.get(b_rev, {}).get("data", {}) - return _diff( - format_dict(old), format_dict(new), with_unchanged=with_unchanged - ) + return _diff(format_dict(old), format_dict(new), with_unchanged=with_unchanged) diff --git a/dvc/repo/params/show.py b/dvc/repo/params/show.py index 317d4c230..2268623b9 100644 --- a/dvc/repo/params/show.py +++ b/dvc/repo/params/show.py @@ -2,15 +2,7 @@ import logging import os from collections import defaultdict from copy import copy -from typing import ( - TYPE_CHECKING, - Callable, - Dict, - Iterable, - List, - Optional, - Tuple, -) +from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Tuple from scmrepo.exceptions import SCMError @@ -20,12 +12,7 @@ from dvc.repo.collect import collect from dvc.scm import NoSCMError from dvc.stage import PipelineStage from dvc.ui import ui -from dvc.utils import ( - as_posix, - error_handler, - errored_revisions, - onerror_collect, -) +from dvc.utils import as_posix, error_handler, errored_revisions, onerror_collect from dvc.utils.collections import ensure_list from dvc.utils.serialize import load_path @@ -43,9 +30,7 @@ def _is_params(dep: "Output"): def _collect_top_level_params(repo): top_params = repo.index._params # pylint: disable=protected-access for dvcfile, params in top_params.items(): - wdir = repo.fs.path.relpath( - repo.fs.path.parent(dvcfile), repo.root_dir - ) + wdir = repo.fs.path.relpath(repo.fs.path.parent(dvcfile), repo.root_dir) for file in params: path = repo.fs.path.join(wdir, as_posix(file)) yield repo.fs.path.normpath(path) @@ -54,7 +39,6 @@ def _collect_top_level_params(repo): def _collect_configs( repo: "Repo", targets=None, deps=False, stages=None ) -> Tuple[List["Output"], List[str]]: - params, fs_paths = collect( repo, targets=targets or [], @@ -67,9 +51,7 @@ def _collect_configs( default_params = repo.fs.path.join( repo.root_dir, ParamsDependency.DEFAULT_PARAMS_FILE ) - if default_params not in all_fs_paths and repo.fs.exists( - default_params - ): + if default_params not in all_fs_paths and repo.fs.exists(default_params): fs_paths.append(default_params) if targets and (deps or stages) and not params: # A target has been provided but it is not used in the stages diff --git a/dvc/repo/plots/__init__.py b/dvc/repo/plots/__init__.py index eb196d45c..b5536a58e 100644 --- a/dvc/repo/plots/__init__.py +++ b/dvc/repo/plots/__init__.py @@ -50,8 +50,7 @@ class PlotMetricTypeError(DvcException): class NotAPlotError(DvcException): def __init__(self, out): super().__init__( - f"'{out}' is not a known plot. Use `dvc plots modify` to turn it " - "into one." + f"'{out}' is not a known plot. Use `dvc plots modify` to turn it into one." ) @@ -310,8 +309,7 @@ def _collect_plots( ) result = { - repo.dvcfs.from_os_path(plot.fs_path): _plot_props(plot) - for plot in plots + repo.dvcfs.from_os_path(plot.fs_path): _plot_props(plot) for plot in plots } result.update({fs_path: {} for fs_path in fs_paths}) return result @@ -352,8 +350,7 @@ def _matches(targets, config_file, plot_id): full_id = get_plot_id(plot_id, config_file) if any( - (re.match(target, plot_id) or re.match(target, full_id)) - for target in targets + (re.match(target, plot_id) or re.match(target, full_id)) for target in targets ): return True return False @@ -373,14 +370,10 @@ def _relpath(fs, path): # and invoking from some subdir `dvcfile.relpath` returns strange long # relative paths # ("../../../../../../dvc.yaml") - investigate - return fs.path.relpath( - fs.path.join("/", fs.from_os_path(path)), fs.path.getcwd() - ) + return fs.path.relpath(fs.path.join("/", fs.from_os_path(path)), fs.path.getcwd()) -def _collect_output_plots( - repo, targets, props, onerror: Optional[Callable] = None -): +def _collect_output_plots(repo, targets, props, onerror: Optional[Callable] = None): fs = repo.dvcfs result: Dict[str, Dict] = {} for plot in repo.index.plots: @@ -497,7 +490,6 @@ def _collect_definitions( onerror: Optional[Callable] = None, **kwargs, ) -> Dict: - result: Dict = defaultdict(dict) props = props or {} @@ -542,9 +534,7 @@ def _collect_definitions( return dict(result) -def unpack_if_dir( - fs, path, props: Dict[str, str], onerror: Optional[Callable] = None -): +def unpack_if_dir(fs, path, props: Dict[str, str], onerror: Optional[Callable] = None): result: Dict[str, Dict] = defaultdict(dict) if fs.isdir(path): unpacked = _unpack_dir_files(fs, path, onerror=onerror) diff --git a/dvc/repo/plots/diff.py b/dvc/repo/plots/diff.py index d09312afa..1fd71c764 100644 --- a/dvc/repo/plots/diff.py +++ b/dvc/repo/plots/diff.py @@ -14,6 +14,4 @@ def _revisions(repo, revs, experiment): def diff(repo, *args, revs=None, experiment=False, **kwargs): if repo.scm.no_commits: return {} - return repo.plots.show( - *args, revs=_revisions(repo, revs, experiment), **kwargs - ) + return repo.plots.show(*args, revs=_revisions(repo, revs, experiment), **kwargs) diff --git a/dvc/repo/push.py b/dvc/repo/push.py index 71786b9a5..f44a9b8ef 100644 --- a/dvc/repo/push.py +++ b/dvc/repo/push.py @@ -29,16 +29,13 @@ def push( # noqa: C901 odb: Optional["ObjectDB"] = None, include_imports=False, ): - worktree_remote: Optional["Remote"] = None _remote = self.cloud.get_remote(name=remote) if _remote.worktree or _remote.fs.version_aware: worktree_remote = _remote pushed = 0 - used_run_cache = ( - self.stage_cache.push(remote, odb=odb) if run_cache else [] - ) + used_run_cache = self.stage_cache.push(remote, odb=odb) if run_cache else [] pushed += len(used_run_cache) if isinstance(targets, str): diff --git a/dvc/repo/reproduce.py b/dvc/repo/reproduce.py index 586087ab0..23aa63b04 100644 --- a/dvc/repo/reproduce.py +++ b/dvc/repo/reproduce.py @@ -34,8 +34,7 @@ def _reproduce_stage(stage: "Stage", **kwargs) -> List["Stage"]: if stage.frozen and not stage.is_import: logger.warning( - "%s is frozen. Its dependencies are" - " not going to be reproduced.", + "%s is frozen. Its dependencies are not going to be reproduced.", stage, ) @@ -58,9 +57,7 @@ def _get_stage_files(stage: "Stage") -> Iterator[str]: if ( not dep.use_scm_ignore and dep.is_in_repo - and not stage.repo.dvcfs.isdvc( - stage.repo.dvcfs.from_os_path(str(dep)) - ) + and not stage.repo.dvcfs.isdvc(stage.repo.dvcfs.from_os_path(str(dep))) ): yield dep.fs_path for out in stage.outs: @@ -205,8 +202,10 @@ def _reproduce_stages( # noqa: C901 except CheckpointKilledError: result.append(stage) logger.warning( - "Checkpoint stage '%s' was interrupted remaining stages in " - "pipeline will not be reproduced.", + ( + "Checkpoint stage '%s' was interrupted remaining stages in" + " pipeline will not be reproduced." + ), stage.addressing, ) logger.warning( diff --git a/dvc/repo/run.py b/dvc/repo/run.py index c44bf2558..9aa669673 100644 --- a/dvc/repo/run.py +++ b/dvc/repo/run.py @@ -19,12 +19,9 @@ def run( no_commit: bool = False, run_cache: bool = True, force: bool = True, - **kwargs + **kwargs, ) -> Union["Stage", "PipelineStage"]: - - kwargs.update( - {"force": force, "params": parse_params(kwargs.get("params", []))} - ) + kwargs.update({"force": force, "params": parse_params(kwargs.get("params", []))}) stage = self.stage.create(**kwargs) if no_exec: diff --git a/dvc/repo/scm_context.py b/dvc/repo/scm_context.py index 82db4068d..6228405e9 100644 --- a/dvc/repo/scm_context.py +++ b/dvc/repo/scm_context.py @@ -26,9 +26,7 @@ logger = logging.getLogger(__name__) class SCMContext: - def __init__( - self, scm: "Base", config: Optional[Dict[str, Any]] = None - ) -> None: + def __init__(self, scm: "Base", config: Optional[Dict[str, Any]] = None) -> None: from funcy import get_in self.scm: "Base" = scm @@ -39,9 +37,7 @@ class SCMContext: self.files_to_track: Set[str] = set() self.quiet: bool = False - def track_file( - self, paths: Union[str, Iterable[str], None] = None - ) -> None: + def track_file(self, paths: Union[str, Iterable[str], None] = None) -> None: """Track file to remind user to track new files or autostage later.""" return self.files_to_track.update(ensure_list(paths)) @@ -134,8 +130,7 @@ class SCMContext: add_cmd = self._make_git_add_cmd(self.files_to_track) logger.info("\nTo track the changes with git, run:\n\n%s", add_cmd) logger.info( - "\nTo enable auto staging, run:\n\n" - "\tdvc config core.autostage true" + "\nTo enable auto staging, run:\n\n\tdvc config core.autostage true" ) self.files_to_track = set() @@ -149,9 +144,7 @@ class SCMContext: self._cm.__exit__(*exc_args) # pylint: disable=no-member -def scm_context( - method, autostage: Optional[bool] = None, quiet: Optional[bool] = None -): +def scm_context(method, autostage: Optional[bool] = None, quiet: Optional[bool] = None): @wraps(method) def run(repo: "Repo", *args, **kw): with repo.scm_context(autostage=autostage, quiet=quiet): diff --git a/dvc/repo/stage.py b/dvc/repo/stage.py index 43d477e18..2856b21f5 100644 --- a/dvc/repo/stage.py +++ b/dvc/repo/stage.py @@ -152,18 +152,9 @@ class StageLoad: stage_data: Stage data to create from (see create_stage and loads_from for more information) """ - from dvc.stage import ( - PipelineStage, - Stage, - create_stage, - restore_fields, - ) + from dvc.stage import PipelineStage, Stage, create_stage, restore_fields from dvc.stage.exceptions import InvalidStageName - from dvc.stage.utils import ( - is_valid_name, - prepare_file_path, - validate_kwargs, - ) + from dvc.stage.utils import is_valid_name, prepare_file_path, validate_kwargs stage_data = validate_kwargs( single_stage=single_stage, fname=fname, **stage_data @@ -178,9 +169,7 @@ class StageLoad: if not (stage_name and is_valid_name(stage_name)): raise InvalidStageName - stage = create_stage( - stage_cls, repo=self.repo, path=path, **stage_data - ) + stage = create_stage(stage_cls, repo=self.repo, path=path, **stage_data) if validate: if not force: from dvc.stage.utils import check_stage_exists @@ -200,9 +189,7 @@ class StageLoad: (see load method below for further details) """ path, name = parse_target(target, isa_glob=glob) - return self.load_all( - path=path, name=name, accept_group=accept_group, glob=glob - ) + return self.load_all(path=path, name=name, accept_group=accept_group, glob=glob) def get_target(self, target: str) -> "Stage": """ @@ -238,7 +225,6 @@ class StageLoad: accept_group: bool = True, glob: bool = False, ) -> Iterable[str]: - if not name: return stages.keys() if accept_group and stages.is_foreach_generated(name): @@ -386,9 +372,7 @@ class StageLoad: target = as_posix(target) - stages, file, _ = _collect_specific_target( - self, target, with_deps, recursive - ) + stages, file, _ = _collect_specific_target(self, target, with_deps, recursive) if not stages: if not (recursive and self.fs.isdir(target)): try: @@ -398,10 +382,7 @@ class StageLoad: pass from dvc.dvcfile import is_valid_filename - from dvc.stage.exceptions import ( - StageFileDoesNotExistError, - StageNotFound, - ) + from dvc.stage.exceptions import StageFileDoesNotExistError, StageNotFound try: stages = self.collect( diff --git a/dvc/repo/status.py b/dvc/repo/status.py index f2c2f83f7..b5deced3e 100644 --- a/dvc/repo/status.py +++ b/dvc/repo/status.py @@ -14,17 +14,15 @@ def _joint_status(pairs): status_info = {} for stage, filter_info in pairs: - if stage.frozen and not ( - stage.is_repo_import or stage.is_versioned_import - ): + if stage.frozen and not (stage.is_repo_import or stage.is_versioned_import): logger.warning( - "%s is frozen. Its dependencies are" - " not going to be shown in the status output.", + ( + "%s is frozen. Its dependencies are" + " not going to be shown in the status output." + ), stage, ) - status_info.update( - stage.status(check_updates=True, filter_info=filter_info) - ) + status_info.update(stage.status(check_updates=True, filter_info=filter_info)) return status_info @@ -32,9 +30,7 @@ def _joint_status(pairs): def _local_status(self, targets=None, with_deps=False, recursive=False): targets = targets or [None] pairs = cat( - self.stage.collect_granular( - t, with_deps=with_deps, recursive=recursive - ) + self.stage.collect_granular(t, with_deps=with_deps, recursive=recursive) for t in targets ) @@ -144,6 +140,4 @@ def status( msg = "The following options are meaningless for local status: {}" raise InvalidArgumentError(msg.format(", ".join(ignored))) - return _local_status( - self, targets, with_deps=with_deps, recursive=recursive - ) + return _local_status(self, targets, with_deps=with_deps, recursive=recursive) diff --git a/dvc/repo/update.py b/dvc/repo/update.py index 798da8b14..ce34cddcf 100644 --- a/dvc/repo/update.py +++ b/dvc/repo/update.py @@ -28,14 +28,10 @@ def update( # noqa: C901 targets = [targets] if to_remote and no_download: - raise InvalidArgumentError( - "--to-remote can't be used with --no-download" - ) + raise InvalidArgumentError("--to-remote can't be used with --no-download") if not to_remote and remote: - raise InvalidArgumentError( - "--remote can't be used without --to-remote" - ) + raise InvalidArgumentError("--remote can't be used without --to-remote") import_stages = set() other_stage_infos: List["StageInfo"] = [] @@ -58,23 +54,17 @@ def update( # noqa: C901 if other_stage_infos: if rev: - raise InvalidArgumentError( - "--rev can't be used with worktree update" - ) + raise InvalidArgumentError("--rev can't be used with worktree update") if no_download: raise InvalidArgumentError( "--no-download can't be used with worktree update" ) if to_remote: - raise InvalidArgumentError( - "--to-remote can't be used with worktree update" - ) + raise InvalidArgumentError("--to-remote can't be used with worktree update") update_worktree_stages( self, other_stage_infos, ) - stages = import_stages | { - stage_info.stage for stage_info in other_stage_infos - } + stages = import_stages | {stage_info.stage for stage_info in other_stage_infos} return list(stages) diff --git a/dvc/repo/worktree.py b/dvc/repo/worktree.py index e935ab93c..cefcef15a 100644 --- a/dvc/repo/worktree.py +++ b/dvc/repo/worktree.py @@ -1,16 +1,6 @@ import logging from functools import partial -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Iterable, - Optional, - Set, - Tuple, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Set, Tuple, Union, cast from funcy import first @@ -69,9 +59,7 @@ def worktree_view_by_remotes( return for remote in remotes: - yield remote, IndexView( - index, view._stage_infos, outs_filter(view, remote) - ) + yield remote, IndexView(index, view._stage_infos, outs_filter(view, remote)) def worktree_view( @@ -95,11 +83,7 @@ def worktree_view( return True def outs_filter(out: "Output") -> bool: - if ( - not out.is_in_repo - or not out.use_cache - or (push and not out.can_push) - ): + if not out.is_in_repo or not out.use_cache or (push and not out.can_push): return False return True @@ -175,9 +159,7 @@ def push_worktree( logger.debug("Pushing worktree changes to '%s'", remote_obj.path) else: old_index = None - logger.debug( - "Pushing version-aware files to '%s'", remote_obj.path - ) + logger.debug("Pushing version-aware files to '%s'", remote_obj.path) if remote_obj.worktree: diff_kwargs: Dict[str, Any] = { @@ -398,9 +380,7 @@ def _update_out_meta( ): from dvc_data.index.save import build_tree - index = _get_update_diff_index( - repo, out, local_index, remote_index, remote - ) + index = _get_update_diff_index(repo, out, local_index, remote_index, remote) _, key = out.index_key entry = index[key] diff --git a/dvc/rwlock.py b/dvc/rwlock.py index f8e1afe91..a1fdb9388 100644 --- a/dvc/rwlock.py +++ b/dvc/rwlock.py @@ -31,8 +31,9 @@ RWLOCK_LOCK = "rwlock.lock" class RWLockFileCorruptedError(DvcException): def __init__(self, path): super().__init__( - "Unable to read RWLock-file '{}'. JSON structure is " - "corrupted".format(relpath(path)) + "Unable to read RWLock-file '{}'. JSON structure is corrupted".format( + relpath(path) + ) ) @@ -103,8 +104,10 @@ def _check_blockers(tmp_dir, lock, info, *, mode, waiters): # noqa: C901 non_existing_pid.add(pid) cmd = blocker["cmd"] logger.warning( - "Process '%s' with (Pid %s), in RWLock-file '%s'" - " had been killed. Auto remove it from the lock file.", + ( + "Process '%s' with (Pid %s), in RWLock-file '%s'" + " had been killed. Auto remove it from the lock file." + ), cmd, pid, relpath(path), @@ -205,10 +208,7 @@ def rwlock(tmp_dir, fs, cmd, read, write, hardlink): info = {"pid": os.getpid(), "cmd": cmd} with _edit_rwlock(tmp_dir, fs, hardlink) as lock: - - _check_blockers( - tmp_dir, lock, info, mode="write", waiters=read + write - ) + _check_blockers(tmp_dir, lock, info, mode="write", waiters=read + write) _check_blockers(tmp_dir, lock, info, mode="read", waiters=write) rchanges = _acquire_read(lock, info, read) diff --git a/dvc/schema.py b/dvc/schema.py index 8cf89ecb2..5b65000b5 100644 --- a/dvc/schema.py +++ b/dvc/schema.py @@ -90,9 +90,7 @@ STAGE_DEFINITION = { Optional(StageParams.PARAM_DESC): str, Optional(StageParams.PARAM_ALWAYS_CHANGED): bool, Optional(StageParams.PARAM_OUTS): [Any(str, OUT_PSTAGE_DETAILED_SCHEMA)], - Optional(StageParams.PARAM_METRICS): [ - Any(str, OUT_PSTAGE_DETAILED_SCHEMA) - ], + Optional(StageParams.PARAM_METRICS): [Any(str, OUT_PSTAGE_DETAILED_SCHEMA)], Optional(StageParams.PARAM_PLOTS): [Any(str, PLOT_PSTAGE_SCHEMA)], } @@ -100,10 +98,7 @@ STAGE_DEFINITION = { def either_or(primary, fallback, fallback_includes=None): def validator(data): schema = primary - if ( - isinstance(data, Mapping) - and set(fallback_includes or []) & data.keys() - ): + if isinstance(data, Mapping) and set(fallback_includes or []) & data.keys(): schema = fallback return Schema(schema)(data) diff --git a/dvc/scm.py b/dvc/scm.py index 865ff909b..0c0c9e838 100644 --- a/dvc/scm.py +++ b/dvc/scm.py @@ -138,16 +138,12 @@ def SCM( with map_scm_exception(): if no_scm: return NoSCM(root_dir, _raise_not_implemented_as=NoSCMError) - return Git( - root_dir, search_parent_directories=search_parent_directories - ) + return Git(root_dir, search_parent_directories=search_parent_directories) class TqdmGit(Tqdm): BAR_FMT = ( - "{desc}|{bar}|" - "{postfix[info]}{n_fmt}/{total_fmt}" - " [{elapsed}, {rate_fmt:>11}]" + "{desc}|{bar}|{postfix[info]}{n_fmt}/{total_fmt} [{elapsed}, {rate_fmt:>11}]" ) def __init__(self, *args, **kwargs): @@ -200,10 +196,7 @@ def resolve_rev(scm: "Git", rev: str) -> str: # `scm` will only resolve git branch and tag names, # if rev is not a sha it may be an abbreviated experiment name if not (rev == "HEAD" or rev.startswith("refs/")): - from dvc.repo.experiments.utils import ( - AmbiguousExpRefInfo, - resolve_name, - ) + from dvc.repo.experiments.utils import AmbiguousExpRefInfo, resolve_name try: ref_infos = resolve_name(scm, rev).get(rev) @@ -275,15 +268,11 @@ def iter_revs( # noqa: C901 if commit_date: from datetime import datetime - commit_datestamp = datetime.strptime( - commit_date, "%Y-%m-%d" - ).timestamp() + commit_datestamp = datetime.strptime(commit_date, "%Y-%m-%d").timestamp() def _time_filter(rev): try: - return ( - scm.resolve_commit(rev).commit_time >= commit_datestamp - ) + return scm.resolve_commit(rev).commit_time >= commit_datestamp except _SCMError: return True diff --git a/dvc/stage/__init__.py b/dvc/stage/__init__.py index 6e3e48391..10d76157a 100644 --- a/dvc/stage/__init__.py +++ b/dvc/stage/__init__.py @@ -21,12 +21,7 @@ from typing import ( from funcy import project from dvc import prompt -from dvc.exceptions import ( - CacheLinkError, - CheckoutError, - DvcException, - MergeError, -) +from dvc.exceptions import CacheLinkError, CheckoutError, DvcException, MergeError from dvc.utils import relpath from dvc.utils.objects import cached_property @@ -199,8 +194,7 @@ class Stage(params.StageParams): if not self.path: raise DvcException( - "Stage does not have any path set " - "and is detached from dvcfile." + "Stage does not have any path set and is detached from dvcfile." ) from dvc.dvcfile import load_file @@ -209,9 +203,7 @@ class Stage(params.StageParams): return self._dvcfile @dvcfile.setter - def dvcfile( - self, dvcfile: Union["ProjectFile", "SingleStageFile"] - ) -> None: + def dvcfile(self, dvcfile: Union["ProjectFile", "SingleStageFile"]) -> None: self._dvcfile = dvcfile def __repr__(self): @@ -433,14 +425,11 @@ class Stage(params.StageParams): if not isinstance(self, PipelineStage) and self.is_data_source: logger.info("'%s' didn't change, skipping", self.addressing) else: - logger.info( - "Stage '%s' didn't change, skipping", self.addressing - ) + logger.info("Stage '%s' didn't change, skipping", self.addressing) return None - msg = ( - "Going to reproduce {stage}. " - "Are you sure you want to continue?".format(stage=self) + msg = "Going to reproduce {stage}. Are you sure you want to continue?".format( + stage=self ) if interactive and not prompt.confirm(msg): @@ -517,10 +506,7 @@ class Stage(params.StageParams): merge_versioned = any( ( out.files is not None - or ( - out.meta is not None - and out.meta.version_id is not None - ) + or (out.meta is not None and out.meta.version_id is not None) ) for out in old_outs.values() ) @@ -588,15 +574,11 @@ class Stage(params.StageParams): self.remove_outs(ignore_remove=False, force=False) if (not self.frozen and self.is_import) or self.is_partial_import: - self._sync_import( - dry, force, kwargs.get("jobs", None), no_download - ) + self._sync_import(dry, force, kwargs.get("jobs", None), no_download) elif not self.frozen and self.cmd: self._run_stage(dry, force, **kwargs) else: - args = ( - ("outputs", "frozen ") if self.frozen else ("data sources", "") - ) + args = ("outputs", "frozen ") if self.frozen else ("data sources", "") logger.info("Verifying %s in %s%s", *args, self) if not dry: self._check_missing_outputs() @@ -712,11 +694,7 @@ class Stage(params.StageParams): ret.append("changed checksum") def already_cached(self) -> bool: - return ( - not self.changed_stage() - and self.deps_cached() - and self.outs_cached() - ) + return not self.changed_stage() and self.deps_cached() and self.outs_cached() def deps_cached(self) -> bool: return all(not dep.changed() for dep in self.deps) @@ -729,8 +707,7 @@ class Stage(params.StageParams): def get_all_files_number(self, filter_info=None) -> int: return sum( - out.get_files_number(filter_info) - for out in self.filter_outs(filter_info) + out.get_files_number(filter_info) for out in self.filter_outs(filter_info) ) def get_used_objs( @@ -753,14 +730,11 @@ class Stage(params.StageParams): if not stage.is_data_source or stage.deps or len(stage.outs) > 1: raise MergeError( - "unable to auto-merge DVC files that weren't " - "created by `dvc add`" + "unable to auto-merge DVC files that weren't created by `dvc add`" ) if ancestor_out and not stage.outs: - raise MergeError( - "unable to auto-merge DVC files with deleted outputs" - ) + raise MergeError("unable to auto-merge DVC files with deleted outputs") def merge(self, ancestor, other, allowed=None) -> None: assert other diff --git a/dvc/stage/cache.py b/dvc/stage/cache.py index 31e3cc59b..6abc87aa8 100644 --- a/dvc/stage/cache.py +++ b/dvc/stage/cache.py @@ -133,9 +133,7 @@ class StageCache: # outputs. cached_stage = self._create_stage(cache, wdir=stage.wdir) - outs_no_cache = [ - out.def_path for out in stage.outs if not out.use_cache - ] + outs_no_cache = [out.def_path for out in stage.outs if not out.use_cache] # NOTE: using copy link to make it look like a git-tracked file with self._cache_type_copy(): @@ -224,11 +222,7 @@ class StageCache: runs = from_fs.path.join(from_odb.path, "runs") http_odb = next( - ( - odb - for odb in (from_odb, to_odb) - if isinstance(odb.fs, HTTPFileSystem) - ), + (odb for odb in (from_odb, to_odb) if isinstance(odb.fs, HTTPFileSystem)), None, ) if http_odb: @@ -264,15 +258,11 @@ class StageCache: return ret def push(self, remote: Optional[str], odb: Optional["ObjectDB"] = None): - dest_odb = odb or self.repo.cloud.get_remote_odb( - remote, "push --run-cache" - ) + dest_odb = odb or self.repo.cloud.get_remote_odb(remote, "push --run-cache") return self.transfer(self.repo.odb.local, dest_odb) def pull(self, remote: Optional[str], odb: Optional["ObjectDB"] = None): - odb = odb or self.repo.cloud.get_remote_odb( - remote, "fetch --run-cache" - ) + odb = odb or self.repo.cloud.get_remote_odb(remote, "fetch --run-cache") return self.transfer(odb, self.repo.odb.local) def get_used_objs(self, used_run_cache, *args, **kwargs): diff --git a/dvc/stage/exceptions.py b/dvc/stage/exceptions.py index 4ed800371..5f9ac50d6 100644 --- a/dvc/stage/exceptions.py +++ b/dvc/stage/exceptions.py @@ -70,10 +70,7 @@ class StageExternalOutputsError(DvcException): class StageUpdateError(DvcException): def __init__(self, path): - super().__init__( - "update is not supported for '{}' that is not an " - "import.".format(path) - ) + super().__init__(f"update is not supported for '{path}' that is not an import.") class MissingDataSource(DvcException): @@ -97,9 +94,7 @@ class StageNotFound(DvcException, KeyError): def __init__(self, file, name): self.file = file.relpath self.name = name - super().__init__( - f"Stage '{self.name}' not found inside '{self.file}' file" - ) + super().__init__(f"Stage '{self.name}' not found inside '{self.file}' file") def __str__(self): # `KeyError` quotes the message diff --git a/dvc/stage/loader.py b/dvc/stage/loader.py index 872af8d0c..ac6a2b3d6 100644 --- a/dvc/stage/loader.py +++ b/dvc/stage/loader.py @@ -88,9 +88,7 @@ class StageLoader(Mapping): item._compute_meta_hash_info_from_files() @classmethod - def load_stage( - cls, dvcfile: "ProjectFile", name, stage_data, lock_data=None - ): + def load_stage(cls, dvcfile: "ProjectFile", name, stage_data, lock_data=None): assert all([name, dvcfile, dvcfile.repo, dvcfile.path]) assert stage_data assert isinstance(stage_data, dict) @@ -161,9 +159,7 @@ class StageLoader(Mapping): if group and keys and name not in self.stages_data: stage.raw_data.generated_from = group - stage.raw_data.parametrized = ( - self.stages_data.get(name, {}) != resolved_stage - ) + stage.raw_data.parametrized = self.stages_data.get(name, {}) != resolved_stage return stage def __iter__(self): @@ -176,9 +172,7 @@ class StageLoader(Mapping): return self.resolver.has_key(name) # noqa: W601 def is_foreach_generated(self, name: str): - return ( - name in self.stages_data and FOREACH_KWD in self.stages_data[name] - ) + return name in self.stages_data and FOREACH_KWD in self.stages_data[name] class SingleStageLoader(Mapping): @@ -201,9 +195,7 @@ class SingleStageLoader(Mapping): ) # during `load`, we remove attributes from stage data, so as to # not duplicate, therefore, for MappingView, we need to deepcopy. - return self.load_stage( - self.dvcfile, deepcopy(self.stage_data), self.stage_text - ) + return self.load_stage(self.dvcfile, deepcopy(self.stage_data), self.stage_text) @classmethod def load_stage( @@ -217,9 +209,7 @@ class SingleStageLoader(Mapping): ) stage = loads_from(Stage, dvcfile.repo, path, wdir, d) stage._stage_text = stage_text # pylint: disable=protected-access - stage.deps = dependency.loadd_from( - stage, d.get(Stage.PARAM_DEPS) or [] - ) + stage.deps = dependency.loadd_from(stage, d.get(Stage.PARAM_DEPS) or []) stage.outs = output.loadd_from(stage, d.get(Stage.PARAM_OUTS) or []) return stage diff --git a/dvc/stage/monitor.py b/dvc/stage/monitor.py index b0869ba2b..b703ea1fd 100644 --- a/dvc/stage/monitor.py +++ b/dvc/stage/monitor.py @@ -49,14 +49,10 @@ class CheckpointTask(MonitorTask): SIGNAL_FILE = "DVC_CHECKPOINT" error_cls = CheckpointKilledError - def __init__( - self, stage: "Stage", callback_func: Callable, proc: subprocess.Popen - ): + def __init__(self, stage: "Stage", callback_func: Callable, proc: subprocess.Popen): super().__init__( stage, - functools.partial( - CheckpointTask._run_callback, stage, callback_func - ), + functools.partial(CheckpointTask._run_callback, stage, callback_func), proc, ) @@ -118,9 +114,7 @@ class Monitor: ) Monitor.kill(task.proc) finally: - logger.debug( - "Removing signal file for '%s' task", task.name - ) + logger.debug("Removing signal file for '%s' task", task.name) os.remove(task.signal_path) if done.wait(Monitor.AWAIT): diff --git a/dvc/stage/run.py b/dvc/stage/run.py index 4f4e0ebf0..18d9acfbc 100644 --- a/dvc/stage/run.py +++ b/dvc/stage/run.py @@ -26,10 +26,7 @@ def _make_cmd(executable, cmd): def warn_if_fish(executable): - if ( - executable is None - or os.path.basename(os.path.realpath(executable)) != "fish" - ): + if executable is None or os.path.basename(os.path.realpath(executable)) != "fish": return logger.warning( @@ -112,7 +109,6 @@ def _run(stage: "Stage", executable, cmd, checkpoint_func, **kwargs): def _get_monitor_tasks(stage, checkpoint_func, proc) -> List[CheckpointTask]: - result = [] if checkpoint_func: result.append(CheckpointTask(stage, checkpoint_func, proc)) @@ -123,9 +119,7 @@ def _get_monitor_tasks(stage, checkpoint_func, proc) -> List[CheckpointTask]: def cmd_run(stage, dry=False, checkpoint_func=None, run_env=None): logger.info("Running stage '%s':", stage.addressing) commands = _enforce_cmd_list(stage.cmd) - kwargs = prepare_kwargs( - stage, checkpoint_func=checkpoint_func, run_env=run_env - ) + kwargs = prepare_kwargs(stage, checkpoint_func=checkpoint_func, run_env=run_env) executable = get_executable() if not dry: diff --git a/dvc/stage/serialize.py b/dvc/stage/serialize.py index 515af5b03..f80b6cd42 100644 --- a/dvc/stage/serialize.py +++ b/dvc/stage/serialize.py @@ -150,9 +150,7 @@ def to_pipeline_file(stage: "PipelineStage"): (stage.PARAM_ALWAYS_CHANGED, stage.always_changed), (stage.PARAM_META, stage.meta), ] - return { - stage.name: OrderedDict([(key, value) for key, value in res if value]) - } + return {stage.name: OrderedDict([(key, value) for key, value in res if value])} def to_single_stage_lockfile(stage: "Stage", **kwargs) -> dict: @@ -167,8 +165,7 @@ def to_single_stage_lockfile(stage: "Stage", **kwargs) -> dict: if obj: obj = cast("Tree", obj) ret[item.PARAM_FILES] = [ - split_file_meta_from_cloud(f) - for f in obj.as_list(with_meta=True) + split_file_meta_from_cloud(f) for f in obj.as_list(with_meta=True) ] else: meta_d = item.meta.to_dict() diff --git a/dvc/stage/utils.py b/dvc/stage/utils.py index 2be41340d..6d2e2ade4 100644 --- a/dvc/stage/utils.py +++ b/dvc/stage/utils.py @@ -79,16 +79,12 @@ def fill_stage_outputs(stage, **kwargs): ) -def fill_stage_dependencies( - stage, deps=None, erepo=None, params=None, fs_config=None -): +def fill_stage_dependencies(stage, deps=None, erepo=None, params=None, fs_config=None): from dvc.dependency import loads_from, loads_params assert not stage.deps stage.deps = [] - stage.deps += loads_from( - stage, deps or [], erepo=erepo, fs_config=fs_config - ) + stage.deps += loads_from(stage, deps or [], erepo=erepo, fs_config=fs_config) stage.deps += loads_params(stage, params or []) @@ -114,8 +110,7 @@ def check_no_externals(stage): str_outs = ", ".join(outs) link = format_link("https://dvc.org/doc/user-guide/managing-external-data") raise StageExternalOutputsError( - f"Output(s) outside of DVC project: {str_outs}. " - f"See {link} for more info." + f"Output(s) outside of DVC project: {str_outs}. See {link} for more info." ) @@ -250,21 +245,14 @@ def prepare_file_path(kwargs) -> str: ) return ( - os.path.basename(os.path.normpath(out)) + DVC_FILE_SUFFIX - if out - else DVC_FILE + os.path.basename(os.path.normpath(out)) + DVC_FILE_SUFFIX if out else DVC_FILE ) -def check_stage_exists( - repo: "Repo", stage: Union["Stage", "PipelineStage"], path: str -): +def check_stage_exists(repo: "Repo", stage: Union["Stage", "PipelineStage"], path: str): from dvc.dvcfile import load_file from dvc.stage import PipelineStage - from dvc.stage.exceptions import ( - DuplicateStageName, - StageFileAlreadyExistsError, - ) + from dvc.stage.exceptions import DuplicateStageName, StageFileAlreadyExistsError dvcfile = load_file(repo, path) if not dvcfile.exists(): @@ -272,9 +260,7 @@ def check_stage_exists( hint = "Use '--force' to overwrite." if not isinstance(stage, PipelineStage): - raise StageFileAlreadyExistsError( - f"'{stage.relpath}' already exists. {hint}" - ) + raise StageFileAlreadyExistsError(f"'{stage.relpath}' already exists. {hint}") if stage.name and stage.name in dvcfile.stages: raise DuplicateStageName( f"Stage '{stage.name}' already exists in '{stage.relpath}'. {hint}" @@ -291,9 +277,7 @@ def validate_kwargs( stage_name = kwargs.get("name") if stage_name and single_stage: - raise InvalidArgumentError( - "`-n|--name` is incompatible with `--single-stage`" - ) + raise InvalidArgumentError("`-n|--name` is incompatible with `--single-stage`") if stage_name and fname: raise InvalidArgumentError( "`--file` is currently incompatible with `-n|--name` " diff --git a/dvc/testing/api_tests.py b/dvc/testing/api_tests.py index 072e2e388..37b8067ad 100644 --- a/dvc/testing/api_tests.py +++ b/dvc/testing/api_tests.py @@ -32,9 +32,7 @@ class TestAPI: with api.open("dir/bar") as fobj: assert fobj.read() == "bar-text" - @pytest.mark.parametrize( - "clear_cache", [True, False], ids=["cache", "no_cache"] - ) + @pytest.mark.parametrize("clear_cache", [True, False], ids=["cache", "no_cache"]) @pytest.mark.parametrize( "fs_kwargs", [ @@ -67,9 +65,7 @@ class TestAPI: remove(dvc.odb.repo.path) if url := fs_kwargs.get("url"): - fs_kwargs["url"] = url.format( - path=tmp_dir, posixpath=tmp_dir.as_posix() - ) + fs_kwargs["url"] = url.format(path=tmp_dir, posixpath=tmp_dir.as_posix()) fs = DVCFileSystem(**fs_kwargs) @@ -77,9 +73,7 @@ class TestAPI: "/.gitignore", "/scripts", "/data" ) assert fs.ls("scripts", detail=False) == ["scripts/script1"] - assert fs.ls("data", detail=False) == M.unordered( - "data/foo", "data/bar" - ) + assert fs.ls("data", detail=False) == M.unordered("data/foo", "data/bar") data_info = M.dict( name="/data", diff --git a/dvc/testing/fixtures.py b/dvc/testing/fixtures.py index 6bd4fff97..dfa608f3e 100644 --- a/dvc/testing/fixtures.py +++ b/dvc/testing/fixtures.py @@ -37,9 +37,7 @@ CACHE: Dict[Tuple[bool, bool, bool], str] = {} @pytest.fixture(scope="session") def make_tmp_dir(tmp_path_factory, request, worker_id): - def make( - name, *, scm=False, dvc=False, subdir=False - ): # pylint: disable=W0621 + def make(name, *, scm=False, dvc=False, subdir=False): # pylint: disable=W0621 from shutil import copytree, ignore_patterns from dvc.repo import Repo @@ -65,9 +63,7 @@ def make_tmp_dir(tmp_path_factory, request, worker_id): if dvc: new_dir.dvc = Repo(str_path) if scm: - new_dir.scm = ( - new_dir.dvc.scm if hasattr(new_dir, "dvc") else Git(str_path) - ) + new_dir.scm = new_dir.dvc.scm if hasattr(new_dir, "dvc") else Git(str_path) request.addfinalizer(new_dir.close) return new_dir @@ -138,9 +134,7 @@ def make_remote(tmp_dir, dvc, make_cloud): # noqa: ARG001 @pytest.fixture -def make_remote_version_aware( - tmp_dir, dvc, make_cloud_version_aware # noqa: ARG001 -): +def make_remote_version_aware(tmp_dir, dvc, make_cloud_version_aware): # noqa: ARG001 def _make_remote(name, typ="local", **kwargs): cloud = make_cloud_version_aware(typ) # pylint: disable=W0621 config = dict(cloud.config) @@ -152,9 +146,7 @@ def make_remote_version_aware( @pytest.fixture -def make_remote_worktree( - tmp_dir, dvc, make_cloud_version_aware # noqa: ARG001 -): +def make_remote_worktree(tmp_dir, dvc, make_cloud_version_aware): # noqa: ARG001 def _make_remote(name, typ="local", **kwargs): cloud = make_cloud_version_aware(typ) # pylint: disable=W0621 config = dict(cloud.config) @@ -249,9 +241,7 @@ def docker_compose_project_name(): @pytest.fixture(scope="session") -def docker_services( - docker_compose_file, docker_compose_project_name, tmp_path_factory -): +def docker_services(docker_compose_file, docker_compose_project_name, tmp_path_factory): # overriding `docker_services` fixture from `pytest_docker` plugin to # only launch docker images once. @@ -260,9 +250,7 @@ def docker_services( # pylint: disable-next=import-error from pytest_docker.plugin import DockerComposeExecutor, Services - executor = DockerComposeExecutor( - docker_compose_file, docker_compose_project_name - ) + executor = DockerComposeExecutor(docker_compose_file, docker_compose_project_name) # making sure we don't accidentally launch docker-compose in parallel, # as it might result in network conflicts. Inspired by: diff --git a/dvc/testing/path_info.py b/dvc/testing/path_info.py index 4f65751c9..af7a6a257 100644 --- a/dvc/testing/path_info.py +++ b/dvc/testing/path_info.py @@ -172,9 +172,7 @@ class URLInfo(_BasePath): return self._base_parts + self._path.parts def replace(self, path=None): - return self.from_parts( # type: ignore[misc] - *self._base_parts, path=path - ) + return self.from_parts(*self._base_parts, path=path) # type: ignore[misc] @cached_property def url(self) -> str: @@ -257,9 +255,7 @@ class URLInfo(_BasePath): other = self.__class__(other) elif self.__class__ != other.__class__: return False - return self._base_parts == other._base_parts and self._path.isin( - other._path - ) + return self._base_parts == other._base_parts and self._path.isin(other._path) class CloudURLInfo(URLInfo): diff --git a/dvc/testing/remote_tests.py b/dvc/testing/remote_tests.py index 1ac6dde60..6d10937ab 100644 --- a/dvc/testing/remote_tests.py +++ b/dvc/testing/remote_tests.py @@ -116,9 +116,7 @@ class TestRemote: class TestRemoteVersionAware: - def test_file( - self, tmp_dir, dvc, remote_version_aware # pylint: disable=W0613 - ): + def test_file(self, tmp_dir, dvc, remote_version_aware): # pylint: disable=W0613 (stage,) = tmp_dir.dvc_gen("foo", "foo") dvc.push() @@ -133,9 +131,7 @@ class TestRemoteVersionAware: dvc.pull() assert (tmp_dir / "foo").read_text() == "foo" - def test_dir( - self, tmp_dir, dvc, remote_version_aware # pylint: disable=W0613 - ): + def test_dir(self, tmp_dir, dvc, remote_version_aware): # pylint: disable=W0613 (stage,) = tmp_dir.dvc_gen( { "data_dir": { @@ -167,9 +163,7 @@ class TestRemoteVersionAware: class TestRemoteWorktree: - def test_file( - self, tmp_dir, dvc, remote_worktree # pylint: disable=W0613 - ): + def test_file(self, tmp_dir, dvc, remote_worktree): # pylint: disable=W0613 (stage,) = tmp_dir.dvc_gen("foo", "foo") dvc.push() @@ -245,9 +239,7 @@ class TestRemoteWorktree: dvc.pull() assert (tmp_dir / "data_dir" / "data").read_text() == "data" - def test_update( - self, tmp_dir, dvc, remote_worktree # pylint: disable=W0613 - ): + def test_update(self, tmp_dir, dvc, remote_worktree): # pylint: disable=W0613 (foo_stage,) = tmp_dir.dvc_gen("foo", "foo") (data_dir_stage,) = tmp_dir.dvc_gen( { diff --git a/dvc/testing/tmp_dir.py b/dvc/testing/tmp_dir.py index 76fa80478..38ef70d0c 100644 --- a/dvc/testing/tmp_dir.py +++ b/dvc/testing/tmp_dir.py @@ -102,9 +102,7 @@ class TmpDir(pathlib.Path): subdir=subdir, ) if scm: - self.scm = ( - self.dvc.scm if hasattr(self, "dvc") else Git(self.fs_path) - ) + self.scm = self.dvc.scm if hasattr(self, "dvc") else Git(self.fs_path) if dvc and hasattr(self, "scm"): self.scm.commit("init dvc") @@ -162,9 +160,7 @@ class TmpDir(pathlib.Path): return os.path.join(os.path.dirname(stage_path), Git.GITIGNORE) gitignores = [ - to_gitignore(s) - for s in output_paths - if os.path.exists(to_gitignore(s)) + to_gitignore(s) for s in output_paths if os.path.exists(to_gitignore(s)) ] return self.scm_add(output_paths + gitignores, commit=msg) @@ -184,9 +180,7 @@ class TmpDir(pathlib.Path): if commit: self.scm.commit(commit) - def add_remote( - self, *, url=None, config=None, name="upstream", default=True - ): + def add_remote(self, *, url=None, config=None, name="upstream", default=True): self._require("dvc") assert bool(url) ^ bool(config) @@ -231,8 +225,7 @@ class TmpDir(pathlib.Path): # rely on exception flow control if self.is_dir(): return { - path.name: path.read_text(*args, **kwargs) - for path in self.iterdir() + path.name: path.read_text(*args, **kwargs) for path in self.iterdir() } kwargs.setdefault("encoding", "utf-8") # type: ignore[call-overload] return super().read_text(*args, **kwargs) diff --git a/dvc/testing/workspace_tests.py b/dvc/testing/workspace_tests.py index 5a9e3e590..4300cd869 100644 --- a/dvc/testing/workspace_tests.py +++ b/dvc/testing/workspace_tests.py @@ -29,9 +29,7 @@ class TestImport: def test_import_dir(self, tmp_dir, dvc, workspace, stage_md5, dir_md5): from dvc.odbmgr import ODBManager - workspace.gen( - {"dir": {"file": "file", "subdir": {"subfile": "subfile"}}} - ) + workspace.gen({"dir": {"file": "file", "subdir": {"subfile": "subfile"}}}) # remove external cache to make sure that we don't need it # to import dirs @@ -44,9 +42,7 @@ class TestImport: assert set(os.listdir(tmp_dir / "dir")) == {"file", "subdir"} assert (tmp_dir / "dir" / "file").read_text() == "file" assert list(os.listdir(tmp_dir / "dir" / "subdir")) == ["subfile"] - assert ( - tmp_dir / "dir" / "subdir" / "subfile" - ).read_text() == "subfile" + assert (tmp_dir / "dir" / "subdir" / "subfile").read_text() == "subfile" assert dvc.status() == {} @@ -70,9 +66,7 @@ class TestImport: def is_object_storage(self): pytest.skip() - def test_import_empty_dir( - self, tmp_dir, dvc, workspace, is_object_storage - ): + def test_import_empty_dir(self, tmp_dir, dvc, workspace, is_object_storage): # prefix based storage services (e.g s3) doesn't have the real concept # of directories. So instead we create an empty file that ends with a # trailing slash in order to actually support this operation @@ -130,26 +124,20 @@ class TestImportURLVersionAware: dvc.pull() assert (tmp_dir / "data_dir" / "subdir" / "file").read_text() == "file" - (remote_version_aware / "data_dir" / "subdir" / "file").write_text( - "modified" - ) + (remote_version_aware / "data_dir" / "subdir" / "file").write_text("modified") (remote_version_aware / "data_dir" / "new_file").write_text("new") assert dvc.status().get("data_dir.dvc") == [ {"changed deps": {"remote://upstream/data_dir": "modified"}}, ] dvc.update(str(tmp_dir / "data_dir.dvc")) - assert ( - tmp_dir / "data_dir" / "subdir" / "file" - ).read_text() == "modified" + assert (tmp_dir / "data_dir" / "subdir" / "file").read_text() == "modified" assert (tmp_dir / "data_dir" / "new_file").read_text() == "new" assert dvc.status() == {} dvc.odb.local.clear() remove(tmp_dir / "data_dir") dvc.pull() - assert ( - tmp_dir / "data_dir" / "subdir" / "file" - ).read_text() == "modified" + assert (tmp_dir / "data_dir" / "subdir" / "file").read_text() == "modified" assert (tmp_dir / "data_dir" / "new_file").read_text() == "new" @@ -188,31 +176,17 @@ class TestAdd: assert dvc.status() == {} + # pylint: disable-next=unused-argument def test_add_dir(self, tmp_dir, dvc, workspace, hash_name, dir_hash_value): - workspace.gen( - {"dir": {"file": "file", "subdir": {"subfile": "subfile"}}} - ) + workspace.gen({"dir": {"file": "file", "subdir": {"subfile": "subfile"}}}) dvc.add("remote://workspace/dir") - assert (tmp_dir / "dir.dvc").read_text() == ( - "outs:\n" - f"- {hash_name}: {dir_hash_value}\n" - " size: 11\n" - " nfiles: 2\n" - " path: remote://workspace/dir\n" - ) - assert ( - workspace / "cache" / dir_hash_value[:2] / dir_hash_value[2:] - ).is_file() + assert (workspace / "cache" / dir_hash_value[:2] / dir_hash_value[2:]).is_file() def match_files(fs, entries, expected): - entries_content = { - (fs.path.normpath(d["path"]), d["isdir"]) for d in entries - } - expected_content = { - (fs.path.normpath(d["path"]), d["isdir"]) for d in expected - } + entries_content = {(fs.path.normpath(d["path"]), d["isdir"]) for d in entries} + expected_content = {(fs.path.normpath(d["path"]), d["isdir"]) for d in expected} assert entries_content == expected_content @@ -229,9 +203,7 @@ class TestLsUrl: ) def test_dir(self, cloud): - cloud.gen( - {"dir/foo": "foo contents", "dir/subdir/bar": "bar contents"} - ) + cloud.gen({"dir/foo": "foo contents", "dir/subdir/bar": "bar contents"}) if not (cloud / "dir").is_dir(): pytest.skip("Cannot create directories on this cloud") fs, _ = parse_external_url(cloud.url, cloud.config) @@ -246,15 +218,11 @@ class TestLsUrl: ) def test_recursive(self, cloud): - cloud.gen( - {"dir/foo": "foo contents", "dir/subdir/bar": "bar contents"} - ) + cloud.gen({"dir/foo": "foo contents", "dir/subdir/bar": "bar contents"}) if not (cloud / "dir").is_dir(): pytest.skip("Cannot create directories on this cloud") fs, _ = parse_external_url(cloud.url, cloud.config) - result = ls_url( - str(cloud / "dir"), config=cloud.config, recursive=True - ) + result = ls_url(str(cloud / "dir"), config=cloud.config, recursive=True) match_files( fs, result, diff --git a/dvc/ui/__init__.py b/dvc/ui/__init__.py index e86a4226c..a261bcb1b 100644 --- a/dvc/ui/__init__.py +++ b/dvc/ui/__init__.py @@ -54,9 +54,7 @@ class Console: self.formatter: Formatter = formatter or Formatter() self._enabled: bool = enable - self._paginate: ContextVar[bool] = ContextVar( - "_paginate", default=False - ) + self._paginate: ContextVar[bool] = ContextVar("_paginate", default=False) def enable(self) -> None: self._enabled = True @@ -196,10 +194,7 @@ class Console: if answer in choices: return answer - self.write( - f"Your response must be one of: {choices}. " - "Please try again." - ) + self.write(f"Your response must be one of: {choices}. Please try again.") def confirm(self, statement: str) -> bool: """Ask the user for confirmation about the specified statement. @@ -234,9 +229,7 @@ class Console: rich_table: bool = False, force: bool = True, pager: bool = False, - header_styles: Optional[ - Union[Dict[str, "Styles"], Sequence["Styles"]] - ] = None, + header_styles: Optional[Union[Dict[str, "Styles"], Sequence["Styles"]]] = None, row_styles: Optional[Sequence["Styles"]] = None, borders: Union[bool, str] = False, ) -> None: @@ -279,18 +272,12 @@ class Console: from dvc.utils import relpath path = Path(file).resolve() - url = ( - relpath(path) - if "microsoft" in uname().release.lower() - else path.as_uri() - ) + url = relpath(path) if "microsoft" in uname().release.lower() else path.as_uri() opened = webbrowser.open(url) if not opened: - ui.error_write( - f"Failed to open {url}. Please try opening it manually." - ) + ui.error_write(f"Failed to open {url}. Please try opening it manually.") return 1 return 0 @@ -308,6 +295,4 @@ if __name__ == "__main__": ui.error("too few arguments.") ui.table([("scores.json", "0.5674")], headers=["Path", "auc"]) - ui.table( - [("scores.json", "0.5674")], headers=["Path", "auc"], markdown=True - ) + ui.table([("scores.json", "0.5674")], headers=["Path", "auc"], markdown=True) diff --git a/dvc/ui/pager.py b/dvc/ui/pager.py index 9f8fa2538..e4c4a038f 100644 --- a/dvc/ui/pager.py +++ b/dvc/ui/pager.py @@ -61,8 +61,7 @@ def find_pager(): if not pager: if os.system(f"({DEFAULT_PAGER}) 2>{os.devnull}") != 0: logger.warning( - "Unable to find `less` in the PATH. Check out " - "%s for more info.", + "Unable to find `less` in the PATH. Check out %s for more info.", format_link("https://man.dvc.org/pipeline/show"), ) else: diff --git a/dvc/ui/prompt.py b/dvc/ui/prompt.py index 71e5dd1d5..671f82bc4 100644 --- a/dvc/ui/prompt.py +++ b/dvc/ui/prompt.py @@ -21,9 +21,7 @@ class RichInputMixin: @classmethod def get_input(cls, console: "RichConsole", *args, **kwargs) -> str: try: - return super().get_input( # type: ignore[misc] - console, *args, **kwargs - ) + return super().get_input(console, *args, **kwargs) # type: ignore[misc] except (KeyboardInterrupt, EOFError): console.print() raise @@ -48,9 +46,7 @@ class Prompt(RichInputMixin, rich.prompt.Prompt): cls, *args: Any, allow_omission: bool = False, - validator: Optional[ - Callable[[str], Union[str, Tuple[str, str]]] - ] = None, + validator: Optional[Callable[[str], Union[str, Tuple[str, str]]]] = None, **kwargs: Any, ) -> str: """Extended to pass validator argument. @@ -79,9 +75,7 @@ class Prompt(RichInputMixin, rich.prompt.Prompt): cls, *args: Any, allow_omission: bool = False, - validator: Optional[ - Callable[[str], Union[str, Tuple[str, str]]] - ] = None, + validator: Optional[Callable[[str], Union[str, Tuple[str, str]]]] = None, **kwargs: Any, ) -> Optional[str]: """Extends `.ask()` to allow skipping/returning None type.""" @@ -95,9 +89,7 @@ class Prompt(RichInputMixin, rich.prompt.Prompt): def __call__( self, *args: Any, - validator: Optional[ - Callable[[str], Union[str, Tuple[str, str]]] - ] = None, + validator: Optional[Callable[[str], Union[str, Tuple[str, str]]]] = None, **kwargs: Any, ) -> str: """Supports validating response and show warning message.""" @@ -106,9 +98,7 @@ class Prompt(RichInputMixin, rich.prompt.Prompt): # `default` values. while True: value = super().__call__(*args, **kwargs) - if validator is None or ( - self.allow_omission and value == self.omit_value - ): + if validator is None or (self.allow_omission and value == self.omit_value): return value try: diff --git a/dvc/ui/table.py b/dvc/ui/table.py index de55c9b44..b13f3d754 100644 --- a/dvc/ui/table.py +++ b/dvc/ui/table.py @@ -50,9 +50,7 @@ def plain_table( @contextmanager -def console_width( - table: "Table", console: "RichConsole", val: int -) -> Iterator[None]: +def console_width(table: "Table", console: "RichConsole", val: int) -> Iterator[None]: # NOTE: rich does not have native support for unlimited width # via pager. we override rich table compression by setting # console width to the full width of the table @@ -110,9 +108,7 @@ def rich_table( stack = ExitStack() if pager: - stack.enter_context( - console_width(table, ui.rich_console, SHOW_MAX_WIDTH) - ) + stack.enter_context(console_width(table, ui.rich_console, SHOW_MAX_WIDTH)) stack.enter_context(ui.pager()) with stack: diff --git a/dvc/updater.py b/dvc/updater.py index 96b7749bf..3b5f3fff9 100644 --- a/dvc/updater.py +++ b/dvc/updater.py @@ -78,9 +78,7 @@ class Updater: info = json.load(fobj) latest = info["version"] except Exception as e: # noqa: BLE001 # pylint: disable=W0703 - logger.debug( - "'%s' is not a valid json: %s", self.updater_file, e - ) + logger.debug("'%s' is not a valid json: %s", self.updater_file, e) self.fetch() return @@ -134,9 +132,7 @@ class Updater: f"You are using dvc version [bold]{current}[/]; " f"however, version [bold]{latest}[/] is available." ) - instruction = ui.rich_text.from_markup( - self._get_update_instructions(pkg=pkg) - ) + instruction = ui.rich_text.from_markup(self._get_update_instructions(pkg=pkg)) return ui.rich_text.assemble( "\n", update_message, "\n", instruction, style=color ) @@ -145,8 +141,7 @@ class Updater: def _get_update_instructions(pkg: Optional[str] = None) -> str: if pkg in ("osxpkg", "exe", "binary"): return ( - "To upgrade, uninstall dvc and reinstall from " - "[blue]https://dvc.org[/]." + "To upgrade, uninstall dvc and reinstall from [blue]https://dvc.org[/]." ) instructions = { @@ -173,9 +168,7 @@ class Updater: enabled = to_bool( Config(validate=False).get("core", {}).get("check_update", "true") ) - logger.debug( - "Check for update is %sabled.", "en" if enabled else "dis" - ) + logger.debug("Check for update is %sabled.", "en" if enabled else "dis") return enabled diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index 6c4f15ebf..508b90137 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -32,9 +32,7 @@ def dict_filter(d, exclude=()): if isinstance(d, list): return [dict_filter(e, exclude) for e in d] - return { - k: dict_filter(v, exclude) for k, v in d.items() if k not in exclude - } + return {k: dict_filter(v, exclude) for k, v in d.items() if k not in exclude} def dict_hash(d, typ, exclude=()): @@ -303,9 +301,7 @@ def resolve_paths(repo, out, always_local=False): "Cannot add files inside symlinked directories to DVC. " "See {} for more information." ).format( - format_link( - "https://dvc.org/doc/user-guide/troubleshooting#add-symlink" - ) + format_link("https://dvc.org/doc/user-guide/troubleshooting#add-symlink") ) raise DvcException(msg) else: @@ -362,18 +358,14 @@ def parse_target( if path: if os.path.basename(path) == LOCK_FILE: raise DvcException( - "Did you mean: `{}`?".format( - target.replace(".lock", ".yaml", 1) - ) + "Did you mean: `{}`?".format(target.replace(".lock", ".yaml", 1)) ) if not name: ret = (target, None) return ret if is_valid_filename(target) else ret[::-1] if not path: - logger.trace( # type: ignore[attr-defined] - "Assuming file to be '%s'", default - ) + logger.trace("Assuming file to be '%s'", default) # type: ignore[attr-defined] return path or default, name diff --git a/dvc/utils/cli_parse.py b/dvc/utils/cli_parse.py index 4972ec53f..d894eebe2 100644 --- a/dvc/utils/cli_parse.py +++ b/dvc/utils/cli_parse.py @@ -25,7 +25,6 @@ def to_path_overrides( path_overrides = defaultdict(list) for path_param in path_params: - path_and_name = path_param.partition("=")[0] if ":" not in path_and_name: override = path_param diff --git a/dvc/utils/collections.py b/dvc/utils/collections.py index 805353d02..9fd5696f8 100644 --- a/dvc/utils/collections.py +++ b/dvc/utils/collections.py @@ -1,15 +1,7 @@ import inspect from collections.abc import Mapping from functools import wraps -from typing import ( - Callable, - Dict, - Iterable, - List, - TypeVar, - Union, - no_type_check, -) +from typing import Callable, Dict, Iterable, List, TypeVar, Union, no_type_check @no_type_check @@ -27,15 +19,12 @@ def apply_diff(src, dest): # noqa: C901 def is_same_type(a, b): return any( - isinstance(a, t) and isinstance(b, t) - for t in [str, Mapping, Seq, bool] + isinstance(a, t) and isinstance(b, t) for t in [str, Mapping, Seq, bool] ) if isinstance(src, Mapping) and isinstance(dest, Mapping): for key, value in src.items(): - if isinstance(value, Container) and is_same_type( - value, dest.get(key) - ): + if isinstance(value, Container) and is_same_type(value, dest.get(key)): apply_diff(value, dest[key]) elif key not in dest or value != dest[key]: dest[key] = value @@ -46,9 +35,7 @@ def apply_diff(src, dest): # noqa: C901 dest[:] = src else: for i, value in enumerate(src): - if isinstance(value, Container) and is_same_type( - value, dest[i] - ): + if isinstance(value, Container) and is_same_type(value, dest[i]): apply_diff(value, dest[i]) elif value != dest[i]: dest[i] = value @@ -164,9 +151,7 @@ def validate(*validators: Callable, post: bool = False): def inner(*args, **kwargs): ba = sig.bind(*args, **kwargs) ba.apply_defaults() - ba.arguments = _NamespacedDict( # type: ignore[assignment] - ba.arguments - ) + ba.arguments = _NamespacedDict(ba.arguments) # type: ignore[assignment] if not post: for validator in validators: diff --git a/dvc/utils/diff.py b/dvc/utils/diff.py index 269bf31a7..d4d6fe618 100644 --- a/dvc/utils/diff.py +++ b/dvc/utils/diff.py @@ -17,11 +17,7 @@ def _parse(raw): def _diff_vals(old, new, with_unchanged): - if ( - isinstance(new, list) - and isinstance(old, list) - and len(old) == len(new) == 1 - ): + if isinstance(new, list) and isinstance(old, list) and len(old) == len(new) == 1: return _diff_vals(old[0], new[0], with_unchanged) if not with_unchanged and old == new: diff --git a/dvc/utils/fs.py b/dvc/utils/fs.py index 2dd89a291..e7390229a 100644 --- a/dvc/utils/fs.py +++ b/dvc/utils/fs.py @@ -16,9 +16,7 @@ logger = logging.getLogger(__name__) class BasePathNotInCheckedPathException(DvcException): def __init__(self, path, base_path): - msg = "Path: {} does not overlap with base path: {}".format( - path, base_path - ) + msg = f"Path: {path} does not overlap with base path: {base_path}" super().__init__(msg) diff --git a/dvc/utils/humanize.py b/dvc/utils/humanize.py index d7c71722d..e70d889f1 100644 --- a/dvc/utils/humanize.py +++ b/dvc/utils/humanize.py @@ -7,9 +7,7 @@ def join(words): return "" return ( - "{before} and {after}".format( - before=", ".join(words[:-1]), after=words[-1] - ) + "{before} and {after}".format(before=", ".join(words[:-1]), after=words[-1]) if len(words) > 1 else words[0] ) @@ -17,9 +15,7 @@ def join(words): def get_summary(stats): status = ( - (state, len(data) if is_seq(data) else data) - for state, data in stats - if data + (state, len(data) if is_seq(data) else data) for state, data in stats if data ) return join( "{} file{} {}".format(num, "s" if num > 1 else "", state) @@ -30,9 +26,7 @@ def get_summary(stats): ELLIPSIS = "…" -def truncate_text( - text: str, max_length: int, with_ellipsis: bool = True -) -> str: +def truncate_text(text: str, max_length: int, with_ellipsis: bool = True) -> str: if with_ellipsis and len(text) > max_length: return text[: max_length - 1] + ELLIPSIS diff --git a/dvc/utils/hydra.py b/dvc/utils/hydra.py index 5674c9696..c83be6f2f 100644 --- a/dvc/utils/hydra.py +++ b/dvc/utils/hydra.py @@ -106,8 +106,7 @@ def get_hydra_sweeps(path_overrides): for override in overrides: if override.value_type == ValueType.GLOB_CHOICE_SWEEP: raise InvalidArgumentError( - f"Glob override '{override.input_line}' " - "is not supported." + f"Glob override '{override.input_line}' is not supported." ) path_sweeps[path] = BasicSweeper.split_arguments(overrides, None)[0] return dict_product(path_sweeps) diff --git a/dvc/utils/plots.py b/dvc/utils/plots.py index 80d5c145a..636a22059 100644 --- a/dvc/utils/plots.py +++ b/dvc/utils/plots.py @@ -1,6 +1,4 @@ def get_plot_id(config_plot_id: str, config_file_path: str = ""): return ( - f"{config_file_path}::{config_plot_id}" - if config_file_path - else config_plot_id + f"{config_file_path}::{config_plot_id}" if config_file_path else config_plot_id ) diff --git a/dvc/utils/serialize/__init__.py b/dvc/utils/serialize/__init__.py index 82fc51d21..bd8bfa75a 100644 --- a/dvc/utils/serialize/__init__.py +++ b/dvc/utils/serialize/__init__.py @@ -10,9 +10,7 @@ from ._yaml import * # noqa, pylint: disable=wildcard-import LOADERS: DefaultDict[str, LoaderFn] = defaultdict( # noqa: F405 lambda: load_yaml # noqa: F405 ) -LOADERS.update( - {".toml": load_toml, ".json": load_json, ".py": load_py} # noqa: F405 -) +LOADERS.update({".toml": load_toml, ".json": load_json, ".py": load_py}) # noqa: F405 def load_path(fs_path, fs): @@ -24,9 +22,7 @@ def load_path(fs_path, fs): DUMPERS: DefaultDict[str, DumperFn] = defaultdict( # noqa: F405 lambda: dump_yaml # noqa: F405 ) -DUMPERS.update( - {".toml": dump_toml, ".json": dump_json, ".py": dump_py} # noqa: F405 -) +DUMPERS.update({".toml": dump_toml, ".json": dump_json, ".py": dump_py}) # noqa: F405 MODIFIERS: DefaultDict[str, ModifierFn] = defaultdict( # noqa: F405 lambda: modify_yaml # noqa: F405 diff --git a/dvc/utils/serialize/_common.py b/dvc/utils/serialize/_common.py index 0088e3cfa..7ab6a30ef 100644 --- a/dvc/utils/serialize/_common.py +++ b/dvc/utils/serialize/_common.py @@ -43,9 +43,7 @@ class ModifierFn(Protocol): class LoaderFn(Protocol): - def __call__( - self, path: "StrPath", fs: Optional["FileSystem"] = None - ) -> Any: + def __call__(self, path: "StrPath", fs: Optional["FileSystem"] = None) -> Any: ... @@ -72,9 +70,7 @@ class EncodingError(ParseError): super().__init__(path, f"is not valid {encoding}") -def _load_data( - path: "StrPath", parser: ParserFn, fs: Optional["FileSystem"] = None -): +def _load_data(path: "StrPath", parser: ParserFn, fs: Optional["FileSystem"] = None): open_fn = fs.open if fs else open encoding = "utf-8" with open_fn(path, encoding=encoding) as fd: # type: ignore[operator] @@ -101,9 +97,7 @@ def _modify_data( dumper: DumpersFn, fs: Optional["FileSystem"] = None, ): - exists_fn = cast( - Callable[["StrPath"], bool], fs.exists if fs else os.path.exists - ) + exists_fn = cast(Callable[["StrPath"], bool], fs.exists if fs else os.path.exists) file_exists = exists_fn(cast(str, path)) data = _load_data(path, parser=parser, fs=fs) if file_exists else {} yield data diff --git a/dvc/utils/serialize/_py.py b/dvc/utils/serialize/_py.py index 945410d8e..babed3589 100644 --- a/dvc/utils/serialize/_py.py +++ b/dvc/utils/serialize/_py.py @@ -39,7 +39,6 @@ def parse_py_for_update(text, path): def _dump(data, stream): - old_params = data[_PARAMS_KEY] new_params = { key: value @@ -98,20 +97,12 @@ def _ast_tree_to_dict(tree, only_self_params=False, lineno=False): for _body in tree.body: try: if isinstance(_body, (ast.Assign, ast.AnnAssign)): - result.update( - _ast_assign_to_dict(_body, only_self_params, lineno) - ) + result.update(_ast_assign_to_dict(_body, only_self_params, lineno)) elif isinstance(_body, ast.ClassDef): + result.update({_body.name: _ast_tree_to_dict(_body, lineno=lineno)}) + elif isinstance(_body, ast.FunctionDef) and _body.name == "__init__": result.update( - {_body.name: _ast_tree_to_dict(_body, lineno=lineno)} - ) - elif ( - isinstance(_body, ast.FunctionDef) and _body.name == "__init__" - ): - result.update( - _ast_tree_to_dict( - _body, only_self_params=True, lineno=lineno - ) + _ast_tree_to_dict(_body, only_self_params=True, lineno=lineno) ) except ValueError: continue diff --git a/dvc/utils/strictyaml.py b/dvc/utils/strictyaml.py index d3676413c..cfa8cfa6a 100644 --- a/dvc/utils/strictyaml.py +++ b/dvc/utils/strictyaml.py @@ -53,9 +53,7 @@ def _prepare_cause(cause: str) -> "RichText": return ui.rich_text(cause, style="bold") -def _prepare_code_snippets( - code: str, start_line: int = 1, **kwargs: Any -) -> "Syntax": +def _prepare_code_snippets(code: str, start_line: int = 1, **kwargs: Any) -> "Syntax": from rich.syntax import Syntax kwargs.setdefault("start_line", start_line) @@ -116,9 +114,7 @@ class YAMLSyntaxError(PrettyDvcException, YAMLFileCorruptedError): lines: List[object] = [] if hasattr(exc, "context"): if exc.context_mark is not None: - lines.append( - prepare_message(str(exc.context), exc.context_mark) - ) + lines.append(prepare_message(str(exc.context), exc.context_mark)) if exc.context_mark is not None and ( exc.problem is None or exc.problem_mark is None @@ -128,9 +124,7 @@ class YAMLSyntaxError(PrettyDvcException, YAMLFileCorruptedError): ): lines.extend([prepare_code(exc.context_mark), ""]) if exc.problem is not None: - lines.append( - prepare_message(str(exc.problem), exc.problem_mark) - ) + lines.append(prepare_message(str(exc.problem), exc.problem_mark)) if exc.problem_mark is not None: lines.append(prepare_code(exc.problem_mark)) diff --git a/dvc/utils/table.py b/dvc/utils/table.py index 31213bd5a..f045b6679 100644 --- a/dvc/utils/table.py +++ b/dvc/utils/table.py @@ -26,10 +26,7 @@ class Table(RichTable): last_collapsed = -1 columns = self.columns for i in range(len(columns) - 1, -1, -1): - if ( - widths[i] == 0 - and columns[i].collapse # type: ignore[attr-defined] - ): + if widths[i] == 0 and columns[i].collapse: # type: ignore[attr-defined] if last_collapsed >= 0: del widths[last_collapsed] del columns[last_collapsed] @@ -64,8 +61,7 @@ class Table(RichTable): handling will be used. """ collapsible = [ - column.collapse # type: ignore[attr-defined] - for column in self.columns + column.collapse for column in self.columns # type: ignore[attr-defined] ] total_width = sum(widths) excess_width = total_width - max_width diff --git a/dvc/utils/threadpool.py b/dvc/utils/threadpool.py index e621c05a5..5d75c2e75 100644 --- a/dvc/utils/threadpool.py +++ b/dvc/utils/threadpool.py @@ -14,7 +14,7 @@ class ThreadPoolExecutor(futures.ThreadPoolExecutor): self, max_workers: Optional[int] = None, cancel_on_error: bool = False, - **kwargs + **kwargs, ): super().__init__(max_workers=max_workers, **kwargs) self._cancel_on_error = cancel_on_error @@ -37,9 +37,7 @@ class ThreadPoolExecutor(futures.ThreadPoolExecutor): it = zip(*iterables) tasks = create_taskset(self.max_workers * 5) while tasks: - done, tasks = futures.wait( - tasks, return_when=futures.FIRST_COMPLETED - ) + done, tasks = futures.wait(tasks, return_when=futures.FIRST_COMPLETED) for fut in done: yield fut.result() tasks.update(create_taskset(len(done))) diff --git a/pyproject.toml b/pyproject.toml index 93f19888b..d99d17e56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -135,7 +135,6 @@ find = {namespaces = false, exclude=["tests", "tests.*"]} write_to = "dvc/_dvc_version.py" [tool.black] -line-length = 79 include = '\.pyi?$' exclude = ''' /( @@ -155,10 +154,9 @@ exclude = ''' [tool.isort] profile = "black" known_first_party = ["dvc", "dvc_data", "dvc_objects", "dvc_render", "dvc_task", "tests"] -line_length = 79 +line_length = 88 [tool.ruff] -line-length = 79 target-version = "py38" show-source = true ignore = ["S101", "PT004", "PT007", "PT019", "SIM105", "SIM108", "SIM110", "SIM111", "SIM117", "TRY003", "TRY200", "TRY300", "TRY301", "PLR0913", "PLR2004", "RUF005"] diff --git a/scripts/build.py b/scripts/build.py index 37302bc31..2bab7372f 100644 --- a/scripts/build.py +++ b/scripts/build.py @@ -10,9 +10,7 @@ innosetup = path / "innosetup" fpm = path / "fpm" parser = argparse.ArgumentParser() -parser.add_argument( - "pkg", choices=["deb", "rpm", "osxpkg", "exe"], help="package type" -) +parser.add_argument("pkg", choices=["deb", "rpm", "osxpkg", "exe"], help="package type") parser.add_argument("--sign-application", default=False, action="store_true") parser.add_argument("--application-id") parser.add_argument("--sign-installer", default=False, action="store_true") @@ -61,9 +59,7 @@ else: if args.sign_installer: if args.pkg != "osxpkg": raise NotImplementedError - if not all( - [args.installer_id, args.apple_id_username, args.apple_id_password] - ): + if not all([args.installer_id, args.apple_id_username, args.apple_id_password]): print("--sign-installer requires --installer-id") sys.exit(1) check_call( @@ -76,9 +72,7 @@ if args.notarize: if args.pkg != "osxpkg": raise NotImplementedError if not all([args.apple_id_username, args.apple_id_password]): - print( - "--notarize requires --apple-id-username and --apple-id-password" - ) + print("--notarize requires --apple-id-username and --apple-id-password") sys.exit(1) check_call( [ diff --git a/scripts/fpm/build.py b/scripts/fpm/build.py index 952462bd6..7b46ad352 100644 --- a/scripts/fpm/build.py +++ b/scripts/fpm/build.py @@ -12,9 +12,7 @@ build = path / "build" install = build / "usr" parser = argparse.ArgumentParser() -parser.add_argument( - "pkg", choices=["deb", "rpm", "osxpkg"], help="package type" -) +parser.add_argument("pkg", choices=["deb", "rpm", "osxpkg"], help="package type") args = parser.parse_args() flags = [ diff --git a/scripts/fpm/notarize.py b/scripts/fpm/notarize.py index bca3e4514..665e5dcb1 100644 --- a/scripts/fpm/notarize.py +++ b/scripts/fpm/notarize.py @@ -14,9 +14,7 @@ parser.add_argument( nargs="?", help="Path to the osxpkg to notarize. If not specified - try to find one.", ) -parser.add_argument( - "--apple-id-username", required=True, help="Apple ID username." -) +parser.add_argument("--apple-id-username", required=True, help="Apple ID username.") parser.add_argument( "--apple-id-password", required=True, diff --git a/scripts/pyinstaller/hooks/hook-dvc.py b/scripts/pyinstaller/hooks/hook-dvc.py index bc8b3c3c2..3c84707b8 100644 --- a/scripts/pyinstaller/hooks/hook-dvc.py +++ b/scripts/pyinstaller/hooks/hook-dvc.py @@ -1,6 +1,4 @@ -from PyInstaller.utils.hooks import ( # pylint:disable=import-error - copy_metadata, -) +from PyInstaller.utils.hooks import copy_metadata # pylint:disable=import-error # needed for `dvc doctor` to show dep versions datas = copy_metadata("adlfs", recursive=True) diff --git a/scripts/pyinstaller/hooks/hook-dvc.tree.gs.py b/scripts/pyinstaller/hooks/hook-dvc.tree.gs.py index 697876a0b..8c36c8471 100644 --- a/scripts/pyinstaller/hooks/hook-dvc.tree.gs.py +++ b/scripts/pyinstaller/hooks/hook-dvc.tree.gs.py @@ -1,6 +1,4 @@ -from PyInstaller.utils.hooks import ( # pylint:disable=import-error - copy_metadata, -) +from PyInstaller.utils.hooks import copy_metadata # pylint:disable=import-error datas = copy_metadata("google-cloud-storage") diff --git a/scripts/pyinstaller/hooks/hook-dvc.utils.flatten.py b/scripts/pyinstaller/hooks/hook-dvc.utils.flatten.py index a4f7faf4e..bc23f359f 100644 --- a/scripts/pyinstaller/hooks/hook-dvc.utils.flatten.py +++ b/scripts/pyinstaller/hooks/hook-dvc.utils.flatten.py @@ -1,5 +1,3 @@ -from PyInstaller.utils.hooks import ( # pylint:disable=import-error - copy_metadata, -) +from PyInstaller.utils.hooks import copy_metadata # pylint:disable=import-error datas = copy_metadata("flatten-dict") diff --git a/scripts/pyinstaller/hooks/hook-google_compute_engine.logger.py b/scripts/pyinstaller/hooks/hook-google_compute_engine.logger.py index 970554d2e..9cc2aec62 100644 --- a/scripts/pyinstaller/hooks/hook-google_compute_engine.logger.py +++ b/scripts/pyinstaller/hooks/hook-google_compute_engine.logger.py @@ -1,6 +1,4 @@ -from PyInstaller.utils.hooks import ( # pylint:disable=import-error - copy_metadata, -) +from PyInstaller.utils.hooks import copy_metadata # pylint:disable=import-error datas = copy_metadata("google-compute-engine") hiddenimports = ["google_cloud_engine"] diff --git a/scripts/pyinstaller/hooks/hook-pydrive2.py b/scripts/pyinstaller/hooks/hook-pydrive2.py index 45dc6aba3..6315db70b 100644 --- a/scripts/pyinstaller/hooks/hook-pydrive2.py +++ b/scripts/pyinstaller/hooks/hook-pydrive2.py @@ -1,5 +1,3 @@ -from PyInstaller.utils.hooks import ( # pylint:disable=import-error - copy_metadata, -) +from PyInstaller.utils.hooks import copy_metadata # pylint:disable=import-error datas = copy_metadata("pydrive2") diff --git a/tests/conftest.py b/tests/conftest.py index 2c49d011d..e9492c94c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -119,10 +119,7 @@ def pytest_runtest_setup(item): for marker in item.iter_markers(): item.config.dvc_config.apply_marker(marker) - if ( - "CI" in os.environ - and item.get_closest_marker("needs_internet") is not None - ): + if "CI" in os.environ and item.get_closest_marker("needs_internet") is not None: # remotes that need internet connection might be flaky, # so we rerun them in case it fails. item.add_marker(pytest.mark.flaky(max_runs=5, min_passes=1)) diff --git a/tests/func/api/test_data.py b/tests/func/api/test_data.py index 8815764ec..651451cd7 100644 --- a/tests/func/api/test_data.py +++ b/tests/func/api/test_data.py @@ -4,14 +4,8 @@ import pytest from funcy import first, get_in from dvc import api -from dvc.exceptions import ( - FileMissingError, - OutputNotFoundError, - PathMissingError, -) -from dvc.testing.api_tests import ( # noqa, pylint: disable=unused-import - TestAPI, -) +from dvc.exceptions import FileMissingError, OutputNotFoundError, PathMissingError +from dvc.testing.api_tests import TestAPI # noqa, pylint: disable=unused-import from dvc.testing.tmp_dir import make_subrepo from dvc.utils.fs import remove @@ -95,13 +89,11 @@ def test_open_scm_controlled(tmp_dir, erepo_dir): def test_open_not_cached(dvc): metric_file = "metric.txt" metric_content = "0.6" - metric_code = "open('{}', 'w').write('{}')".format( - metric_file, metric_content - ) + metric_code = f"open('{metric_file}', 'w').write('{metric_content}')" dvc.run( single_stage=True, metrics_no_cache=[metric_file], - cmd=(f'python -c "{metric_code}"'), + cmd=f'python -c "{metric_code}"', ) with api.open(metric_file) as fd: @@ -160,13 +152,9 @@ def test_read_with_subrepos(tmp_dir, scm, local_cloud, local_repo): subrepo_path = os.path.join("dir", "subrepo") assert api.read("foo.txt", repo=repo_path) == "foo.txt" + assert api.read(os.path.join(subrepo_path, "lorem"), repo=repo_path) == "lorem" assert ( - api.read(os.path.join(subrepo_path, "lorem"), repo=repo_path) - == "lorem" - ) - assert ( - api.read(os.path.join(subrepo_path, "dvc-file"), repo=repo_path) - == "dvc-file" + api.read(os.path.join(subrepo_path, "dvc-file"), repo=repo_path) == "dvc-file" ) assert ( api.read(os.path.join(subrepo_path, "dir", "file.txt"), repo=repo_path) @@ -176,9 +164,7 @@ def test_read_with_subrepos(tmp_dir, scm, local_cloud, local_repo): def test_get_url_granular(tmp_dir, dvc, cloud): tmp_dir.add_remote(config=cloud.config) - tmp_dir.dvc_gen( - {"dir": {"foo": "foo", "bar": "bar", "nested": {"file": "file"}}} - ) + tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar", "nested": {"file": "file"}}}) expected_url = (cloud / "5f" / "c28ea78987408341668eba6525ebd1.dir").url assert api.get_url("dir") == expected_url @@ -197,19 +183,13 @@ def test_get_url_subrepos(tmp_dir, scm, local_cloud): subrepo = tmp_dir / "subrepo" make_subrepo(subrepo, scm, config=local_cloud.config) with subrepo.chdir(): - subrepo.dvc_gen( - {"dir": {"foo": "foo"}, "bar": "bar"}, commit="add files" - ) + subrepo.dvc_gen({"dir": {"foo": "foo"}, "bar": "bar"}, commit="add files") subrepo.dvc.push() - expected_url = os.fspath( - local_cloud / "ac" / "bd18db4cc2f85cedef654fccc4a4d8" - ) + expected_url = os.fspath(local_cloud / "ac" / "bd18db4cc2f85cedef654fccc4a4d8") assert api.get_url(os.path.join("subrepo", "dir", "foo")) == expected_url - expected_url = os.fspath( - local_cloud / "37" / "b51d194a7513e45b56f6524f2d51f2" - ) + expected_url = os.fspath(local_cloud / "37" / "b51d194a7513e45b56f6524f2d51f2") assert api.get_url("subrepo/bar") == expected_url diff --git a/tests/func/experiments/test_checkpoints.py b/tests/func/experiments/test_checkpoints.py index ad89a1c09..35d6fa8ab 100644 --- a/tests/func/experiments/test_checkpoints.py +++ b/tests/func/experiments/test_checkpoints.py @@ -14,9 +14,7 @@ from dvc.scm import InvalidRemoteSCMRepo, RevError @pytest.mark.parametrize("links", ["reflink,copy", "hardlink,symlink"]) -def test_new_checkpoint( - tmp_dir, scm, dvc, checkpoint_stage, mocker, workspace, links -): +def test_new_checkpoint(tmp_dir, scm, dvc, checkpoint_stage, mocker, workspace, links): with dvc.config.edit() as conf: conf["cache"]["type"] = links @@ -40,9 +38,7 @@ def test_new_checkpoint( assert scm.get_ref(EXEC_APPLY) == exp assert scm.get_ref(EXEC_CHECKPOINT) == exp if workspace: - assert (tmp_dir / "foo").read_text().strip() == str( - checkpoint_stage.iterations - ) + assert (tmp_dir / "foo").read_text().strip() == str(checkpoint_stage.iterations) assert (tmp_dir / "metrics.yaml").read_text().strip() == "foo: 2" @@ -87,9 +83,7 @@ def test_resume_checkpoint( assert scm.get_ref(EXEC_CHECKPOINT) == exp -def test_reset_checkpoint( - tmp_dir, scm, dvc, checkpoint_stage, caplog, workspace -): +def test_reset_checkpoint(tmp_dir, scm, dvc, checkpoint_stage, caplog, workspace): dvc.experiments.run(checkpoint_stage.addressing, tmp_dir=not workspace) results = dvc.experiments.run( @@ -164,9 +158,7 @@ def test_resume_branch(tmp_dir, scm, dvc, checkpoint_stage, workspace): ) -def test_resume_non_head_checkpoint( - tmp_dir, scm, dvc, checkpoint_stage, workspace -): +def test_resume_non_head_checkpoint(tmp_dir, scm, dvc, checkpoint_stage, workspace): orig_head = scm.get_rev() results = dvc.experiments.run( checkpoint_stage.addressing, params=["foo=2"], tmp_dir=not workspace @@ -177,9 +169,7 @@ def test_resume_non_head_checkpoint( rev = list(scm.branch_revs(checkpoint_head, orig_head))[-1] dvc.experiments.apply(rev) - with pytest.raises( - DvcException, match="Nothing to do for unchanged checkpoint" - ): + with pytest.raises(DvcException, match="Nothing to do for unchanged checkpoint"): dvc.experiments.run(checkpoint_stage.addressing, tmp_dir=not workspace) results = dvc.experiments.run( @@ -263,10 +253,7 @@ def test_auto_push_self_remote( root_dir = str(tmp_dir) monkeypatch.setenv(DVC_EXP_GIT_REMOTE, root_dir) monkeypatch.setenv(DVC_EXP_AUTO_PUSH, "true") - assert ( - dvc.experiments.run(checkpoint_stage.addressing, params=["foo=2"]) - != {} - ) + assert dvc.experiments.run(checkpoint_stage.addressing, params=["foo=2"]) != {} with caplog.at_level(logging.WARNING, logger="dvc.repo.experiments"): assert ( @@ -277,9 +264,7 @@ def test_auto_push_self_remote( ) -def test_tmp_dir_failed_checkpoint( - tmp_dir, scm, dvc, failed_checkpoint_stage, caplog -): +def test_tmp_dir_failed_checkpoint(tmp_dir, scm, dvc, failed_checkpoint_stage, caplog): baseline = scm.get_rev() results = dvc.experiments.run( diff --git a/tests/func/experiments/test_experiments.py b/tests/func/experiments/test_experiments.py index 4caf3ce94..0e742854b 100644 --- a/tests/func/experiments/test_experiments.py +++ b/tests/func/experiments/test_experiments.py @@ -112,9 +112,7 @@ def test_apply(tmp_dir, scm, dvc, exp_stage): from dvc.exceptions import InvalidArgumentError from dvc.repo.experiments.exceptions import ApplyConflictError - results = dvc.experiments.run( - exp_stage.addressing, params=["foo=2"], tmp_dir=True - ) + results = dvc.experiments.run(exp_stage.addressing, params=["foo=2"], tmp_dir=True) exp_a = first(results) results = dvc.experiments.run( @@ -151,9 +149,7 @@ def test_apply(tmp_dir, scm, dvc, exp_stage): def test_apply_failed(tmp_dir, scm, dvc, failed_exp_stage, test_queue): from dvc.exceptions import InvalidArgumentError - dvc.experiments.run( - failed_exp_stage.addressing, params=["foo=2"], queue=True - ) + dvc.experiments.run(failed_exp_stage.addressing, params=["foo=2"], queue=True) exp_rev = dvc.experiments.scm.resolve_rev(f"{CELERY_STASH}@{{0}}") dvc.experiments.run( @@ -218,7 +214,6 @@ def test_apply_untracked(tmp_dir, scm, dvc, exp_stage): def test_get_baseline(tmp_dir, scm, dvc, exp_stage): - init_rev = scm.get_rev() assert dvc.experiments.get_baseline(init_rev) is None @@ -267,9 +262,11 @@ def test_update_py_params(tmp_dir, scm, dvc, test_queue, copy_script): tmp_dir.gen( "params.py", - "INT = 1\nFLOAT = 0.001\nDICT = {'a': 1}\n\n" - "class Train:\n seed = 2020\n\n" - "class Klass:\n def __init__(self):\n self.a = 111\n", + ( + "INT = 1\nFLOAT = 0.001\nDICT = {'a': 1}\n\n" + "class Train:\n seed = 2020\n\n" + "class Klass:\n def __init__(self):\n self.a = 111\n" + ), ) stage = dvc.run( cmd="python copy.py params.py metrics.py", @@ -322,9 +319,7 @@ def test_update_py_params(tmp_dir, scm, dvc, test_queue, copy_script): scm.commit("init") with pytest.raises(PythonFileCorruptedError): - dvc.experiments.run( - stage.addressing, params=["params.py:INT=2a"], tmp_dir=True - ) + dvc.experiments.run(stage.addressing, params=["params.py:INT=2a"], tmp_dir=True) def test_detached_parent(tmp_dir, scm, dvc, exp_stage, mocker): @@ -352,9 +347,7 @@ def test_branch(tmp_dir, scm, dvc, exp_stage): scm.branch("branch-exists") - results = dvc.experiments.run( - exp_stage.addressing, params=["foo=2"], name="foo" - ) + results = dvc.experiments.run(exp_stage.addressing, params=["foo=2"], name="foo") exp_a = first(results) ref_a = dvc.experiments.get_branch_by_rev(exp_a) @@ -369,9 +362,7 @@ def test_branch(tmp_dir, scm, dvc, exp_stage): assert scm.resolve_rev(name) == exp_a tmp_dir.scm_gen({"new_file": "new_file"}, commit="new baseline") - results = dvc.experiments.run( - exp_stage.addressing, params=["foo=2"], name="foo" - ) + results = dvc.experiments.run(exp_stage.addressing, params=["foo=2"], name="foo") exp_b = first(results) ref_b = dvc.experiments.get_branch_by_rev(exp_b) @@ -500,9 +491,7 @@ def test_subdir(tmp_dir, scm, dvc, workspace): name="copy-file", no_exec=True, ) - scm.add( - [subdir / "dvc.yaml", subdir / "copy.py", subdir / "params.yaml"] - ) + scm.add([subdir / "dvc.yaml", subdir / "copy.py", subdir / "params.yaml"]) scm.commit("init") results = dvc.experiments.run( @@ -590,9 +579,7 @@ def test_run_celery(tmp_dir, scm, dvc, exp_stage, mocker): def test_run_metrics(tmp_dir, scm, dvc, exp_stage, mocker): from dvc.cli import main - mocker.patch.object( - dvc.experiments, "run", return_value={"abc123": "abc123"} - ) + mocker.patch.object(dvc.experiments, "run", return_value={"abc123": "abc123"}) show_mock = mocker.patch.object(dvc.metrics, "show", return_value={}) main(["exp", "run", "-m"]) @@ -638,9 +625,7 @@ def test_fix_exp_head(tmp_dir, scm, tail): "params, target", itertools.product(("foo: 1", "foo: 2"), (True, False)), ) -def test_modified_data_dep( - tmp_dir, scm, dvc, workspace, params, target, copy_script -): +def test_modified_data_dep(tmp_dir, scm, dvc, workspace, params, target, copy_script): tmp_dir.dvc_gen("data", "data") tmp_dir.gen("params.yaml", "foo: 1") exp_stage = dvc.run( diff --git a/tests/func/experiments/test_init.py b/tests/func/experiments/test_init.py index 5e200e547..e41e20373 100644 --- a/tests/func/experiments/test_init.py +++ b/tests/func/experiments/test_init.py @@ -61,8 +61,8 @@ def test_when_stage_already_exists_with_same_name(tmp_dir, dvc, interactive): defaults=CmdExperimentsInit.DEFAULTS, ) assert ( - str(exc.value) == "Stage 'train' already exists in 'dvc.yaml'. " - "Use '--force' to overwrite." + str(exc.value) + == "Stage 'train' already exists in 'dvc.yaml'. Use '--force' to overwrite." ) @@ -95,9 +95,7 @@ def test_creates_params_file_by_default(tmp_dir, dvc, interactive): def test_with_a_custom_name(tmp_dir, dvc): init(dvc, name="custom", overrides={"cmd": "cmd"}) - assert (tmp_dir / "dvc.yaml").parse() == { - "stages": {"custom": {"cmd": "cmd"}} - } + assert (tmp_dir / "dvc.yaml").parse() == {"stages": {"custom": {"cmd": "cmd"}}} def test_init_with_no_defaults_non_interactive(tmp_dir, scm, dvc): @@ -154,9 +152,7 @@ def test_when_params_is_omitted_in_interactive_mode(tmp_dir, scm, dvc): (tmp_dir / "params.yaml").dump({"foo": 1}) inp = io.StringIO("python script.py\nscript.py\ndata\nn") - init( - dvc, interactive=True, stream=inp, defaults=CmdExperimentsInit.DEFAULTS - ) + init(dvc, interactive=True, stream=inp, defaults=CmdExperimentsInit.DEFAULTS) assert (tmp_dir / "dvc.yaml").parse() == { "stages": { @@ -184,9 +180,7 @@ def test_init_interactive_params_validation(tmp_dir, dvc, capsys): (tmp_dir / "params.yaml").dump({"foo": 1}) inp = io.StringIO("python script.py\nscript.py\ndata\ndata") - init( - dvc, stream=inp, interactive=True, defaults=CmdExperimentsInit.DEFAULTS - ) + init(dvc, stream=inp, interactive=True, defaults=CmdExperimentsInit.DEFAULTS) assert (tmp_dir / "dvc.yaml").parse() == { "stages": { @@ -304,9 +298,7 @@ def test_init_default(tmp_dir, scm, dvc, interactive, overrides, inp, capsys): ( True, {}, - io.StringIO( - "python script.py\nscript.py\ndata\nparams.yaml\nmodels\ny" - ), + io.StringIO("python script.py\nscript.py\ndata\nparams.yaml\nmodels\ny"), ), ( True, @@ -326,9 +318,7 @@ def test_init_default(tmp_dir, scm, dvc, interactive, overrides, inp, capsys): "interactive-cmd-models-provided", ], ) -def test_init_interactive_live( - tmp_dir, scm, dvc, interactive, overrides, inp, capsys -): +def test_init_interactive_live(tmp_dir, scm, dvc, interactive, overrides, inp, capsys): overrides["live"] = "dvclive" (tmp_dir / "params.yaml").dump({"foo": {"bar": 1}}) @@ -350,9 +340,7 @@ def test_init_interactive_live( ], "outs": ["models"], "params": [{"params.yaml": None}], - "plots": [ - {os.path.join("dvclive", "plots"): {"cache": False}} - ], + "plots": [{os.path.join("dvclive", "plots"): {"cache": False}}], } } } @@ -419,9 +407,7 @@ def test_init_with_type_checkpoint_and_models_plots_provided( (True, io.StringIO()), ], ) -def test_init_with_type_default_and_live_provided( - tmp_dir, dvc, interactive, inp -): +def test_init_with_type_default_and_live_provided(tmp_dir, dvc, interactive, inp): (tmp_dir / "params.yaml").dump({"foo": 1}) init( dvc, @@ -457,9 +443,7 @@ def test_init_with_type_default_and_live_provided( (True, io.StringIO()), ], ) -def test_init_with_live_and_metrics_plots_provided( - tmp_dir, dvc, interactive, inp -): +def test_init_with_live_and_metrics_plots_provided(tmp_dir, dvc, interactive, inp): (tmp_dir / "params.yaml").dump({"foo": 1}) init( dvc, diff --git a/tests/func/experiments/test_remote.py b/tests/func/experiments/test_remote.py index 0cd4b88d4..b8bcc7189 100644 --- a/tests/func/experiments/test_remote.py +++ b/tests/func/experiments/test_remote.py @@ -38,12 +38,8 @@ def test_push(tmp_dir, scm, dvc, git_upstream, exp_stage, use_url): assert git_upstream.tmp_dir.scm.get_ref(str(ref_info1)) == exp1 -@pytest.mark.parametrize( - "all_,rev,result3", [(True, False, True), (False, True, None)] -) -def test_push_args( - tmp_dir, scm, dvc, git_upstream, exp_stage, all_, rev, result3 -): +@pytest.mark.parametrize("all_,rev,result3", [(True, False, True), (False, True, None)]) +def test_push_args(tmp_dir, scm, dvc, git_upstream, exp_stage, all_, rev, result3): remote = git_upstream.url baseline = scm.get_rev() @@ -88,18 +84,14 @@ def test_push_diverged(tmp_dir, scm, dvc, git_upstream, exp_stage): def test_push_checkpoint(tmp_dir, scm, dvc, git_upstream, checkpoint_stage): - results = dvc.experiments.run( - checkpoint_stage.addressing, params=["foo=2"] - ) + results = dvc.experiments.run(checkpoint_stage.addressing, params=["foo=2"]) exp_a = first(results) ref_info_a = first(exp_refs_by_rev(scm, exp_a)) dvc.experiments.push(git_upstream.remote, [ref_info_a.name], force=True) assert git_upstream.tmp_dir.scm.get_ref(str(ref_info_a)) == exp_a - results = dvc.experiments.run( - checkpoint_stage.addressing, checkpoint_resume=exp_a - ) + results = dvc.experiments.run(checkpoint_stage.addressing, checkpoint_resume=exp_a) exp_b = first(results) ref_info_b = first(exp_refs_by_rev(scm, exp_b)) @@ -114,16 +106,12 @@ def test_push_ambiguous_name(tmp_dir, scm, dvc, git_upstream, exp_stage): remote = git_upstream.remote - results = dvc.experiments.run( - exp_stage.addressing, params=["foo=2"], name="foo" - ) + results = dvc.experiments.run(exp_stage.addressing, params=["foo=2"], name="foo") exp_a = first(results) ref_info_a = first(exp_refs_by_rev(scm, exp_a)) tmp_dir.scm_gen("new", "new", commit="new") - results = dvc.experiments.run( - exp_stage.addressing, params=["foo=3"], name="foo" - ) + results = dvc.experiments.run(exp_stage.addressing, params=["foo=3"], name="foo") exp_b = first(results) ref_info_b = first(exp_refs_by_rev(scm, exp_b)) @@ -206,12 +194,8 @@ def test_pull(tmp_dir, scm, dvc, git_downstream, exp_stage, use_url): assert git_downstream.tmp_dir.scm.get_ref(str(ref_info1)) == exp1 -@pytest.mark.parametrize( - "all_,rev,result3", [(True, False, True), (False, True, None)] -) -def test_pull_args( - tmp_dir, scm, dvc, git_downstream, exp_stage, all_, rev, result3 -): +@pytest.mark.parametrize("all_,rev,result3", [(True, False, True), (False, True, None)]) +def test_pull_args(tmp_dir, scm, dvc, git_downstream, exp_stage, all_, rev, result3): baseline = scm.get_rev() results = dvc.experiments.run(exp_stage.addressing, params=["foo=1"]) @@ -259,9 +243,7 @@ def test_pull_diverged(tmp_dir, scm, dvc, git_downstream, exp_stage): def test_pull_checkpoint(tmp_dir, scm, dvc, git_downstream, checkpoint_stage): - results = dvc.experiments.run( - checkpoint_stage.addressing, params=["foo=2"] - ) + results = dvc.experiments.run(checkpoint_stage.addressing, params=["foo=2"]) exp_a = first(results) ref_info_a = first(exp_refs_by_rev(scm, exp_a)) @@ -269,9 +251,7 @@ def test_pull_checkpoint(tmp_dir, scm, dvc, git_downstream, checkpoint_stage): downstream_exp.pull(git_downstream.remote, [ref_info_a.name], force=True) assert git_downstream.tmp_dir.scm.get_ref(str(ref_info_a)) == exp_a - results = dvc.experiments.run( - checkpoint_stage.addressing, checkpoint_resume=exp_a - ) + results = dvc.experiments.run(checkpoint_stage.addressing, checkpoint_resume=exp_a) exp_b = first(results) ref_info_b = first(exp_refs_by_rev(scm, exp_b)) @@ -282,16 +262,12 @@ def test_pull_checkpoint(tmp_dir, scm, dvc, git_downstream, checkpoint_stage): def test_pull_ambiguous_name(tmp_dir, scm, dvc, git_downstream, exp_stage): from dvc.exceptions import InvalidArgumentError - results = dvc.experiments.run( - exp_stage.addressing, params=["foo=2"], name="foo" - ) + results = dvc.experiments.run(exp_stage.addressing, params=["foo=2"], name="foo") exp_a = first(results) ref_info_a = first(exp_refs_by_rev(scm, exp_a)) tmp_dir.scm_gen("new", "new", commit="new") - results = dvc.experiments.run( - exp_stage.addressing, params=["foo=3"], name="foo" - ) + results = dvc.experiments.run(exp_stage.addressing, params=["foo=3"], name="foo") exp_b = first(results) ref_info_b = first(exp_refs_by_rev(scm, exp_b)) @@ -315,9 +291,7 @@ def test_push_pull_cache( from tests.func.test_diff import digest remote = git_upstream.remote - results = dvc.experiments.run( - checkpoint_stage.addressing, params=["foo=2"] - ) + results = dvc.experiments.run(checkpoint_stage.addressing, params=["foo=2"]) exp = first(results) ref_info = first(exp_refs_by_rev(scm, exp)) diff --git a/tests/func/experiments/test_remove.py b/tests/func/experiments/test_remove.py index 43fed7d72..e2fc557e4 100644 --- a/tests/func/experiments/test_remove.py +++ b/tests/func/experiments/test_remove.py @@ -11,9 +11,7 @@ def test_remove_experiments_by_ref(tmp_dir, scm, dvc, exp_stage, caplog): ref_name_list = [] for i in range(queue_length): - results = dvc.experiments.run( - exp_stage.addressing, params=[f"foo={i}"] - ) + results = dvc.experiments.run(exp_stage.addressing, params=[f"foo={i}"]) ref_info = first(exp_refs_by_rev(scm, first(results))) ref_info_list.append(ref_info) ref_name_list.append(str(ref_info)) @@ -24,9 +22,7 @@ def test_remove_experiments_by_ref(tmp_dir, scm, dvc, exp_stage, caplog): assert scm.get_ref(ref_name_list[1]) is not None assert scm.get_ref(ref_name_list[2]) is not None - assert set(dvc.experiments.remove(ref_name_list[:2])) == set( - ref_name_list[:2] - ) + assert set(dvc.experiments.remove(ref_name_list[:2])) == set(ref_name_list[:2]) assert scm.get_ref(ref_name_list[0]) is None assert scm.get_ref(ref_name_list[1]) is None assert scm.get_ref(ref_name_list[2]) is not None @@ -35,13 +31,9 @@ def test_remove_experiments_by_ref(tmp_dir, scm, dvc, exp_stage, caplog): def test_remove_all_queued_experiments(tmp_dir, scm, dvc, exp_stage): queue_length = 3 for i in range(queue_length): - dvc.experiments.run( - exp_stage.addressing, params=[f"foo={i}"], queue=True - ) + dvc.experiments.run(exp_stage.addressing, params=[f"foo={i}"], queue=True) - results = dvc.experiments.run( - exp_stage.addressing, params=[f"foo={queue_length}"] - ) + results = dvc.experiments.run(exp_stage.addressing, params=[f"foo={queue_length}"]) ref_info = first(exp_refs_by_rev(scm, first(results))) assert len(dvc.experiments.stash_revs) == queue_length @@ -75,9 +67,11 @@ def test_remove_special_queued_experiments(tmp_dir, scm, dvc, exp_stage): assert scm.get_ref(str(ref_info2)) is not None rev2 = queue_revs["queue2"] - assert set( - dvc.experiments.remove(["queue1", rev2[:5], str(ref_info1)]) - ) == {"queue1", rev2[:5], str(ref_info1)} + assert set(dvc.experiments.remove(["queue1", rev2[:5], str(ref_info1)])) == { + "queue1", + rev2[:5], + str(ref_info1), + } assert len(list(dvc.experiments.celery_queue.iter_queued())) == 1 assert scm.get_ref(str(ref_info1)) is None assert scm.get_ref(str(ref_info2)) is not None @@ -110,9 +104,7 @@ def test_remove_remote(tmp_dir, scm, dvc, exp_stage, git_upstream, use_url): ref_info_list = [] exp_list = [] for i in range(3): - results = dvc.experiments.run( - exp_stage.addressing, params=[f"foo={i}"] - ) + results = dvc.experiments.run(exp_stage.addressing, params=[f"foo={i}"]) exp = first(results) exp_list.append(exp) ref_info = first(exp_refs_by_rev(scm, exp)) @@ -127,9 +119,7 @@ def test_remove_remote(tmp_dir, scm, dvc, exp_stage, git_upstream, use_url): assert git_upstream.tmp_dir.scm.get_ref(str(ref_info_list[0])) is None assert git_upstream.tmp_dir.scm.get_ref(str(ref_info_list[1])) is None - assert ( - git_upstream.tmp_dir.scm.get_ref(str(ref_info_list[2])) == exp_list[2] - ) + assert git_upstream.tmp_dir.scm.get_ref(str(ref_info_list[2])) == exp_list[2] def test_remove_experiments_by_rev(tmp_dir, scm, dvc, exp_stage): diff --git a/tests/func/experiments/test_save.py b/tests/func/experiments/test_save.py index faf3af601..52b1c0f5c 100644 --- a/tests/func/experiments/test_save.py +++ b/tests/func/experiments/test_save.py @@ -1,10 +1,7 @@ import pytest from funcy import first -from dvc.repo.experiments.exceptions import ( - ExperimentExistsError, - InvalidArgumentError, -) +from dvc.repo.experiments.exceptions import ExperimentExistsError, InvalidArgumentError from dvc.repo.experiments.utils import exp_refs_by_rev from dvc.scm import resolve_rev diff --git a/tests/func/experiments/test_set_params.py b/tests/func/experiments/test_set_params.py index dc79e8e36..75b37073c 100644 --- a/tests/func/experiments/test_set_params.py +++ b/tests/func/experiments/test_set_params.py @@ -136,6 +136,4 @@ def test_hydra_sweep_cant_use_name(tmp_dir, params_repo, dvc): InvalidArgumentError, match="Sweep overrides can't be used alongside `--name`", ): - dvc.experiments.run( - params=["db=mysql,postgresql"], queue=True, name="foo" - ) + dvc.experiments.run(params=["db=mysql,postgresql"], queue=True, name="foo") diff --git a/tests/func/experiments/test_show.py b/tests/func/experiments/test_show.py index 787a89d5c..12057878e 100644 --- a/tests/func/experiments/test_show.py +++ b/tests/func/experiments/test_show.py @@ -9,18 +9,10 @@ from funcy import first, get_in from scmrepo.exceptions import SCMError from dvc.cli import main -from dvc.repo.experiments.executor.base import ( - BaseExecutor, - ExecutorInfo, - TaskStatus, -) +from dvc.repo.experiments.executor.base import BaseExecutor, ExecutorInfo, TaskStatus from dvc.repo.experiments.queue.base import QueueEntry from dvc.repo.experiments.refs import CELERY_STASH, ExpRefInfo -from dvc.repo.experiments.utils import ( - EXEC_PID_DIR, - EXEC_TMP_DIR, - exp_refs_by_rev, -) +from dvc.repo.experiments.utils import EXEC_PID_DIR, EXEC_TMP_DIR, exp_refs_by_rev from dvc.utils import relpath from dvc.utils.serialize import YAMLFileCorruptedError @@ -105,9 +97,7 @@ def test_show_experiment(tmp_dir, scm, dvc, exp_stage, workspace): scm.gitpython.repo.rev_parse(baseline_rev).committed_date ) - dvc.experiments.run( - exp_stage.addressing, params=["foo=2"], tmp_dir=not workspace - ) + dvc.experiments.run(exp_stage.addressing, params=["foo=2"], tmp_dir=not workspace) results = dvc.experiments.show() expected_baseline = { @@ -180,9 +170,7 @@ def test_show_failed_experiment(tmp_dir, scm, dvc, failed_exp_stage): scm.gitpython.repo.rev_parse(baseline_rev).committed_date ) - dvc.experiments.run( - failed_exp_stage.addressing, params=["foo=2"], queue=True - ) + dvc.experiments.run(failed_exp_stage.addressing, params=["foo=2"], queue=True) exp_rev = dvc.experiments.scm.resolve_rev(f"{CELERY_STASH}@{{0}}") dvc.experiments.run(run_all=True) experiments = dvc.experiments.show()[baseline_rev] @@ -233,9 +221,7 @@ def test_show_failed_experiment(tmp_dir, scm, dvc, failed_exp_stage): @pytest.mark.vscode @pytest.mark.parametrize("workspace", [True, False]) -def test_show_checkpoint( - tmp_dir, scm, dvc, checkpoint_stage, capsys, workspace -): +def test_show_checkpoint(tmp_dir, scm, dvc, checkpoint_stage, capsys, workspace): baseline_rev = scm.get_rev() results = dvc.experiments.run( checkpoint_stage.addressing, params=["foo=2"], tmp_dir=not workspace @@ -272,9 +258,7 @@ def test_show_checkpoint( @pytest.mark.vscode @pytest.mark.parametrize("workspace", [True, False]) -def test_show_checkpoint_branch( - tmp_dir, scm, dvc, checkpoint_stage, capsys, workspace -): +def test_show_checkpoint_branch(tmp_dir, scm, dvc, checkpoint_stage, capsys, workspace): results = dvc.experiments.run( checkpoint_stage.addressing, params=["foo=2"], tmp_dir=not workspace ) @@ -412,13 +396,9 @@ def test_show_sort(tmp_dir, scm, dvc, exp_stage, caplog): assert main(["exp", "show", "--no-pager", "--sort-by=foo"]) != 0 assert "Ambiguous sort column" in caplog.text - assert ( - main(["exp", "show", "--no-pager", "--sort-by=params.yaml:foo"]) == 0 - ) + assert main(["exp", "show", "--no-pager", "--sort-by=params.yaml:foo"]) == 0 - assert ( - main(["exp", "show", "--no-pager", "--sort-by=metrics.yaml:foo"]) == 0 - ) + assert main(["exp", "show", "--no-pager", "--sort-by=metrics.yaml:foo"]) == 0 @pytest.mark.vscode @@ -437,9 +417,7 @@ def test_show_running_workspace( pid_dir = os.path.join(dvc.tmp_dir, EXEC_TMP_DIR, EXEC_PID_DIR) lock_file = relpath(os.path.join(dvc.tmp_dir, RWLOCK_FILE), str(tmp_dir)) - info = make_executor_info( - location=BaseExecutor.DEFAULT_LOCATION, status=status - ) + info = make_executor_info(location=BaseExecutor.DEFAULT_LOCATION, status=status) pidfile = os.path.join( pid_dir, "workspace", @@ -464,12 +442,16 @@ def test_show_running_workspace( "metrics": {"metrics.yaml": {"data": {"foo": 1}}}, "params": {"params.yaml": {"data": {"foo": 1}}}, "outs": {}, - "status": "Running" - if status == TaskStatus.RUNNING and pid_exists - else "Success", - "executor": info.location - if status == TaskStatus.RUNNING and pid_exists - else None, + "status": ( + "Running" + if status == TaskStatus.RUNNING and pid_exists + else "Success" + ), + "executor": ( + info.location + if status == TaskStatus.RUNNING and pid_exists + else None + ), "timestamp": None, } } @@ -486,11 +468,9 @@ def test_show_running_workspace( pid = LOCK_CONTENTS["read"]["data/MNIST"][0]["pid"] assert ( f"Process '{cmd}' with (Pid {pid}), in RWLock-file " - f"'{lock_file}' had been killed." - ) in caplog.text - assert ( - f"Delete corrupted RWLock-file '{lock_file}'" - ) in caplog.text + f"'{lock_file}' had been killed." in caplog.text + ) + assert f"Delete corrupted RWLock-file '{lock_file}'" in caplog.text def test_show_running_tempdir(tmp_dir, scm, dvc, exp_stage, mocker): @@ -570,9 +550,7 @@ def test_show_running_checkpoint(tmp_dir, scm, dvc, checkpoint_stage, mocker): from dvc.repo.experiments.executor.local import TempDirExecutor baseline_rev = scm.get_rev() - dvc.experiments.run( - checkpoint_stage.addressing, params=["foo=2"], queue=True - ) + dvc.experiments.run(checkpoint_stage.addressing, params=["foo=2"], queue=True) queue = dvc.experiments.celery_queue entries = list(queue.iter_queued()) @@ -600,9 +578,7 @@ def test_show_running_checkpoint(tmp_dir, scm, dvc, checkpoint_stage, mocker): os.makedirs(os.path.dirname(pidfile), exist_ok=True) (tmp_dir / pidfile).dump_json(info.asdict()) - mocker.patch.object( - BaseExecutor, "fetch_exps", return_value=[str(exp_ref)] - ) + mocker.patch.object(BaseExecutor, "fetch_exps", return_value=[str(exp_ref)]) results = dvc.experiments.show() @@ -730,9 +706,7 @@ def test_show_only_changed(tmp_dir, dvc, scm, capsys, copy_script): assert "metrics.yaml:goobar" in cap.out -def test_show_parallel_coordinates( - tmp_dir, dvc, scm, mocker, capsys, copy_script -): +def test_show_parallel_coordinates(tmp_dir, dvc, scm, mocker, capsys, copy_script): from dvc.commands.experiments import show webbroser_open = mocker.patch("webbrowser.open") @@ -1039,9 +1013,7 @@ def test_show_completed_error(tmp_dir, scm, dvc, exp_stage, mocker): @pytest.mark.vscode def test_show_checkpoint_error(tmp_dir, scm, dvc, checkpoint_stage, mocker): baseline_rev = scm.get_rev() - results = dvc.experiments.run( - checkpoint_stage.addressing, params=["foo=2"] - ) + results = dvc.experiments.run(checkpoint_stage.addressing, params=["foo=2"]) exp_rev = first(results) exp_ref = str(first(exp_refs_by_rev(scm, exp_rev))) diff --git a/tests/func/experiments/test_utils.py b/tests/func/experiments/test_utils.py index 7b75b87e0..0585bad92 100644 --- a/tests/func/experiments/test_utils.py +++ b/tests/func/experiments/test_utils.py @@ -40,9 +40,7 @@ def test_post_to_studio(tmp_dir, dvc, scm, exp_stage, mocker, monkeypatch): monkeypatch.setenv(env.STUDIO_TOKEN, "STUDIO_TOKEN") baseline_sha = scm.get_rev() - exp_rev = first( - dvc.experiments.run(exp_stage.addressing, params=["foo=1"]) - ) + exp_rev = first(dvc.experiments.run(exp_stage.addressing, params=["foo=1"])) name = dvc.experiments.get_exact_name([exp_rev])[exp_rev] assert mocked_post.call_count == 2 diff --git a/tests/func/machine/test_machine_config.py b/tests/func/machine/test_machine_config.py index 80a39f6b6..0e5b775fb 100644 --- a/tests/func/machine/test_machine_config.py +++ b/tests/func/machine/test_machine_config.py @@ -56,9 +56,7 @@ def test_machine_modify_startup_script(tmp_dir, dvc, machine_config): ("instance_hdd_size", "BIG", "expected int"), ], ) -def test_machine_modify_fail( - tmp_dir, dvc, machine_config, caplog, slot, value, msg -): +def test_machine_modify_fail(tmp_dir, dvc, machine_config, caplog, slot, value, msg): assert main(["machine", "modify", "foo", slot, value]) == 251 assert (tmp_dir / ".dvc" / "config").read_text() == machine_config assert msg in caplog.text @@ -110,9 +108,7 @@ def test_machine_list(tmp_dir, dvc, capsys): assert "bar azure" in out -def test_machine_rename_success( - tmp_dir, scm, dvc, machine_config, capsys, mocker -): +def test_machine_rename_success(tmp_dir, scm, dvc, machine_config, capsys, mocker): config_file = tmp_dir / ".dvc" / "config" mocker.patch.object( @@ -128,12 +124,8 @@ def test_machine_rename_success( cap = capsys.readouterr() assert "Rename machine 'foo' to 'bar'." in cap.out assert config_file.read_text() == machine_config.replace("foo", "bar") - assert not ( - tmp_dir / ".dvc" / "tmp" / "machine" / "terraform" / "foo" - ).exists() - assert ( - tmp_dir / ".dvc" / "tmp" / "machine" / "terraform" / "bar" - ).exists() + assert not (tmp_dir / ".dvc" / "tmp" / "machine" / "terraform" / "foo").exists() + assert (tmp_dir / ".dvc" / "tmp" / "machine" / "terraform" / "bar").exists() def test_machine_rename_none_exist(tmp_dir, scm, dvc, caplog): @@ -154,9 +146,7 @@ def test_machine_rename_exist(tmp_dir, scm, dvc, caplog): assert "Machine 'bar' already exists." in caplog.text -def test_machine_rename_error( - tmp_dir, scm, dvc, machine_config, caplog, mocker -): +def test_machine_rename_error(tmp_dir, scm, dvc, machine_config, caplog, mocker): config_file = tmp_dir / ".dvc" / "config" os.makedirs(tmp_dir / ".dvc" / "tmp" / "machine" / "terraform" / "foo") diff --git a/tests/func/machine/test_machine_status.py b/tests/func/machine/test_machine_status.py index cb935bb64..d4980ff0a 100644 --- a/tests/func/machine/test_machine_status.py +++ b/tests/func/machine/test_machine_status.py @@ -4,20 +4,19 @@ from tests.utils import console_width def test_status(tmp_dir, scm, dvc, machine_config, machine_instance, capsys): - assert main(["machine", "add", "bar", "aws"]) == 0 with console_width(ui.rich_console, 255): assert main(["machine", "status"]) == 0 cap = capsys.readouterr() assert ( "name instance status cloud instance_ip " - "instance_type instance_hdd_size instance_gpu" - ) in cap.out + "instance_type instance_hdd_size instance_gpu" in cap.out + ) assert ( "bar - offline - - " - "- - -" - ) in cap.out + "- - -" in cap.out + ) assert ( "foo num_1 running aws 123.123.123.123 " - "m 35 None" - ) in cap.out + "m 35 None" in cap.out + ) diff --git a/tests/func/metrics/test_diff.py b/tests/func/metrics/test_diff.py index eff5e77a3..8cb3aa89c 100644 --- a/tests/func/metrics/test_diff.py +++ b/tests/func/metrics/test_diff.py @@ -26,9 +26,7 @@ def test_metrics_diff_yaml(tmp_dir, scm, dvc, run_copy_metrics): def _gen(val): metrics = {"a": {"b": {"c": val, "d": 1, "e": str(val)}}} (tmp_dir / "m_temp.yaml").dump(metrics) - run_copy_metrics( - "m_temp.yaml", "m.yaml", metrics=["m.yaml"], commit=str(val) - ) + run_copy_metrics("m_temp.yaml", "m.yaml", metrics=["m.yaml"], commit=str(val)) _gen(1) _gen(2) @@ -43,9 +41,7 @@ def test_metrics_diff_json(tmp_dir, scm, dvc, run_copy_metrics): def _gen(val): metrics = {"a": {"b": {"c": val, "d": 1, "e": str(val)}}} (tmp_dir / "m_temp.json").dump(metrics) - run_copy_metrics( - "m_temp.json", "m.json", metrics=["m.json"], commit=str(val) - ) + run_copy_metrics("m_temp.json", "m.json", metrics=["m.json"], commit=str(val)) _gen(1) _gen(2) @@ -59,9 +55,7 @@ def test_metrics_diff_json_unchanged(tmp_dir, scm, dvc, run_copy_metrics): def _gen(val): metrics = {"a": {"b": {"c": val, "d": 1, "e": str(val)}}} (tmp_dir / "m_temp.json").dump(metrics) - run_copy_metrics( - "m_temp.json", "m.json", metrics=["m.json"], commit=str(val) - ) + run_copy_metrics("m_temp.json", "m.json", metrics=["m.json"], commit=str(val)) _gen(1) _gen(2) @@ -191,8 +185,8 @@ def test_metrics_diff_cli(tmp_dir, scm, dvc, run_copy_metrics, caplog, capsys): captured = capsys.readouterr() - assert captured.out == ( - "Path Metric HEAD~2 workspace Change\n" + assert ( + captured.out == "Path Metric HEAD~2 workspace Change\n" "m.yaml foo 1.23457 3.45679 2.22222\n" ) @@ -206,9 +200,7 @@ def test_metrics_diff_non_metrics(tmp_dir, scm, dvc): _gen(3) result = dvc.metrics.diff(targets=["some_file.yaml"], a_rev="HEAD~2") - assert result == { - "some_file.yaml": {"foo": {"old": 1, "new": 3, "diff": 2}} - } + assert result == {"some_file.yaml": {"foo": {"old": 1, "new": 3, "diff": 2}}} @pytest.mark.parametrize( @@ -230,21 +222,15 @@ def test_diff_top_level_metrics(tmp_dir, dvc, scm, dvcfile, metrics_file): metrics_file.dump({"foo": 5}) assert dvc.metrics.diff() == { - relpath(directory / metrics_file): { - "foo": {"diff": 2, "new": 5, "old": 3} - } + relpath(directory / metrics_file): {"foo": {"diff": 2, "new": 5, "old": 3}} } -def test_metrics_diff_active_branch_unchanged( - tmp_dir, scm, dvc, run_copy_metrics -): +def test_metrics_diff_active_branch_unchanged(tmp_dir, scm, dvc, run_copy_metrics): def _gen(val): metrics = {"a": {"b": {"c": val, "d": 1, "e": str(val)}}} (tmp_dir / "m_temp.yaml").dump(metrics) - run_copy_metrics( - "m_temp.yaml", "m.yaml", metrics=["m.yaml"], commit=str(val) - ) + run_copy_metrics("m_temp.yaml", "m.yaml", metrics=["m.yaml"], commit=str(val)) _gen(1) _gen(2) diff --git a/tests/func/metrics/test_show.py b/tests/func/metrics/test_show.py index 9816037b7..1bc9dcb7b 100644 --- a/tests/func/metrics/test_show.py +++ b/tests/func/metrics/test_show.py @@ -12,12 +12,8 @@ from dvc.utils.serialize import JSONFileCorruptedError, YAMLFileCorruptedError def test_show_simple(tmp_dir, dvc, run_copy_metrics): tmp_dir.gen("metrics_t.yaml", "1.1") - run_copy_metrics( - "metrics_t.yaml", "metrics.yaml", metrics=["metrics.yaml"] - ) - assert dvc.metrics.show() == { - "": {"data": {"metrics.yaml": {"data": 1.1}}} - } + run_copy_metrics("metrics_t.yaml", "metrics.yaml", metrics=["metrics.yaml"]) + assert dvc.metrics.show() == {"": {"data": {"metrics.yaml": {"data": 1.1}}}} def test_show_simple_from_subdir(tmp_dir, dvc, run_copy_metrics): @@ -35,22 +31,16 @@ def test_show_simple_from_subdir(tmp_dir, dvc, run_copy_metrics): expected_path = os.path.join("..", "subdir", "metrics.yaml") with subdir.chdir(): - assert dvc.metrics.show() == { - "": {"data": {expected_path: {"data": 1.1}}} - } + assert dvc.metrics.show() == {"": {"data": {expected_path: {"data": 1.1}}}} subdir2 = tmp_dir / "subdir2" subdir2.mkdir() with subdir2.chdir(): - assert dvc.metrics.show() == { - "": {"data": {expected_path: {"data": 1.1}}} - } + assert dvc.metrics.show() == {"": {"data": {expected_path: {"data": 1.1}}}} def test_show(tmp_dir, dvc, run_copy_metrics): tmp_dir.gen("metrics_t.yaml", "foo: 1.1") - run_copy_metrics( - "metrics_t.yaml", "metrics.yaml", metrics=["metrics.yaml"] - ) + run_copy_metrics("metrics_t.yaml", "metrics.yaml", metrics=["metrics.yaml"]) assert dvc.metrics.show() == { "": {"data": {"metrics.yaml": {"data": {"foo": 1.1}}}} } @@ -58,9 +48,7 @@ def test_show(tmp_dir, dvc, run_copy_metrics): def test_show_toml(tmp_dir, dvc, run_copy_metrics): tmp_dir.gen("metrics_t.toml", "[foo]\nbar = 1.2") - run_copy_metrics( - "metrics_t.toml", "metrics.toml", metrics=["metrics.toml"] - ) + run_copy_metrics("metrics_t.toml", "metrics.toml", metrics=["metrics.toml"]) assert dvc.metrics.show() == { "": {"data": {"metrics.toml": {"data": {"foo": {"bar": 1.2}}}}} } @@ -68,15 +56,10 @@ def test_show_toml(tmp_dir, dvc, run_copy_metrics): def test_show_targets(tmp_dir, dvc, run_copy_metrics): tmp_dir.gen("metrics_t.yaml", "foo: 1.1") - run_copy_metrics( - "metrics_t.yaml", "metrics.yaml", metrics=["metrics.yaml"] - ) + run_copy_metrics("metrics_t.yaml", "metrics.yaml", metrics=["metrics.yaml"]) expected = {"": {"data": {"metrics.yaml": {"data": {"foo": 1.1}}}}} assert dvc.metrics.show(targets=["metrics.yaml"]) == expected - assert ( - dvc.metrics.show(targets=(tmp_dir / "metrics.yaml").fs_path) - == expected - ) + assert dvc.metrics.show(targets=(tmp_dir / "metrics.yaml").fs_path) == expected def test_show_multiple(tmp_dir, dvc, run_copy_metrics): @@ -85,9 +68,7 @@ def test_show_multiple(tmp_dir, dvc, run_copy_metrics): run_copy_metrics("foo_temp", "foo", fname="foo.dvc", metrics=["foo"]) run_copy_metrics("baz_temp", "baz", fname="baz.dvc", metrics=["baz"]) assert dvc.metrics.show() == { - "": { - "data": {"foo": {"data": {"foo": 1}}, "baz": {"data": {"baz": 2}}} - } + "": {"data": {"foo": {"data": {"foo": 1}}, "baz": {"data": {"baz": 2}}}} } @@ -144,9 +125,7 @@ def test_show_subrepo_with_preexisting_tags(tmp_dir, scm): def test_missing_cache(tmp_dir, dvc, run_copy_metrics): tmp_dir.gen("metrics_t.yaml", "1.1") - run_copy_metrics( - "metrics_t.yaml", "metrics.yaml", metrics=["metrics.yaml"] - ) + run_copy_metrics("metrics_t.yaml", "metrics.yaml", metrics=["metrics.yaml"]) # This one should be skipped stage = run_copy_metrics( @@ -205,24 +184,16 @@ def test_show_non_metric_branch(tmp_dir, scm, use_dvc): def test_non_metric_and_recurisve_show(tmp_dir, dvc, run_copy_metrics): - tmp_dir.gen( - {"metrics_t.yaml": "foo: 1.1", "metrics": {"metric1.yaml": "bar: 1.2"}} - ) + tmp_dir.gen({"metrics_t.yaml": "foo: 1.1", "metrics": {"metric1.yaml": "bar: 1.2"}}) metric2 = os.fspath(tmp_dir / "metrics" / "metric2.yaml") run_copy_metrics("metrics_t.yaml", metric2, metrics=[metric2]) - assert dvc.metrics.show( - targets=["metrics_t.yaml", "metrics"], recursive=True - ) == { + assert dvc.metrics.show(targets=["metrics_t.yaml", "metrics"], recursive=True) == { "": { "data": { - os.path.join("metrics", "metric1.yaml"): { - "data": {"bar": 1.2} - }, - os.path.join("metrics", "metric2.yaml"): { - "data": {"foo": 1.1} - }, + os.path.join("metrics", "metric1.yaml"): {"data": {"bar": 1.2}}, + os.path.join("metrics", "metric2.yaml"): {"data": {"foo": 1.1}}, "metrics_t.yaml": {"data": {"foo": 1.1}}, } } @@ -250,9 +221,7 @@ def test_show_malformed_metric(tmp_dir, scm, dvc, caplog): tmp_dir.gen("metric.json", '{"m":1') assert isinstance( - dvc.metrics.show(targets=["metric.json"])[""]["data"]["metric.json"][ - "error" - ], + dvc.metrics.show(targets=["metric.json"])[""]["data"]["metric.json"]["error"], JSONFileCorruptedError, ) @@ -266,9 +235,7 @@ def test_show_no_metrics_files(tmp_dir, dvc, caplog): @pytest.mark.parametrize("clear_before_run", [True, False]) -def test_metrics_show_overlap( - tmp_dir, dvc, run_copy_metrics, clear_before_run -): +def test_metrics_show_overlap(tmp_dir, dvc, run_copy_metrics, clear_before_run): data_dir = tmp_dir / "data" data_dir.mkdir() @@ -307,9 +274,7 @@ def test_metrics_show_overlap( ("metrics.yaml", ["workspace", "data", "metrics.yaml", "error"]), ), ) -def test_log_errors( - tmp_dir, scm, dvc, capsys, run_copy_metrics, file, error_path -): +def test_log_errors(tmp_dir, scm, dvc, capsys, run_copy_metrics, file, error_path): tmp_dir.gen("metrics_t.yaml", "m: 1.1") run_copy_metrics( "metrics_t.yaml", @@ -329,6 +294,5 @@ def test_log_errors( assert isinstance(get_in(result, error_path), YAMLFileCorruptedError) assert ( - "DVC failed to load some metrics for following revisions: 'workspace'." - in error + "DVC failed to load some metrics for following revisions: 'workspace'." in error ) diff --git a/tests/func/params/test_diff.py b/tests/func/params/test_diff.py index 00b20311d..a698665a6 100644 --- a/tests/func/params/test_diff.py +++ b/tests/func/params/test_diff.py @@ -40,18 +40,14 @@ def test_diff_dirty(tmp_dir, scm, dvc): tmp_dir.scm_gen("params.yaml", "foo: baz", commit="baz") tmp_dir.gen("params.yaml", "foo: qux") - assert dvc.params.diff() == { - "params.yaml": {"foo": {"old": "baz", "new": "qux"}} - } + assert dvc.params.diff() == {"params.yaml": {"foo": {"old": "baz", "new": "qux"}}} def test_diff_new(tmp_dir, scm, dvc): tmp_dir.gen("params.yaml", "foo: bar") dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True) - assert dvc.params.diff() == { - "params.yaml": {"foo": {"old": None, "new": "bar"}} - } + assert dvc.params.diff() == {"params.yaml": {"foo": {"old": None, "new": "bar"}}} def test_diff_deleted(tmp_dir, scm, dvc): @@ -62,9 +58,7 @@ def test_diff_deleted(tmp_dir, scm, dvc): (tmp_dir / "params.yaml").unlink() - assert dvc.params.diff() == { - "params.yaml": {"foo": {"old": "bar", "new": None}} - } + assert dvc.params.diff() == {"params.yaml": {"foo": {"old": "bar", "new": None}}} def test_diff_list(tmp_dir, scm, dvc): @@ -165,9 +159,7 @@ def test_vars_shows_on_params_diff(tmp_dir, scm, dvc): param_data["vars"]["model1"]["epoch"] = 20 (tmp_dir / params_file).dump(param_data) assert dvc.params.diff() == { - "test_params.yaml": { - "vars.model1.epoch": {"new": 20, "old": 15, "diff": 5} - } + "test_params.yaml": {"vars.model1.epoch": {"new": 20, "old": 15, "diff": 5}} } data_dir = tmp_dir / "data" @@ -263,9 +255,7 @@ def test_diff_top_level_params(tmp_dir, dvc, scm, dvcfile, params_file): params_file.dump({"foo": 5}) assert dvc.params.diff() == { - relpath(directory / params_file): { - "foo": {"diff": 2, "new": 5, "old": 3} - } + relpath(directory / params_file): {"foo": {"diff": 2, "new": 5, "old": 3}} } diff --git a/tests/func/params/test_show.py b/tests/func/params/test_show.py index effa3b890..4a5e1f482 100644 --- a/tests/func/params/test_show.py +++ b/tests/func/params/test_show.py @@ -25,22 +25,14 @@ def test_show_targets(tmp_dir, dvc): dvc.run(cmd="echo params.yaml", params=["foo"], single_stage=True) expected = {"": {"data": {"params.yaml": {"data": {"foo": "bar"}}}}} assert dvc.params.show(targets=["params.yaml"]) == expected - assert ( - dvc.params.show(targets=(tmp_dir / "params.yaml").fs_path) == expected - ) + assert dvc.params.show(targets=(tmp_dir / "params.yaml").fs_path) == expected def test_show_toml(tmp_dir, dvc): tmp_dir.gen("params.toml", "[foo]\nbar = 42\nbaz = [1, 2]\n") - dvc.run( - cmd="echo params.toml", params=["params.toml:foo"], single_stage=True - ) + dvc.run(cmd="echo params.toml", params=["params.toml:foo"], single_stage=True) assert dvc.params.show() == { - "": { - "data": { - "params.toml": {"data": {"foo": {"bar": 42, "baz": [1, 2]}}} - } - } + "": {"data": {"params.toml": {"data": {"foo": {"bar": 42, "baz": [1, 2]}}}}} } @@ -108,31 +100,21 @@ def test_show_branch(tmp_dir, scm, dvc): def test_pipeline_params(tmp_dir, scm, dvc, run_copy): - tmp_dir.gen( - {"foo": "foo", "params.yaml": "foo: bar\nxyz: val\nabc: ignore"} - ) + tmp_dir.gen({"foo": "foo", "params.yaml": "foo: bar\nxyz: val\nabc: ignore"}) run_copy("foo", "bar", name="copy-foo-bar", params=["foo,xyz"]) scm.add(["params.yaml", PROJECT_FILE]) scm.commit("add stage") - tmp_dir.scm_gen( - "params.yaml", "foo: baz\nxyz: val\nabc: ignore", commit="baz" - ) - tmp_dir.scm_gen( - "params.yaml", "foo: qux\nxyz: val\nabc: ignore", commit="qux" - ) + tmp_dir.scm_gen("params.yaml", "foo: baz\nxyz: val\nabc: ignore", commit="baz") + tmp_dir.scm_gen("params.yaml", "foo: qux\nxyz: val\nabc: ignore", commit="qux") assert dvc.params.show(revs=["master"], deps=True) == { - "master": { - "data": {"params.yaml": {"data": {"foo": "qux", "xyz": "val"}}} - } + "master": {"data": {"params.yaml": {"data": {"foo": "qux", "xyz": "val"}}}} } assert dvc.params.show(revs=["master"]) == { "master": { "data": { - "params.yaml": { - "data": {"abc": "ignore", "foo": "qux", "xyz": "val"} - } + "params.yaml": {"data": {"abc": "ignore", "foo": "qux", "xyz": "val"}} } } } @@ -144,11 +126,7 @@ def test_show_no_repo(tmp_dir): dvc = Repo(uninitialized=True) assert dvc.params.show(targets=["params_file.yaml"]) == { - "": { - "data": { - "params_file.yaml": {"data": {"foo": "bar", "xyz": "val"}} - } - } + "": {"data": {"params_file.yaml": {"data": {"foo": "bar", "xyz": "val"}}}} } @@ -184,10 +162,7 @@ def test_log_errors(tmp_dir, scm, dvc, capsys, file, error_path): assert isinstance( reduce(operator.getitem, error_path, result), YAMLFileCorruptedError ) - assert ( - "DVC failed to load some parameters for following revisions: 'v1'." - in error - ) + assert "DVC failed to load some parameters for following revisions: 'v1'." in error @pytest.mark.parametrize("file", ["params.yaml", "other_params.yaml"]) @@ -205,9 +180,7 @@ def test_show_without_targets_specified(tmp_dir, dvc, scm, file): def test_deps_multi_stage(tmp_dir, scm, dvc, run_copy): - tmp_dir.gen( - {"foo": "foo", "params.yaml": "foo: bar\nxyz: val\nabc: ignore"} - ) + tmp_dir.gen({"foo": "foo", "params.yaml": "foo: bar\nxyz: val\nabc: ignore"}) run_copy("foo", "bar", name="copy-foo-bar", params=["foo"]) run_copy("foo", "bar1", name="copy-foo-bar-1", params=["xyz"]) @@ -215,16 +188,12 @@ def test_deps_multi_stage(tmp_dir, scm, dvc, run_copy): scm.commit("add stage") assert dvc.params.show(revs=["master"], deps=True) == { - "master": { - "data": {"params.yaml": {"data": {"foo": "bar", "xyz": "val"}}} - } + "master": {"data": {"params.yaml": {"data": {"foo": "bar", "xyz": "val"}}}} } def test_deps_with_targets(tmp_dir, scm, dvc, run_copy): - tmp_dir.gen( - {"foo": "foo", "params.yaml": "foo: bar\nxyz: val\nabc: ignore"} - ) + tmp_dir.gen({"foo": "foo", "params.yaml": "foo: bar\nxyz: val\nabc: ignore"}) run_copy("foo", "bar", name="copy-foo-bar", params=["foo"]) run_copy("foo", "bar1", name="copy-foo-bar-1", params=["xyz"]) diff --git a/tests/func/parsing/test_errors.py b/tests/func/parsing/test_errors.py index 559e734c4..a733e4696 100644 --- a/tests/func/parsing/test_errors.py +++ b/tests/func/parsing/test_errors.py @@ -29,8 +29,7 @@ def test_vars_interpolation_errors(tmp_dir, dvc, vars_): definition.resolve() assert ( - str(exc_info.value) - == "failed to parse 'stages.build.vars' in 'dvc.yaml': " + str(exc_info.value) == "failed to parse 'stages.build.vars' in 'dvc.yaml': " "interpolating is not allowed" ) @@ -44,8 +43,9 @@ def test_failed_to_interpolate(tmp_dir, dvc): with pytest.raises(ResolveError) as exc_info: definition.resolve() - assert escape_ansi(str(exc_info.value)) == ( - "failed to parse 'stages.build.cmd' in 'dvc.yaml':\n" + assert ( + escape_ansi(str(exc_info.value)) + == "failed to parse 'stages.build.cmd' in 'dvc.yaml':\n" "${models.foo.}\n" " ^\n" "ParseException: Expected end of text, found '.'" @@ -64,8 +64,8 @@ def test_local_vars_params_file_not_exist(tmp_dir, dvc): with pytest.raises(ResolveError) as exc_info: definition.resolve() - assert str(exc_info.value) == ( - "failed to parse stage 'build' in 'dvc.yaml': " + assert ( + str(exc_info.value) == "failed to parse stage 'build' in 'dvc.yaml': " "'not_existing_params.yaml' does not exist" ) assert not definition.context @@ -81,8 +81,8 @@ def test_specified_key_does_not_exist(tmp_dir, dvc): with pytest.raises(ResolveError) as exc_info: definition.resolve() - assert str(exc_info.value) == ( - "failed to parse 'stages.build.cmd' in 'dvc.yaml': " + assert ( + str(exc_info.value) == "failed to parse 'stages.build.cmd' in 'dvc.yaml': " "Could not find 'models.foobar'" ) assert definition.context == {"models": {"foo": "foo"}} @@ -94,10 +94,12 @@ def test_specified_key_does_not_exist(tmp_dir, dvc): ("${models[foobar]}", " Could not find 'models.foobar'"), ( "${models.foo]}", - "\n${models.foo]}\n" - " ^\n" - "ParseException: Expected end of text, found ']'" - " (at char 12), (line:1, col:13)", + ( + "\n${models.foo]}\n" + " ^\n" + "ParseException: Expected end of text, found ']'" + " (at char 12), (line:1, col:13)" + ), ), ], ) @@ -124,8 +126,8 @@ def test_interpolate_non_string(tmp_dir, dvc): with pytest.raises(ResolveError) as exc_info: definition.resolve() - assert str(exc_info.value) == ( - "failed to parse 'stages.build.outs' in 'dvc.yaml':\n" + assert ( + str(exc_info.value) == "failed to parse 'stages.build.outs' in 'dvc.yaml':\n" "Cannot interpolate data of type 'dict'" ) assert definition.context == {"models": {}} @@ -141,8 +143,8 @@ def test_interpolate_nested_iterable(tmp_dir, dvc): with pytest.raises(ResolveError) as exc_info: definition.resolve() - assert str(exc_info.value) == ( - "failed to parse 'stages.build.cmd' in 'dvc.yaml':\n" + assert ( + str(exc_info.value) == "failed to parse 'stages.build.cmd' in 'dvc.yaml':\n" "Cannot interpolate nested iterable in 'list'" ) @@ -159,8 +161,8 @@ def test_partial_vars_doesnot_exist(tmp_dir, dvc): with pytest.raises(ResolveError) as exc_info: definition.resolve() - assert str(exc_info.value) == ( - "failed to parse stage 'build' in 'dvc.yaml': " + assert ( + str(exc_info.value) == "failed to parse stage 'build' in 'dvc.yaml': " "could not find 'sub3' in 'test_params.yaml'" ) assert not definition.context @@ -174,8 +176,9 @@ def test_foreach_data_syntax_error(tmp_dir, dvc): with pytest.raises(ResolveError) as exc_info: definition.resolve_all() - assert escape_ansi(str(exc_info.value)) == ( - "failed to parse 'stages.build.foreach' in 'dvc.yaml':\n" + assert ( + escape_ansi(str(exc_info.value)) + == "failed to parse 'stages.build.foreach' in 'dvc.yaml':\n" "${syntax.[error}\n" " ^\n" "ParseException: Expected end of text, found '.'" @@ -188,8 +191,8 @@ def test_foreach_data_key_does_not_exists(tmp_dir, dvc, key): definition = make_foreach_def(tmp_dir, "build", embrace(key), {}) with pytest.raises(ResolveError) as exc_info: definition.resolve_all() - assert str(exc_info.value) == ( - "failed to parse 'stages.build.foreach' in 'dvc.yaml': " + assert ( + str(exc_info.value) == "failed to parse 'stages.build.foreach' in 'dvc.yaml': " f"Could not find '{key}'" ) @@ -198,14 +201,13 @@ def test_foreach_data_key_does_not_exists(tmp_dir, dvc, key): "foreach_data", ["${foo}", "${dct.model1}", "${lst.0}", "foobar"] ) def test_foreach_data_expects_list_or_dict(tmp_dir, dvc, foreach_data): - context = Context( - {"foo": "bar", "dct": {"model1": "a-out"}, "lst": ["foo", "bar"]} - ) + context = Context({"foo": "bar", "dct": {"model1": "a-out"}, "lst": ["foo", "bar"]}) definition = make_foreach_def(tmp_dir, "build", foreach_data, {}, context) with pytest.raises(ResolveError) as exc_info: definition.resolve_all() - assert str(exc_info.value) == ( - "failed to resolve 'stages.build.foreach' in 'dvc.yaml': " + assert ( + str(exc_info.value) + == "failed to resolve 'stages.build.foreach' in 'dvc.yaml': " "expected list/dictionary, got str" ) @@ -218,9 +220,7 @@ def test_foreach_data_expects_list_or_dict(tmp_dir, dvc, foreach_data): ({"key": 5}, "key is"), ], ) -def test_foreach_overwriting_item_in_list( - tmp_dir, dvc, caplog, global_data, where -): +def test_foreach_overwriting_item_in_list(tmp_dir, dvc, caplog, global_data, where): context = Context(global_data) definition = make_foreach_def( tmp_dir, "build", {"model1": 10, "model2": 5}, {}, context @@ -242,8 +242,9 @@ def test_foreach_do_syntax_errors(tmp_dir, dvc): with pytest.raises(ResolveError) as exc_info: definition.resolve_all() - assert escape_ansi(str(exc_info.value)) == ( - "failed to parse 'stages.build.cmd' in 'dvc.yaml':\n" + assert ( + escape_ansi(str(exc_info.value)) + == "failed to parse 'stages.build.cmd' in 'dvc.yaml':\n" "${syntax.[error}\n" " ^\n" "ParseException: Expected end of text, found '.'" @@ -274,8 +275,9 @@ def test_foreach_do_definition_item_does_not_exist(tmp_dir, dvc, key, loc): with pytest.raises(ResolveError) as exc_info: definition.resolve_all() - assert str(exc_info.value) == ( - f"failed to parse '{loc}' in 'dvc.yaml': Could not find '{key}'" + assert ( + str(exc_info.value) + == f"failed to parse '{loc}' in 'dvc.yaml': Could not find '{key}'" ) # should have no `item` and `key` even though it failed to resolve. @@ -313,8 +315,8 @@ def test_item_key_in_generated_stage_vars(tmp_dir, dvc, redefine, from_file): message = str(exc_info.value) assert ( "failed to parse stage 'build@model1' in 'dvc.yaml': " - "attempted to modify reserved" - ) in message + "attempted to modify reserved" in message + ) key_or_keys = "keys" if len(redefine) > 1 else "key" assert f"{key_or_keys} {join(redefine)}" in message @@ -335,7 +337,6 @@ def test_foreach_wdir_key_does_not_exist(tmp_dir, dvc): definition.resolve_all() assert ( str(exc_info.value) - == "failed to parse 'stages.build@foo.wdir' in 'dvc.yaml': " - "Could not find 'ite'" + == "failed to parse 'stages.build@foo.wdir' in 'dvc.yaml': Could not find 'ite'" ) assert definition.context == {"models": ["foo", "bar"]} diff --git a/tests/func/parsing/test_foreach.py b/tests/func/parsing/test_foreach.py index 87ee97f74..98e5d872a 100644 --- a/tests/func/parsing/test_foreach.py +++ b/tests/func/parsing/test_foreach.py @@ -180,9 +180,7 @@ def test_mixed_vars_for_foreach_data(tmp_dir, dvc): (tmp_dir / "params.yaml").dump({"models": {"model1": "foo"}}) (tmp_dir / "test_params.yaml").dump({"models": {"model2": "bar"}}) - resolver = DataResolver( - dvc, tmp_dir.fs_path, {"vars": ["test_params.yaml"]} - ) + resolver = DataResolver(dvc, tmp_dir.fs_path, {"vars": ["test_params.yaml"]}) data = {"foreach": "${models}", "do": {"cmd": "echo ${item}"}} definition = ForeachDefinition(resolver, resolver.context, "build", data) @@ -205,9 +203,7 @@ def test_mixed_vars_for_foreach_data_2(tmp_dir, dvc): {"models": {"model1": {"epochs": 5}, "model2": {"epochs": 10}}}, ) - resolver = DataResolver( - dvc, tmp_dir.fs_path, {"vars": ["test_params.yaml"]} - ) + resolver = DataResolver(dvc, tmp_dir.fs_path, {"vars": ["test_params.yaml"]}) data = { "foreach": "${models}", "do": {"cmd": "echo ${item.thresh} ${item.epochs}"}, @@ -316,12 +312,8 @@ def test_foreach_with_imported_vars(tmp_dir, dvc, local_import): } -@pytest.mark.parametrize( - "local_import", ["params.yaml", "params.yaml:train,prepare"] -) -def test_foreach_with_interpolated_wdir_and_local_vars( - tmp_dir, dvc, local_import -): +@pytest.mark.parametrize("local_import", ["params.yaml", "params.yaml:train,prepare"]) +def test_foreach_with_interpolated_wdir_and_local_vars(tmp_dir, dvc, local_import): (tmp_dir / "params.yaml").dump({"models": {"model1": {"thresh": "foo"}}}) for i in range(5): diff --git a/tests/func/parsing/test_interpolated_entry.py b/tests/func/parsing/test_interpolated_entry.py index 0fa56a59c..e734629db 100644 --- a/tests/func/parsing/test_interpolated_entry.py +++ b/tests/func/parsing/test_interpolated_entry.py @@ -8,12 +8,7 @@ from dvc.parsing import DEFAULT_PARAMS_FILE, DataResolver from dvc.parsing.context import recurse_not_a_node from dvc.parsing.interpolate import escape_str -from . import ( - CONTEXT_DATA, - RESOLVED_DVC_YAML_DATA, - TEMPLATED_DVC_YAML_DATA, - USED_VARS, -) +from . import CONTEXT_DATA, RESOLVED_DVC_YAML_DATA, TEMPLATED_DVC_YAML_DATA, USED_VARS def assert_stage_equal(d1, d2): @@ -31,9 +26,7 @@ def assert_stage_equal(d1, d2): def test_simple(tmp_dir, dvc): (tmp_dir / DEFAULT_PARAMS_FILE).dump(CONTEXT_DATA) - resolver = DataResolver( - dvc, tmp_dir.fs_path, deepcopy(TEMPLATED_DVC_YAML_DATA) - ) + resolver = DataResolver(dvc, tmp_dir.fs_path, deepcopy(TEMPLATED_DVC_YAML_DATA)) assert_stage_equal(resolver.resolve(), deepcopy(RESOLVED_DVC_YAML_DATA)) assert resolver.tracked_vars == { "stage1": {DEFAULT_PARAMS_FILE: USED_VARS["stage1"]}, @@ -135,9 +128,7 @@ def test_with_templated_wdir(tmp_dir, dvc): } } } - (tmp_dir / DEFAULT_PARAMS_FILE).dump( - {"dict": {"bar": "bar", "ws": "data"}} - ) + (tmp_dir / DEFAULT_PARAMS_FILE).dump({"dict": {"bar": "bar", "ws": "data"}}) data_dir = tmp_dir / "data" data_dir.mkdir() (data_dir / DEFAULT_PARAMS_FILE).dump({"dict": {"foo": "foo"}}) @@ -310,13 +301,16 @@ def test_cmd_dict(tmp_dir, dvc, bool_config, list_config): { "stages": { "stage1": { - "cmd": "python script.py" - " --foo foo --bar 2" - f" --string {escape_str(string)}" - f" --mixed_quote_string {escape_str(mixed_quote_string)}" - f"{bool_resolved}" - f"{list_resolved}" - " --nested.foo foo" + "cmd": ( + "python script.py" + " --foo foo --bar 2" + f" --string {escape_str(string)}" + " --mixed_quote_string" + f" {escape_str(mixed_quote_string)}" + f"{bool_resolved}" + f"{list_resolved}" + " --nested.foo foo" + ) } } }, diff --git a/tests/func/parsing/test_resolver.py b/tests/func/parsing/test_resolver.py index dd151e402..5f1443437 100644 --- a/tests/func/parsing/test_resolver.py +++ b/tests/func/parsing/test_resolver.py @@ -45,9 +45,7 @@ def test_vars_interpolation_errors(tmp_dir, dvc, vars_): ) -@pytest.mark.parametrize( - "vars_", [{}, {"vars": []}, {"vars": [DEFAULT_PARAMS_FILE]}] -) +@pytest.mark.parametrize("vars_", [{}, {"vars": []}, {"vars": [DEFAULT_PARAMS_FILE]}]) def test_default_params_file(tmp_dir, dvc, vars_): (tmp_dir / DEFAULT_PARAMS_FILE).dump(DATA) resolver = DataResolver(dvc, tmp_dir.fs_path, vars_) @@ -100,8 +98,8 @@ def test_global_overwrite_error_on_imports(tmp_dir, dvc): with pytest.raises(ResolveError) as exc_info: DataResolver(dvc, tmp_dir.fs_path, d) - assert str(exc_info.value) == ( - "failed to parse 'vars' in 'dvc.yaml':\n" + assert ( + str(exc_info.value) == "failed to parse 'vars' in 'dvc.yaml':\n" "cannot redefine 'models.bar' from 'params.json' " "as it already exists in 'params.yaml'" ) @@ -114,8 +112,8 @@ def test_global_overwrite_vars(tmp_dir, dvc): with pytest.raises(ResolveError) as exc_info: DataResolver(dvc, tmp_dir.fs_path, d) - assert str(exc_info.value) == ( - "failed to parse 'vars' in 'dvc.yaml':\n" + assert ( + str(exc_info.value) == "failed to parse 'vars' in 'dvc.yaml':\n" "cannot redefine 'models.bar' from 'vars[0]' " "as it already exists in 'params.yaml'" ) @@ -128,8 +126,8 @@ def test_local_declared_vars_overwrite(tmp_dir, dvc): with pytest.raises(ResolveError) as exc_info: DataResolver(dvc, tmp_dir.fs_path, d) - assert str(exc_info.value) == ( - "failed to parse 'vars' in 'dvc.yaml':\n" + assert ( + str(exc_info.value) == "failed to parse 'vars' in 'dvc.yaml':\n" "cannot redefine 'bar' from 'vars[1]' " "as it already exists in 'vars[0]'" ) @@ -140,8 +138,8 @@ def test_specified_params_file_not_exist(tmp_dir, dvc): with pytest.raises(ResolveError) as exc_info: DataResolver(dvc, tmp_dir.fs_path, d) - assert str(exc_info.value) == ( - "failed to parse 'vars' in 'dvc.yaml': " + assert ( + str(exc_info.value) == "failed to parse 'vars' in 'dvc.yaml': " "'not_existing_params.yaml' does not exist" ) @@ -182,8 +180,8 @@ def test_local_overwrite_error(tmp_dir, dvc, vars_, loc): with pytest.raises(ResolveError) as exc_info: resolver.resolve() - assert str(exc_info.value) == ( - "failed to parse stage 'build' in 'dvc.yaml':\n" + assert ( + str(exc_info.value) == "failed to parse stage 'build' in 'dvc.yaml':\n" f"cannot redefine 'models.bar' from '{loc}' " "as it already exists in 'params.yaml'" ) diff --git a/tests/func/plots/test_diff.py b/tests/func/plots/test_diff.py index ec163bf7c..cf2b44cb5 100644 --- a/tests/func/plots/test_diff.py +++ b/tests/func/plots/test_diff.py @@ -31,9 +31,9 @@ def test_diff_dirty(tmp_dir, scm, dvc, run_copy_metrics): diff_result, "workspace", "definitions", file="", endkey="data" ) == {"metric.json": props} assert get_plot(diff_result, "HEAD", file="metric.json") == metric_head - assert get_plot( - diff_result, "HEAD", "definitions", file="", endkey="data" - ) == {"metric.json": props} + assert get_plot(diff_result, "HEAD", "definitions", file="", endkey="data") == { + "metric.json": props + } metric_2 = [{"y": 7}, {"y": 8}] (tmp_dir / "metric.json").dump_json(metric_2, sort_keys=True) diff --git a/tests/func/plots/test_modify.py b/tests/func/plots/test_modify.py index 566953517..26c40add6 100644 --- a/tests/func/plots/test_modify.py +++ b/tests/func/plots/test_modify.py @@ -18,9 +18,7 @@ def test_plots_modify_existing_template( name="copy-metrics", single_stage=False, ) - dvc.plots.modify( - "metric.json", props={"template": relpath(custom_template)} - ) + dvc.plots.modify("metric.json", props={"template": relpath(custom_template)}) stage = stage.reload() assert stage.outs[0].plot == {"template": relpath(custom_template)} @@ -38,9 +36,7 @@ def test_plots_modify_should_not_change_lockfile( ) (tmp_dir / LOCK_FILE).unlink() - dvc.plots.modify( - "metric.json", props={"template": relpath(custom_template)} - ) + dvc.plots.modify("metric.json", props={"template": relpath(custom_template)}) assert not (tmp_dir / LOCK_FILE).exists() diff --git a/tests/func/plots/test_show.py b/tests/func/plots/test_show.py index 2e52bbe97..b9ed6e350 100644 --- a/tests/func/plots/test_show.py +++ b/tests/func/plots/test_show.py @@ -88,9 +88,7 @@ def test_show_non_plot(tmp_dir, scm, use_dvc): assert get_plot(plots, "workspace", file="metric.json") == metric -def test_show_non_plot_and_plot_with_params( - tmp_dir, scm, dvc, run_copy_metrics -): +def test_show_non_plot_and_plot_with_params(tmp_dir, scm, dvc, run_copy_metrics): metric = [{"first_val": 100, "val": 2}, {"first_val": 200, "val": 3}] (tmp_dir / "metric.json").dump_json(metric, sort_keys=True) run_copy_metrics( @@ -106,10 +104,7 @@ def test_show_non_plot_and_plot_with_params( assert get_plot(result, "workspace", file="metric.json") == metric assert get_plot(result, "workspace", file="metric2.json") == metric - assert ( - get_plot(result, "workspace", file="metric2.json", endkey="props") - == props - ) + assert get_plot(result, "workspace", file="metric2.json", endkey="props") == props def test_show_from_subdir(tmp_dir, dvc, capsys): @@ -137,8 +132,7 @@ def test_plots_show_non_existing(tmp_dir, dvc, capsys): cap = capsys.readouterr() assert ( - "DVC failed to load some plots for following revisions: 'workspace'" - in cap.err + "DVC failed to load some plots for following revisions: 'workspace'" in cap.err ) @@ -234,9 +228,7 @@ def test_ignore_parsing_error(tmp_dir, dvc, run_copy_metrics): ), ), ) -def test_log_errors( - tmp_dir, scm, dvc, run_copy_metrics, file, path_kwargs, capsys -): +def test_log_errors(tmp_dir, scm, dvc, run_copy_metrics, file, path_kwargs, capsys): metric = [{"val": 2}, {"val": 3}] (tmp_dir / "metric_t.yaml").dump(metric) run_copy_metrics( @@ -256,15 +248,12 @@ def test_log_errors( assert isinstance(get_plot(result, **path_kwargs), YAMLFileCorruptedError) assert ( - "DVC failed to load some plots for following revisions: 'workspace'." - in error + "DVC failed to load some plots for following revisions: 'workspace'." in error ) @pytest.mark.parametrize("ext", ["jpg", "svg"]) -def test_plots_binary( - tmp_dir, scm, dvc, run_copy_metrics, custom_template, ext -): +def test_plots_binary(tmp_dir, scm, dvc, run_copy_metrics, custom_template, ext): file1 = f"image.{ext}" file2 = f"plot.{ext}" with open(file1, "wb") as fd: @@ -330,9 +319,7 @@ def test_collect_non_existing_dir(tmp_dir, dvc, run_copy_metrics): remove(subdir_stage.outs[0].fs_path) result = dvc.plots.show() - assert get_plot( - result, "workspace", typ="definitions", file="", endkey="error" - ) + assert get_plot(result, "workspace", typ="definitions", file="", endkey="error") # make sure others gets loaded assert get_plot(result, "workspace", file="plot.json") == metric diff --git a/tests/func/test_add.py b/tests/func/test_add.py index 6997688ce..4074090be 100644 --- a/tests/func/test_add.py +++ b/tests/func/test_add.py @@ -30,10 +30,7 @@ from dvc.output import ( OutputIsStageFileError, ) from dvc.stage import Stage -from dvc.stage.exceptions import ( - StageExternalOutputsError, - StagePathNotFoundError, -) +from dvc.stage.exceptions import StageExternalOutputsError, StagePathNotFoundError from dvc.testing.workspace_tests import TestAdd from dvc.utils import LARGE_DIR_SIZE from dvc.utils.fs import path_isin @@ -140,9 +137,7 @@ def test_warn_about_large_directories_recursive_add(tmp_dir, dvc, capsys): nc=colorama.Style.RESET_ALL, ) - tmp_dir.gen( - "large-dir", {f"{i}": f"{i}" for i in range(LARGE_DIR_SIZE + 1)} - ) + tmp_dir.gen("large-dir", {f"{i}": f"{i}" for i in range(LARGE_DIR_SIZE + 1)}) assert main(["add", "--recursive", "large-dir"]) == 0 assert warning in capsys.readouterr()[1] @@ -376,9 +371,7 @@ def test_should_update_state_entry_for_file_after_add(mocker, dvc, tmp_dir): assert file_md5_counter.mock.call_count == 1 -def test_should_update_state_entry_for_directory_after_add( - mocker, dvc, tmp_dir -): +def test_should_update_state_entry_for_directory_after_add(mocker, dvc, tmp_dir): file_md5_counter = mocker.spy(dvc_data.hashfile.hash, "file_md5") tmp_dir.gen({"data/data": "foo", "data/data_sub/sub_data": "foo"}) @@ -395,9 +388,7 @@ def test_should_update_state_entry_for_directory_after_add( assert file_md5_counter.mock.call_count == 6 ls = "dir" if os.name == "nt" else "ls" - ret = main( - ["run", "--single-stage", "-d", "data", "{} {}".format(ls, "data")] - ) + ret = main(["run", "--single-stage", "-d", "data", "{} {}".format(ls, "data")]) assert ret == 0 assert file_md5_counter.mock.call_count == 8 @@ -457,9 +448,7 @@ def test_should_place_stage_in_data_dir_if_repository_below_symlink( assert (tmp_dir / "data" / "foo.dvc").exists() -def test_should_throw_proper_exception_on_corrupted_stage_file( - caplog, tmp_dir, dvc -): +def test_should_throw_proper_exception_on_corrupted_stage_file(caplog, tmp_dir, dvc): tmp_dir.gen({"foo": "foo", "bar": " bar"}) assert main(["add", "foo"]) == 0 @@ -468,9 +457,7 @@ def test_should_throw_proper_exception_on_corrupted_stage_file( caplog.clear() assert main(["add", "bar"]) == 1 - expected_error = ( - "unable to read: 'foo.dvc', YAML file structure is corrupted" - ) + expected_error = "unable to read: 'foo.dvc', YAML file structure is corrupted" assert expected_error in caplog.text @@ -644,9 +631,7 @@ def test_should_not_checkout_when_adding_cached_copy(tmp_dir, dvc, mocker): ("copy", "symlink", system.is_symlink), ], ) -def test_should_relink_on_repeated_add( - link, new_link, link_test_func, tmp_dir, dvc -): +def test_should_relink_on_repeated_add(link, new_link, link_test_func, tmp_dir, dvc): dvc.config["cache"]["type"] = link tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"}) @@ -1098,9 +1083,7 @@ def test_add_on_not_existing_file_should_not_remove_stage_file(tmp_dir, dvc): "dvc.stage.Stage.commit", ], ) -def test_add_does_not_remove_stage_file_on_failure( - tmp_dir, dvc, mocker, target -): +def test_add_does_not_remove_stage_file_on_failure(tmp_dir, dvc, mocker, target): (stage,) = tmp_dir.dvc_gen("foo", "foo") tmp_dir.gen("foo", "foobar") # update file dvcfile_contents = (tmp_dir / stage.path).read_text() diff --git a/tests/func/test_analytics.py b/tests/func/test_analytics.py index c122f61fd..1ce40e259 100644 --- a/tests/func/test_analytics.py +++ b/tests/func/test_analytics.py @@ -31,9 +31,7 @@ def mock_daemon(mocker): def func(argv): return main(["daemon", *argv]) - return mocker.patch( - "dvc.daemon.daemon", mocker.MagicMock(side_effect=func) - ) + return mocker.patch("dvc.daemon.daemon", mocker.MagicMock(side_effect=func)) def test_collect_and_send_report(mocker, dvc, mock_daemon): diff --git a/tests/func/test_check_ignore.py b/tests/func/test_check_ignore.py index 80e21494a..3e00d0a34 100644 --- a/tests/func/test_check_ignore.py +++ b/tests/func/test_check_ignore.py @@ -87,15 +87,11 @@ def test_check_ignore_out_side_repo(tmp_dir, dvc): def test_check_ignore_sub_repo(tmp_dir, dvc, capsys): - tmp_dir.gen( - {DvcIgnore.DVCIGNORE_FILE: "other", "dir": {".dvc": {}, "foo": "bar"}} - ) + tmp_dir.gen({DvcIgnore.DVCIGNORE_FILE: "other", "dir": {".dvc": {}, "foo": "bar"}}) assert main(["check-ignore", "-d", os.path.join("dir", "foo")]) == 0 out, _ = capsys.readouterr() - assert ( - "in sub_repo:{}\t{}".format("dir", os.path.join("dir", "foo")) in out - ) + assert "in sub_repo:{}\t{}".format("dir", os.path.join("dir", "foo")) in out def test_check_sub_dir_ignore_file(tmp_dir, dvc, capsys): @@ -136,9 +132,7 @@ def test_check_ignore_details_all(tmp_dir, dvc, capsys): @pytest.mark.parametrize( "file,ret,output", [("ignored", 0, True), ("not_ignored", 1, False)] ) -def test_check_ignore_stdin_mode( - tmp_dir, dvc, file, ret, output, capsys, mocker -): +def test_check_ignore_stdin_mode(tmp_dir, dvc, file, ret, output, capsys, mocker): tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "ignored") mocker.patch("builtins.input", side_effect=[file, ""]) diff --git a/tests/func/test_checkout.py b/tests/func/test_checkout.py index be28cec87..8bb51a008 100644 --- a/tests/func/test_checkout.py +++ b/tests/func/test_checkout.py @@ -324,9 +324,7 @@ def test_checkout_suggest_git(tmp_dir, dvc, scm): assert e.value.__cause__.__cause__ is None -def test_checkout_target_recursive_should_not_remove_other_used_files( - tmp_dir, dvc -): +def test_checkout_target_recursive_should_not_remove_other_used_files(tmp_dir, dvc): tmp_dir.dvc_gen({"foo": "foo", "bar": "bar", "data": {"file": "file"}}) assert main(["checkout", "-R", "data"]) == 0 assert (tmp_dir / "foo").exists() @@ -375,9 +373,7 @@ def test_checkout_moved_cache_dir_with_symlinks(tmp_dir, dvc): assert system.is_symlink(os.path.join("data", "file")) new_data_link = os.path.realpath(os.path.join("data", "file")) - assert relpath(old_foo_link, old_cache_dir) == relpath( - new_foo_link, new_cache_dir - ) + assert relpath(old_foo_link, old_cache_dir) == relpath(new_foo_link, new_cache_dir) assert relpath(old_data_link, old_cache_dir) == relpath( new_data_link, new_cache_dir @@ -386,9 +382,7 @@ def test_checkout_moved_cache_dir_with_symlinks(tmp_dir, dvc): def test_checkout_no_checksum(tmp_dir, dvc): tmp_dir.gen("file", "file content") - stage = dvc.run( - outs=["file"], no_exec=True, cmd="somecmd", single_stage=True - ) + stage = dvc.run(outs=["file"], no_exec=True, cmd="somecmd", single_stage=True) with pytest.raises(CheckoutError): dvc.checkout([stage.path], force=True) @@ -566,9 +560,7 @@ def test_stats_on_removed_file_from_tracked_dir(tmp_dir, dvc, scm): assert dvc.checkout() == empty_checkout -def test_stats_on_show_changes_does_not_show_summary( - tmp_dir, dvc, scm, capsys -): +def test_stats_on_show_changes_does_not_show_summary(tmp_dir, dvc, scm, capsys): tmp_dir.dvc_gen( {"dir": {"subdir": {"file": "file"}}, "other": "other"}, commit="initial", @@ -767,9 +759,7 @@ def test_checkout_executable(tmp_dir, dvc): def test_checkout_partial(tmp_dir, dvc): - tmp_dir.dvc_gen( - {"data": {"foo": "foo", "bar": "bar", "sub_dir": {"baz": "baz"}}} - ) + tmp_dir.dvc_gen({"data": {"foo": "foo", "bar": "bar", "sub_dir": {"baz": "baz"}}}) data_dir = tmp_dir / "data" shutil.rmtree(data_dir) @@ -837,9 +827,7 @@ def test_checkout_partial_unchanged(tmp_dir, dvc): def test_checkout_partial_subdir(tmp_dir, dvc): - tmp_dir.dvc_gen( - {"data": {"foo": "foo", "sub_dir": {"bar": "bar", "baz": "baz"}}} - ) + tmp_dir.dvc_gen({"data": {"foo": "foo", "sub_dir": {"bar": "bar", "baz": "baz"}}}) data_dir = tmp_dir / "data" sub_dir = data_dir / "sub_dir" diff --git a/tests/func/test_commit.py b/tests/func/test_commit.py index 89355e53f..23d271e6e 100644 --- a/tests/func/test_commit.py +++ b/tests/func/test_commit.py @@ -119,9 +119,7 @@ def test_commit_changed_md5(tmp_dir, dvc): def test_commit_no_exec(tmp_dir, dvc): tmp_dir.gen({"dep": "dep", "out": "out"}) - stage = dvc.run( - name="my", cmd="mycmd", deps=["dep"], outs=["out"], no_exec=True - ) + stage = dvc.run(name="my", cmd="mycmd", deps=["dep"], outs=["out"], no_exec=True) assert dvc.status(stage.path) dvc.commit(stage.path, force=True) @@ -140,9 +138,7 @@ def test_commit_granular_output(tmp_dir, dvc): assert not list(cache.glob("*/*")) dvc.commit("foo") - assert list(cache.glob("*/*")) == [ - cache / "d3" / "b07384d113edec49eaa6238ad5ff00" - ] + assert list(cache.glob("*/*")) == [cache / "d3" / "b07384d113edec49eaa6238ad5ff00"] def test_commit_granular_output_file(tmp_dir, dvc): @@ -208,9 +204,7 @@ def test_commit_granular_dir(tmp_dir, dvc): def test_commit_no_exec_missing_dep(tmp_dir, dvc): - stage = dvc.run( - name="my", cmd="mycmd", deps=["dep"], outs=["out"], no_exec=True - ) + stage = dvc.run(name="my", cmd="mycmd", deps=["dep"], outs=["out"], no_exec=True) assert dvc.status(stage.path) with pytest.raises(DependencyDoesNotExistError): diff --git a/tests/func/test_config.py b/tests/func/test_config.py index 8dfd01371..fec065661 100644 --- a/tests/func/test_config.py +++ b/tests/func/test_config.py @@ -213,14 +213,12 @@ def test_list_bad_args(tmp_dir, dvc, caplog, args): assert main(["config", "--list", *args]) == 1 assert ( "-l/--list can't be used together with any of these options: " - "-u/--unset, name, value" - ) in caplog.text + "-u/--unset, name, value" in caplog.text + ) def test_set_invalid_key(dvc): - with pytest.raises( # noqa: PT012 - ConfigError, match=r"extra keys not allowed" - ): + with pytest.raises(ConfigError, match=r"extra keys not allowed"): # noqa: PT012 with dvc.config.edit() as conf: conf["core"]["invalid_key"] = "value" @@ -276,9 +274,7 @@ def test_load_relative_paths(dvc, field, remote_url): # load config and check that it contains what we expect # (relative paths are evaluated correctly) cfg = Config(dvc_dir) - assert cfg["remote"]["test"][field] == os.path.join( - dvc_dir, "..", "file.txt" - ) + assert cfg["remote"]["test"][field] == os.path.join(dvc_dir, "..", "file.txt") def test_config_gdrive_fields(tmp_dir, dvc): @@ -293,9 +289,7 @@ def test_config_gdrive_fields(tmp_dir, dvc): def test_config_remote(tmp_dir, dvc, capsys): (tmp_dir / ".dvc" / "config").write_text( - "['remote \"myremote\"']\n" - " url = s3://bucket/path\n" - " region = myregion\n" + "['remote \"myremote\"']\n url = s3://bucket/path\n region = myregion\n" ) assert main(["config", "remote.myremote.url"]) == 0 @@ -309,25 +303,14 @@ def test_config_remote(tmp_dir, dvc, capsys): def test_config_show_origin_single(tmp_dir, dvc, capsys): (tmp_dir / ".dvc" / "config").write_text( - "['remote \"myremote\"']\n" - " url = s3://bucket/path\n" - " region = myregion\n" + "['remote \"myremote\"']\n url = s3://bucket/path\n region = myregion\n" ) - assert ( - main(["config", "--show-origin", "--project", "remote.myremote.url"]) - == 0 - ) + assert main(["config", "--show-origin", "--project", "remote.myremote.url"]) == 0 out, _ = capsys.readouterr() - assert ( - "{}\t{}\n".format(os.path.join(".dvc", "config"), "s3://bucket/path") - in out - ) + assert "{}\t{}\n".format(os.path.join(".dvc", "config"), "s3://bucket/path") in out - assert ( - main(["config", "--show-origin", "--local", "remote.myremote.url"]) - == 251 - ) + assert main(["config", "--show-origin", "--local", "remote.myremote.url"]) == 251 assert main(["config", "--list", "--project", "--show-origin"]) == 0 out, _ = capsys.readouterr() @@ -342,9 +325,7 @@ def test_config_show_origin_single(tmp_dir, dvc, capsys): def test_config_show_origin_merged(tmp_dir, dvc, capsys): (tmp_dir / ".dvc" / "config").write_text( - "['remote \"myremote\"']\n" - " url = s3://bucket/path\n" - " region = myregion\n" + "['remote \"myremote\"']\n url = s3://bucket/path\n region = myregion\n" ) (tmp_dir / ".dvc" / "config.local").write_text( diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index 5f3aca21f..38da3d1db 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -9,9 +9,7 @@ import dvc_data from dvc.cli import main from dvc.external_repo import clean_repos from dvc.stage.exceptions import StageNotFound -from dvc.testing.remote_tests import ( # noqa, pylint: disable=unused-import - TestRemote, -) +from dvc.testing.remote_tests import TestRemote # noqa, pylint: disable=unused-import from dvc.utils.fs import remove from dvc_data.hashfile.db import HashFileDB from dvc_data.hashfile.db.local import LocalHashFileDB @@ -45,8 +43,7 @@ def test_cloud_cli(tmp_dir, dvc, remote, mocker): assert os.path.isfile(cache_dir) assert oids_exist.called assert all( - _kwargs["jobs"] == jobs - for (_args, _kwargs) in oids_exist.call_args_list + _kwargs["jobs"] == jobs for (_args, _kwargs) in oids_exist.call_args_list ) dvc.odb.local.clear() @@ -58,8 +55,7 @@ def test_cloud_cli(tmp_dir, dvc, remote, mocker): assert os.path.isfile(cache_dir) assert oids_exist.called assert all( - _kwargs["jobs"] == jobs - for (_args, _kwargs) in oids_exist.call_args_list + _kwargs["jobs"] == jobs for (_args, _kwargs) in oids_exist.call_args_list ) oids_exist.reset_mock() @@ -72,8 +68,7 @@ def test_cloud_cli(tmp_dir, dvc, remote, mocker): assert os.path.isdir("data_dir") assert oids_exist.called assert all( - _kwargs["jobs"] == jobs - for (_args, _kwargs) in oids_exist.call_args_list + _kwargs["jobs"] == jobs for (_args, _kwargs) in oids_exist.call_args_list ) with open(cache, encoding="utf-8") as fd: @@ -90,17 +85,14 @@ def test_cloud_cli(tmp_dir, dvc, remote, mocker): # NOTE: check if remote gc works correctly on directories assert main(["gc", "-cw", "-f", *args]) == 0 assert _list_oids_traverse.called - assert all( - _kwargs["jobs"] == 2 for (_args, _kwargs) in oids_exist.call_args_list - ) + assert all(_kwargs["jobs"] == 2 for (_args, _kwargs) in oids_exist.call_args_list) shutil.move(dvc.odb.local.path, dvc.odb.local.path + ".back") assert main(["fetch", *args]) == 0 assert oids_exist.called assert all( - _kwargs["jobs"] == jobs - for (_args, _kwargs) in oids_exist.call_args_list + _kwargs["jobs"] == jobs for (_args, _kwargs) in oids_exist.call_args_list ) oids_exist.reset_mock() @@ -112,8 +104,7 @@ def test_cloud_cli(tmp_dir, dvc, remote, mocker): assert os.path.isdir("data_dir") assert oids_exist.called assert all( - _kwargs["jobs"] == jobs - for (_args, _kwargs) in oids_exist.call_args_list + _kwargs["jobs"] == jobs for (_args, _kwargs) in oids_exist.call_args_list ) @@ -195,9 +186,7 @@ def test_missing_cache(tmp_dir, dvc, local_remote, caplog): assert bar not in caplog.text -def test_verify_hashes( - tmp_dir, scm, dvc, mocker, tmp_path_factory, local_remote -): +def test_verify_hashes(tmp_dir, scm, dvc, mocker, tmp_path_factory, local_remote): tmp_dir.dvc_gen({"file": "file1 content"}, commit="add file") tmp_dir.dvc_gen({"dir": {"subfile": "file2 content"}}, commit="add dir") dvc.push() @@ -276,9 +265,7 @@ def test_pull_external_dvc_imports(tmp_dir, dvc, scm, erepo_dir): def test_pull_partial_import(tmp_dir, dvc, local_workspace): local_workspace.gen("file", "file content") dst = tmp_dir / "file" - stage = dvc.imp_url( - "remote://workspace/file", os.fspath(dst), no_download=True - ) + stage = dvc.imp_url("remote://workspace/file", os.fspath(dst), no_download=True) result = dvc.pull("file") assert result["fetched"] == 1 @@ -287,9 +274,7 @@ def test_pull_partial_import(tmp_dir, dvc, local_workspace): assert stage.outs[0].get_hash().value == "d10b4c3ff123b26dc068d43a8bef2d23" -def test_pull_external_dvc_imports_mixed( - tmp_dir, dvc, scm, erepo_dir, local_remote -): +def test_pull_external_dvc_imports_mixed(tmp_dir, dvc, scm, erepo_dir, local_remote): with erepo_dir.chdir(): erepo_dir.dvc_gen("foo", "foo", commit="first") os.remove("foo") @@ -319,9 +304,7 @@ def clean(outs, dvc=None): def recurse_list_dir(d): return [ - os.path.join(root, f) - for root, _, filenames in os.walk(d) - for f in filenames + os.path.join(root, f) for root, _, filenames in os.walk(d) for f in filenames ] diff --git a/tests/func/test_data_status.py b/tests/func/test_data_status.py index 2aa34cd99..e5a263c12 100644 --- a/tests/func/test_data_status.py +++ b/tests/func/test_data_status.py @@ -171,9 +171,7 @@ def test_skip_uncached_pipeline_outputs(tmp_dir, dvc, run_copy_metrics): name="copy-metrics", ) assert dvc.data_status() == EMPTY_STATUS - assert ( - dvc.data_status(granular=True, untracked_files="all") == EMPTY_STATUS - ) + assert dvc.data_status(granular=True, untracked_files="all") == EMPTY_STATUS def test_outs_with_no_hashes(M, tmp_dir, dvc, scm): @@ -215,9 +213,7 @@ def test_subdir(M, tmp_dir, scm): assert dvc.data_status(granular=True, untracked_files="all") == { **EMPTY_STATUS, "git": M.dict(), - "unchanged": M.unordered( - "bar", join("dir", ""), join("dir", "foo") - ), + "unchanged": M.unordered("bar", join("dir", ""), join("dir", "foo")), "untracked": ["untracked"], } @@ -228,9 +224,7 @@ def test_untracked_newly_added_files(M, tmp_dir, dvc, scm): expected = { **EMPTY_STATUS, - "untracked": M.unordered( - join("dir", "foo"), join("dir", "bar"), "foobar" - ), + "untracked": M.unordered(join("dir", "foo"), join("dir", "bar"), "foobar"), "git": M.dict(), } assert dvc.data_status(untracked_files="all") == expected @@ -254,9 +248,7 @@ def test_missing_cache_workspace_exists(M, tmp_dir, dvc, scm): **EMPTY_STATUS, "untracked": M.unordered("foobar.dvc", "dir.dvc", ".gitignore"), "committed": {"added": M.unordered("foobar", join("dir", ""))}, - "uncommitted": { - "unknown": M.unordered(join("dir", "foo"), join("dir", "bar")) - }, + "uncommitted": {"unknown": M.unordered(join("dir", "foo"), join("dir", "bar"))}, "not_in_cache": M.unordered( "foobar", join("dir", ""), @@ -307,9 +299,7 @@ def test_git_committed_missing_cache_workspace_exists(M, tmp_dir, dvc, scm): "foobar", join("dir", ""), ), - "uncommitted": { - "unknown": M.unordered(join("dir", "foo"), join("dir", "bar")) - }, + "uncommitted": {"unknown": M.unordered(join("dir", "foo"), join("dir", "bar"))}, "git": M.dict(), "unchanged": M.unordered("foobar", join("dir", "")), } @@ -361,9 +351,7 @@ def test_partial_missing_cache(M, tmp_dir, dvc, scm): def test_missing_dir_object_from_head(M, tmp_dir, dvc, scm): - (stage,) = tmp_dir.dvc_gen( - {"dir": {"foo": "foo", "bar": "bar"}}, commit="add dir" - ) + (stage,) = tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar"}}, commit="add dir") remove("dir") tmp_dir.dvc_gen({"dir": {"foobar": "foobar"}}) odb = dvc.odb.repo diff --git a/tests/func/test_diff.py b/tests/func/test_diff.py index fc1c10023..50a866dd9 100644 --- a/tests/func/test_diff.py +++ b/tests/func/test_diff.py @@ -247,9 +247,7 @@ def test_diff_no_cache(tmp_dir, scm, dvc): tmp_dir.dvc_gen({"dir": {"file": "file content"}}, commit="first") scm.tag("v1") - tmp_dir.dvc_gen( - {"dir": {"file": "modified file content"}}, commit="second" - ) + tmp_dir.dvc_gen({"dir": {"file": "modified file content"}}, commit="second") scm.tag("v2") remove(dvc.odb.local.path) @@ -298,9 +296,7 @@ def test_diff_dirty(tmp_dir, scm, dvc): }, {"hash": "86d049de17c76ac44cdcac146042ec9b", "path": "new_file"}, ], - "deleted": [ - {"hash": "7f0b6bb0b7e951b7fd2b2a4a326297e1", "path": "file"} - ], + "deleted": [{"hash": "7f0b6bb0b7e951b7fd2b2a4a326297e1", "path": "file"}], "modified": [ { "hash": { @@ -413,9 +409,7 @@ def test_targets_single_file_in_dir(tmp_dir, scm, dvc): def test_targets_two_files_in_dir(tmp_dir, scm, dvc): setup_targets_test(tmp_dir) - assert dvc.diff( - targets=[os.path.join("dir", "1"), os.path.join("dir", "2")] - ) == { + assert dvc.diff(targets=[os.path.join("dir", "1"), os.path.join("dir", "2")]) == { "added": [ {"path": os.path.join("dir", "1"), "hash": digest("1")}, {"path": os.path.join("dir", "2"), "hash": digest("2")}, @@ -582,9 +576,7 @@ def test_diff_rename_file(tmp_dir, scm, dvc, commit_last): last_commit_msg = None a_rev = "HEAD" - paths = tmp_dir.gen( - {"dir": {"file": "text1", "subdir": {"file2": "text2"}}} - ) + paths = tmp_dir.gen({"dir": {"file": "text1", "subdir": {"file2": "text2"}}}) tmp_dir.dvc_add(paths, commit="commit #1") (tmp_dir / "dir" / "file").replace(tmp_dir / "dir" / "subdir" / "file3") diff --git a/tests/func/test_dvcfile.py b/tests/func/test_dvcfile.py index 42cb781ba..fb8bd9d37 100644 --- a/tests/func/test_dvcfile.py +++ b/tests/func/test_dvcfile.py @@ -211,9 +211,7 @@ def test_remove_stage_lockfile(tmp_dir, dvc, run_copy): lock_file = dvc_file._lockfile assert dvc_file.exists() assert lock_file.exists() - assert {"copy-bar-foobar", "copy-foo-bar"} == set( - lock_file.load()["stages"].keys() - ) + assert {"copy-bar-foobar", "copy-foo-bar"} == set(lock_file.load()["stages"].keys()) lock_file.remove_stage(stage) assert ["copy-bar-foobar"] == list(lock_file.load()["stages"].keys()) @@ -303,9 +301,7 @@ def test_remove_stage_preserves_comment(tmp_dir, dvc, run_copy): ) -def test_remove_stage_removes_dvcfiles_if_no_stages_left( - tmp_dir, dvc, run_copy -): +def test_remove_stage_removes_dvcfiles_if_no_stages_left(tmp_dir, dvc, run_copy): tmp_dir.gen("foo", "foo") run_copy("foo", "bar", name="run_copy") @@ -404,9 +400,7 @@ def test_dvcfile_load_dump_stage_with_desc_meta(tmp_dir, dvc): stage = dvc.stage.load_one(name="stage1") assert stage.meta == {"key1": "value1", "key2": "value2"} assert stage.desc == "stage desc" - assert stage.outs[0].annot == Annotation( - desc="bar desc", meta={"key": "value"} - ) + assert stage.outs[0].annot == Annotation(desc="bar desc", meta={"key": "value"}) # sanity check stage.dump() @@ -438,10 +432,7 @@ def test_dvcfile_load_dump_stage_with_desc_meta(tmp_dir, dvc): def test_dvcfile_load_with_plots(tmp_dir, dvc, data): (tmp_dir / "dvc.yaml").dump(data) plots = list(dvc.plots.collect()) - top_level_plots = plots[0]["workspace"]["definitions"]["data"]["dvc.yaml"][ - "data" - ] + top_level_plots = plots[0]["workspace"]["definitions"]["data"]["dvc.yaml"]["data"] assert all( - name in top_level_plots - for name in ("path/to/plot", "path/to/another/plot") + name in top_level_plots for name in ("path/to/plot", "path/to/another/plot") ) diff --git a/tests/func/test_external_repo.py b/tests/func/test_external_repo.py index 133637e5b..34dfb7953 100644 --- a/tests/func/test_external_repo.py +++ b/tests/func/test_external_repo.py @@ -139,9 +139,7 @@ def test_shallow_clone_branch(erepo_dir, mocker): with repo.open_by_relpath("file") as fd: assert fd.read() == "branch" - clone_spy.assert_called_with( - url, ANY, shallow_branch="branch", progress=ANY - ) + clone_spy.assert_called_with(url, ANY, shallow_branch="branch", progress=ANY) path, _ = CLONES[url] CLONES[url] = (path, True) diff --git a/tests/func/test_gc.py b/tests/func/test_gc.py index b081a0610..d5f264dd1 100644 --- a/tests/func/test_gc.py +++ b/tests/func/test_gc.py @@ -127,9 +127,7 @@ def test_gc_no_unpacked_dir(tmp_dir, dvc): dvc.status() os.remove("dir.dvc") - unpackeddir = ( - dir_stages[0].outs[0].cache_path + LocalHashFileDB.UNPACKED_DIR_SUFFIX - ) + unpackeddir = dir_stages[0].outs[0].cache_path + LocalHashFileDB.UNPACKED_DIR_SUFFIX # older (pre 1.0) versions of dvc used to generate this dir shutil.copytree("dir", unpackeddir) @@ -183,8 +181,8 @@ def test_gc_without_workspace(tmp_dir, dvc, caplog, cloud): assert ( "Either of `-w|--workspace`, `-a|--all-branches`, `-T|--all-tags` " "`--all-experiments`, `--all-commits`, `--date` or `--rev` " - "needs to be set." - ) in caplog.text + "needs to be set." in caplog.text + ) def test_gc_with_possible_args_positive(tmp_dir, dvc): @@ -215,9 +213,7 @@ def test_gc_cloud_remove_order(tmp_dir, scm, dvc, tmp_path_factory, mocker): dvc.remove(dir2.relpath) dvc.gc(workspace=True) - mocked_remove = mocker.patch.object( - LocalFileSystem, "remove", autospec=True - ) + mocked_remove = mocker.patch.object(LocalFileSystem, "remove", autospec=True) dvc.gc(workspace=True, cloud=True) assert len(mocked_remove.mock_calls) == 4 # Unpacked dir should be the first removed @@ -263,21 +259,15 @@ def test_gc_external_output(tmp_dir, dvc, workspace): foo_hash = foo_stage.outs[0].hash_info.value bar_hash = bar_stage.outs[0].hash_info.value - assert ( - workspace / "cache" / foo_hash[:2] / foo_hash[2:] - ).read_text() == "foo" - assert ( - workspace / "cache" / bar_hash[:2] / bar_hash[2:] - ).read_text() == "bar" + assert (workspace / "cache" / foo_hash[:2] / foo_hash[2:]).read_text() == "foo" + assert (workspace / "cache" / bar_hash[:2] / bar_hash[2:]).read_text() == "bar" (tmp_dir / "foo.dvc").unlink() dvc.gc(workspace=True) assert not (workspace / "cache" / foo_hash[:2] / foo_hash[2:]).exists() - assert ( - workspace / "cache" / bar_hash[:2] / bar_hash[2:] - ).read_text() == "bar" + assert (workspace / "cache" / bar_hash[:2] / bar_hash[2:]).read_text() == "bar" def test_gc_all_experiments(tmp_dir, scm, dvc): @@ -300,9 +290,7 @@ def test_gc_all_experiments(tmp_dir, scm, dvc): # all_experiments will include the experiment commit (baz) plus baseline # commit (bar) - assert not ( - tmp_dir / ".dvc" / "cache" / foo_hash[:2] / foo_hash[2:] - ).exists() + assert not (tmp_dir / ".dvc" / "cache" / foo_hash[:2] / foo_hash[2:]).exists() assert ( tmp_dir / ".dvc" / "cache" / bar_hash[:2] / bar_hash[2:] ).read_text() == "bar" @@ -341,9 +329,7 @@ def test_date(tmp_dir, scm, dvc): dvc.gc(commit_date=datestamp) assert _count_files(dvc.odb.local.path) == 1 - assert dvc.odb.local.exists( - "9ae73c65f418e6f79ceb4f0e4a4b98d5" # "modified" - ) + assert dvc.odb.local.exists("9ae73c65f418e6f79ceb4f0e4a4b98d5") # "modified" tmp_dir.dvc_gen("testfile", "modified, again", commit="modify") @@ -351,6 +337,4 @@ def test_date(tmp_dir, scm, dvc): dvc.gc(commit_date=datestamp) assert _count_files(dvc.odb.local.path) == 2 assert dvc.odb.local.exists("9ae73c65f418e6f79ceb4f0e4a4b98d5") - assert dvc.odb.local.exists( - "3bcf3b1be3e794a97a5a6b93a005784c" # "modified, again" - ) + assert dvc.odb.local.exists("3bcf3b1be3e794a97a5a6b93a005784c") # "modified, again" diff --git a/tests/func/test_get.py b/tests/func/test_get.py index b1d83342e..aed930977 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -24,9 +24,7 @@ def test_get_repo_file(tmp_dir, erepo_dir): def test_get_repo_file_replace_without_confirmation(tmp_dir, erepo_dir): with erepo_dir.chdir(): erepo_dir.dvc_gen("file", "contents", commit="create file") - erepo_dir.dvc_gen( - "file2", "something different", commit="create file2" - ) + erepo_dir.dvc_gen("file2", "something different", commit="create file2") Repo.get(os.fspath(erepo_dir), "file", "file_imported") # getting another file with a name that already exists in Repo. @@ -67,9 +65,7 @@ def test_get_git_dir(tmp_dir, erepo): src = "some_directory" dst = "some_directory_imported" - erepo.scm_gen( - {src: {"dir": {"file.txt": "hello"}}}, commit="add a regular dir" - ) + erepo.scm_gen({src: {"dir": {"file.txt": "hello"}}}, commit="add a regular dir") Repo.get(os.fspath(erepo), src, dst) @@ -123,9 +119,7 @@ def test_get_full_dvc_path(tmp_dir, erepo_dir, tmp_path_factory): erepo_dir.dvc.add(os.fspath(external_data), external=True) erepo_dir.scm_add("ext_data.dvc", commit="add external data") - Repo.get( - os.fspath(erepo_dir), os.fspath(external_data), "ext_data_imported" - ) + Repo.get(os.fspath(erepo_dir), os.fspath(external_data), "ext_data_imported") assert (tmp_dir / "ext_data_imported").read_text() == "ext_data" @@ -210,9 +204,7 @@ def test_get_file_from_dir(tmp_dir, erepo_dir): assert (tmp_dir / "subdir" / "foo").read_text() == "foo" assert (tmp_dir / "subdir" / "bar").read_text() == "bar" - Repo.get( - os.fspath(erepo_dir), os.path.join("dir", "subdir", "foo"), out="X" - ) + Repo.get(os.fspath(erepo_dir), os.path.join("dir", "subdir", "foo"), out="X") assert (tmp_dir / "X").read_text() == "foo" @@ -252,9 +244,7 @@ def test_get_url_git_only_repo(tmp_dir, scm, caplog): assert "failed to show URL" in caplog.text -def test_get_pipeline_tracked_outs( - tmp_dir, dvc, scm, git_dir, run_copy, local_remote -): +def test_get_pipeline_tracked_outs(tmp_dir, dvc, scm, git_dir, run_copy, local_remote): from dvc.dvcfile import LOCK_FILE, PROJECT_FILE tmp_dir.gen("foo", "foo") diff --git a/tests/func/test_ignore.py b/tests/func/test_ignore.py index 49cf9dce5..e77100621 100644 --- a/tests/func/test_ignore.py +++ b/tests/func/test_ignore.py @@ -225,9 +225,7 @@ def test_ignore_external(tmp_dir, scm, dvc, tmp_path_factory): (ext_dir / "tmp" / "file").fs_path, } assert dvc.dvcignore.is_ignored_dir(os.fspath(ext_dir / "tmp")) is False - assert ( - dvc.dvcignore.is_ignored_file(os.fspath(ext_dir / "y.backup")) is False - ) + assert dvc.dvcignore.is_ignored_file(os.fspath(ext_dir / "y.backup")) is False def test_ignore_resurface_subrepo(tmp_dir, scm, dvc): @@ -295,9 +293,7 @@ def test_ignore_file_in_parent_path( tmp_dir.gen(DvcIgnore.DVCIGNORE_FILE, "\n".join(pattern_list)) dvc._reset() result = dvc.dvcignore.find(dvc.fs, tmp_dir / "dir") - assert set(result) == { - (tmp_dir / relpath).fs_path for relpath in result_set - } + assert set(result) == {(tmp_dir / relpath).fs_path for relpath in result_set} # If there is a separator at the end of the pattern then the pattern @@ -330,9 +326,7 @@ def test_ignore_directory(tmp_dir, dvc): tmp_dir.gen({"dir": {DvcIgnore.DVCIGNORE_FILE: "fortz"}}) dvc._reset() result = dvc.dvcignore.find(dvc.fs, tmp_dir / "dir") - assert set(result) == { - (tmp_dir / "dir" / DvcIgnore.DVCIGNORE_FILE).fs_path - } + assert set(result) == {(tmp_dir / "dir" / DvcIgnore.DVCIGNORE_FILE).fs_path} def test_multi_ignore_file(tmp_dir, dvc, monkeypatch): @@ -367,12 +361,8 @@ def test_pattern_trie_fs(tmp_dir, dvc): dvc._reset() dvcignore = dvc.dvcignore - ignore_pattern_top = dvcignore._get_trie_pattern( - os.fspath(tmp_dir / "top") - ) - ignore_pattern_other = dvcignore._get_trie_pattern( - os.fspath(tmp_dir / "other") - ) + ignore_pattern_top = dvcignore._get_trie_pattern(os.fspath(tmp_dir / "top")) + ignore_pattern_other = dvcignore._get_trie_pattern(os.fspath(tmp_dir / "other")) ignore_pattern_first = dvcignore._get_trie_pattern( os.fspath(tmp_dir / "top" / "first") ) diff --git a/tests/func/test_import.py b/tests/func/test_import.py index 24b758596..09eacc91e 100644 --- a/tests/func/test_import.py +++ b/tests/func/test_import.py @@ -59,9 +59,7 @@ def test_import_cached_file(mocker, erepo_dir, tmp_dir, dvc, scm, monkeypatch): (tmp_dir / dst).unlink() remote_exception = NoRemoteError("dvc import") - mocker.patch.object( - dvc.cloud, "get_remote_odb", side_effect=remote_exception - ) + mocker.patch.object(dvc.cloud, "get_remote_odb", side_effect=remote_exception) tmp_dir.dvc.imp(os.fspath(erepo_dir), src, dst) assert (tmp_dir / dst).is_file() @@ -130,9 +128,7 @@ def test_import_file_from_dir(tmp_dir, scm, dvc, erepo_dir): assert (tmp_dir / "subdir" / "bar").read_text() == "bar" assert (tmp_dir / "subdir.dvc").exists() - dvc.imp( - os.fspath(erepo_dir), os.path.join("dir", "subdir", "foo"), out="X" - ) + dvc.imp(os.fspath(erepo_dir), os.path.join("dir", "subdir", "foo"), out="X") assert (tmp_dir / "X").read_text() == "foo" assert (tmp_dir / "X.dvc").exists() @@ -168,9 +164,7 @@ def test_import_non_cached(erepo_dir, tmp_dir, dvc, scm): cmd=f"echo hello > {src}", outs_no_cache=[src], single_stage=True ) - erepo_dir.scm_add( - [os.fspath(erepo_dir / src)], commit="add a non-cached out" - ) + erepo_dir.scm_add([os.fspath(erepo_dir / src)], commit="add a non-cached out") stage = tmp_dir.dvc.imp(os.fspath(erepo_dir), src, dst) @@ -236,9 +230,7 @@ def test_import_no_download(tmp_dir, scm, dvc, erepo_dir): def test_pull_import_no_download(tmp_dir, scm, dvc, erepo_dir): with erepo_dir.chdir(): erepo_dir.scm_gen(os.path.join("foo", "bar"), b"bar", commit="add bar") - erepo_dir.dvc_gen( - os.path.join("foo", "baz"), b"baz contents", commit="add baz" - ) + erepo_dir.dvc_gen(os.path.join("foo", "baz"), b"baz contents", commit="add baz") size = ( len(b"bar") + len(b"baz contents") @@ -252,9 +244,7 @@ def test_pull_import_no_download(tmp_dir, scm, dvc, erepo_dir): stage = load_file(dvc, "foo_imported.dvc").stage - assert ( - stage.outs[0].hash_info.value == "bdb8641831d8fcb03939637e09011c21.dir" - ) + assert stage.outs[0].hash_info.value == "bdb8641831d8fcb03939637e09011c21.dir" assert stage.outs[0].meta.size == size assert stage.outs[0].meta.nfiles == 3 @@ -313,9 +303,7 @@ def test_pull_imported_directory_stage(tmp_dir, dvc, erepo_dir): def test_pull_wildcard_imported_directory_stage(tmp_dir, dvc, erepo_dir): with erepo_dir.chdir(): - erepo_dir.dvc_gen( - {"dir123": {"foo": "foo content"}}, commit="create dir" - ) + erepo_dir.dvc_gen({"dir123": {"foo": "foo content"}}, commit="create dir") dvc.imp(os.fspath(erepo_dir), "dir123", "dir_imported123") @@ -341,9 +329,7 @@ def test_push_wildcard_from_bare_git_repo( }, commit="initial", ) - erepo_dir.dvc.push( - [os.path.join(os.fspath(erepo_dir), "dire*")], glob=True - ) + erepo_dir.dvc.push([os.path.join(os.fspath(erepo_dir), "dire*")], glob=True) erepo_dir.scm.gitpython.repo.create_remote("origin", os.fspath(tmp_dir)) erepo_dir.scm.gitpython.repo.remote("origin").push("master") @@ -356,9 +342,7 @@ def test_push_wildcard_from_bare_git_repo( dvc_repo.dvc.imp(os.fspath(tmp_dir), "dir123") -def test_download_error_pulling_imported_stage( - mocker, tmp_dir, dvc, erepo_dir -): +def test_download_error_pulling_imported_stage(mocker, tmp_dir, dvc, erepo_dir): with erepo_dir.chdir(): erepo_dir.dvc_gen("foo", "foo content", commit="create foo") dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported") @@ -448,9 +432,7 @@ def test_pull_no_rev_lock(erepo_dir, tmp_dir, dvc): assert (tmp_dir / "foo_imported").read_text() == "contents" -def test_import_from_bare_git_repo( - tmp_dir, make_tmp_dir, erepo_dir, local_cloud -): +def test_import_from_bare_git_repo(tmp_dir, make_tmp_dir, erepo_dir, local_cloud): Git.init(tmp_dir.fs_path, bare=True).close() erepo_dir.add_remote(config=local_cloud.config) @@ -542,9 +524,7 @@ def test_granular_import_from_subrepos(tmp_dir, dvc, erepo_dir): @pytest.mark.parametrize("is_dvc", [True, False]) @pytest.mark.parametrize("files", [{"foo": "foo"}, {"dir": {"bar": "bar"}}]) -def test_pull_imported_stage_from_subrepos( - tmp_dir, dvc, erepo_dir, is_dvc, files -): +def test_pull_imported_stage_from_subrepos(tmp_dir, dvc, erepo_dir, is_dvc, files): subrepo = erepo_dir / "subrepo" make_subrepo(subrepo, erepo_dir.scm) gen = subrepo.dvc_gen if is_dvc else subrepo.scm_gen @@ -666,9 +646,7 @@ def test_circular_import(tmp_dir, dvc, scm, erepo_dir): with erepo_dir.chdir(): with pytest.raises(CircularImportError): - erepo_dir.dvc.imp( - os.fspath(tmp_dir), "dir_imported", "circular_import" - ) + erepo_dir.dvc.imp(os.fspath(tmp_dir), "dir_imported", "circular_import") @pytest.mark.parametrize("paths", ([], ["dir"])) @@ -717,8 +695,6 @@ def test_import_with_annotations(M, tmp_dir, scm, dvc, erepo_dir): # try to selectively update/overwrite some annotations annot = {**annot, "type": "t2"} - stage = dvc.imp( - os.fspath(erepo_dir), "foo", "foo", no_exec=True, type="t2" - ) + stage = dvc.imp(os.fspath(erepo_dir), "foo", "foo", no_exec=True, type="t2") assert stage.outs[0].annot == Annotation(**annot) assert (tmp_dir / "foo.dvc").parse() == M.dict(outs=[M.dict(**annot)]) diff --git a/tests/func/test_import_url.py b/tests/func/test_import_url.py index bb2b502c9..4380afe7c 100644 --- a/tests/func/test_import_url.py +++ b/tests/func/test_import_url.py @@ -61,9 +61,7 @@ def test_import_filename(tmp_dir, dvc, cloud): (tmp_dir / "sub").mkdir() path = tmp_dir / "sub" / "bar.dvc" - ret = main( - ["import-url", "--file", path.fs_path, external_source.fs_path, "out"] - ) + ret = main(["import-url", "--file", path.fs_path, external_source.fs_path, "out"]) assert ret == 0 assert path.exists() @@ -173,9 +171,7 @@ def test_import_url_preserve_fields(tmp_dir, dvc): ) -def test_import_url_to_remote_single_file( - tmp_dir, dvc, workspace, local_remote -): +def test_import_url_to_remote_single_file(tmp_dir, dvc, workspace, local_remote): workspace.gen("foo", "foo") url = "remote://workspace/foo" @@ -190,9 +186,7 @@ def test_import_url_to_remote_single_file( assert len(stage.outs) == 1 hash_info = stage.outs[0].hash_info - with open( - local_remote.oid_to_path(hash_info.value), encoding="utf-8" - ) as fobj: + with open(local_remote.oid_to_path(hash_info.value), encoding="utf-8") as fobj: assert fobj.read() == "foo" assert stage.outs[0].meta.size == len("foo") @@ -219,9 +213,7 @@ def test_import_url_to_remote_directory(tmp_dir, dvc, workspace, local_remote): assert len(stage.outs) == 1 hash_info = stage.outs[0].hash_info - with open( - local_remote.oid_to_path(hash_info.value), encoding="utf-8" - ) as stream: + with open(local_remote.oid_to_path(hash_info.value), encoding="utf-8") as stream: file_parts = json.load(stream) assert len(file_parts) == 3 @@ -232,15 +224,11 @@ def test_import_url_to_remote_directory(tmp_dir, dvc, workspace, local_remote): } for file_part in file_parts: - with open( - local_remote.oid_to_path(file_part["md5"]), encoding="utf-8" - ) as fobj: + with open(local_remote.oid_to_path(file_part["md5"]), encoding="utf-8") as fobj: assert fobj.read() == file_part["relpath"] -def test_import_url_to_remote_absolute( - tmp_dir, make_tmp_dir, dvc, local_remote -): +def test_import_url_to_remote_absolute(tmp_dir, make_tmp_dir, dvc, local_remote): tmp_abs_dir = make_tmp_dir("abs") tmp_foo = tmp_abs_dir / "foo" tmp_foo.write_text("foo") @@ -276,9 +264,7 @@ def test_import_url_to_remote_status(tmp_dir, dvc, local_cloud, local_remote): def test_import_url_no_download(tmp_dir, scm, dvc, local_workspace): local_workspace.gen("file", "file content") dst = tmp_dir / "file" - stage = dvc.imp_url( - "remote://workspace/file", os.fspath(dst), no_download=True - ) + stage = dvc.imp_url("remote://workspace/file", os.fspath(dst), no_download=True) assert stage.deps[0].hash_info.value == "d10b4c3ff123b26dc068d43a8bef2d23" diff --git a/tests/func/test_init.py b/tests/func/test_init.py index 2649fa6ab..e53b7d3a8 100644 --- a/tests/func/test_init.py +++ b/tests/func/test_init.py @@ -93,8 +93,8 @@ def test_subdir_init_no_option(tmp_dir, scm, monkeypatch, caplog): "{} is not tracked by any supported SCM tool (e.g. Git). " "Use `--no-scm` if you don't want to use any SCM or " "`--subdir` if initializing inside a subdirectory of a parent SCM " - "repository.".format(os.fspath(tmp_dir / "subdir")) - ) in caplog.text + "repository.".format(os.fspath(tmp_dir / "subdir")) in caplog.text + ) def test_gen_dvcignore(tmp_dir): @@ -118,4 +118,5 @@ def test_init_when_ignored_by_git(tmp_dir, scm, caplog): "for example, by adding '!.dvc' to .gitignore.".format( dvc_dir=tmp_dir / DvcRepo.DVC_DIR ) - ) in caplog.text + in caplog.text + ) diff --git a/tests/func/test_install.py b/tests/func/test_install.py index f4f5626d9..cae020185 100644 --- a/tests/func/test_install.py +++ b/tests/func/test_install.py @@ -43,8 +43,8 @@ class TestInstall: with pytest.raises(DvcException) as exc_info: # noqa: PT011 dvc.install() - assert escape_ansi(str(exc_info.value)) == ( - "Hook 'post-checkout' already exists. " + assert ( + escape_ansi(str(exc_info.value)) == "Hook 'post-checkout' already exists. " "Please refer to for more info." ) @@ -77,9 +77,7 @@ class TestInstall: tmp_dir.dvc_gen("file", "file_content", "commit message") file_checksum = file_md5("file", dvc.fs) - expected_storage_path = ( - storage_path / file_checksum[:2] / file_checksum[2:] - ) + expected_storage_path = storage_path / file_checksum[:2] / file_checksum[2:] scm.gitpython.repo.clone(os.fspath(git_remote)) scm.gitpython.repo.create_remote("origin", os.fspath(git_remote)) @@ -109,9 +107,7 @@ def test_merge_driver_no_ancestor(tmp_dir, scm, dvc): dvc.install() (tmp_dir / ".gitattributes").write_text("*.dvc merge=dvc") - scm.gitpython.repo.git.merge( - "one", m="merged", no_gpg_sign=True, no_signoff=True - ) + scm.gitpython.repo.git.merge("one", m="merged", no_gpg_sign=True, no_signoff=True) # NOTE: dvc shouldn't checkout automatically as it might take a long time assert (tmp_dir / "data").read_text() == {"bar": "bar"} @@ -146,9 +142,7 @@ def test_merge_driver(tmp_dir, scm, dvc): dvc.install() (tmp_dir / ".gitattributes").write_text("*.dvc merge=dvc") - scm.gitpython.repo.git.merge( - "one", m="merged", no_gpg_sign=True, no_signoff=True - ) + scm.gitpython.repo.git.merge("one", m="merged", no_gpg_sign=True, no_signoff=True) # NOTE: dvc shouldn't checkout automatically as it might take a long time assert (tmp_dir / "data").read_text() == {"master": "master", "two": "two"} diff --git a/tests/func/test_lock.py b/tests/func/test_lock.py index 8049c085e..669145b30 100644 --- a/tests/func/test_lock.py +++ b/tests/func/test_lock.py @@ -34,9 +34,7 @@ def test_unlock_lock_failed(tmp_dir, dvc, mocker): def test_unlock_unlocked_raises(): lock = Lock("lock") - with pytest.raises( - DvcException, match="Unlock called on an unlocked lock" - ): + with pytest.raises(DvcException, match="Unlock called on an unlocked lock"): lock.unlock() diff --git a/tests/func/test_lockfile.py b/tests/func/test_lockfile.py index 1f148b03d..38097fe13 100644 --- a/tests/func/test_lockfile.py +++ b/tests/func/test_lockfile.py @@ -49,12 +49,8 @@ def test_deps_outs_are_sorted_by_path(tmp_dir, dvc, run_head): assert list(lock.keys()) == ["cmd", "deps", "outs"] # `path` key appear first and then the `md5` - assert all( - list(dep.keys()) == ["path", "md5", "size"] for dep in lock["deps"] - ) - assert all( - list(out.keys()) == ["path", "md5", "size"] for out in lock["outs"] - ) + assert all(list(dep.keys()) == ["path", "md5", "size"] for dep in lock["deps"]) + assert all(list(out.keys()) == ["path", "md5", "size"] for out in lock["outs"]) # deps are always sorted by the file path naming assert list(map(itemgetter("path"), lock["deps"])) == sorted(deps) @@ -65,9 +61,7 @@ def test_deps_outs_are_sorted_by_path(tmp_dir, dvc, run_head): ] -def test_order_is_preserved_when_pipeline_order_changes( - tmp_dir, dvc, run_head -): +def test_order_is_preserved_when_pipeline_order_changes(tmp_dir, dvc, run_head): tmp_dir.gen(FS_STRUCTURE) deps = ["foo", "bar", "foobar"] stage = run_head(*deps, name="copy-first-line") @@ -188,9 +182,7 @@ def test_can_read_v1_lockfile(tmp_dir, dvc, v1_repo_lock): } -def test_migrates_v1_lockfile_to_v2_during_dump( - tmp_dir, dvc, v1_repo_lock, caplog -): +def test_migrates_v1_lockfile_to_v2_during_dump(tmp_dir, dvc, v1_repo_lock, caplog): caplog.clear() with caplog.at_level(logging.INFO, logger="dvc.dvcfile"): assert dvc.reproduce() diff --git a/tests/func/test_ls.py b/tests/func/test_ls.py index 59885e9cd..e24db4a61 100644 --- a/tests/func/test_ls.py +++ b/tests/func/test_ls.py @@ -158,10 +158,7 @@ def test_ls_repo_with_path_dir_dvc_only_empty(tmp_dir, dvc, scm): assert Repo.ls(os.fspath(tmp_dir), path="folder", dvc_only=True) == [] - assert ( - Repo.ls(os.fspath(tmp_dir), path="empty_dvc_folder", dvc_only=True) - == [] - ) + assert Repo.ls(os.fspath(tmp_dir), path="empty_dvc_folder", dvc_only=True) == [] def test_ls_repo_with_path_subdir(tmp_dir, dvc, scm): @@ -196,9 +193,7 @@ def test_ls_repo_with_path_subdir_dvc_only_recursive(tmp_dir, dvc, scm): path = os.path.join("data", "subcontent") files = Repo.ls(os.fspath(tmp_dir), path, dvc_only=True, recursive=True) - match_files( - files, ((("data.xml",), True), (("statistics", "data.csv"), True)) - ) + match_files(files, ((("data.xml",), True), (("statistics", "data.csv"), True))) def test_ls_repo_with_path_file_out(tmp_dir, dvc, scm): @@ -555,9 +550,7 @@ def test_subrepo(dvc_top_level, erepo): dvc_files = {"dvc_dir", "foo.txt", "foo.txt.dvc", "dvc_dir.dvc"} common_outputs = git_tracked_outputs | extras | dvc_files - top_level_outputs = ( - common_outputs if dvc_top_level else git_tracked_outputs - ) + top_level_outputs = common_outputs if dvc_top_level else git_tracked_outputs assert _list_files(erepo) == top_level_outputs assert _list_files(erepo, "scm_dir") == {"ipsum"} if dvc_top_level: diff --git a/tests/func/test_merge_driver.py b/tests/func/test_merge_driver.py index 7b673328b..865b036fe 100644 --- a/tests/func/test_merge_driver.py +++ b/tests/func/test_merge_driver.py @@ -90,16 +90,13 @@ def test_merge(tmp_dir, dvc, ancestor, our, their, merged): {"foo": "foo"}, {"foo": "bar"}, {"foo": "baz"}, - ("unable to auto-merge the following paths:\nfoo"), + "unable to auto-merge the following paths:\nfoo", ), ( {"common": "common", "foo": "foo"}, {"common": "common", "bar": "bar"}, {"baz": "baz"}, - ( - "unable to auto-merge the following paths:\n" - "both deleted: ('foo',)" - ), + "unable to auto-merge the following paths:\nboth deleted: ('foo',)", ), ], ) @@ -130,9 +127,7 @@ def test_merge_conflict(tmp_dir, dvc, ancestor, our, their, error, caplog): def test_merge_different_output_options(tmp_dir, dvc, caplog): (tmp_dir / "ancestor").touch() - (tmp_dir / "our").write_text( - "outs:\n- md5: f123456789.dir\n path: path\n" - ) + (tmp_dir / "our").write_text("outs:\n- md5: f123456789.dir\n path: path\n") (tmp_dir / "their").write_text( "outs:\n- md5: f987654321.dir\n path: path\n cache: false\n" @@ -161,9 +156,7 @@ def test_merge_different_output_options(tmp_dir, dvc, caplog): def test_merge_file(tmp_dir, dvc, caplog): (tmp_dir / "ancestor").touch() - (tmp_dir / "our").write_text( - "outs:\n- md5: f123456789.dir\n path: path\n" - ) + (tmp_dir / "our").write_text("outs:\n- md5: f123456789.dir\n path: path\n") (tmp_dir / "their").write_text("outs:\n- md5: f987654321\n path: path\n") diff --git a/tests/func/test_odb.py b/tests/func/test_odb.py index 16dfb0a5d..fc2c303b2 100644 --- a/tests/func/test_odb.py +++ b/tests/func/test_odb.py @@ -186,9 +186,7 @@ def test_shared_cache(tmp_dir, dvc, group): assert not os.path.exists(cache_dir) - tmp_dir.dvc_gen( - {"file": "file content", "dir": {"file2": "file 2 content"}} - ) + tmp_dir.dvc_gen({"file": "file content", "dir": {"file2": "file 2 content"}}) file_mode = oct(0o444) dir_mode = oct(0o2775 if group else (0o777 & ~system.umask)) diff --git a/tests/func/test_remote.py b/tests/func/test_remote.py index 28297cc51..d4cb18408 100644 --- a/tests/func/test_remote.py +++ b/tests/func/test_remote.py @@ -60,9 +60,7 @@ def test_referencing_other_remotes(dvc): def test_remove_default(tmp_dir, dvc): remote = "mys3" - assert ( - main(["remote", "add", "--default", remote, "s3://bucket/name"]) == 0 - ) + assert main(["remote", "add", "--default", remote, "s3://bucket/name"]) == 0 assert main(["remote", "modify", remote, "profile", "default"]) == 0 assert main(["config", "--local", "core.remote", remote]) == 0 @@ -138,26 +136,18 @@ def test_dir_hash_should_be_key_order_agnostic(tmp_dir, dvc): path = (tmp_dir / "data").fs_path - tree = Tree.from_list( - [{"relpath": "1", "md5": "1"}, {"relpath": "2", "md5": "2"}] - ) + tree = Tree.from_list([{"relpath": "1", "md5": "1"}, {"relpath": "2", "md5": "2"}]) tree.digest() - with patch( - "dvc_data.hashfile.build._build_tree", return_value=(None, tree) - ): + with patch("dvc_data.hashfile.build._build_tree", return_value=(None, tree)): _, _, obj = build(dvc.odb.local, path, dvc.odb.local.fs, "md5") hash1 = obj.hash_info # remove the raw dir obj to force building the tree on the next build call dvc.odb.local.fs.remove(dvc.odb.local.oid_to_path(hash1.as_raw().value)) - tree = Tree.from_list( - [{"md5": "1", "relpath": "1"}, {"md5": "2", "relpath": "2"}] - ) + tree = Tree.from_list([{"md5": "1", "relpath": "1"}, {"md5": "2", "relpath": "2"}]) tree.digest() - with patch( - "dvc_data.hashfile.build._build_tree", return_value=(None, tree) - ): + with patch("dvc_data.hashfile.build._build_tree", return_value=(None, tree)): _, _, obj = build(dvc.odb.local, path, dvc.odb.local.fs, "md5") hash2 = obj.hash_info @@ -254,12 +244,8 @@ def test_modify_missing_remote(tmp_dir, dvc): def test_remote_modify_local_on_repo_config(tmp_dir, dvc): assert main(["remote", "add", "myremote", "http://example.com/path"]) == 0 - assert ( - main(["remote", "modify", "myremote", "user", "xxx", "--local"]) == 0 - ) - assert dvc.config.load_one("local")["remote"]["myremote"] == { - "user": "xxx" - } + assert main(["remote", "modify", "myremote", "user", "xxx", "--local"]) == 0 + assert dvc.config.load_one("local")["remote"]["myremote"] == {"user": "xxx"} assert dvc.config.load_one("repo")["remote"]["myremote"] == { "url": "http://example.com/path" } @@ -311,14 +297,9 @@ def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker, local_remote): def test_remote_modify_validation(dvc): remote_name = "drive" unsupported_config = "unsupported_config" + assert main(["remote", "add", "-d", remote_name, "gdrive://test/test"]) == 0 assert ( - main(["remote", "add", "-d", remote_name, "gdrive://test/test"]) == 0 - ) - assert ( - main( - ["remote", "modify", remote_name, unsupported_config, "something"] - ) - == 251 + main(["remote", "modify", remote_name, unsupported_config, "something"]) == 251 ) config = configobj.ConfigObj(dvc.config.files["repo"]) assert unsupported_config not in config[f'remote "{remote_name}"'] @@ -329,20 +310,14 @@ def test_remote_modify_unset(dvc): config = configobj.ConfigObj(dvc.config.files["repo"]) assert config['remote "myremote"'] == {"url": "gdrive://test/test"} - assert ( - main(["remote", "modify", "myremote", "gdrive_client_id", "something"]) - == 0 - ) + assert main(["remote", "modify", "myremote", "gdrive_client_id", "something"]) == 0 config = configobj.ConfigObj(dvc.config.files["repo"]) assert config['remote "myremote"'] == { "url": "gdrive://test/test", "gdrive_client_id": "something", } - assert ( - main(["remote", "modify", "myremote", "gdrive_client_id", "--unset"]) - == 0 - ) + assert main(["remote", "modify", "myremote", "gdrive_client_id", "--unset"]) == 0 config = configobj.ConfigObj(dvc.config.files["repo"]) assert config['remote "myremote"'] == {"url": "gdrive://test/test"} diff --git a/tests/func/test_remove.py b/tests/func/test_remove.py index 40ad64110..21d6453cb 100644 --- a/tests/func/test_remove.py +++ b/tests/func/test_remove.py @@ -4,10 +4,7 @@ import pytest from dvc.cli import main from dvc.fs import system -from dvc.stage.exceptions import ( - StageFileDoesNotExistError, - StageFileIsNotDvcFileError, -) +from dvc.stage.exceptions import StageFileDoesNotExistError, StageFileIsNotDvcFileError from dvc.utils.fs import remove from dvc_objects.errors import ObjectDBError from tests.utils import get_gitignore_content diff --git a/tests/func/test_repo_index.py b/tests/func/test_repo_index.py index 1ff764ce8..a890df741 100644 --- a/tests/func/test_repo_index.py +++ b/tests/func/test_repo_index.py @@ -184,9 +184,7 @@ def test_used_objs(tmp_dir, scm, dvc, run_copy, rev): assert index.used_objs() == {None: set(expected_objs)} assert index.used_objs("dir") == {None: set(expected_objs[1:])} assert index.used_objs(".", recursive=True) == {None: set(expected_objs)} - assert index.used_objs("copy-foo-bar", with_deps=True) == { - None: {expected_objs[0]} - } + assert index.used_objs("copy-foo-bar", with_deps=True) == {None: {expected_objs[0]}} def test_view_granular_dir(tmp_dir, scm, dvc, run_copy): @@ -260,9 +258,7 @@ def test_view_stage_filter(tmp_dir, scm, dvc, run_copy): None, stage_filter=lambda s: getattr(s, "name", "").startswith("copy") ) assert set(view.stages) == {stage2} - assert {out.fs_path for out in view.outs} == { - out.fs_path for out in stage2.outs - } + assert {out.fs_path for out in view.outs} == {out.fs_path for out in stage2.outs} def test_view_outs_filter(tmp_dir, scm, dvc, run_copy): @@ -273,9 +269,7 @@ def test_view_outs_filter(tmp_dir, scm, dvc, run_copy): view = index.targets_view(None, outs_filter=lambda o: o.def_path == "foo") assert set(view.stages) == {stage1, stage2} - assert {out.fs_path for out in view.outs} == { - out.fs_path for out in stage1.outs - } + assert {out.fs_path for out in view.outs} == {out.fs_path for out in stage1.outs} def test_view_combined_filter(tmp_dir, scm, dvc, run_copy): @@ -298,9 +292,7 @@ def test_view_combined_filter(tmp_dir, scm, dvc, run_copy): outs_filter=lambda o: o.def_path == "bar", ) assert set(view.stages) == {stage2} - assert {out.fs_path for out in view.outs} == { - out.fs_path for out in stage2.outs - } + assert {out.fs_path for out in view.outs} == {out.fs_path for out in stage2.outs} def test_view_brancher(tmp_dir, scm, dvc): diff --git a/tests/func/test_repro.py b/tests/func/test_repro.py index 9784caeda..203cecb79 100644 --- a/tests/func/test_repro.py +++ b/tests/func/test_repro.py @@ -8,11 +8,7 @@ import pytest from dvc.cli import main from dvc.dvcfile import DVC_FILE, load_file -from dvc.exceptions import ( - CyclicGraphError, - ReproductionError, - StagePathAsOutputError, -) +from dvc.exceptions import CyclicGraphError, ReproductionError, StagePathAsOutputError from dvc.fs import LocalFileSystem, system from dvc.output import Output from dvc.stage import Stage @@ -221,9 +217,7 @@ def test_repro_dep_under_dir(tmp_dir, dvc, copy_script, run_stage): assert filecmp.cmp("file1", "foo", shallow=False) -def test_repro_dep_dir_with_outputs_under_it( - tmp_dir, dvc, copy_script, run_stage -): +def test_repro_dep_dir_with_outputs_under_it(tmp_dir, dvc, copy_script, run_stage): tmp_dir.gen("foo", "foo") file_stage, _ = tmp_dir.dvc_gen( {"data/file": "file", "data/sub": {"foo": "foo", "bar": "bar"}} @@ -348,8 +342,9 @@ def test_repro_dry_no_exec(tmp_dir, dvc): idir, "-o", odir, - "python -c 'import shutil; " - 'shutil.copytree("{}", "{}")\''.format(idir, odir), + 'python -c \'import shutil; shutil.copytree("{}", "{}")\''.format( + idir, odir + ), ] ) assert ret == 0 @@ -362,9 +357,7 @@ def test_repro_dry_no_exec(tmp_dir, dvc): "--file", DVC_FILE, *deps, - "ls {}".format( - " ".join(dep for i, dep in enumerate(deps) if i % 2) - ), + "ls {}".format(" ".join(dep for i, dep in enumerate(deps) if i % 2)), ] ) assert ret == 0 @@ -829,9 +822,7 @@ def test_repro_all_pipelines(mocker, dvc, run_stage): dvc.state = StateNoop() - mock_reproduce = mocker.patch.object( - Stage, "reproduce", side_effect=stages - ) + mock_reproduce = mocker.patch.object(Stage, "reproduce", side_effect=stages) ret = main(["repro", "--all-pipelines"]) assert ret == 0 assert mock_reproduce.call_count == 4 @@ -898,9 +889,7 @@ class TestReproAlreadyCached: assert spy_checkout.call_count == 0 -def test_should_display_metrics_on_repro_with_metrics_option( - caplog, capsys, dvc -): +def test_should_display_metrics_on_repro_with_metrics_option(caplog, capsys, dvc): metrics_file = "metrics_file" metrics_value = 0.123489015 ret = main( @@ -919,9 +908,7 @@ def test_should_display_metrics_on_repro_with_metrics_option( from dvc.dvcfile import DVC_FILE_SUFFIX - ret = main( - ["repro", "--force", "--metrics", metrics_file + DVC_FILE_SUFFIX] - ) + ret = main(["repro", "--force", "--metrics", metrics_file + DVC_FILE_SUFFIX]) assert ret == 0 expected_metrics_display = f"Path\n{metrics_file} {metrics_value}\n" @@ -995,13 +982,9 @@ def repro_dir(tmp_dir, dvc, run_copy): stages["last_stage"] = stage # Unrelated are to verify that reproducing `dir` will not trigger them too + assert run_copy(os.fspath(origin_copy), "unrelated1", single_stage=True) is not None assert ( - run_copy(os.fspath(origin_copy), "unrelated1", single_stage=True) - is not None - ) - assert ( - run_copy(os.fspath(dir_file_path), "unrelated2", single_stage=True) - is not None + run_copy(os.fspath(dir_file_path), "unrelated2", single_stage=True) is not None ) return stages @@ -1111,9 +1094,7 @@ def test_recursive_repro_on_stage_file(dvc, repro_dir): def test_dvc_formatting_retained(tmp_dir, dvc, run_copy): tmp_dir.dvc_gen("foo", "foo content") - stage = run_copy( - "foo", "foo_copy", fname="foo_copy.dvc", single_stage=True - ) + stage = run_copy("foo", "foo_copy", fname="foo_copy.dvc", single_stage=True) stage_path = tmp_dir / stage.relpath # Add comments and custom formatting to DVC-file @@ -1161,12 +1142,8 @@ def test_downstream(dvc): # A # assert main(["run", "--single-stage", "-o", "A", "echo A>A"]) == 0 - assert ( - main(["run", "--single-stage", "-d", "A", "-o", "B", "echo B>B"]) == 0 - ) - assert ( - main(["run", "--single-stage", "-d", "A", "-o", "C", "echo C>C"]) == 0 - ) + assert main(["run", "--single-stage", "-d", "A", "-o", "B", "echo B>B"]) == 0 + assert main(["run", "--single-stage", "-d", "A", "-o", "C", "echo C>C"]) == 0 assert ( main( [ @@ -1184,9 +1161,7 @@ def test_downstream(dvc): == 0 ) assert main(["run", "--single-stage", "-o", "G", "echo G>G"]) == 0 - assert ( - main(["run", "--single-stage", "-d", "G", "-o", "F", "echo F>F"]) == 0 - ) + assert main(["run", "--single-stage", "-d", "G", "-o", "F", "echo F>F"]) == 0 assert ( main( [ @@ -1245,10 +1220,6 @@ def test_repro_when_cmd_changes(tmp_dir, dvc, run_copy, mocker): data["cmd"] = " ".join(stage.cmd.split()) # change cmd spacing by two (tmp_dir / stage.path).dump(data) - assert dvc.status([stage.addressing]) == { - stage.addressing: ["changed checksum"] - } + assert dvc.status([stage.addressing]) == {stage.addressing: ["changed checksum"]} assert dvc.reproduce(stage.addressing)[0] == stage - m.assert_called_once_with( - stage, checkpoint_func=None, dry=False, run_env=None - ) + m.assert_called_once_with(stage, checkpoint_func=None, dry=False, run_env=None) diff --git a/tests/func/test_repro_multistage.py b/tests/func/test_repro_multistage.py index 955f84df3..d190a3b79 100644 --- a/tests/func/test_repro_multistage.py +++ b/tests/func/test_repro_multistage.py @@ -48,14 +48,9 @@ def test_downstream(tmp_dir, dvc): # assert main(["run", "-n", "A-gen", "-o", "A", "echo A>A"]) == 0 assert main(["run", "-n", "B-gen", "-d", "A", "-o", "B", "echo B>B"]) == 0 + assert main(["run", "--single-stage", "-d", "A", "-o", "C", "echo C>C"]) == 0 assert ( - main(["run", "--single-stage", "-d", "A", "-o", "C", "echo C>C"]) == 0 - ) - assert ( - main( - ["run", "-n", "D-gen", "-d", "B", "-d", "C", "-o", "D", "echo D>D"] - ) - == 0 + main(["run", "-n", "D-gen", "-d", "B", "-d", "C", "-o", "D", "echo D>D"]) == 0 ) assert main(["run", "--single-stage", "-o", "G", "echo G>G"]) == 0 assert main(["run", "-n", "F-gen", "-d", "G", "-o", "F", "echo F>F"]) == 0 @@ -84,9 +79,7 @@ def test_downstream(tmp_dir, dvc): # / # B # - evaluation = dvc.reproduce( - PROJECT_FILE + ":B-gen", downstream=True, force=True - ) + evaluation = dvc.reproduce(PROJECT_FILE + ":B-gen", downstream=True, force=True) assert len(evaluation) == 3 assert isinstance(evaluation[0], PipelineStage) @@ -102,9 +95,7 @@ def test_downstream(tmp_dir, dvc): # B, C should be run (in any order) before D # See https://github.com/iterative/dvc/issues/3602 - evaluation = dvc.reproduce( - PROJECT_FILE + ":A-gen", downstream=True, force=True - ) + evaluation = dvc.reproduce(PROJECT_FILE + ":A-gen", downstream=True, force=True) assert len(evaluation) == 5 assert isinstance(evaluation[0], PipelineStage) @@ -144,14 +135,10 @@ def test_repro_when_cmd_changes(tmp_dir, dvc, run_copy, mocker): assert dvc.status([target]) == {target: ["changed command"]} assert dvc.reproduce(target)[0] == stage - m.assert_called_once_with( - stage, checkpoint_func=None, dry=False, run_env=None - ) + m.assert_called_once_with(stage, checkpoint_func=None, dry=False, run_env=None) -def test_repro_when_new_deps_is_added_in_dvcfile( - tmp_dir, dvc, run_copy, copy_script -): +def test_repro_when_new_deps_is_added_in_dvcfile(tmp_dir, dvc, run_copy, copy_script): from dvc.dvcfile import load_file tmp_dir.gen({"foo": "foo", "bar": "bar"}) @@ -399,9 +386,7 @@ def test_repro_list_of_commands_in_order(tmp_dir, dvc, multiline): @pytest.mark.parametrize("multiline", [True, False]) -def test_repro_list_of_commands_raise_and_stops_after_failure( - tmp_dir, dvc, multiline -): +def test_repro_list_of_commands_raise_and_stops_after_failure(tmp_dir, dvc, multiline): cmd = ["echo foo>foo", "failed_command", "echo baz>bar"] if multiline: cmd = "\n".join(cmd) diff --git a/tests/func/test_run_cache.py b/tests/func/test_run_cache.py index ed5d88092..aa9d2ec6e 100644 --- a/tests/func/test_run_cache.py +++ b/tests/func/test_run_cache.py @@ -82,14 +82,12 @@ def test_outs_no_cache_deactivate_run_cache(tmp_dir, dvc, out_type, run_cache): cmd="cp foo bar && cp foo goo", outs=["goo"], name="copy-foo-bar", - **{out_type: ["bar"]} + **{out_type: ["bar"]}, ) assert os.path.isdir(dvc.stage_cache.cache_dir) == run_cache -def test_memory_for_multiple_runs_of_same_stage( - tmp_dir, dvc, run_copy, mocker -): +def test_memory_for_multiple_runs_of_same_stage(tmp_dir, dvc, run_copy, mocker): tmp_dir.gen("foo", "foo") assert not os.path.exists(dvc.stage_cache.cache_dir) run_copy("foo", "bar", name="copy-foo-bar") diff --git a/tests/func/test_run_multistage.py b/tests/func/test_run_multistage.py index 802756ac3..40af2d747 100644 --- a/tests/func/test_run_multistage.py +++ b/tests/func/test_run_multistage.py @@ -198,9 +198,7 @@ def test_run_dump_on_multistage(tmp_dir, dvc, run_head): } -@pytest.mark.parametrize( - "char", ["@:", "#", "$", ":", "/", "\\", ".", ";", ","] -) +@pytest.mark.parametrize("char", ["@:", "#", "$", ":", "/", "\\", ".", ";", ","]) def test_run_with_invalid_stage_name(run_copy, char): with pytest.raises(InvalidStageName): run_copy("foo", "bar", name=f"copy_name-{char}") @@ -246,9 +244,7 @@ def test_run_params_default(tmp_dir, dvc): } data, _ = stage.dvcfile._load() - assert data["stages"]["read_params"]["params"] == [ - "nested.nested1.nested2" - ] + assert data["stages"]["read_params"]["params"] == ["nested.nested1.nested2"] def test_run_params_custom_file(tmp_dir, dvc): @@ -269,9 +265,7 @@ def test_run_params_custom_file(tmp_dir, dvc): } data, _ = stage.dvcfile._load() - assert data["stages"]["read_params"]["params"] == [ - {"params2.yaml": ["lists"]} - ] + assert data["stages"]["read_params"]["params"] == [{"params2.yaml": ["lists"]}] def test_run_params_no_exec(tmp_dir, dvc): @@ -290,9 +284,7 @@ def test_run_params_no_exec(tmp_dir, dvc): assert not stage.dvcfile._lockfile.exists() data, _ = stage.dvcfile._load() - assert data["stages"]["read_params"]["params"] == [ - {"params2.yaml": ["lists"]} - ] + assert data["stages"]["read_params"]["params"] == [{"params2.yaml": ["lists"]}] @pytest.mark.parametrize( @@ -345,9 +337,7 @@ def test_run_overwrite_preserves_meta_and_comment(tmp_dir, dvc, run_copy): assert run_copy("foo1", "bar1", name="copy-foo-bar", force=True) - assert (tmp_dir / PROJECT_FILE).read_text() == text.format( - src="foo1", dest="bar1" - ) + assert (tmp_dir / PROJECT_FILE).read_text() == text.format(src="foo1", dest="bar1") def test_run_external_outputs( diff --git a/tests/func/test_run_single_stage.py b/tests/func/test_run_single_stage.py index 97a3d85e5..11aa6e798 100644 --- a/tests/func/test_run_single_stage.py +++ b/tests/func/test_run_single_stage.py @@ -629,9 +629,7 @@ def test_rerun_deterministic_ignore_cache(tmp_dir, run_copy, mocker): def test_rerun_callback(dvc): def run_callback(force=False): - return dvc.run( - cmd="echo content > out", force=force, single_stage=True - ) + return dvc.run(cmd="echo content > out", force=force, single_stage=True) assert run_callback() is not None with pytest.raises(StageFileAlreadyExistsError): @@ -723,9 +721,7 @@ def test_run_persist(tmp_dir, dvc, outs_command): assert not (tmp_dir / "file").exists() -def test_should_raise_on_overlapping_output_paths( - tmp_dir, dvc, append_foo_script -): +def test_should_raise_on_overlapping_output_paths(tmp_dir, dvc, append_foo_script): tmp_dir.gen("data", {"foo": "foo", "bar": "bar"}) ret = main(["add", "data"]) assert ret == 0 @@ -847,9 +843,7 @@ def test_metrics_dir(tmp_dir, dvc, caplog, run_copy_metrics, metrics_type): tmp_dir.gen({"dir": {"file": "content"}}) with caplog.at_level(logging.DEBUG, "dvc"): run_copy_metrics("dir", "dir_metric", **copyargs) - assert ( - "directory 'dir_metric' cannot be used as metrics." in caplog.messages - ) + assert "directory 'dir_metric' cannot be used as metrics." in caplog.messages def test_run_force_preserves_comments_and_meta(tmp_dir, dvc, run_copy): diff --git a/tests/func/test_stage.py b/tests/func/test_stage.py index 657c028dc..209c8f8c1 100644 --- a/tests/func/test_stage.py +++ b/tests/func/test_stage.py @@ -109,9 +109,7 @@ def test_default_wdir_ignored_in_checksum(tmp_dir, dvc): def test_external_remote_output_resolution(tmp_dir, dvc, make_remote): tmp_path = make_remote("tmp", default=False) - tmp_dir.add_remote( - url="remote://tmp/storage", name="storage", default=False - ) + tmp_dir.add_remote(url="remote://tmp/storage", name="storage", default=False) storage = tmp_path / "storage" storage.mkdir() file_path = storage / "file" @@ -126,9 +124,7 @@ def test_external_remote_output_resolution(tmp_dir, dvc, make_remote): def test_external_remote_dependency_resolution(tmp_dir, dvc, make_remote): tmp_path = make_remote("tmp", default=False) - tmp_dir.add_remote( - url="remote://tmp/storage", name="storage", default=False - ) + tmp_dir.add_remote(url="remote://tmp/storage", name="storage", default=False) storage = tmp_path / "storage" storage.mkdir() file_path = storage / "file" @@ -168,13 +164,7 @@ def test_md5_ignores_annotations(tmp_dir, dvc): stage = dvc.stage.load_one("foo.dvc") assert compute_md5(stage) == "1822617147b53ae6f9eb4b3c87c0b6f3" assert ( - dict_md5( - { - "outs": [ - {"md5": "d3b07384d113edec49eaa6238ad5ff00", "path": "foo"} - ] - } - ) + dict_md5({"outs": [{"md5": "d3b07384d113edec49eaa6238ad5ff00", "path": "foo"}]}) == "1822617147b53ae6f9eb4b3c87c0b6f3" ) @@ -248,9 +238,7 @@ def test_collect_symlink(tmp_dir, dvc, with_deps): data_link = tmp_dir / "data_link" data_link.symlink_to("data") stage = list( - dvc.stage.collect( - target=str(data_link / "foo.dvc"), with_deps=with_deps - ) + dvc.stage.collect(target=str(data_link / "foo.dvc"), with_deps=with_deps) )[0] assert stage.addressing == f"{foo_path}.dvc" diff --git a/tests/func/test_stage_load.py b/tests/func/test_stage_load.py index b5238ab37..c6053dc91 100644 --- a/tests/func/test_stage_load.py +++ b/tests/func/test_stage_load.py @@ -46,9 +46,7 @@ def test_collect(tmp_dir, scm, dvc, run_copy): "foobar", "foo", } - assert collect_outs("dvc.yaml:copy-foo-foobar", recursive=True) == { - "foobar" - } + assert collect_outs("dvc.yaml:copy-foo-foobar", recursive=True) == {"foobar"} assert collect_outs("copy-foo-foobar") == {"foobar"} assert collect_outs("copy-foo-foobar", with_deps=True) == {"foobar", "foo"} assert collect_outs("copy-foo-foobar", recursive=True) == {"foobar"} @@ -71,9 +69,7 @@ def test_collect_dir_recursive(tmp_dir, dvc, run_head): } -def test_collect_with_not_existing_output_or_stage_name( - tmp_dir, dvc, run_copy -): +def test_collect_with_not_existing_output_or_stage_name(tmp_dir, dvc, run_copy): with pytest.raises(StageFileDoesNotExistError): dvc.stage.collect("some_file") tmp_dir.dvc_gen("foo", "foo") @@ -84,9 +80,7 @@ def test_collect_with_not_existing_output_or_stage_name( def test_stages(tmp_dir, dvc): def collect_stages(): - return { - stage.relpath for stage in Repo(os.fspath(tmp_dir)).index.stages - } + return {stage.relpath for stage in Repo(os.fspath(tmp_dir)).index.stages} tmp_dir.dvc_gen({"file": "a", "dir/file": "b", "dir/subdir/file": "c"}) @@ -109,16 +103,12 @@ def stages(tmp_dir, run_copy): "lorem-generate": stage2, "copy-foo-bar": run_copy("foo", "bar", single_stage=True), "copy-bar-foobar": run_copy("bar", "foobar", name="copy-bar-foobar"), - "copy-lorem-ipsum": run_copy( - "lorem", "ipsum", name="copy-lorem-ipsum" - ), + "copy-lorem-ipsum": run_copy("lorem", "ipsum", name="copy-lorem-ipsum"), } def test_collect_not_a_group_stage_with_group_flag(tmp_dir, dvc, stages): - assert set(dvc.stage.collect("copy-bar-foobar")) == { - stages["copy-bar-foobar"] - } + assert set(dvc.stage.collect("copy-bar-foobar")) == {stages["copy-bar-foobar"]} assert set(dvc.stage.collect("copy-bar-foobar", with_deps=True)) == { stages["copy-bar-foobar"], stages["copy-foo-bar"], @@ -127,9 +117,7 @@ def test_collect_not_a_group_stage_with_group_flag(tmp_dir, dvc, stages): assert set(dvc.stage.collect_granular("copy-bar-foobar")) == { (stages["copy-bar-foobar"], None) } - assert set( - dvc.stage.collect_granular("copy-bar-foobar", with_deps=True) - ) == { + assert set(dvc.stage.collect_granular("copy-bar-foobar", with_deps=True)) == { (stages["copy-bar-foobar"], None), (stages["copy-foo-bar"], None), (stages["foo-generate"], None), @@ -139,9 +127,7 @@ def test_collect_not_a_group_stage_with_group_flag(tmp_dir, dvc, stages): def test_collect_generated(tmp_dir, dvc): d = { "vars": [{"vars": [1, 2, 3, 4, 5]}], - "stages": { - "build": {"foreach": "${vars}", "do": {"cmd": "echo ${item}"}} - }, + "stages": {"build": {"foreach": "${vars}", "do": {"cmd": "echo ${item}"}}}, } (tmp_dir / "dvc.yaml").dump(d) @@ -152,40 +138,31 @@ def test_collect_generated(tmp_dir, dvc): assert set(dvc.stage.collect("build")) == all_stages assert set(dvc.stage.collect("build", with_deps=True)) == all_stages assert set(dvc.stage.collect("build*", glob=True)) == all_stages - assert ( - set(dvc.stage.collect("build*", glob=True, with_deps=True)) - == all_stages - ) + assert set(dvc.stage.collect("build*", glob=True, with_deps=True)) == all_stages stages_info = {(stage, None) for stage in all_stages} assert set(dvc.stage.collect_granular("build")) == stages_info - assert ( - set(dvc.stage.collect_granular("build", with_deps=True)) == stages_info - ) + assert set(dvc.stage.collect_granular("build", with_deps=True)) == stages_info def test_collect_glob(tmp_dir, dvc, stages): assert set(dvc.stage.collect("copy*", glob=True)) == { stages[key] for key in ["copy-bar-foobar", "copy-lorem-ipsum"] } - assert set( - dvc.stage.collect("copy-lorem*", glob=True, with_deps=True) - ) == {stages[key] for key in ["copy-lorem-ipsum", "lorem-generate"]} + assert set(dvc.stage.collect("copy-lorem*", glob=True, with_deps=True)) == { + stages[key] for key in ["copy-lorem-ipsum", "lorem-generate"] + } def test_collect_granular_with_no_target(tmp_dir, dvc, stages): - assert set(map(itemgetter(0), dvc.stage.collect_granular())) == set( - stages.values() + assert set(map(itemgetter(0), dvc.stage.collect_granular())) == set(stages.values()) + assert list(map(itemgetter(1), dvc.stage.collect_granular())) == [None] * len( + stages ) - assert list(map(itemgetter(1), dvc.stage.collect_granular())) == [ - None - ] * len(stages) def test_collect_granular_with_target(tmp_dir, dvc, stages): - assert dvc.stage.collect_granular("bar.dvc") == [ - (stages["copy-foo-bar"], None) - ] + assert dvc.stage.collect_granular("bar.dvc") == [(stages["copy-foo-bar"], None)] assert dvc.stage.collect_granular(PROJECT_FILE) == [ (stages["copy-bar-foobar"], None), (stages["copy-lorem-ipsum"], None), @@ -287,21 +264,15 @@ def test_collect_granular_priority_on_collision(tmp_dir, dvc, run_copy): stage2 = run_copy("foo", "bar", name="dir") assert dvc.stage.collect_granular("dir") == [(stage2, None)] - assert dvc.stage.collect_granular("dir", recursive=True) == [ - (stage1, None) - ] + assert dvc.stage.collect_granular("dir", recursive=True) == [(stage1, None)] remove(tmp_dir / "dir") assert dvc.stage.collect_granular("dir") == [(stage2, None)] - assert dvc.stage.collect_granular("dir", recursive=True) == [ - (stage2, None) - ] + assert dvc.stage.collect_granular("dir", recursive=True) == [(stage2, None)] -def test_collect_granular_collision_output_dir_stage_name( - tmp_dir, dvc, run_copy -): +def test_collect_granular_collision_output_dir_stage_name(tmp_dir, dvc, run_copy): stage1, *_ = tmp_dir.dvc_gen({"dir": {"foo": "foo"}, "foo": "foo"}) stage3 = run_copy("foo", "bar", name="dir") @@ -317,9 +288,9 @@ def test_collect_granular_not_existing_stage_name(tmp_dir, dvc, run_copy): (stage,) = tmp_dir.dvc_gen("copy-foo-bar", "copy-foo-bar") run_copy("foo", "bar", name="copy-foo-bar") - assert dvc.stage.collect_granular( - "copy-foo-bar.dvc:stage_name_not_needed" - ) == [(stage, None)] + assert dvc.stage.collect_granular("copy-foo-bar.dvc:stage_name_not_needed") == [ + (stage, None) + ] with pytest.raises(StageNotFound): dvc.stage.collect_granular("dvc.yaml:does-not-exist") @@ -335,9 +306,9 @@ def test_get_stages(tmp_dir, dvc, run_copy): assert set(dvc.stage.load_all()) == {stage1, stage2} assert set(dvc.stage.load_all(path=PROJECT_FILE)) == {stage1, stage2} assert set(dvc.stage.load_all(name="copy-bar-foobar")) == {stage2} - assert set( - dvc.stage.load_all(path=PROJECT_FILE, name="copy-bar-foobar") - ) == {stage2} + assert set(dvc.stage.load_all(path=PROJECT_FILE, name="copy-bar-foobar")) == { + stage2 + } with pytest.raises(StageFileDoesNotExistError): dvc.stage.load_all(path=relpath(tmp_dir / ".." / PROJECT_FILE)) @@ -429,15 +400,11 @@ def test_collect_repo_callback(tmp_dir, dvc, mocker): assert isinstance(exc, YAMLValidationError) -def test_gitignored_file_try_collect_granular_for_data_files( - tmp_dir, dvc, scm -): +def test_gitignored_file_try_collect_granular_for_data_files(tmp_dir, dvc, scm): (stage,) = tmp_dir.dvc_gen({"data": {"foo": "foo", "bar": "bar"}}) path = os.path.join("data", "foo") - assert dvc.stage.collect_granular(path) == [ - (stage, os.path.join(tmp_dir, path)) - ] + assert dvc.stage.collect_granular(path) == [(stage, os.path.join(tmp_dir, path))] scm.ignore(stage.path) dvc._reset() diff --git a/tests/func/test_state.py b/tests/func/test_state.py index 15d96db36..b113597a7 100644 --- a/tests/func/test_state.py +++ b/tests/func/test_state.py @@ -78,9 +78,7 @@ def test_state_dir_config(make_tmp_dir, dvc): index_dir = str(make_tmp_dir("tmp_index")) repo = Repo(config={"state": {"dir": index_dir}}) - assert os.path.dirname(repo.state.tmp_dir) == os.path.join( - index_dir, ".dvc" - ) + assert os.path.dirname(repo.state.tmp_dir) == os.path.join(index_dir, ".dvc") assert re.match( r"^test_state_dir_config0-([0-9a-f]+)$", os.path.basename(repo.state.tmp_dir), diff --git a/tests/func/test_status.py b/tests/func/test_status.py index a9f3cd30c..9cd89d28e 100644 --- a/tests/func/test_status.py +++ b/tests/func/test_status.py @@ -23,9 +23,7 @@ def test_quiet(tmp_dir, dvc, capsys): def test_implied_cloud(dvc, mocker): - mock_status = mocker.patch( - "dvc.repo.status._cloud_status", return_value=True - ) + mock_status = mocker.patch("dvc.repo.status._cloud_status", return_value=True) main(["status", "--remote", "something"]) assert mock_status.called @@ -43,9 +41,7 @@ def test_status_non_dvc_repo_import(tmp_dir, dvc, git_dir): git_dir.scm_gen("file", "second version", commit="update file") (status,) = dvc.status(["file.dvc"])["file.dvc"] - assert status == { - "changed deps": {f"file ({git_dir})": "update available"} - } + assert status == {"changed deps": {f"file ({git_dir})": "update available"}} def test_status_before_and_after_dvc_init(tmp_dir, dvc, git_dir): @@ -109,11 +105,7 @@ def test_status_recursive(tmp_dir, dvc): {"changed outs": {os.path.join("dir", "file"): "not in cache"}} ], os.path.join("dir", "subdir", "file2.dvc"): [ - { - "changed outs": { - os.path.join("dir", "subdir", "file2"): "not in cache" - } - } + {"changed outs": {os.path.join("dir", "subdir", "file2"): "not in cache"}} ], } @@ -129,9 +121,7 @@ def test_status_outputs(tmp_dir, dvc): tmp_dir.gen({"alice": "new alice", "bob": "new bob"}) assert dvc.status(targets=["alice_bob"]) == { - "alice_bob": [ - {"changed outs": {"alice": "modified", "bob": "modified"}} - ] + "alice_bob": [{"changed outs": {"alice": "modified", "bob": "modified"}}] } assert dvc.status(targets=["alice"]) == { @@ -140,12 +130,8 @@ def test_status_outputs(tmp_dir, dvc): def test_params_without_targets(tmp_dir, dvc): - dvc.stage.add( - name="test", cmd="echo params.yaml", params=[{"params.yaml": None}] - ) - assert dvc.status() == { - "test": [{"changed deps": {"params.yaml": "deleted"}}] - } + dvc.stage.add(name="test", cmd="echo params.yaml", params=[{"params.yaml": None}]) + assert dvc.status() == {"test": [{"changed deps": {"params.yaml": "deleted"}}]} (tmp_dir / "params.yaml").touch() assert dvc.status() == {"test": [{"changed deps": {"params.yaml": "new"}}]} @@ -155,17 +141,13 @@ def test_params_without_targets(tmp_dir, dvc): # and be able to distinguish between no-lock-entry and empty-lock-entry. assert (tmp_dir / "dvc.lock").parse() == { "schema": "2.0", - "stages": { - "test": {"cmd": "echo params.yaml", "params": {"params.yaml": {}}} - }, + "stages": {"test": {"cmd": "echo params.yaml", "params": {"params.yaml": {}}}}, } assert dvc.status() == {} (tmp_dir / "params.yaml").dump({"foo": "foo", "bar": "bar"}) assert dvc.status() == { - "test": [ - {"changed deps": {"params.yaml": {"bar": "new", "foo": "new"}}} - ] + "test": [{"changed deps": {"params.yaml": {"bar": "new", "foo": "new"}}}] } dvc.commit("test", force=True) diff --git a/tests/func/test_unprotect.py b/tests/func/test_unprotect.py index 4abd6be85..f2d661612 100644 --- a/tests/func/test_unprotect.py +++ b/tests/func/test_unprotect.py @@ -6,9 +6,7 @@ def test_unprotect(tmp_dir, dvc): dvc.odb.local.cache_types = ["hardlink"] dvc.add("foo") - cache = os.path.join( - ".dvc", "cache", "ac", "bd18db4cc2f85cedef654fccc4a4d8" - ) + cache = os.path.join(".dvc", "cache", "ac", "bd18db4cc2f85cedef654fccc4a4d8") assert not os.access("foo", os.W_OK) assert not os.access(cache, os.W_OK) diff --git a/tests/func/test_update.py b/tests/func/test_update.py index ce27d53df..96a954926 100644 --- a/tests/func/test_update.py +++ b/tests/func/test_update.py @@ -49,18 +49,10 @@ def test_update_import(tmp_dir, dvc, erepo_dir, cached): assert dvc.status() == { "dir.dvc": [ - { - "changed deps": { - f"dir ({os.fspath(erepo_dir)})": "update available" - } - } + {"changed deps": {f"dir ({os.fspath(erepo_dir)})": "update available"}} ], "version.dvc": [ - { - "changed deps": { - f"version ({os.fspath(erepo_dir)})": "update available" - } - } + {"changed deps": {f"version ({os.fspath(erepo_dir)})": "update available"}} ], } @@ -98,9 +90,7 @@ def test_update_import_after_remote_updates_to_dvc(tmp_dir, dvc, erepo_dir): new_rev = None with erepo_dir.branch("branch", new=False), erepo_dir.chdir(): erepo_dir.scm.gitpython.repo.index.remove(["version"]) - erepo_dir.dvc_gen( - "version", "updated", commit="upgrade to DVC tracking" - ) + erepo_dir.dvc_gen("version", "updated", commit="upgrade to DVC tracking") new_rev = erepo_dir.scm.get_rev() assert old_rev != new_rev @@ -143,11 +133,7 @@ def test_update_before_and_after_dvc_init(tmp_dir, dvc, git_dir): assert dvc.status([stage.path]) == { "file.dvc": [ - { - "changed deps": { - f"file ({os.fspath(git_dir)})": "update available" - } - } + {"changed deps": {f"file ({os.fspath(git_dir)})": "update available"}} ] } @@ -219,9 +205,7 @@ def test_update_import_url(tmp_dir, dvc, workspace): @pytest.mark.parametrize("outs_exist", (False, True)) -def test_update_import_url_no_download( - tmp_dir, dvc, workspace, outs_exist, mocker -): +def test_update_import_url_no_download(tmp_dir, dvc, workspace, outs_exist, mocker): workspace.gen("file", "file content") dst = tmp_dir / "imported_file" @@ -321,12 +305,8 @@ def test_update_recursive(tmp_dir, dvc, erepo_dir): ) assert (tmp_dir / os.path.join("dir", "foo1")).read_text() == "text1" - assert ( - tmp_dir / os.path.join("dir", "subdir", "foo2") - ).read_text() == "text2" - assert ( - tmp_dir / os.path.join("dir", "subdir", "foo3") - ).read_text() == "text3" + assert (tmp_dir / os.path.join("dir", "subdir", "foo2")).read_text() == "text2" + assert (tmp_dir / os.path.join("dir", "subdir", "foo3")).read_text() == "text3" assert stage1.deps[0].def_repo["rev_lock"] == old_rev assert stage2.deps[0].def_repo["rev_lock"] == old_rev @@ -457,14 +437,10 @@ def test_update_import_url_to_remote_directory_changed_contents( def test_update_import_url_to_remote_directory_same_hash( tmp_dir, dvc, local_workspace, local_remote ): - local_workspace.gen( - {"data": {"foo": "foo", "bar": {"baz": "baz"}, "same": "same"}} - ) + local_workspace.gen({"data": {"foo": "foo", "bar": {"baz": "baz"}, "same": "same"}}) stage = dvc.imp_url("remote://workspace/data", to_remote=True) - local_workspace.gen( - {"data": {"foo": "baz", "bar": {"baz": "foo"}, "same": "same"}} - ) + local_workspace.gen({"data": {"foo": "baz", "bar": {"baz": "foo"}, "same": "same"}}) stage = dvc.update(stage.path, to_remote=True) dvc.pull("data") diff --git a/tests/func/test_used_objs.py b/tests/func/test_used_objs.py index 29d16a44e..ae1045265 100644 --- a/tests/func/test_used_objs.py +++ b/tests/func/test_used_objs.py @@ -29,18 +29,14 @@ import pytest ), ], ) -def test_from_gitfs_when_pwd_not_in_root( - tmp_dir, scm, dvc, stage_wdir, cwd, target -): +def test_from_gitfs_when_pwd_not_in_root(tmp_dir, scm, dvc, stage_wdir, cwd, target): path = tmp_dir.joinpath(stage_wdir).resolve() path.mkdir(parents=True, exist_ok=True) wdir = tmp_dir.joinpath(cwd).resolve() wdir.mkdir(parents=True, exist_ok=True) (path / "dvc.yaml").write_text( - json.dumps( - {"stages": {"train": {"cmd": "echo foo > foo", "outs": ["foo"]}}} - ) + json.dumps({"stages": {"train": {"cmd": "echo foo > foo", "outs": ["foo"]}}}) ) path.gen({"foo": "foo"}) dvc.commit(None, force=True) diff --git a/tests/func/utils/test_strict_yaml.py b/tests/func/utils/test_strict_yaml.py index 6ea8d9e73..b32b35557 100644 --- a/tests/func/utils/test_strict_yaml.py +++ b/tests/func/utils/test_strict_yaml.py @@ -343,9 +343,7 @@ def fixed_width_term(mocker): ) -@pytest.mark.parametrize( - "text, expected", examples.values(), ids=examples.keys() -) +@pytest.mark.parametrize("text, expected", examples.values(), ids=examples.keys()) def test_exceptions( tmp_dir, dvc, @@ -365,9 +363,7 @@ def test_exceptions( # strip whitespace on the right: output is always left-justified # by rich.syntax.Syntax: - for expected_line, err_line in zip( - expected.splitlines(), err.splitlines() - ): + for expected_line, err_line in zip(expected.splitlines(), err.splitlines()): assert expected_line == err_line.rstrip(" ") @@ -431,10 +427,7 @@ def test_fallback_exception_message(tmp_dir, dvc, mocker, caplog): dvc_file = tmp_dir / "dvc.yaml" dvc_file.write_text(MAPPING_VALUES_NOT_ALLOWED) assert main(["stage", "list"]) != 0 - assert ( - "unable to read: 'dvc.yaml', " - "YAML file structure is corrupted" in caplog.text - ) + assert "unable to read: 'dvc.yaml', YAML file structure is corrupted" in caplog.text caplog.clear() # validation error diff --git a/tests/integration/plots/test_plots.py b/tests/integration/plots/test_plots.py index 193c58871..d0683911b 100644 --- a/tests/integration/plots/test_plots.py +++ b/tests/integration/plots/test_plots.py @@ -18,9 +18,7 @@ JSON_OUT = "vis_data" def call(capsys, subcommand="show"): capsys.readouterr() - assert ( - main(["plots", subcommand, "--json", "-o", JSON_OUT, "--split"]) == 0 - ) + assert main(["plots", subcommand, "--json", "-o", JSON_OUT, "--split"]) == 0 split_json_out, _ = capsys.readouterr() split_json_result = json.loads(split_json_out) @@ -50,9 +48,7 @@ def extract_vega_specs(html_path, plots_ids): reader = BeautifulSoup(content, features="html.parser") for plot_id in plots_ids: script = _remove_blanks( - reader.find( - "div", id=Renderer.remove_special_chars(plot_id) - ).script.text + reader.find("div", id=Renderer.remove_special_chars(plot_id)).script.text ) result[plot_id] = json.loads( script.split("; vegaEmbed")[0].replace("var spec = ", "") @@ -94,9 +90,7 @@ def verify_image(path, version, filename, content, html_path, json_result): # there should be no absolute paths in produced HTML # TODO uncomment once dvc-render is adjusted # assert str(path) not in html_content - assert ( - path / "dvc_plots" / "static" / output_name - ).read_bytes() == content + assert (path / "dvc_plots" / "static" / output_name).read_bytes() == content def _remove_blanks(text: str): @@ -127,14 +121,9 @@ def verify_vega( assert json_result[0]["content"]["data"]["values"] assert html_result["data"]["values"] - assert ( - split_json_result[0]["content"]["data"]["values"] - == "" - ) + assert split_json_result[0]["content"]["data"]["values"] == "" - def _assert_templates_equal( - html_template, filled_template, split_template - ): + def _assert_templates_equal(html_template, filled_template, split_template): # besides data, json and split json should be equal path = ["data", "values"] tmp1 = deepcopy(html_template) @@ -161,20 +150,12 @@ def verify_vega_props(plot_id, json_result, title, x, y, **kwargs): try: # TODO confusion_matrix_plot - need to find better way of asserting # encoding as its place is not constant in vega - plot_x = dpath.util.get( - data, ["content", "spec", "encoding", "x", "field"] - ) - plot_y = dpath.util.get( - data, ["content", "spec", "encoding", "y", "field"] - ) + plot_x = dpath.util.get(data, ["content", "spec", "encoding", "x", "field"]) + plot_y = dpath.util.get(data, ["content", "spec", "encoding", "y", "field"]) except KeyError: # default plot - plot_x = dpath.util.get( - data, ["content", "layer", 0, "encoding", "x", "field"] - ) - plot_y = dpath.util.get( - data, ["content", "layer", 0, "encoding", "y", "field"] - ) + plot_x = dpath.util.get(data, ["content", "layer", 0, "encoding", "x", "field"]) + plot_y = dpath.util.get(data, ["content", "layer", 0, "encoding", "y", "field"]) assert plot_x == x assert plot_y == y @@ -198,17 +179,13 @@ def test_no_plots(tmp_dir, scm, dvc, capsys): @pytest.mark.vscode -def test_repo_with_plots( - tmp_dir, scm, dvc, capsys, run_copy_metrics, repo_with_plots -): +def test_repo_with_plots(tmp_dir, scm, dvc, capsys, run_copy_metrics, repo_with_plots): repo_state = repo_with_plots() image_v1, linear_v1, confusion_v1, confusion_props = next(repo_state) html_path, json_result, split_json_result = call(capsys) - html_result = extract_vega_specs( - html_path, ["linear.json", "confusion.json"] - ) + html_result = extract_vega_specs(html_path, ["linear.json", "confusion.json"]) assert json_result["linear.json"][0]["content"]["data"][ "values" @@ -240,17 +217,13 @@ def test_repo_with_plots( }, }, ) - assert html_result["confusion.json"]["data"][ - "values" - ] == _update_datapoints( + assert html_result["confusion.json"]["data"]["values"] == _update_datapoints( confusion_v1, { REVISION_FIELD: "workspace", }, ) - verify_image( - tmp_dir, "workspace", "image.png", image_v1, html_path, json_result - ) + verify_image(tmp_dir, "workspace", "image.png", image_v1, html_path, json_result) for plot in ["linear.json", "confusion.json"]: verify_vega( @@ -265,16 +238,10 @@ def test_repo_with_plots( image_v2, linear_v2, confusion_v2, confusion_props = next(repo_state) html_path, json_result, split_json_result = call(capsys, subcommand="diff") - html_result = extract_vega_specs( - html_path, ["linear.json", "confusion.json"] - ) + html_result = extract_vega_specs(html_path, ["linear.json", "confusion.json"]) - verify_image( - tmp_dir, "workspace", "image.png", image_v2, html_path, json_result - ) - verify_image( - tmp_dir, "HEAD", "image.png", image_v1, html_path, json_result - ) + verify_image(tmp_dir, "workspace", "image.png", image_v2, html_path, json_result) + verify_image(tmp_dir, "HEAD", "image.png", image_v1, html_path, json_result) for plot in ["linear.json", "confusion.json"]: verify_vega( @@ -287,9 +254,7 @@ def test_repo_with_plots( path = tmp_dir / "subdir" path.mkdir() with path.chdir(): - html_path, json_result, split_json_result = call( - capsys, subcommand="diff" - ) + html_path, json_result, split_json_result = call(capsys, subcommand="diff") html_result = extract_vega_specs( html_path, ["../linear.json", "../confusion.json"], @@ -315,9 +280,7 @@ def test_repo_with_plots( }, }, ) - assert html_result["../linear.json"]["data"][ - "values" - ] == _update_datapoints( + assert html_result["../linear.json"]["data"]["values"] == _update_datapoints( linear_v2, { REVISION_FIELD: "workspace", @@ -349,9 +312,7 @@ def test_repo_with_plots( }, }, ) - assert html_result["../confusion.json"]["data"][ - "values" - ] == _update_datapoints( + assert html_result["../confusion.json"]["data"]["values"] == _update_datapoints( confusion_v2, { REVISION_FIELD: "workspace", @@ -430,10 +391,7 @@ def test_config_output_dir(tmp_dir, dvc, capsys): assert (subdir / "index.html").is_file() cli_arg_subdir = tmp_dir / "cli_option" - assert ( - main(["plots", "show", "-o", os.fspath(cli_arg_subdir), "metric.json"]) - == 0 - ) + assert main(["plots", "show", "-o", os.fspath(cli_arg_subdir), "metric.json"]) == 0 out, _ = capsys.readouterr() assert cli_arg_subdir.as_uri() in out @@ -468,7 +426,5 @@ def test_repo_with_config_plots(tmp_dir, capsys, repo_with_config_plots): }, ) - assert ( - html_result["dvc.yaml::linear_train_vs_test"]["data"]["values"] == ble - ) + assert html_result["dvc.yaml::linear_train_vs_test"]["data"]["values"] == ble # TODO check json results once vscode is able to handle flexible plots diff --git a/tests/integration/plots/test_repo_plots_api.py b/tests/integration/plots/test_repo_plots_api.py index f7b0064a9..252b4909f 100644 --- a/tests/integration/plots/test_repo_plots_api.py +++ b/tests/integration/plots/test_repo_plots_api.py @@ -10,10 +10,7 @@ def test_api(tmp_dir, dvc, repo_with_plots): workspace_data = next(dvc.plots.collect()) - assert ( - get_plot(workspace_data, "workspace", file="image.png", endkey="props") - == {} - ) + assert get_plot(workspace_data, "workspace", file="image.png", endkey="props") == {} image_source = get_plot( workspace_data, "workspace", file="image.png", endkey="data_source" ) @@ -30,9 +27,7 @@ def test_api(tmp_dir, dvc, repo_with_plots): assert linear_source() == {"data": linear_v1} assert ( - get_plot( - workspace_data, "workspace", file="confusion.json", endkey="props" - ) + get_plot(workspace_data, "workspace", file="confusion.json", endkey="props") == confusion_params ) confusion_source = get_plot( @@ -49,10 +44,7 @@ def test_api(tmp_dir, dvc, repo_with_plots): workspace_data = next(data_generator) - assert ( - get_plot(workspace_data, "workspace", file="image.png", endkey="props") - == {} - ) + assert get_plot(workspace_data, "workspace", file="image.png", endkey="props") == {} image_source = get_plot( workspace_data, "workspace", file="image.png", endkey="data_source" ) @@ -69,9 +61,7 @@ def test_api(tmp_dir, dvc, repo_with_plots): assert linear_source() == {"data": linear_v2} assert ( - get_plot( - workspace_data, "workspace", file="confusion.json", endkey="props" - ) + get_plot(workspace_data, "workspace", file="confusion.json", endkey="props") == confusion_params ) confusion_source = get_plot( @@ -86,9 +76,7 @@ def test_api(tmp_dir, dvc, repo_with_plots): head_data = next(data_generator) assert get_plot(head_data, "HEAD", file="image.png", endkey="props") == {} - image_source = get_plot( - head_data, "HEAD", file="image.png", endkey="data_source" - ) + image_source = get_plot(head_data, "HEAD", file="image.png", endkey="data_source") assert callable(image_source) assert image_source() == {"data": image_v1} @@ -126,8 +114,6 @@ def test_api_with_config_plots(tmp_dir, dvc, capsys, repo_with_config_plots): ) for file in plots_state["data"]: - data_source = get_plot( - plots_data, "workspace", file=file, endkey="data_source" - ) + data_source = get_plot(plots_data, "workspace", file=file, endkey="data_source") assert callable(data_source) assert data_source() == {"data": plots_state["data"][file]} diff --git a/tests/unit/cli/test_main.py b/tests/unit/cli/test_main.py index fbe6ef7d2..076e7e41f 100644 --- a/tests/unit/cli/test_main.py +++ b/tests/unit/cli/test_main.py @@ -22,11 +22,11 @@ def test_state_pickle_errors_are_correctly_raised(tmp_dir, caplog, mocker): assert main() == 255 assert ( - f"Could not open pickled 'md5s' cache.\n" + "Could not open pickled 'md5s' cache.\n" f"Remove the '{path.relative_to(tmp_dir)}' directory " "and then retry this command.\n" - "See for more information." - ) in caplog.text + "See for more information." in caplog.text + ) @pytest.mark.parametrize( @@ -37,9 +37,7 @@ def test_state_pickle_errors_are_correctly_raised(tmp_dir, caplog, mocker): ("conda", "conda install -c conda-forge dvc-proto"), ], ) -def test_remote_missing_deps_are_correctly_reported( - tmp_dir, caplog, mocker, pkg, msg -): +def test_remote_missing_deps_are_correctly_reported(tmp_dir, caplog, mocker, pkg, msg): error = RemoteMissingDepsError(FileSystem(), "proto", "proto://", ["deps"]) mocker.patch("dvc.utils.pkg.PKG", pkg) mocker.patch( diff --git a/tests/unit/command/ls/test_ls.py b/tests/unit/command/ls/test_ls.py index 3bc114368..480d517df 100644 --- a/tests/unit/command/ls/test_ls.py +++ b/tests/unit/command/ls/test_ls.py @@ -18,42 +18,32 @@ def _test_cli(mocker, *args): def test_list(mocker): url = "local_dir" m = _test_cli(mocker, url) - m.assert_called_once_with( - url, None, recursive=False, rev=None, dvc_only=False - ) + m.assert_called_once_with(url, None, recursive=False, rev=None, dvc_only=False) def test_list_recursive(mocker): url = "local_dir" m = _test_cli(mocker, url, "-R") - m.assert_called_once_with( - url, None, recursive=True, rev=None, dvc_only=False - ) + m.assert_called_once_with(url, None, recursive=True, rev=None, dvc_only=False) def test_list_git_ssh_rev(mocker): url = "git@github.com:repo" m = _test_cli(mocker, url, "--rev", "123") - m.assert_called_once_with( - url, None, recursive=False, rev="123", dvc_only=False - ) + m.assert_called_once_with(url, None, recursive=False, rev="123", dvc_only=False) def test_list_targets(mocker): url = "local_dir" target = "subdir" m = _test_cli(mocker, url, target) - m.assert_called_once_with( - url, target, recursive=False, rev=None, dvc_only=False - ) + m.assert_called_once_with(url, target, recursive=False, rev=None, dvc_only=False) def test_list_outputs_only(mocker): url = "local_dir" m = _test_cli(mocker, url, None, "--dvc-only") - m.assert_called_once_with( - url, None, recursive=False, rev=None, dvc_only=True - ) + m.assert_called_once_with(url, None, recursive=False, rev=None, dvc_only=True) def test_show_json(mocker, capsys): @@ -75,9 +65,7 @@ def test_show_colors(mocker, capsys, monkeypatch): assert cli_args.func == CmdList cmd = cli_args.func(cli_args) - monkeypatch.setenv( - "LS_COLORS", "ex=01;32:rs=0:di=01;34:*.xml=01;31:*.dvc=01;33:" - ) + monkeypatch.setenv("LS_COLORS", "ex=01;32:rs=0:di=01;34:*.xml=01;31:*.dvc=01;33:") result = [ {"isdir": False, "isexec": 0, "isout": False, "path": ".dvcignore"}, {"isdir": False, "isexec": 0, "isout": False, "path": ".gitignore"}, diff --git a/tests/unit/command/ls/test_ls_colors.py b/tests/unit/command/ls/test_ls_colors.py index 5e7e804a8..f3145bb8b 100644 --- a/tests/unit/command/ls/test_ls_colors.py +++ b/tests/unit/command/ls/test_ls_colors.py @@ -26,9 +26,7 @@ def test_ls_colors_out_dir(): def test_ls_colors_out_exec(): ls_colors = LsColors(LsColors.default) - assert ( - colorize(ls_colors)("script.sh", "eo") == "\x1b[01;32mscript.sh\x1b[0m" - ) + assert colorize(ls_colors)("script.sh", "eo") == "\x1b[01;32mscript.sh\x1b[0m" def test_ls_colors_out_ext(): @@ -48,9 +46,7 @@ def test_ls_colors_dir(): def test_ls_colors_exec(): ls_colors = LsColors(LsColors.default) - assert ( - colorize(ls_colors)("script.sh", "e") == "\x1b[01;32mscript.sh\x1b[0m" - ) + assert colorize(ls_colors)("script.sh", "e") == "\x1b[01;32mscript.sh\x1b[0m" def test_ls_colors_ext(): @@ -68,6 +64,4 @@ def test_ls_repo_with_custom_color_env_defined(monkeypatch): assert colorizer("README.md") == "README.md" assert colorizer("data", "d") == "\x1b[01;34mdata\x1b[0m" assert colorizer("structure.xml") == "\x1b[01;31mstructure.xml\x1b[0m" - assert ( - colorizer("structure.xml.dvc") == "\x1b[01;33mstructure.xml.dvc\x1b[0m" - ) + assert colorizer("structure.xml.dvc") == "\x1b[01;33mstructure.xml.dvc\x1b[0m" diff --git a/tests/unit/command/test_add.py b/tests/unit/command/test_add.py index 69581587d..e5efd31ee 100644 --- a/tests/unit/command/test_add.py +++ b/tests/unit/command/test_add.py @@ -82,9 +82,7 @@ def test_add_to_remote(mocker): def test_add_to_remote_invalid_combinations(mocker, caplog): - cli_args = parse_args( - ["add", "s3://bucket/foo", "s3://bucket/bar", "--to-remote"] - ) + cli_args = parse_args(["add", "s3://bucket/foo", "s3://bucket/bar", "--to-remote"]) assert cli_args.func == CmdAdd cmd = cli_args.func(cli_args) @@ -104,9 +102,7 @@ def test_add_to_remote_invalid_combinations(mocker, caplog): def test_add_to_cache_invalid_combinations(mocker, caplog): - cli_args = parse_args( - ["add", "s3://bucket/foo", "s3://bucket/bar", "-o", "foo"] - ) + cli_args = parse_args(["add", "s3://bucket/foo", "s3://bucket/bar", "-o", "foo"]) assert cli_args.func == CmdAdd cmd = cli_args.func(cli_args) diff --git a/tests/unit/command/test_checkout.py b/tests/unit/command/test_checkout.py index bb1807fa2..79f988da0 100644 --- a/tests/unit/command/test_checkout.py +++ b/tests/unit/command/test_checkout.py @@ -3,9 +3,7 @@ from dvc.commands.checkout import CmdCheckout, log_changes def test_checkout(tmp_dir, dvc, mocker): - cli_args = parse_args( - ["checkout", "foo.dvc", "bar.dvc", "--relink", "--with-deps"] - ) + cli_args = parse_args(["checkout", "foo.dvc", "bar.dvc", "--relink", "--with-deps"]) assert cli_args.func == CmdCheckout cmd = cli_args.func(cli_args) diff --git a/tests/unit/command/test_config.py b/tests/unit/command/test_config.py index 8af45f739..c9ba34aaa 100644 --- a/tests/unit/command/test_config.py +++ b/tests/unit/command/test_config.py @@ -23,9 +23,7 @@ def test_config_formatter(): ) -@pytest.mark.parametrize( - "name", ["way.too.long", "no_option", "remote.way.too.long"] -) +@pytest.mark.parametrize("name", ["way.too.long", "no_option", "remote.way.too.long"]) def test_config_bad_name(name): with pytest.raises(DvcParserError): parse_args(["config", name]) diff --git a/tests/unit/command/test_dag.py b/tests/unit/command/test_dag.py index dde967160..e97561246 100644 --- a/tests/unit/command/test_dag.py +++ b/tests/unit/command/test_dag.py @@ -2,13 +2,7 @@ import networkx as nx import pytest from dvc.cli import main, parse_args -from dvc.commands.dag import ( - CmdDAG, - _build, - _show_ascii, - _show_dot, - _show_mermaid, -) +from dvc.commands.dag import CmdDAG, _build, _show_ascii, _show_dot, _show_mermaid @pytest.mark.parametrize( @@ -47,12 +41,8 @@ def repo(tmp_dir, dvc): tmp_dir.dvc_gen("a", "a") tmp_dir.dvc_gen("b", "b") - dvc.run( - no_exec=True, deps=["a", "c"], outs=["d", "e"], cmd="cmd1", name="1" - ) - dvc.run( - no_exec=True, deps=["b", "c"], outs=["f", "g"], cmd="cmd2", name="2" - ) + dvc.run(no_exec=True, deps=["a", "c"], outs=["d", "e"], cmd="cmd1", name="1") + dvc.run(no_exec=True, deps=["b", "c"], outs=["f", "g"], cmd="cmd2", name="2") dvc.run( no_exec=True, deps=["a", "b", "c"], @@ -113,9 +103,7 @@ def test_build_full_outs(repo, granular): def test_show_ascii(repo): - assert [ - line.rstrip() for line in _show_ascii(repo.index.graph).splitlines() - ] == [ + assert [line.rstrip() for line in _show_ascii(repo.index.graph).splitlines()] == [ " +----------------+ +----------------+", # noqa: E501 " | stage: 'a.dvc' | | stage: 'b.dvc' |", # noqa: E501 " *+----------------+**** +----------------+", # noqa: E501 @@ -154,9 +142,7 @@ def test_show_dot(repo): "strict digraph {", "}", ] - actual = sorted( - line.rstrip() for line in _show_dot(repo.index.graph).splitlines() - ) + actual = sorted(line.rstrip() for line in _show_dot(repo.index.graph).splitlines()) assert actual == expected @@ -195,9 +181,7 @@ def test_show_dot_properly_escapes(): def test_show_mermaid(repo): - assert [ - line.rstrip() for line in _show_mermaid(repo.index.graph).splitlines() - ] == [ + assert [line.rstrip() for line in _show_mermaid(repo.index.graph).splitlines()] == [ "flowchart TD", "\tnode1[\"stage: '1'\"]", "\tnode2[\"stage: '2'\"]", @@ -219,9 +203,7 @@ def test_show_mermaid_markdown(repo, dvc, capsys, mocker): capsys.readouterr() assert main(["dag", "--md"]) == 0 - assert [ - line.rstrip() for line in capsys.readouterr().out.splitlines() - ] == [ + assert [line.rstrip() for line in capsys.readouterr().out.splitlines()] == [ "```mermaid", "flowchart TD", "\tnode1[\"stage: '1'\"]", diff --git a/tests/unit/command/test_data_status.py b/tests/unit/command/test_data_status.py index 6c66c811f..f971e00dd 100644 --- a/tests/unit/command/test_data_status.py +++ b/tests/unit/command/test_data_status.py @@ -33,9 +33,7 @@ def mocked_status(): def test_cli(dvc, mocker, mocked_status): - status = mocker.patch( - "dvc.repo.Repo.data_status", return_value=mocked_status - ) + status = mocker.patch("dvc.repo.Repo.data_status", return_value=mocked_status) cli_args = parse_args( [ diff --git a/tests/unit/command/test_diff.py b/tests/unit/command/test_diff.py index 4dd6a1aad..6319594ee 100644 --- a/tests/unit/command/test_diff.py +++ b/tests/unit/command/test_diff.py @@ -81,13 +81,13 @@ def test_show_hash(mocker, capsys): out, _ = capsys.readouterr() assert ( - "Deleted:\n" - " XXXXXXXX " + os.path.join("data", "") + "\n" - " 00000000 " + os.path.join("data", "bar") + "\n" - " 11111111 " + os.path.join("data", "foo") + "\n" - "\n" - "Renamed:\n" - " 11111111 " + "Deleted:\n XXXXXXXX " + + os.path.join("data", "") + + "\n 00000000 " + + os.path.join("data", "bar") + + "\n 11111111 " + + os.path.join("data", "foo") + + "\n\nRenamed:\n 11111111 " + os.path.join("data", "file_old") + " -> " + os.path.join("data", "file_new") @@ -270,9 +270,7 @@ def test_show_markdown_with_hash(capsys): {"path": os.path.join("data", "foo"), "hash": "11111111"}, {"path": os.path.join("data", "bar"), "hash": "00000000"}, ], - "modified": [ - {"path": "file", "hash": {"old": "AAAAAAAA", "new": "BBBBBBBB"}} - ], + "modified": [{"path": "file", "hash": {"old": "AAAAAAAA", "new": "BBBBBBBB"}}], "added": [{"path": "file", "hash": "00000000"}], "renamed": [ { @@ -358,6 +356,6 @@ def test_hide_missing(mocker, capsys): "Renamed:\n" " file_old -> file_new\n" "\n" - "files summary: 1 added, 1 renamed" - ) in out + "files summary: 1 added, 1 renamed" in out + ) assert "not in cache" not in out diff --git a/tests/unit/command/test_experiments.py b/tests/unit/command/test_experiments.py index 12294bd41..d5ab2dc40 100644 --- a/tests/unit/command/test_experiments.py +++ b/tests/unit/command/test_experiments.py @@ -69,12 +69,8 @@ def test_experiments_diff_revs(mocker, capsys): mocker.patch( "dvc.repo.experiments.diff.diff", return_value={ - "params": { - "params.yaml": {"foo": {"diff": 1, "old": 1, "new": 2}} - }, - "metrics": { - "metrics.yaml": {"foo": {"diff": 1, "old": 1, "new": 2}} - }, + "params": {"params.yaml": {"foo": {"diff": 1, "old": 1, "new": 2}}}, + "metrics": {"metrics.yaml": {"foo": {"diff": 1, "old": 1, "new": 2}}}, }, ) @@ -528,9 +524,7 @@ def test_show_experiments_csv(capsys): }, } - show_experiments( - all_experiments, precision=None, fill_value="", iso=True, csv=True - ) + show_experiments(all_experiments, precision=None, fill_value="", iso=True, csv=True) cap = capsys.readouterr() assert ( "Experiment,rev,typ,Created,parent,State,scores.json:" @@ -545,13 +539,11 @@ def test_show_experiments_csv(capsys): ) assert ( "master,b05eecc,baseline,2021-08-02T16:48:14,,,3000,1," - "0.5325162867864254,0.9106964878520005,3000,1,20170428,100,2" - in cap.out + "0.5325162867864254,0.9106964878520005,3000,1,20170428,100,2" in cap.out ) assert ( "exp-44136,ae99936,branch_base,2021-08-31T14:56:55,,Running," - "3000,1,0.5843640011189556,0.9544670443829399,3000,1,20170428,100,36" - in cap.out + "3000,1,0.5843640011189556,0.9544670443829399,3000,1,20170428,100,36" in cap.out ) @@ -564,9 +556,7 @@ def test_show_experiments_md(capsys): "params": {"params.yaml": {"data": {"foo": 1}}}, "status": "Success", "executor": None, - "metrics": { - "scores.json": {"data": {"bar": 0.9544670443829399}} - }, + "metrics": {"scores.json": {"data": {"bar": 0.9544670443829399}}}, } } }, @@ -698,9 +688,7 @@ def test_show_experiments_sort_by(capsys, sort_order): @pytest.mark.parametrize("extra_args", [(), ("--run",)]) def test_experiments_init(dvc, scm, mocker, capsys, extra_args): stage = mocker.Mock(outs=[], addressing="train") - m = mocker.patch( - "dvc.repo.experiments.init.init", return_value=(stage, [], []) - ) + m = mocker.patch("dvc.repo.experiments.init.init", return_value=(stage, [], [])) runner = mocker.patch("dvc.repo.experiments.run.run", return_value=0) cli_args = parse_args(["exp", "init", *extra_args, "cmd"]) cmd = cli_args.func(cli_args) @@ -734,9 +722,7 @@ def test_experiments_init_config(dvc, scm, mocker): conf["exp"] = {"code": "new_src", "models": "new_models"} stage = mocker.Mock(outs=[]) - m = mocker.patch( - "dvc.repo.experiments.init.init", return_value=(stage, [], []) - ) + m = mocker.patch("dvc.repo.experiments.init.init", return_value=(stage, [], [])) cli_args = parse_args(["exp", "init", "cmd"]) cmd = cli_args.func(cli_args) @@ -763,9 +749,7 @@ def test_experiments_init_config(dvc, scm, mocker): def test_experiments_init_explicit(dvc, mocker): stage = mocker.Mock(outs=[]) - m = mocker.patch( - "dvc.repo.experiments.init.init", return_value=(stage, [], []) - ) + m = mocker.patch("dvc.repo.experiments.init.init", return_value=(stage, [], [])) cli_args = parse_args(["exp", "init", "--explicit", "cmd"]) cmd = cli_args.func(cli_args) @@ -797,9 +781,7 @@ def test_experiments_init_cmd_not_required_for_interactive_mode(dvc, mocker): assert isinstance(cmd, CmdExperimentsInit) stage = mocker.Mock(outs=[]) - m = mocker.patch( - "dvc.repo.experiments.init.init", return_value=(stage, [], []) - ) + m = mocker.patch("dvc.repo.experiments.init.init", return_value=(stage, [], [])) assert cmd.run() == 0 assert called_once_with_subset(m, ANY(Repo), interactive=True) @@ -853,9 +835,7 @@ def test_experiments_init_extra_args(extra_args, expected_kw, mocker): assert isinstance(cmd, CmdExperimentsInit) stage = mocker.Mock(outs=[]) - m = mocker.patch( - "dvc.repo.experiments.init.init", return_value=(stage, [], []) - ) + m = mocker.patch("dvc.repo.experiments.init.init", return_value=(stage, [], [])) assert cmd.run() == 0 assert called_once_with_subset(m, ANY(Repo), **expected_kw) @@ -912,16 +892,12 @@ def test_show_experiments_pcp(tmp_dir, mocker): "params": {"params.yaml": {"data": {"foo": 1}}}, "status": "Success", "executor": None, - "metrics": { - "scores.json": {"data": {"bar": 0.9544670443829399}} - }, + "metrics": {"scores.json": {"data": {"bar": 0.9544670443829399}}}, } } }, } - experiments_table = mocker.patch( - "dvc.commands.experiments.show.experiments_table" - ) + experiments_table = mocker.patch("dvc.commands.experiments.show.experiments_table") td = experiments_table.return_value show_experiments(all_experiments, pcp=True) diff --git a/tests/unit/command/test_get.py b/tests/unit/command/test_get.py index 306e2ce2a..5fa99c2b2 100644 --- a/tests/unit/command/test_get.py +++ b/tests/unit/command/test_get.py @@ -23,15 +23,11 @@ def test_get(mocker): assert cmd.run() == 0 - m.assert_called_once_with( - "repo_url", path="src", out="out", rev="version", jobs=4 - ) + m.assert_called_once_with("repo_url", path="src", out="out", rev="version", jobs=4) def test_get_url(mocker, capsys): - cli_args = parse_args( - ["get", "repo_url", "src", "--rev", "version", "--show-url"] - ) + cli_args = parse_args(["get", "repo_url", "src", "--rev", "version", "--show-url"]) assert cli_args.func == CmdGet cmd = cli_args.func(cli_args) diff --git a/tests/unit/command/test_imp_url.py b/tests/unit/command/test_imp_url.py index 63970e423..b862f1a8d 100644 --- a/tests/unit/command/test_imp_url.py +++ b/tests/unit/command/test_imp_url.py @@ -69,7 +69,6 @@ def test_failed_import_url(mocker, caplog): ], ) def test_import_url_no_exec_download_flags(mocker, flag, expected): - cli_args = parse_args( [ "import-url", @@ -100,7 +99,7 @@ def test_import_url_no_exec_download_flags(mocker, flag, expected): labels=None, meta=None, version_aware=False, - **expected + **expected, ) @@ -159,14 +158,11 @@ def test_import_url_to_remote_invalid_combination(dvc, mocker, caplog, flag): cmd = cli_args.func(cli_args) with caplog.at_level(logging.ERROR, logger="dvc"): assert cmd.run() == 1 - expected_msg = ( - "--no-exec/--no-download cannot be combined with --to-remote" - ) + expected_msg = "--no-exec/--no-download cannot be combined with --to-remote" assert expected_msg in caplog.text def test_import_url_to_remote_flag(dvc, mocker, caplog): - cli_args = parse_args( ["import-url", "s3://bucket/foo", "bar", "--remote", "remote"] ) diff --git a/tests/unit/command/test_machine.py b/tests/unit/command/test_machine.py index cdb37cfdb..11a3e681b 100644 --- a/tests/unit/command/test_machine.py +++ b/tests/unit/command/test_machine.py @@ -56,9 +56,7 @@ def test_create(tmp_dir, dvc, mocker): assert cli_args.func == CmdMachineCreate cmd = cli_args.func(cli_args) - m = mocker.patch.object( - cmd.repo.machine, "create", autospec=True, return_value=0 - ) + m = mocker.patch.object(cmd.repo.machine, "create", autospec=True, return_value=0) assert cmd.run() == 0 m.assert_called_once_with("foo") @@ -70,17 +68,13 @@ def test_status(tmp_dir, scm, dvc, mocker): assert cli_args.func == CmdMachineStatus cmd = cli_args.func(cli_args) - m = mocker.patch.object( - cmd.repo.machine, "status", autospec=True, return_value=[] - ) + m = mocker.patch.object(cmd.repo.machine, "status", autospec=True, return_value=[]) assert cmd.run() == 0 m.assert_called_once_with("foo") cli_args = parse_args(["machine", "status"]) cmd = cli_args.func(cli_args) - m = mocker.patch.object( - cmd.repo.machine, "status", autospec=True, return_value=[] - ) + m = mocker.patch.object(cmd.repo.machine, "status", autospec=True, return_value=[]) assert cmd.run() == 0 assert m.call_count == 2 m.assert_has_calls([call("foo"), call("myaws")]) @@ -91,9 +85,7 @@ def test_destroy(tmp_dir, dvc, mocker): assert cli_args.func == CmdMachineDestroy cmd = cli_args.func(cli_args) - m = mocker.patch.object( - cmd.repo.machine, "destroy", autospec=True, return_value=0 - ) + m = mocker.patch.object(cmd.repo.machine, "destroy", autospec=True, return_value=0) assert cmd.run() == 0 m.assert_called_once_with("foo") @@ -168,7 +160,6 @@ def test_rename(tmp_dir, scm, dvc): ], ) def test_help_message(tmp_dir, scm, dvc, cmd, use_config, capsys): - try: parse_args(["machine", cmd, "--help"]) except SystemExit: diff --git a/tests/unit/command/test_metrics.py b/tests/unit/command/test_metrics.py index 8c00e3905..57b88f7ae 100644 --- a/tests/unit/command/test_metrics.py +++ b/tests/unit/command/test_metrics.py @@ -25,9 +25,7 @@ def test_metrics_diff(dvc, mocker, capsys): cmd = cli_args.func(cli_args) diff = {"metrics.yaml": {"": {"old": 1, "new": 3}}} - metrics_diff = mocker.patch( - "dvc.repo.metrics.diff.diff", return_value=diff - ) + metrics_diff = mocker.patch("dvc.repo.metrics.diff.diff", return_value=diff) show_diff_mock = mocker.patch("dvc.compare.show_diff") assert cmd.run() == 0 @@ -75,9 +73,7 @@ def test_metrics_diff_json(dvc, mocker, capsys): cmd = cli_args.func(cli_args) diff = {"metrics.yaml": {"": {"old": 1, "new": 3}}} - metrics_diff = mocker.patch( - "dvc.repo.metrics.diff.diff", return_value=diff - ) + metrics_diff = mocker.patch("dvc.repo.metrics.diff.diff", return_value=diff) show_diff_mock = mocker.patch("dvc.compare.show_diff") assert cmd.run() == 0 diff --git a/tests/unit/command/test_params.py b/tests/unit/command/test_params.py index 8f2d91cfe..31eb2b629 100644 --- a/tests/unit/command/test_params.py +++ b/tests/unit/command/test_params.py @@ -69,9 +69,7 @@ def test_params_diff_from_cli(dvc, mocker): def test_params_diff_show_json(dvc, mocker, capsys): cli_args = parse_args(["params", "diff", "HEAD~10", "HEAD~1", "--json"]) cmd = cli_args.func(cli_args) - mocker.patch( - "dvc.repo.params.diff.diff", return_value={"params.yaml": {"a": "b"}} - ) + mocker.patch("dvc.repo.params.diff.diff", return_value={"params.yaml": {"a": "b"}}) show_diff_mock = mocker.patch("dvc.compare.show_diff") assert cmd.run() == 0 diff --git a/tests/unit/command/test_plots.py b/tests/unit/command/test_plots.py index 68255ec92..f6a543f4b 100644 --- a/tests/unit/command/test_plots.py +++ b/tests/unit/command/test_plots.py @@ -61,9 +61,7 @@ def test_plots_diff(dvc, mocker, plots_data): cmd = cli_args.func(cli_args) m = mocker.patch("dvc.repo.plots.diff.diff", return_value=plots_data) - render_mock = mocker.patch( - "dvc_render.render_html", return_value="html_path" - ) + render_mock = mocker.patch("dvc_render.render_html", return_value="html_path") assert cmd.run() == 0 @@ -108,9 +106,7 @@ def test_plots_show_vega(dvc, mocker, plots_data): return_value=plots_data, config_files=None, ) - render_mock = mocker.patch( - "dvc_render.render_html", return_value="html_path" - ) + render_mock = mocker.patch("dvc_render.render_html", return_value="html_path") assert cmd.run() == 0 @@ -181,9 +177,7 @@ def test_plots_diff_open_WSL(tmp_dir, dvc, mocker, plots_data): mocked_uname_result.release = "microsoft" mocker.patch("platform.uname", return_value=mocked_uname_result) - cli_args = parse_args( - ["plots", "diff", "--targets", "plots.csv", "--open"] - ) + cli_args = parse_args(["plots", "diff", "--targets", "plots.csv", "--open"]) cmd = cli_args.func(cli_args) mocker.patch("dvc.repo.plots.diff.diff", return_value=plots_data) @@ -196,9 +190,7 @@ def test_plots_diff_open_WSL(tmp_dir, dvc, mocker, plots_data): def test_plots_diff_open_failed(tmp_dir, dvc, mocker, capsys, plots_data): mocked_open = mocker.patch("webbrowser.open", return_value=False) - cli_args = parse_args( - ["plots", "diff", "--targets", "plots.csv", "--open"] - ) + cli_args = parse_args(["plots", "diff", "--targets", "plots.csv", "--open"]) cmd = cli_args.func(cli_args) mocker.patch("dvc.repo.plots.diff.diff", return_value=plots_data) @@ -207,8 +199,7 @@ def test_plots_diff_open_failed(tmp_dir, dvc, mocker, capsys, plots_data): mocked_open.assert_called_once_with(expected_url.as_uri()) error_message = ( - f"Failed to open {expected_url.as_uri()}. " - "Please try opening it manually." + f"Failed to open {expected_url.as_uri()}. Please try opening it manually." ) out, err = capsys.readouterr() @@ -233,9 +224,7 @@ def test_plots_diff_open_failed(tmp_dir, dvc, mocker, capsys, plots_data): def test_plots_path_is_quoted_and_resolved_properly( tmp_dir, dvc, mocker, capsys, output, expected_url_path, plots_data ): - cli_args = parse_args( - ["plots", "diff", "--targets", "datafile", "--out", output] - ) + cli_args = parse_args(["plots", "diff", "--targets", "datafile", "--out", output]) cmd = cli_args.func(cli_args) mocker.patch("dvc.repo.plots.diff.diff", return_value=plots_data) @@ -276,25 +265,17 @@ def test_should_pass_template_dir(tmp_dir, dvc, mocker, capsys): ) -@pytest.mark.parametrize( - "output", ("some_out", os.path.join("to", "subdir"), None) -) +@pytest.mark.parametrize("output", ("some_out", os.path.join("to", "subdir"), None)) def test_should_call_render(tmp_dir, mocker, capsys, plots_data, output): - cli_args = parse_args( - ["plots", "diff", "--targets", "plots.csv", "--out", output] - ) + cli_args = parse_args(["plots", "diff", "--targets", "plots.csv", "--out", output]) cmd = cli_args.func(cli_args) mocker.patch("dvc.repo.plots.diff.diff", return_value=plots_data) output = output or "dvc_plots" index_path = tmp_dir / output / "index.html" renderers = mocker.MagicMock() - mocker.patch( - "dvc.render.match.match_defs_renderers", return_value=renderers - ) - render_mock = mocker.patch( - "dvc_render.render_html", return_value=index_path - ) + mocker.patch("dvc.render.match.match_defs_renderers", return_value=renderers) + render_mock = mocker.patch("dvc_render.render_html", return_value=index_path) assert cmd.run() == 0 @@ -329,9 +310,7 @@ def test_plots_diff_json(dvc, mocker, capsys): mocker.patch("dvc.repo.plots.diff.diff", return_value=data) renderers = mocker.MagicMock() - mocker.patch( - "dvc.render.match.match_defs_renderers", return_value=renderers - ) + mocker.patch("dvc.render.match.match_defs_renderers", return_value=renderers) render_mock = mocker.patch("dvc_render.render_html") show_json_mock = mocker.patch("dvc.commands.plots._show_json") @@ -347,9 +326,7 @@ def test_plots_diff_json(dvc, mocker, capsys): "target,expected_out,expected_rtn", (("t1", "\"{'t1'}\"", 0), (None, "t1\nt2", 0), ("t3", "", 1)), ) -def test_plots_templates( - dvc, mocker, capsys, target, expected_out, expected_rtn -): +def test_plots_templates(dvc, mocker, capsys, target, expected_out, expected_rtn): t1 = mocker.Mock() t1.DEFAULT_NAME = "t1" t1.DEFAULT_CONTENT = "{'t1'}" diff --git a/tests/unit/command/test_queue.py b/tests/unit/command/test_queue.py index 8d138d1e5..0d9d85fc8 100644 --- a/tests/unit/command/test_queue.py +++ b/tests/unit/command/test_queue.py @@ -163,7 +163,6 @@ def test_experiments_stop(dvc, scm, mocker): ], ) def test_worker_status(dvc, scm, worker_status, output, mocker, capsys): - cli_args = parse_args( [ "queue", diff --git a/tests/unit/command/test_repro.py b/tests/unit/command/test_repro.py index 162c9add2..bd30d0afe 100644 --- a/tests/unit/command/test_repro.py +++ b/tests/unit/command/test_repro.py @@ -25,9 +25,7 @@ def test_default_arguments(dvc, mocker): cmd = CmdRepro(parse_args(["repro"])) mocker.patch.object(cmd.repo, "reproduce") cmd.run() - cmd.repo.reproduce.assert_called_with( - **common_arguments, **repro_arguments - ) + cmd.repo.reproduce.assert_called_with(**common_arguments, **repro_arguments) def test_downstream(dvc, mocker): diff --git a/tests/unit/data/db/test_local.py b/tests/unit/data/db/test_local.py index 50063c2c9..4ca9d5223 100644 --- a/tests/unit/data/db/test_local.py +++ b/tests/unit/data/db/test_local.py @@ -66,9 +66,7 @@ def test_is_protected(tmp_dir, dvc, link_name): def test_protect_ignore_errors(tmp_dir, dvc, mocker, err): tmp_dir.gen("foo", "foo") - mock_chmod = mocker.patch( - "os.chmod", side_effect=OSError(err, "something") - ) + mock_chmod = mocker.patch("os.chmod", side_effect=OSError(err, "something")) dvc.odb.local.protect("foo") assert mock_chmod.called @@ -77,9 +75,7 @@ def test_protect_ignore_errors(tmp_dir, dvc, mocker, err): def test_set_exec_ignore_errors(tmp_dir, dvc, mocker, err): tmp_dir.gen("foo", "foo") - mock_chmod = mocker.patch( - "os.chmod", side_effect=OSError(err, "something") - ) + mock_chmod = mocker.patch("os.chmod", side_effect=OSError(err, "something")) dvc.odb.local.set_exec("foo") assert mock_chmod.called @@ -93,9 +89,7 @@ def test_staging_file(tmp_dir, dvc): fs = LocalFileSystem() local_odb = dvc.odb.local - staging_odb, _, obj = build( - local_odb, (tmp_dir / "foo").fs_path, fs, "md5" - ) + staging_odb, _, obj = build(local_odb, (tmp_dir / "foo").fs_path, fs, "md5") assert not local_odb.exists(obj.hash_info.value) assert staging_odb.exists(obj.hash_info.value) @@ -121,9 +115,7 @@ def test_staging_dir(tmp_dir, dvc): fs = LocalFileSystem() local_odb = dvc.odb.local - staging_odb, _, obj = build( - local_odb, (tmp_dir / "dir").fs_path, fs, "md5" - ) + staging_odb, _, obj = build(local_odb, (tmp_dir / "dir").fs_path, fs, "md5") assert not local_odb.exists(obj.hash_info.value) assert staging_odb.exists(obj.hash_info.value) @@ -132,9 +124,7 @@ def test_staging_dir(tmp_dir, dvc): check(local_odb, obj) check(staging_odb, obj) - transfer( - staging_odb, local_odb, {obj.hash_info}, shallow=False, hardlink=True - ) + transfer(staging_odb, local_odb, {obj.hash_info}, shallow=False, hardlink=True) check(local_odb, obj) check(staging_odb, obj) diff --git a/tests/unit/dependency/test_params.py b/tests/unit/dependency/test_params.py index a99e24a18..e247a66bd 100644 --- a/tests/unit/dependency/test_params.py +++ b/tests/unit/dependency/test_params.py @@ -77,10 +77,7 @@ def test_loads_params_without_any_specific_targets(dvc): ([3], "Only list of str/dict is supported. Got: 'int'"), ( [{"b_file": "cat"}], - ( - "Expected list of params for custom params file " - "'b_file', got 'str'." - ), + "Expected list of params for custom params file 'b_file', got 'str'.", ), ], ) @@ -125,9 +122,7 @@ def test_read_params_unsupported_format(tmp_dir, dvc): def test_read_params_nested(tmp_dir, dvc): - dump_yaml( - DEFAULT_PARAMS_FILE, {"some": {"path": {"foo": ["val1", "val2"]}}} - ) + dump_yaml(DEFAULT_PARAMS_FILE, {"some": {"path": {"foo": ["val1", "val2"]}}}) dep = ParamsDependency(Stage(dvc), None, ["some.path.foo"]) assert dep.read_params() == {"some.path.foo": ["val1", "val2"]} @@ -158,15 +153,17 @@ def test_read_params_py(tmp_dir, dvc): parameters_file = "parameters.py" tmp_dir.gen( parameters_file, - "INT: int = 5\n" - "FLOAT = 0.001\n" - "STR = 'abc'\n" - "BOOL: bool = True\n" - "DICT = {'a': 1}\n" - "LIST = [1, 2, 3]\n" - "SET = {4, 5, 6}\n" - "TUPLE = (10, 100)\n" - "NONE = None\n", + ( + "INT: int = 5\n" + "FLOAT = 0.001\n" + "STR = 'abc'\n" + "BOOL: bool = True\n" + "DICT = {'a': 1}\n" + "LIST = [1, 2, 3]\n" + "SET = {4, 5, 6}\n" + "TUPLE = (10, 100)\n" + "NONE = None\n" + ), ) dep = ParamsDependency( Stage(dvc), @@ -195,9 +192,7 @@ def test_read_params_py(tmp_dir, dvc): "NONE": None, } - tmp_dir.gen( - parameters_file, "class Train:\n foo = 'val1'\n bar = 'val2'\n" - ) + tmp_dir.gen(parameters_file, "class Train:\n foo = 'val1'\n bar = 'val2'\n") dep = ParamsDependency(Stage(dvc), parameters_file, ["Train.foo"]) assert dep.read_params() == {"Train.foo": "val1"} @@ -206,14 +201,16 @@ def test_read_params_py(tmp_dir, dvc): tmp_dir.gen( parameters_file, - "x = 4\n" - "config.x = 3\n" - "class Klass:\n" - " def __init__(self):\n" - " self.a = 'val1'\n" - " container.a = 2\n" - " self.container.a = 1\n" - " a = 'val2'\n", + ( + "x = 4\n" + "config.x = 3\n" + "class Klass:\n" + " def __init__(self):\n" + " self.a = 'val1'\n" + " container.a = 2\n" + " self.container.a = 1\n" + " a = 'val2'\n" + ), ) dep = ParamsDependency(Stage(dvc), parameters_file, ["x", "Klass.a"]) assert dep.read_params() == {"x": 4, "Klass.a": "val1"} diff --git a/tests/unit/fs/test_azure.py b/tests/unit/fs/test_azure.py index 3626ea6af..3d8b67255 100644 --- a/tests/unit/fs/test_azure.py +++ b/tests/unit/fs/test_azure.py @@ -19,8 +19,7 @@ def test_strip_protocol_env_var(monkeypatch, dvc): def test_strip_protocol(dvc): assert ( - AzureFileSystem._strip_protocol(f"azure://{container_name}") - == container_name + AzureFileSystem._strip_protocol(f"azure://{container_name}") == container_name ) @@ -38,16 +37,10 @@ def test_azure_login_methods(): with pytest.raises(AzureAuthError): get_login_method({}) + assert get_login_method({"connection_string": "test"}) == "connection string" + assert get_login_method({"account_name": "test"}).startswith("default credentials") assert ( - get_login_method({"connection_string": "test"}) == "connection string" - ) - assert get_login_method({"account_name": "test"}).startswith( - "default credentials" - ) - assert ( - get_login_method( - {"account_name": "test", "allow_anonymous_login": True} - ) + get_login_method({"account_name": "test", "allow_anonymous_login": True}) == "anonymous login" ) @@ -73,8 +66,7 @@ def test_azure_login_methods(): == "account key" ) assert ( - get_login_method({"account_name": "test", "sas_token": "test"}) - == "SAS token" + get_login_method({"account_name": "test", "sas_token": "test"}) == "SAS token" ) assert ( get_login_method( diff --git a/tests/unit/fs/test_data.py b/tests/unit/fs/test_data.py index 6bf64739d..ccb5af2be 100644 --- a/tests/unit/fs/test_data.py +++ b/tests/unit/fs/test_data.py @@ -226,9 +226,7 @@ def test_get_hash_file(tmp_dir, dvc): def test_get_hash_dir(tmp_dir, dvc, mocker): - tmp_dir.dvc_gen( - {"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}} - ) + tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}}) fs = DataFileSystem(index=dvc.index.data["repo"]) hash_file_spy = mocker.spy(dvc_data.hashfile.hash, "hash_file") assert fs.info("dir")["md5"] == "8761c4e9acad696bee718615e23e22db.dir" @@ -236,16 +234,12 @@ def test_get_hash_dir(tmp_dir, dvc, mocker): def test_get_hash_granular(tmp_dir, dvc): - tmp_dir.dvc_gen( - {"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}} - ) + tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}}) fs = DataFileSystem(index=dvc.index.data["repo"]) subdir = "dir/subdir" assert fs.info(subdir).get("md5") is None _, _, obj = build(dvc.odb.local, subdir, fs, "md5", dry_run=True) - assert obj.hash_info == HashInfo( - "md5", "af314506f1622d107e0ed3f14ec1a3b5.dir" - ) + assert obj.hash_info == HashInfo("md5", "af314506f1622d107e0ed3f14ec1a3b5.dir") data = posixpath.join(subdir, "data") assert fs.info(data)["md5"] == "8d777f385d3dfec8815d20f7496026dc" _, _, obj = build(dvc.odb.local, data, fs, "md5", dry_run=True) diff --git a/tests/unit/fs/test_dvc.py b/tests/unit/fs/test_dvc.py index b523b477b..26d896ef0 100644 --- a/tests/unit/fs/test_dvc.py +++ b/tests/unit/fs/test_dvc.py @@ -123,9 +123,7 @@ def test_isdir_isfile(tmp_dir, dvc): def test_exists_isdir_isfile_dirty(tmp_dir, dvc): - tmp_dir.dvc_gen( - {"datafile": "data", "datadir": {"foo": "foo", "bar": "bar"}} - ) + tmp_dir.dvc_gen({"datafile": "data", "datadir": {"foo": "foo", "bar": "bar"}}) fs = DVCFileSystem(repo=dvc) shutil.rmtree(tmp_dir / "datadir") @@ -335,9 +333,7 @@ def test_subrepos(tmp_dir, scm, dvc, mocker): with subrepo1.chdir(): subrepo1.dvc_gen({"foo": "foo", "dir1": {"bar": "bar"}}, commit="FOO") with subrepo2.chdir(): - subrepo2.dvc_gen( - {"lorem": "lorem", "dir2": {"ipsum": "ipsum"}}, commit="BAR" - ) + subrepo2.dvc_gen({"lorem": "lorem", "dir2": {"ipsum": "ipsum"}}, commit="BAR") dvc._reset() fs = DVCFileSystem(repo=dvc, subrepos=True) @@ -414,9 +410,7 @@ def test_subrepo_walk(tmp_dir, scm, dvc, dvcfiles, extra_expected): with subrepo1.chdir(): subrepo1.dvc_gen({"foo": "foo", "dir1": {"bar": "bar"}}, commit="FOO") with subrepo2.chdir(): - subrepo2.dvc_gen( - {"lorem": "lorem", "dir2": {"ipsum": "ipsum"}}, commit="BAR" - ) + subrepo2.dvc_gen({"lorem": "lorem", "dir2": {"ipsum": "ipsum"}}, commit="BAR") # using fs that does not have dvcignore dvc._reset() @@ -506,36 +500,26 @@ def test_get_hash_cached_file(tmp_dir, dvc, mocker): def test_get_hash_cached_dir(tmp_dir, dvc, mocker): - tmp_dir.dvc_gen( - {"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}} - ) + tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}}) fs = DVCFileSystem(repo=dvc) expected = "8761c4e9acad696bee718615e23e22db.dir" assert fs.info("dir").get("md5") is None _, _, obj = build(dvc.odb.local, "dir", fs, "md5") - assert obj.hash_info == HashInfo( - "md5", "8761c4e9acad696bee718615e23e22db.dir" - ) + assert obj.hash_info == HashInfo("md5", "8761c4e9acad696bee718615e23e22db.dir") shutil.rmtree(tmp_dir / "dir") assert fs.info("dir")["md5"] == expected _, _, obj = build(dvc.odb.local, "dir", fs, "md5") - assert obj.hash_info == HashInfo( - "md5", "8761c4e9acad696bee718615e23e22db.dir" - ) + assert obj.hash_info == HashInfo("md5", "8761c4e9acad696bee718615e23e22db.dir") def test_get_hash_cached_granular(tmp_dir, dvc, mocker): - tmp_dir.dvc_gen( - {"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}} - ) + tmp_dir.dvc_gen({"dir": {"foo": "foo", "bar": "bar", "subdir": {"data": "data"}}}) fs = DVCFileSystem(repo=dvc) subdir = "dir/subdir" assert fs.info(subdir).get("md5") is None _, _, obj = build(dvc.odb.local, subdir, fs, "md5") - assert obj.hash_info == HashInfo( - "md5", "af314506f1622d107e0ed3f14ec1a3b5.dir" - ) + assert obj.hash_info == HashInfo("md5", "af314506f1622d107e0ed3f14ec1a3b5.dir") assert fs.info(posixpath.join(subdir, "data")).get("md5") is None _, _, obj = build(dvc.odb.local, posixpath.join(subdir, "data"), fs, "md5") assert obj.hash_info == HashInfo("md5", "8d777f385d3dfec8815d20f7496026dc") @@ -560,9 +544,7 @@ def test_get_hash_mixed_dir(tmp_dir, scm, dvc): fs = DVCFileSystem(repo=dvc) _, _, obj = build(dvc.odb.local, "dir", fs, "md5") - assert obj.hash_info == HashInfo( - "md5", "e1d9e8eae5374860ae025ec84cfd85c7.dir" - ) + assert obj.hash_info == HashInfo("md5", "e1d9e8eae5374860ae025ec84cfd85c7.dir") def test_get_hash_dirty_file(tmp_dir, dvc): @@ -601,9 +583,7 @@ def test_get_hash_dirty_dir(tmp_dir, dvc): fs = DVCFileSystem(repo=dvc) _, meta, obj = build(dvc.odb.local, "dir", fs, "md5") - assert obj.hash_info == HashInfo( - "md5", "ba75a2162ca9c29acecb7957105a0bc2.dir" - ) + assert obj.hash_info == HashInfo("md5", "ba75a2162ca9c29acecb7957105a0bc2.dir") assert meta.nfiles == 3 @@ -643,17 +623,11 @@ def test_walk_nested_subrepos(tmp_dir, dvc, scm, traverse_subrepos): if repo_dir != tmp_dir else "/" ) - expected[repo_dir_path] = set( - scm_files.keys() | dvc_files.keys() | extras - ) + expected[repo_dir_path] = set(scm_files.keys() | dvc_files.keys() | extras) # files inside a dvc directory - expected[posixpath.join(repo_dir_path, f"dvc-{base}")] = { - f"ipsum-{base}" - } + expected[posixpath.join(repo_dir_path, f"dvc-{base}")] = {f"ipsum-{base}"} # files inside a git directory - expected[posixpath.join(repo_dir_path, f"dir-{base}")] = { - f"bar-{base}" - } + expected[posixpath.join(repo_dir_path, f"dir-{base}")] = {f"bar-{base}"} if traverse_subrepos: # update subrepos @@ -662,8 +636,6 @@ def test_walk_nested_subrepos(tmp_dir, dvc, scm, traverse_subrepos): actual = {} fs = DVCFileSystem(repo=dvc) - for root, dirs, files in fs.walk( - "/", ignore_subrepos=not traverse_subrepos - ): + for root, dirs, files in fs.walk("/", ignore_subrepos=not traverse_subrepos): actual[root] = set(dirs + files) assert expected == actual diff --git a/tests/unit/fs/test_s3.py b/tests/unit/fs/test_s3.py index 16ec659a6..782dcef91 100644 --- a/tests/unit/fs/test_s3.py +++ b/tests/unit/fs/test_s3.py @@ -105,8 +105,7 @@ def test_grants(dvc): extra_args = fs.fs_args["s3_additional_kwargs"] assert ( - extra_args["GrantRead"] - == "id=read-permission-id,id=other-read-permission-id" + extra_args["GrantRead"] == "id=read-permission-id,id=other-read-permission-id" ) assert extra_args["GrantReadACP"] == "id=read-acp-permission-id" assert extra_args["GrantWriteACP"] == "id=write-acp-permission-id" diff --git a/tests/unit/fs/test_tree.py b/tests/unit/fs/test_tree.py index 95fe7f586..1f16eb259 100644 --- a/tests/unit/fs/test_tree.py +++ b/tests/unit/fs/test_tree.py @@ -7,9 +7,7 @@ from dvc.fs import get_cloud_fs def test_get_cloud_fs(tmp_dir, dvc): tmp_dir.add_remote(name="base", url="s3://bucket/path", default=False) tmp_dir.add_remote(name="first", url="remote://base/first", default=False) - tmp_dir.add_remote( - name="second", url="remote://first/second", default=False - ) + tmp_dir.add_remote(name="second", url="remote://first/second", default=False) base = "bucket/path" first = f"{base}/first" @@ -24,9 +22,7 @@ def test_get_cloud_fs(tmp_dir, dvc): def test_get_cloud_fs_validate(tmp_dir, dvc): - tmp_dir.add_remote( - name="base", url="ssh://example.com/path", default=False - ) + tmp_dir.add_remote(name="base", url="ssh://example.com/path", default=False) tmp_dir.add_remote( name="first", config={"url": "remote://base/first", "type": "symlink"}, diff --git a/tests/unit/output/test_annotations.py b/tests/unit/output/test_annotations.py index 1486a2d3c..74f6312a0 100644 --- a/tests/unit/output/test_annotations.py +++ b/tests/unit/output/test_annotations.py @@ -17,9 +17,7 @@ def test_annotation_to_dict(kwargs): def test_annotation_update(): annot = Annotation(desc="desc", labels=["label1", "label2"]) - annot.update( - labels=["label"], type="type", new="new", meta={"key": "value"} - ) + annot.update(labels=["label"], type="type", new="new", meta={"key": "value"}) assert vars(annot) == { "desc": "desc", diff --git a/tests/unit/output/test_load.py b/tests/unit/output/test_load.py index 8a7cc3d63..0504fdca2 100644 --- a/tests/unit/output/test_load.py +++ b/tests/unit/output/test_load.py @@ -102,9 +102,7 @@ def test_load_from_pipeline_error_on_typ(dvc, typ): @pytest.mark.parametrize("key", [3, "list".split()]) def test_load_from_pipeline_illegal_type(dvc, key): stage = Stage(dvc) - with pytest.raises( - ValueError, match=f"'{type(key).__name__}' not supported." - ): + with pytest.raises(ValueError, match=f"'{type(key).__name__}' not supported."): output.load_from_pipeline(stage, [key], "outs") with pytest.raises( ValueError, diff --git a/tests/unit/proc/test_manager.py b/tests/unit/proc/test_manager.py index 6ec9f3bf9..f01a7f8ff 100644 --- a/tests/unit/proc/test_manager.py +++ b/tests/unit/proc/test_manager.py @@ -5,10 +5,7 @@ import sys import pytest -from dvc.proc.exceptions import ( - ProcessNotTerminatedError, - UnsupportedSignalError, -) +from dvc.proc.exceptions import ProcessNotTerminatedError, UnsupportedSignalError from dvc.proc.manager import ProcessManager from dvc.proc.process import ProcessInfo diff --git a/tests/unit/proc/test_process.py b/tests/unit/proc/test_process.py index 53cd05fe1..a70987d24 100644 --- a/tests/unit/proc/test_process.py +++ b/tests/unit/proc/test_process.py @@ -42,9 +42,7 @@ def test_wait(tmp_dir, mocker): from dvc.proc.exceptions import TimeoutExpired proc = ManagedProcess("/bin/foo") - proc._proc.wait = mocker.Mock( - side_effect=subprocess.TimeoutExpired("/bin/foo", 5) - ) + proc._proc.wait = mocker.Mock(side_effect=subprocess.TimeoutExpired("/bin/foo", 5)) with pytest.raises(TimeoutExpired): proc.wait(5) diff --git a/tests/unit/remote/test_webdav.py b/tests/unit/remote/test_webdav.py index 416556793..cb622e477 100644 --- a/tests/unit/remote/test_webdav.py +++ b/tests/unit/remote/test_webdav.py @@ -67,9 +67,7 @@ def test_token(): def test_ask_password(mocker): - ask_password_mocked = mocker.patch( - "dvc_webdav.ask_password", return_value="pass" - ) + ask_password_mocked = mocker.patch("dvc_webdav.ask_password", return_value="pass") host = "host" # it should not ask for password as password is set @@ -103,9 +101,7 @@ def test_custom_auth_header(): def test_ask_password_custom_auth_header(mocker): - ask_password_mocked = mocker.patch( - "dvc_webdav.ask_password", return_value="pass" - ) + ask_password_mocked = mocker.patch("dvc_webdav.ask_password", return_value="pass") host = "host" # it should not ask for password as password is set @@ -123,9 +119,7 @@ def test_ask_password_custom_auth_header(mocker): config.pop("password") fs = WebDAVFileSystem(**config) - assert issubset( - {"headers": {custom_auth_header: "pass"}, "auth": None}, fs.fs_args - ) + assert issubset({"headers": {custom_auth_header: "pass"}, "auth": None}, fs.fs_args) ask_password_mocked.assert_called_once_with(host, custom_auth_header) diff --git a/tests/unit/remote/test_webhdfs.py b/tests/unit/remote/test_webhdfs.py index a14bcdc88..77b3ea039 100644 --- a/tests/unit/remote/test_webhdfs.py +++ b/tests/unit/remote/test_webhdfs.py @@ -42,9 +42,7 @@ def test_init(dvc, webhdfs_config): def test_verify_ssl(dvc, webhdfs_config, monkeypatch, mocker): mock_session = mocker.create_autospec(requests.Session) - monkeypatch.setattr( - requests, "Session", mocker.Mock(return_value=mock_session) - ) + monkeypatch.setattr(requests, "Session", mocker.Mock(return_value=mock_session)) # can't have token at the same time as user or proxy_to del webhdfs_config["token"] fs = WebHDFSFileSystem(**webhdfs_config) diff --git a/tests/unit/render/test_convert.py b/tests/unit/render/test_convert.py index 1fbf26b02..44b83acb2 100644 --- a/tests/unit/render/test_convert.py +++ b/tests/unit/render/test_convert.py @@ -1,10 +1,4 @@ -from dvc.render import ( - REVISION_FIELD, - REVISIONS_KEY, - SRC_FIELD, - TYPE_KEY, - VERSION_FIELD, -) +from dvc.render import REVISION_FIELD, REVISIONS_KEY, SRC_FIELD, TYPE_KEY, VERSION_FIELD from dvc.render.convert import to_json @@ -56,9 +50,7 @@ def test_to_json_vega(mocker): def test_to_json_vega_split(mocker): vega_renderer = mocker.MagicMock() vega_renderer.TYPE = "vega" - vega_renderer.get_filled_template.return_value = ( - '{"this": "is split vega"}' - ) + vega_renderer.get_filled_template.return_value = '{"this": "is split vega"}' vega_renderer.datapoints = [ { "x": 1, diff --git a/tests/unit/render/test_match.py b/tests/unit/render/test_match.py index 0b3362e96..02316559a 100644 --- a/tests/unit/render/test_match.py +++ b/tests/unit/render/test_match.py @@ -1,9 +1,5 @@ from dvc.render import VERSION_FIELD -from dvc.render.match import ( - PlotsData, - _squash_plots_properties, - match_defs_renderers, -) +from dvc.render.match import PlotsData, _squash_plots_properties, match_defs_renderers def test_group_definitions(): @@ -12,9 +8,7 @@ def test_group_definitions(): "v1": { "definitions": { "data": { - "config_file_1": { - "data": {"plot_id_1": {}, "plot_id_2": {}} - }, + "config_file_1": {"data": {"plot_id_1": {}, "plot_id_2": {}}}, "config_file_2": {"data": {"plot_id_3": {}}}, } } @@ -57,9 +51,7 @@ def test_match_renderers(mocker): }, }, "sources": { - "data": { - "file.json": {"data": [{"x": 1, "y": 1}, {"x": 2, "y": 2}]} - } + "data": {"file.json": {"data": [{"x": 1, "y": 1}, {"x": 2, "y": 2}]}} }, }, "errored_revision": { diff --git a/tests/unit/repo/experiments/conftest.py b/tests/unit/repo/experiments/conftest.py index e016cea5c..4f9dff1f7 100644 --- a/tests/unit/repo/experiments/conftest.py +++ b/tests/unit/repo/experiments/conftest.py @@ -79,10 +79,7 @@ def checkpoint_stage(tmp_dir, scm, dvc, mocker): tmp_dir.gen("checkpoint.py", CHECKPOINT_SCRIPT) tmp_dir.gen("params.yaml", "foo: 1") stage = dvc.run( - cmd=( - f"python checkpoint.py foo {DEFAULT_ITERATIONS} " - "params.yaml metrics.yaml" - ), + cmd=f"python checkpoint.py foo {DEFAULT_ITERATIONS} params.yaml metrics.yaml", metrics_no_cache=["metrics.yaml"], params=["foo"], checkpoints=["foo"], @@ -103,10 +100,7 @@ def failed_checkpoint_stage(tmp_dir, scm, dvc, mocker): tmp_dir.gen("checkpoint.py", FAILED_CHECKPOINT_SCRIPT) tmp_dir.gen("params.yaml", "foo: 1") stage = dvc.run( - cmd=( - f"python checkpoint.py foo {DEFAULT_ITERATIONS+2} " - "params.yaml metrics.yaml" - ), + cmd=f"python checkpoint.py foo {DEFAULT_ITERATIONS+2} params.yaml metrics.yaml", metrics_no_cache=["metrics.yaml"], params=["foo"], checkpoints=["foo"], diff --git a/tests/unit/repo/experiments/queue/test_celery.py b/tests/unit/repo/experiments/queue/test_celery.py index 74bfa91df..42c0ad490 100644 --- a/tests/unit/repo/experiments/queue/test_celery.py +++ b/tests/unit/repo/experiments/queue/test_celery.py @@ -28,7 +28,6 @@ def test_shutdown(test_queue, mocker): def test_shutdown_with_kill(test_queue, mocker): - mock_entry_foo = mocker.Mock(stash_rev="af12de") mock_entry_foo.name = "foo" mock_entry_bar = mocker.Mock(stash_rev="bar") @@ -51,7 +50,6 @@ def test_shutdown_with_kill(test_queue, mocker): def test_post_run_after_kill(test_queue): - from celery import chain sig_bar = test_queue.proc.run_signature( @@ -80,7 +78,6 @@ def test_post_run_after_kill(test_queue): @pytest.mark.parametrize("force", [True, False]) def test_celery_queue_kill(test_queue, mocker, force): - mock_entry_foo = mocker.Mock(stash_rev="foo") mock_entry_bar = mocker.Mock(stash_rev="bar") mock_entry_foobar = mocker.Mock(stash_rev="foobar") @@ -133,9 +130,7 @@ def test_celery_queue_kill(test_queue, mocker, force): "kill" if force else "interrupt", side_effect=mocker.MagicMock(side_effect=kill_function), ) - with pytest.raises( - CannotKillTasksError, match="Task 'foobar' is initializing," - ): + with pytest.raises(CannotKillTasksError, match="Task 'foobar' is initializing,"): test_queue.kill(["bar", "foo", "foobar"], force=force) assert kill_mock.called_once_with(mock_entry_foo.stash_rev) assert kill_mock.called_once_with(mock_entry_bar.stash_rev) @@ -145,7 +140,6 @@ def test_celery_queue_kill(test_queue, mocker, force): @pytest.mark.parametrize("force", [True, False]) def test_celery_queue_kill_invalid(test_queue, mocker, force): - mock_entry_foo = mocker.Mock(stash_rev="foo") mock_entry_bar = mocker.Mock(stash_rev="bar") @@ -170,7 +164,6 @@ def test_celery_queue_kill_invalid(test_queue, mocker, force): @pytest.mark.parametrize("status", ["FAILURE", "SUCCESS"]) def test_queue_iter_done_task(test_queue, mocker, status): - mock_entry = mocker.Mock(stash_rev=_foo.name) result = mocker.Mock(status=status) @@ -182,9 +175,7 @@ def test_queue_iter_done_task(test_queue, mocker, status): ) if status == "FAILURE": - assert list(test_queue.iter_failed()) == [ - QueueDoneResult(mock_entry, None) - ] + assert list(test_queue.iter_failed()) == [QueueDoneResult(mock_entry, None)] elif status == "SUCCESS": with pytest.raises(DvcException, match="Invalid experiment"): diff --git a/tests/unit/repo/experiments/queue/test_remove.py b/tests/unit/repo/experiments/queue/test_remove.py index 8bb4165b2..e7c6221d6 100644 --- a/tests/unit/repo/experiments/queue/test_remove.py +++ b/tests/unit/repo/experiments/queue/test_remove.py @@ -4,7 +4,6 @@ from dvc.repo.experiments.queue.base import QueueDoneResult def test_remove_queued(test_queue, mocker): - queued_test = ["queue1", "queue2", "queue3"] stash_dict = {} @@ -21,9 +20,7 @@ def test_remove_queued(test_queue, mocker): msg_iter = [(msg_dict[name], entry_dict[name]) for name in queued_test] entry_iter = [entry_dict[name] for name in queued_test] - stash = mocker.patch.object( - test_queue, "stash", return_value=mocker.Mock() - ) + stash = mocker.patch.object(test_queue, "stash", return_value=mocker.Mock()) stash.stash_revs = stash_dict mocker.patch.object(test_queue, "_iter_queued", return_value=msg_iter) mocker.patch.object(test_queue, "iter_queued", return_value=entry_iter) @@ -57,30 +54,21 @@ def test_remove_done(test_queue, mocker): msg_dict = {} entry_dict = {} for name in concat(failed_test, success_test): - msg_dict[name] = mocker.Mock( - delivery_tag=f"msg_{name}", headers={"id": 0} - ) + msg_dict[name] = mocker.Mock(delivery_tag=f"msg_{name}", headers={"id": 0}) entry_dict[name] = mocker.Mock(stash_rev=name) entry_dict[name].name = name msg_iter = [ - (msg_dict[name], entry_dict[name]) - for name in concat(failed_test, success_test) + (msg_dict[name], entry_dict[name]) for name in concat(failed_test, success_test) ] done_iter = [ QueueDoneResult(entry_dict[name], None) for name in concat(failed_test, success_test) ] - failed_iter = [ - QueueDoneResult(entry_dict[name], None) for name in failed_test - ] - success_iter = [ - QueueDoneResult(entry_dict[name], None) for name in success_test - ] + failed_iter = [QueueDoneResult(entry_dict[name], None) for name in failed_test] + success_iter = [QueueDoneResult(entry_dict[name], None) for name in success_test] - stash = mocker.patch.object( - test_queue, "failed_stash", return_value=mocker.Mock() - ) + stash = mocker.patch.object(test_queue, "failed_stash", return_value=mocker.Mock()) stash.stash_revs = stash_dict mocker.patch.object(test_queue, "_iter_processed", return_value=msg_iter) mocker.patch.object(test_queue, "iter_done", return_value=done_iter) @@ -88,9 +76,7 @@ def test_remove_done(test_queue, mocker): mocker.patch.object(test_queue, "iter_failed", return_value=failed_iter) mocker.patch("celery.result.AsyncResult", return_value=mocker.Mock()) - remove_revs_mocker = mocker.patch.object( - test_queue.failed_stash, "remove_revs" - ) + remove_revs_mocker = mocker.patch.object(test_queue.failed_stash, "remove_revs") purge_mocker = mocker.patch.object(test_queue.celery, "purge") assert test_queue.remove(["failed3", "success2"]) == [ @@ -103,9 +89,9 @@ def test_remove_done(test_queue, mocker): remove_revs_mocker.reset_mock() purge_mocker.reset_mock() - assert set(test_queue.clear(success=True, failed=True)) == set( - failed_test - ) | set(success_test) + assert set(test_queue.clear(success=True, failed=True)) == set(failed_test) | set( + success_test + ) purge_mocker.assert_has_calls( [ call("msg_failed1"), diff --git a/tests/unit/repo/experiments/test_executor_status.py b/tests/unit/repo/experiments/test_executor_status.py index b086b75e8..c396b88a1 100644 --- a/tests/unit/repo/experiments/test_executor_status.py +++ b/tests/unit/repo/experiments/test_executor_status.py @@ -4,11 +4,7 @@ import pytest from dvc.exceptions import ReproductionError from dvc.repo.experiments.executor.base import ExecutorInfo, TaskStatus -from dvc.repo.experiments.queue.tasks import ( - cleanup_exp, - collect_exp, - setup_exp, -) +from dvc.repo.experiments.queue.tasks import cleanup_exp, collect_exp, setup_exp def test_celery_queue_success_status(dvc, scm, test_queue, exp_stage): @@ -23,9 +19,7 @@ def test_celery_queue_success_status(dvc, scm, test_queue, exp_stage): assert executor_info.status == TaskStatus.PREPARING cmd = ["dvc", "exp", "exec-run", "--infofile", infofile] - proc_dict = test_queue.proc.run_signature( - cmd, name=queue_entry.stash_rev - )() + proc_dict = test_queue.proc.run_signature(cmd, name=queue_entry.stash_rev)() executor_info = ExecutorInfo.load_json(infofile) assert executor_info.status == TaskStatus.SUCCESS @@ -70,9 +64,7 @@ def test_workspace_executor_success_status(dvc, scm, exp_stage, queue_type): executor_info = ExecutorInfo.load_json(infofile) assert executor_info.status == TaskStatus.SUCCESS if exec_result.ref_info: - workspace_queue.collect_executor( - dvc.experiments, executor, exec_result - ) + workspace_queue.collect_executor(dvc.experiments, executor, exec_result) executor.cleanup(infofile) if queue_type == "tempdir_queue": @@ -86,9 +78,7 @@ def test_workspace_executor_success_status(dvc, scm, exp_stage, queue_type): "queue_type", ["workspace_queue", "tempdir_queue"], ) -def test_workspace_executor_failed_status( - dvc, scm, failed_exp_stage, queue_type -): +def test_workspace_executor_failed_status(dvc, scm, failed_exp_stage, queue_type): queue = getattr(dvc.experiments, queue_type) queue.put( params={"params.yaml": ["foo=1"]}, diff --git a/tests/unit/repo/experiments/test_remove.py b/tests/unit/repo/experiments/test_remove.py index 9592484ab..41753234e 100644 --- a/tests/unit/repo/experiments/test_remove.py +++ b/tests/unit/repo/experiments/test_remove.py @@ -42,8 +42,7 @@ def test_remove_done_tasks(dvc, test_queue, scm, mocker): ) assert ( - dvc.experiments.remove(failed_test + success_test) - == failed_test + success_test + dvc.experiments.remove(failed_test + success_test) == failed_test + success_test ) remove_tasks_mocker.assert_called_once_with( diff --git a/tests/unit/repo/experiments/test_utils.py b/tests/unit/repo/experiments/test_utils.py index ba7079511..1b6839ba0 100644 --- a/tests/unit/repo/experiments/test_utils.py +++ b/tests/unit/repo/experiments/test_utils.py @@ -47,7 +47,6 @@ def test_resolve_exp_ref(tmp_dir, scm, git_upstream, name_only, use_url): ], ) def test_run_check_ref_format(scm, name, result): - ref = ExpRefInfo("abc123", name) if result: check_ref_format(scm, ref) diff --git a/tests/unit/repo/plots/test_diff.py b/tests/unit/repo/plots/test_diff.py index 04e5b1938..eca5d109d 100644 --- a/tests/unit/repo/plots/test_diff.py +++ b/tests/unit/repo/plots/test_diff.py @@ -30,9 +30,7 @@ def test_revisions(mocker, arg_revisions, is_dirty, expected_revisions): (["v1", "v2"], None, ["v1", "v2"]), ], ) -def test_revisions_experiment( - mocker, arg_revisions, baseline, expected_revisions -): +def test_revisions_experiment(mocker, arg_revisions, baseline, expected_revisions): mock_scm = mocker.Mock() mock_scm.configure_mock( **{"is_dirty.return_value": False, "get_ref.return_value": None} diff --git a/tests/unit/repo/test_repo.py b/tests/unit/repo/test_repo.py index efb1ead44..74fa4a5db 100644 --- a/tests/unit/repo/test_repo.py +++ b/tests/unit/repo/test_repo.py @@ -22,9 +22,7 @@ def test_is_dvc_internal(dvc): ], ) def test_find_outs_by_path(tmp_dir, dvc, path): - (stage,) = tmp_dir.dvc_gen( - {"dir": {"subdir": {"file": "file"}, "other": "other"}} - ) + (stage,) = tmp_dir.dvc_gen({"dir": {"subdir": {"file": "file"}, "other": "other"}}) outs = dvc.find_outs_by_path(path, strict=False) assert len(outs) == 1 diff --git a/tests/unit/repo/test_reproduce.py b/tests/unit/repo/test_reproduce.py index 200521aa1..fbf622f25 100644 --- a/tests/unit/repo/test_reproduce.py +++ b/tests/unit/repo/test_reproduce.py @@ -9,18 +9,10 @@ def test_number_reproduces(tmp_dir, dvc, mocker): ) tmp_dir.dvc_gen({"pre-foo": "pre-foo"}) - dvc.run( - single_stage=True, deps=["pre-foo"], outs=["foo"], cmd="echo foo > foo" - ) - dvc.run( - single_stage=True, deps=["foo"], outs=["bar"], cmd="echo bar > bar" - ) - dvc.run( - single_stage=True, deps=["foo"], outs=["baz"], cmd="echo baz > baz" - ) - dvc.run( - single_stage=True, deps=["bar"], outs=["boop"], cmd="echo boop > boop" - ) + dvc.run(single_stage=True, deps=["pre-foo"], outs=["foo"], cmd="echo foo > foo") + dvc.run(single_stage=True, deps=["foo"], outs=["bar"], cmd="echo bar > bar") + dvc.run(single_stage=True, deps=["foo"], outs=["baz"], cmd="echo baz > baz") + dvc.run(single_stage=True, deps=["bar"], outs=["boop"], cmd="echo boop > boop") reproduce_stage_mock.reset_mock() diff --git a/tests/unit/repo/test_scm_context.py b/tests/unit/repo/test_scm_context.py index 546be9b33..52e3b10d6 100644 --- a/tests/unit/repo/test_scm_context.py +++ b/tests/unit/repo/test_scm_context.py @@ -21,7 +21,7 @@ def scm_context(request, mocker): **{ "ignore_remove.return_value": ".gitignore", "ignore.return_value": ".gitignore", - } + }, ) ) @@ -92,9 +92,7 @@ def test_scm_context_clears_ignores_on_error(scm_context): @pytest.mark.parametrize("autostage", [True, False]) @pytest.mark.parametrize("quiet", [True, False]) -def test_scm_context_on_no_files_to_track( - caplog, scm_context, autostage, quiet -): +def test_scm_context_on_no_files_to_track(caplog, scm_context, autostage, quiet): with scm_context(autostage=autostage, quiet=quiet): pass diff --git a/tests/unit/stage/test_cache.py b/tests/unit/stage/test_cache.py index 6dc5ef24f..faefff8eb 100644 --- a/tests/unit/stage/test_cache.py +++ b/tests/unit/stage/test_cache.py @@ -9,7 +9,7 @@ def test_stage_cache(tmp_dir, dvc, mocker): tmp_dir.gen("dep", "dep") tmp_dir.gen( "script.py", - ('open("out", "w+").write("out"); '), + 'open("out", "w+").write("out"); ', ) stage = dvc.run( cmd="python script.py", @@ -55,7 +55,7 @@ def test_stage_cache_params(tmp_dir, dvc, mocker): tmp_dir.gen("myparams.yaml", "baz: 3\nqux: 4") tmp_dir.gen( "script.py", - ('open("out", "w+").write("out"); '), + 'open("out", "w+").write("out"); ', ) stage = dvc.run( cmd="python script.py", @@ -100,7 +100,7 @@ def test_stage_cache_wdir(tmp_dir, dvc, mocker): tmp_dir.gen("dep", "dep") tmp_dir.gen( "script.py", - ('open("out", "w+").write("out"); '), + 'open("out", "w+").write("out"); ', ) tmp_dir.gen({"wdir": {}}) stage = dvc.run( diff --git a/tests/unit/stage/test_loader_pipeline_file.py b/tests/unit/stage/test_loader_pipeline_file.py index e5fd93bd8..a4cfe4f7f 100644 --- a/tests/unit/stage/test_loader_pipeline_file.py +++ b/tests/unit/stage/test_loader_pipeline_file.py @@ -26,9 +26,7 @@ def lock_data(): def test_fill_from_lock_deps_outs(dvc, lock_data): - stage = create_stage( - PipelineStage, dvc, PROJECT_FILE, deps=["foo"], outs=["bar"] - ) + stage = create_stage(PipelineStage, dvc, PROJECT_FILE, deps=["foo"], outs=["bar"]) for item in chain(stage.deps, stage.outs): assert not item.hash_info @@ -88,9 +86,7 @@ def test_fill_from_lock_params(dvc, lock_data): StageLoader.fill_from_lock(stage, lock_data) assert params_deps[0].hash_info.value == lock_data["params"]["params.yaml"] - assert ( - params_deps[1].hash_info.value == lock_data["params"]["myparams.yaml"] - ) + assert params_deps[1].hash_info.value == lock_data["params"]["myparams.yaml"] def test_fill_from_lock_missing_params_section(dvc, lock_data): @@ -140,9 +136,7 @@ def test_fill_from_lock_use_appropriate_checksum(dvc, lock_data): def test_fill_from_lock_with_missing_sections(dvc, lock_data): - stage = create_stage( - PipelineStage, dvc, PROJECT_FILE, deps=["foo"], outs=["bar"] - ) + stage = create_stage(PipelineStage, dvc, PROJECT_FILE, deps=["foo"], outs=["bar"]) lock = deepcopy(lock_data) del lock["deps"] StageLoader.fill_from_lock(stage, lock) @@ -157,9 +151,7 @@ def test_fill_from_lock_with_missing_sections(dvc, lock_data): def test_fill_from_lock_empty_data(dvc): - stage = create_stage( - PipelineStage, dvc, PROJECT_FILE, deps=["foo"], outs=["bar"] - ) + stage = create_stage(PipelineStage, dvc, PROJECT_FILE, deps=["foo"], outs=["bar"]) StageLoader.fill_from_lock(stage, None) assert not stage.deps[0].hash_info assert not stage.outs[0].hash_info diff --git a/tests/unit/stage/test_serialize_pipeline_file.py b/tests/unit/stage/test_serialize_pipeline_file.py index 3c57560ec..810f143dc 100644 --- a/tests/unit/stage/test_serialize_pipeline_file.py +++ b/tests/unit/stage/test_serialize_pipeline_file.py @@ -153,9 +153,7 @@ def test_plot_props(dvc): def test_frozen(dvc): - stage = create_stage( - PipelineStage, dvc, outs=["output"], deps=["input"], **kwargs - ) + stage = create_stage(PipelineStage, dvc, outs=["output"], deps=["input"], **kwargs) assert stage.PARAM_FROZEN not in to_pipeline_file(stage)["something"] stage = create_stage(PipelineStage, dvc, **kwargs, frozen=True) @@ -163,18 +161,11 @@ def test_frozen(dvc): def test_always_changed(dvc): - stage = create_stage( - PipelineStage, dvc, outs=["output"], deps=["input"], **kwargs - ) - assert ( - stage.PARAM_ALWAYS_CHANGED not in to_pipeline_file(stage)["something"] - ) + stage = create_stage(PipelineStage, dvc, outs=["output"], deps=["input"], **kwargs) + assert stage.PARAM_ALWAYS_CHANGED not in to_pipeline_file(stage)["something"] stage = create_stage(PipelineStage, dvc, **kwargs, always_changed=True) - assert ( - to_pipeline_file(stage)["something"][stage.PARAM_ALWAYS_CHANGED] - is True - ) + assert to_pipeline_file(stage)["something"][stage.PARAM_ALWAYS_CHANGED] is True def test_order(dvc): @@ -199,9 +190,7 @@ def test_order(dvc): ] -@pytest.mark.parametrize( - "typ", ["outs", "metrics", "plots", "params", "deps", None] -) +@pytest.mark.parametrize("typ", ["outs", "metrics", "plots", "params", "deps", None]) def test_order_deps_outs(dvc, typ): all_types = ["deps", "params", "outs", "metrics", "plots"] all_types = [item for item in all_types if item != typ] diff --git a/tests/unit/stage/test_serialize_pipeline_lock.py b/tests/unit/stage/test_serialize_pipeline_lock.py index 342c89ec3..ff64121ec 100644 --- a/tests/unit/stage/test_serialize_pipeline_lock.py +++ b/tests/unit/stage/test_serialize_pipeline_lock.py @@ -7,9 +7,7 @@ from dvc.dvcfile import PROJECT_FILE from dvc.schema import LOCK_FILE_STAGE_SCHEMA, LOCKFILE_STAGES_SCHEMA from dvc.stage import PipelineStage, create_stage from dvc.stage.serialize import DEFAULT_PARAMS_FILE, to_lockfile -from dvc.stage.serialize import ( - to_single_stage_lockfile as _to_single_stage_lockfile, -) +from dvc.stage.serialize import to_single_stage_lockfile as _to_single_stage_lockfile from dvc.stage.utils import split_params_deps from dvc_data.hashfile.hash_info import HashInfo @@ -41,9 +39,7 @@ def test_lock_deps(dvc): def test_lock_deps_order(dvc): - stage = create_stage( - PipelineStage, dvc, deps=["input1", "input0"], **kwargs - ) + stage = create_stage(PipelineStage, dvc, deps=["input1", "input0"], **kwargs) stage.deps[0].hash_info = HashInfo("md5", "md-one1") stage.deps[1].hash_info = HashInfo("md5", "md-zer0") assert to_single_stage_lockfile(stage) == OrderedDict( @@ -61,9 +57,7 @@ def test_lock_deps_order(dvc): def test_lock_params(dvc): - stage = create_stage( - PipelineStage, dvc, params=["lorem.ipsum", "abc"], **kwargs - ) + stage = create_stage(PipelineStage, dvc, params=["lorem.ipsum", "abc"], **kwargs) stage.deps[0].hash_info = HashInfo( "params", {"lorem.ipsum": {"lorem1": 1, "lorem2": 2}, "abc": 3} ) @@ -82,7 +76,7 @@ def test_lock_params_file_sorted(dvc): {"myparams.yaml": ["foo", "foobar"]}, {"a-params-file.yaml": ["bar", "barr"]}, ], - **kwargs + **kwargs, ) stage.deps[0].hash_info = HashInfo( "params", {"lorem.ipsum": {"lorem1": 1, "lorem2": 2}, "abc": 3} @@ -97,9 +91,7 @@ def test_lock_params_file_sorted(dvc): [ ( DEFAULT_PARAMS_FILE, - OrderedDict( - [("abc", 3), ("lorem.ipsum", {"lorem1": 1, "lorem2": 2})] - ), + OrderedDict([("abc", 3), ("lorem.ipsum", {"lorem1": 1, "lorem2": 2})]), ), ( "a-params-file.yaml", @@ -114,9 +106,7 @@ def test_lock_params_file_sorted(dvc): def test_lock_params_no_values_filled(dvc): - stage = create_stage( - PipelineStage, dvc, params=["lorem.ipsum", "abc"], **kwargs - ) + stage = create_stage(PipelineStage, dvc, params=["lorem.ipsum", "abc"], **kwargs) assert to_single_stage_lockfile(stage) == {"cmd": "command"} @@ -129,9 +119,7 @@ def test_lock_params_no_values_filled(dvc): ], ) def test_lock_params_without_targets(dvc, info, expected): - stage = create_stage( - PipelineStage, dvc, params=[{"params.yaml": None}], **kwargs - ) + stage = create_stage(PipelineStage, dvc, params=[{"params.yaml": None}], **kwargs) stage.deps[0].fill_values(info) assert to_single_stage_lockfile(stage) == { "cmd": "command", @@ -177,9 +165,7 @@ def test_lock_outs_isexec(dvc, typ): @pytest.mark.parametrize("typ", ["plots", "metrics", "outs"]) def test_lock_outs_order(dvc, typ): - stage = create_stage( - PipelineStage, dvc, **{typ: ["input1", "input0"]}, **kwargs - ) + stage = create_stage(PipelineStage, dvc, **{typ: ["input1", "input0"]}, **kwargs) stage.outs[0].hash_info = HashInfo("md5", "md-one1") stage.outs[1].hash_info = HashInfo("md5", "md-zer0") assert to_single_stage_lockfile(stage) == OrderedDict( @@ -197,20 +183,14 @@ def test_lock_outs_order(dvc, typ): def test_dump_nondefault_hash(dvc): - stage = create_stage( - PipelineStage, dvc, deps=["s3://dvc-temp/file"], **kwargs - ) + stage = create_stage(PipelineStage, dvc, deps=["s3://dvc-temp/file"], **kwargs) stage.deps[0].hash_info = HashInfo("md5", "value") assert to_single_stage_lockfile(stage) == OrderedDict( [ ("cmd", "command"), ( "deps", - [ - OrderedDict( - [("path", "s3://dvc-temp/file"), ("md5", "value")] - ) - ], + [OrderedDict([("path", "s3://dvc-temp/file"), ("md5", "value")])], ), ] ) @@ -223,7 +203,7 @@ def test_order(dvc): deps=["input"], outs=["output"], params=["foo-param"], - **kwargs + **kwargs, ) params, deps = split_params_deps(stage) diff --git a/tests/unit/stage/test_stage.py b/tests/unit/stage/test_stage.py index 3585543f7..739527ae9 100644 --- a/tests/unit/stage/test_stage.py +++ b/tests/unit/stage/test_stage.py @@ -138,7 +138,5 @@ def test_env(dvc, mocker): return {"foo": str(out)} mocker.patch.object(stage, "_read_env", mock_read_env) - with pytest.raises( - DvcException, match="Conflicting values for env variable" - ): + with pytest.raises(DvcException, match="Conflicting values for env variable"): _ = stage.env() diff --git a/tests/unit/stage/test_utils.py b/tests/unit/stage/test_utils.py index 86d845653..363efffbe 100644 --- a/tests/unit/stage/test_utils.py +++ b/tests/unit/stage/test_utils.py @@ -16,8 +16,6 @@ def test_resolve_paths(): assert path == os.path.abspath(file_path) assert wdir == os.path.abspath(p) - path, wdir = resolve_paths( - fs=localfs, path=file_path, wdir="../../some-dir" - ) + path, wdir = resolve_paths(fs=localfs, path=file_path, wdir="../../some-dir") assert path == os.path.abspath(file_path) assert wdir == os.path.abspath("some-dir") diff --git a/tests/unit/test_analytics.py b/tests/unit/test_analytics.py index 8adb1bc4c..63268aad2 100644 --- a/tests/unit/test_analytics.py +++ b/tests/unit/test_analytics.py @@ -110,9 +110,7 @@ def test_is_enabled(dvc, config, result, monkeypatch, tmp_global_dir): ("true", "false", False), # we checking if env is set ], ) -def test_is_enabled_env_neg( - dvc, config, env, result, monkeypatch, tmp_global_dir -): +def test_is_enabled_env_neg(dvc, config, env, result, monkeypatch, tmp_global_dir): # reset DVC_TEST env var, which affects `is_enabled()` monkeypatch.delenv("DVC_TEST") monkeypatch.delenv("DVC_NO_ANALYTICS", raising=False) diff --git a/tests/unit/test_api.py b/tests/unit/test_api.py index ef54604e5..28cd8ecf5 100644 --- a/tests/unit/test_api.py +++ b/tests/unit/test_api.py @@ -7,9 +7,7 @@ def test_open_raises_error_if_no_context(tmp_dir, dvc): tmp_dir.dvc_gen("foo", "foo-text") fd = api.open("foo") - with pytest.raises( - AttributeError, match="should be used in a with statement." - ): + with pytest.raises(AttributeError, match="should be used in a with statement."): fd.read() diff --git a/tests/unit/test_compare.py b/tests/unit/test_compare.py index 7fd0e7aeb..abcacc26e 100644 --- a/tests/unit/test_compare.py +++ b/tests/unit/test_compare.py @@ -67,11 +67,7 @@ def test_do_not_show_changes(): def test_diff_table_precision(): - diff = { - "metrics.json": { - "a.b.c": {"old": 1.1234, "new": 2.2345, "diff": 3.3456} - } - } + diff = {"metrics.json": {"a.b.c": {"old": 1.1234, "new": 2.2345, "diff": 3.3456}}} td = diff_table(diff, title="Metric", precision=3) assert td.as_dict() == [ { @@ -85,11 +81,7 @@ def test_diff_table_precision(): def test_diff_table_rounding(): - diff = { - "metrics.json": { - "a.b.c": {"old": 1.1234, "new": 2.2345, "diff": 3.3456} - } - } + diff = {"metrics.json": {"a.b.c": {"old": 1.1234, "new": 2.2345, "diff": 3.3456}}} td = diff_table(diff, title="Metric", precision=3, round_digits=True) assert td.as_dict() == [ { @@ -109,7 +101,7 @@ def test_diff_unsupported_diff_message(extra, expected): td = diff_table( {"metrics.json": {"": {"old": "1", "new": "2"}}}, title="Metric", - **extra + **extra, ) assert td.as_dict() == [ { @@ -189,9 +181,7 @@ def test_diff_falsey_values(): [([2, 3], "[2, 3]"), ({"foo": 3, "bar": 3}, "{'foo': 3, 'bar': 3}")], ) def test_diff_list(composite, expected): - td = diff_table( - {"params.yaml": {"a.b.c": {"old": 1, "new": composite}}}, "Param" - ) + td = diff_table({"params.yaml": {"a.b.c": {"old": 1, "new": composite}}}, "Param") assert td.as_dict() == [ { "Path": "params.yaml", @@ -279,9 +269,7 @@ def test_metrics_show_with_valid_falsey_values(): { "branch_1": { "data": { - "metrics.json": { - "data": {"a": 0, "b": {"ad": 0.0, "bc": 0.0}} - } + "metrics.json": {"data": {"a": 0, "b": {"ad": 0.0, "bc": 0.0}}} } } }, @@ -303,9 +291,7 @@ def test_metrics_show_with_no_revision(): { "branch_1": { "data": { - "metrics.json": { - "data": {"a": 0, "b": {"ad": 0.0, "bc": 0.0}} - } + "metrics.json": {"data": {"a": 0, "b": {"ad": 0.0, "bc": 0.0}}} } } }, @@ -321,23 +307,17 @@ def test_metrics_show_with_non_dict_values(): {"branch_1": {"data": {"metrics.json": {"data": 1}}}}, all_branches=True, ) - assert td.as_dict() == [ - {"Revision": "branch_1", "Path": "metrics.json", "": "1"} - ] + assert td.as_dict() == [{"Revision": "branch_1", "Path": "metrics.json", "": "1"}] def test_metrics_show_with_multiple_revision(): td = metrics_table( { "branch_1": { - "data": { - "metrics.json": {"data": {"a": 1, "b": {"ad": 1, "bc": 2}}} - } + "data": {"metrics.json": {"data": {"a": 1, "b": {"ad": 1, "bc": 2}}}} }, "branch_2": { - "data": { - "metrics.json": {"data": {"a": 1, "b": {"ad": 3, "bc": 4}}} - } + "data": {"metrics.json": {"data": {"a": 1, "b": {"ad": 3, "bc": 4}}}} }, }, all_branches=True, @@ -365,12 +345,8 @@ def test_metrics_show_with_one_revision_multiple_paths(): { "branch_1": { "data": { - "metrics.json": { - "data": {"a": 1, "b": {"ad": 0.1, "bc": 1.03}} - }, - "metrics_1.json": { - "data": {"a": 2.3, "b": {"ad": 6.5, "bc": 7.9}} - }, + "metrics.json": {"data": {"a": 1, "b": {"ad": 0.1, "bc": 1.03}}}, + "metrics_1.json": {"data": {"a": 2.3, "b": {"ad": 6.5, "bc": 7.9}}}, } } }, @@ -398,14 +374,10 @@ def test_metrics_show_with_different_metrics_header(): td = metrics_table( { "branch_1": { - "data": { - "metrics.json": {"data": {"b": {"ad": 1, "bc": 2}, "c": 4}} - } + "data": {"metrics.json": {"data": {"b": {"ad": 1, "bc": 2}, "c": 4}}} }, "branch_2": { - "data": { - "metrics.json": {"data": {"a": 1, "b": {"ad": 3, "bc": 4}}} - } + "data": {"metrics.json": {"data": {"a": 1, "b": {"ad": 3, "bc": 4}}}} }, }, all_branches=True, @@ -455,9 +427,7 @@ def test_metrics_show_precision(): } ] - td = metrics_table( - metrics, all_branches=True, precision=4, round_digits=True - ) + td = metrics_table(metrics, all_branches=True, precision=4, round_digits=True) assert td.as_dict() == [ { "Revision": "branch_1", @@ -502,15 +472,11 @@ def test_metrics_show_default(capsys): show_metrics( metrics={ "branch_1": { - "data": { - "metrics.json": {"data": {"b": {"ad": 1, "bc": 2}, "c": 4}} - }, + "data": {"metrics.json": {"data": {"b": {"ad": 1, "bc": 2}, "c": 4}}}, "error": Exception("Failed just a little bit"), }, "branch_2": { - "data": { - "metrics.json": {"data": {"a": 1, "b": {"ad": 3, "bc": 4}}} - } + "data": {"metrics.json": {"data": {"a": 1, "b": {"ad": 3, "bc": 4}}}} }, }, all_branches=True, @@ -529,14 +495,10 @@ def test_metrics_show_markdown(capsys): show_metrics( metrics={ "branch_1": { - "data": { - "metrics.json": {"data": {"b": {"ad": 1, "bc": 2}, "c": 4}} - } + "data": {"metrics.json": {"data": {"b": {"ad": 1, "bc": 2}, "c": 4}}} }, "branch_2": { - "data": { - "metrics.json": {"data": {"a": 1, "b": {"ad": 3, "bc": 4}}} - } + "data": {"metrics.json": {"data": {"a": 1, "b": {"ad": 3, "bc": 4}}}} }, "branch_3": {"error": YAMLFileCorruptedError("failed")}, }, diff --git a/tests/unit/test_dvcfile.py b/tests/unit/test_dvcfile.py index a252987b0..11641388c 100644 --- a/tests/unit/test_dvcfile.py +++ b/tests/unit/test_dvcfile.py @@ -9,10 +9,7 @@ from dvc.dvcfile import ( load_file, ) from dvc.stage import PipelineStage -from dvc.stage.exceptions import ( - StageFileDoesNotExistError, - StageFileIsNotDvcFileError, -) +from dvc.stage.exceptions import StageFileDoesNotExistError, StageFileIsNotDvcFileError from dvc.utils.fs import remove from dvc.utils.serialize import EncodingError from dvc.utils.strictyaml import YAMLValidationError @@ -33,9 +30,7 @@ def test_pipelines_file(path): assert isinstance(file_obj, ProjectFile) -@pytest.mark.parametrize( - "path", ["Dvcfile", "stage.dvc", "../models/stage.dvc"] -) +@pytest.mark.parametrize("path", ["Dvcfile", "stage.dvc", "../models/stage.dvc"]) def test_pipelines_single_stage_file(path): file_obj = load_file(object(), path) assert isinstance(file_obj, SingleStageFile) @@ -75,9 +70,7 @@ def test_stage_load_on_invalid_data(tmp_dir, dvc, file): def test_dump_stage(tmp_dir, dvc): - stage = PipelineStage( - dvc, cmd="command", name="stage_name", path="dvc.yaml" - ) + stage = PipelineStage(dvc, cmd="command", name="stage_name", path="dvc.yaml") dvcfile = load_file(dvc, "dvc.yaml") dvcfile.dump(stage, update_lock=False, update_pipeline=False) diff --git a/tests/unit/test_external_repo.py b/tests/unit/test_external_repo.py index a78e907fb..e80a1b37c 100644 --- a/tests/unit/test_external_repo.py +++ b/tests/unit/test_external_repo.py @@ -79,12 +79,9 @@ def test_subrepo_is_constructed_properly( assert repo.odb.local.cache_types == ["symlink"] assert subrepo.odb.local.cache_types == ["symlink"] - assert ( - subrepo.config["remote"]["auto-generated-upstream"]["url"] - == local_cache - ) + assert subrepo.config["remote"]["auto-generated-upstream"]["url"] == local_cache if root_is_dvc: main_cache = tmp_dir.dvc.odb.local.path - assert repo.config["remote"]["auto-generated-upstream"][ - "url" - ] == str(main_cache) + assert repo.config["remote"]["auto-generated-upstream"]["url"] == str( + main_cache + ) diff --git a/tests/unit/test_hashinfo.py b/tests/unit/test_hashinfo.py index c7da09f33..b07f854a4 100644 --- a/tests/unit/test_hashinfo.py +++ b/tests/unit/test_hashinfo.py @@ -2,9 +2,7 @@ from dvc_data.hashfile.hash_info import HashInfo def test_as_raw(): - hash_info = HashInfo( - "md5", "a1d0c6e83f027327d8461063f4ac58a6.dir", "objname" - ) + hash_info = HashInfo("md5", "a1d0c6e83f027327d8461063f4ac58a6.dir", "objname") raw = hash_info.as_raw() diff --git a/tests/unit/test_ignore.py b/tests/unit/test_ignore.py index becd49fec..a30361913 100644 --- a/tests/unit/test_ignore.py +++ b/tests/unit/test_ignore.py @@ -174,10 +174,7 @@ def mock_dvcignore(dvcignore_path, patterns): ), ], ) -def test_match_ignore_from_file( - file_to_ignore_relpath, patterns, expected_match -): - +def test_match_ignore_from_file(file_to_ignore_relpath, patterns, expected_match): dvcignore_path = os.path.join( os.path.sep, "full", "path", "to", "ignore", "file", ".dvcignore" ) @@ -186,8 +183,7 @@ def test_match_ignore_from_file( ignore_file = mock_dvcignore(dvcignore_path, patterns) assert ( - ignore_file.matches(dvcignore_dirname, file_to_ignore_relpath) - == expected_match + ignore_file.matches(dvcignore_dirname, file_to_ignore_relpath) == expected_match ) diff --git a/tests/unit/test_info.py b/tests/unit/test_info.py index b45ea4f88..f6103dacb 100644 --- a/tests/unit/test_info.py +++ b/tests/unit/test_info.py @@ -30,9 +30,7 @@ def find_supported_remotes(string): if not line.startswith("\t"): break - remote_name, _, raw_dependencies = ( - line.strip().strip(",").partition(" ") - ) + remote_name, _, raw_dependencies = line.strip().strip(",").partition(" ") remotes[remote_name] = { dependency: version for dependency, _, version in [ @@ -87,9 +85,7 @@ def test_info_in_broken_git_repo(tmp_dir, dvc, scm, caplog): def test_caches(tmp_dir, dvc, caplog): - tmp_dir.add_remote( - name="sshcache", url="ssh://example.com/path", default=False - ) + tmp_dir.add_remote(name="sshcache", url="ssh://example.com/path", default=False) with tmp_dir.dvc.config.edit() as conf: conf["cache"]["ssh"] = "sshcache" @@ -108,9 +104,7 @@ def test_remotes_empty(tmp_dir, dvc, caplog): def test_remotes(tmp_dir, dvc, caplog): tmp_dir.add_remote(name="server", url="ssh://localhost", default=False) - tmp_dir.add_remote( - name="r1", url="azure://example.com/path", default=False - ) + tmp_dir.add_remote(name="r1", url="azure://example.com/path", default=False) tmp_dir.add_remote(name="r2", url="remote://server/path", default=False) dvc_info = get_dvc_info() diff --git a/tests/unit/test_interpolate.py b/tests/unit/test_interpolate.py index 530f63576..5829013b2 100644 --- a/tests/unit/test_interpolate.py +++ b/tests/unit/test_interpolate.py @@ -96,6 +96,4 @@ def test_resolve_collection(): def test_resolve_unicode(): context = Context({"नेपाली": {"चिया": ["चि", "या"]}}) assert context.resolve_str("${नेपाली.चिया[0]}${नेपाली.चिया[1]}") == "चिया" - assert ( - context.resolve_str("${नेपाली[चिया][0]}${नेपाली[चिया][1]}") == "चिया" - ) + assert context.resolve_str("${नेपाली[चिया][0]}${नेपाली[चिया][1]}") == "चिया" diff --git a/tests/unit/test_lockfile.py b/tests/unit/test_lockfile.py index ff42a775a..5bbdf7c51 100644 --- a/tests/unit/test_lockfile.py +++ b/tests/unit/test_lockfile.py @@ -69,9 +69,7 @@ def test_load_when_lockfile_does_not_exist(tmp_dir, dvc): { "s1": { "cmd": "command", - "outs": [ - {"md5": "checksum", "path": "path", "random": "value"} - ], + "outs": [{"md5": "checksum", "path": "path", "random": "value"}], } }, {"s1": {"cmd": "command", "deps": [{"md5": "checksum"}]}}, diff --git a/tests/unit/test_tabular_data.py b/tests/unit/test_tabular_data.py index 697b6b41e..be1326aac 100644 --- a/tests/unit/test_tabular_data.py +++ b/tests/unit/test_tabular_data.py @@ -277,9 +277,7 @@ def test_dropna_subset(axis, expected): ) def test_drop_duplicates(axis, expected, ignore_empty): td = TabularData(["col-1", "col-2", "col-3"], fill_value="-") - td.extend( - [["foo"], ["foo", "foo"], ["foo", "foo"], ["foo", "bar", "foobar"]] - ) + td.extend([["foo"], ["foo", "foo"], ["foo", "foo"], ["foo", "bar", "foobar"]]) assert list(td) == [ ["foo", "-", "-"], @@ -384,9 +382,7 @@ def test_to_parallel_coordinates(tmp_dir, mocker): td.extend([["foo", "0.1"], ["bar", "2"]]) write = mocker.patch("dvc_render.html.render_html") - renderer_class = mocker.patch( - "dvc_render.plotly.ParallelCoordinatesRenderer" - ) + renderer_class = mocker.patch("dvc_render.plotly.ParallelCoordinatesRenderer") renderer = renderer_class.return_value td.to_parallel_coordinates(output_path="foo") diff --git a/tests/unit/test_updater.py b/tests/unit/test_updater.py index 8a5319fbb..b74394a56 100644 --- a/tests/unit/test_updater.py +++ b/tests/unit/test_updater.py @@ -94,10 +94,8 @@ def test_check_updates(mocker, capsys, updater, current, latest, notify): updater.check() out, err = capsys.readouterr() expected_message = ( - ( - f"You are using dvc version {current}; " - f"however, version {latest} is available.\n" - ) + f"You are using dvc version {current}; " + f"however, version {latest} is available.\n" if notify else "" ) @@ -123,9 +121,7 @@ def test_check_refetches_each_day(mock_tty, updater, caplog, mocker): mock_time.assert_called() -def test_check_fetches_on_invalid_data_format( - mock_tty, updater, caplog, mocker -): +def test_check_fetches_on_invalid_data_format(mock_tty, updater, caplog, mocker): updater.current = "0.0.5" with open(updater.updater_file, "w+", encoding="utf-8") as f: f.write('"{"version: "0.0.6"') @@ -169,13 +165,17 @@ def test_check(mocker, updater): ), ( None, - "Find the latest release at " - "https://github.com/iterative/dvc/releases/latest.", + ( + "Find the latest release at " + "https://github.com/iterative/dvc/releases/latest." + ), ), ( "unknown", - "Find the latest release at " - "https://github.com/iterative/dvc/releases/latest.", + ( + "Find the latest release at " + "https://github.com/iterative/dvc/releases/latest." + ), ), ], ) diff --git a/tests/unit/ui/test_pager.py b/tests/unit/ui/test_pager.py index 0d9984ca1..48b71a66b 100644 --- a/tests/unit/ui/test_pager.py +++ b/tests/unit/ui/test_pager.py @@ -1,14 +1,7 @@ import pytest from dvc.env import DVC_PAGER -from dvc.ui.pager import ( - DEFAULT_PAGER, - LESS, - PAGER_ENV, - find_pager, - make_pager, - pager, -) +from dvc.ui.pager import DEFAULT_PAGER, LESS, PAGER_ENV, find_pager, make_pager, pager @pytest.fixture(autouse=True) @@ -32,9 +25,7 @@ def test_find_pager_uses_custom_pager_when_dvc_pager_env_var_is_defined( assert find_pager() == "my-pager" -def test_find_pager_uses_custom_pager_when_pager_env_is_defined( - mocker, monkeypatch -): +def test_find_pager_uses_custom_pager_when_pager_env_is_defined(mocker, monkeypatch): monkeypatch.setenv(PAGER_ENV, "my-pager") mocker.patch("sys.stdout.isatty", return_value=True) @@ -56,24 +47,20 @@ def test_find_pager_fails_to_find_any_pager(mocker): @pytest.mark.parametrize("env", [DVC_PAGER, PAGER_ENV, None]) -def test_dvc_sets_default_options_on_less_without_less_env( - mocker, monkeypatch, env -): +def test_dvc_sets_default_options_on_less_without_less_env(mocker, monkeypatch, env): if env: monkeypatch.setenv(env, "less") mocker.patch("sys.stdout.isatty", return_value=True) mocker.patch("os.system", return_value=0) - assert find_pager() == ( - "less --quit-if-one-screen --RAW-CONTROL-CHARS" - " --chop-long-lines --no-init" + assert ( + find_pager() + == "less --quit-if-one-screen --RAW-CONTROL-CHARS --chop-long-lines --no-init" ) @pytest.mark.parametrize("env", [DVC_PAGER, PAGER_ENV, None]) -def test_dvc_sets_some_options_on_less_if_less_env_defined( - mocker, monkeypatch, env -): +def test_dvc_sets_some_options_on_less_if_less_env_defined(mocker, monkeypatch, env): if env: monkeypatch.setenv(env, "less") mocker.patch("sys.stdout.isatty", return_value=True) diff --git a/tests/unit/ui/test_prompt.py b/tests/unit/ui/test_prompt.py index 30dd59437..dbeb9dc84 100644 --- a/tests/unit/ui/test_prompt.py +++ b/tests/unit/ui/test_prompt.py @@ -9,9 +9,7 @@ from dvc.ui.prompt import Confirm, InvalidResponse, Prompt @pytest.mark.parametrize("side_effect", [EOFError, KeyboardInterrupt]) @pytest.mark.parametrize("prompt_cls", [Prompt, Confirm]) -def test_prompt_interrupt_message_not_interleaved( - prompt_cls, side_effect, mocker -): +def test_prompt_interrupt_message_not_interleaved(prompt_cls, side_effect, mocker): """Test that the interrupt message is not interleaved with the prompt.""" prompt = "should print exception in the next line" console = RichConsole(file=io.StringIO()) @@ -27,9 +25,7 @@ def test_prompt_interrupt_message_not_interleaved( def test_prompt_str(): console = RichConsole(file=io.StringIO()) assert ( - Prompt.ask( - "What is your name?", console=console, stream=io.StringIO("foo") - ) + Prompt.ask("What is your name?", console=console, stream=io.StringIO("foo")) == "foo" ) assert console.file.getvalue() == "What is your name?: " @@ -43,8 +39,7 @@ def test_prompt_disallows_empty_response(mocker): stream=io.StringIO("\nfoo"), ) expected = ( - "What is your name?: Response required. Please try again.\n" - "What is your name?: " + "What is your name?: Response required. Please try again.\nWhat is your name?: " ) assert console.file.getvalue() == expected diff --git a/tests/unit/ui/test_table.py b/tests/unit/ui/test_table.py index da528cd62..8218c22ad 100644 --- a/tests/unit/ui/test_table.py +++ b/tests/unit/ui/test_table.py @@ -80,19 +80,22 @@ def test_rich_simple(capsys): ) # not able to test the actual style for now captured = capsys.readouterr() - assert [ - row.strip() for row in captured.out.splitlines() if row.strip() - ] == ["first second", "foo bar", "foo1 bar1", "foo2 bar2"] + assert [row.strip() for row in captured.out.splitlines() if row.strip()] == [ + "first second", + "foo bar", + "foo1 bar1", + "foo2 bar2", + ] def test_rich_headerless(capsys): - ui.table( - [("foo", "bar"), ("foo1", "bar1"), ("foo2", "bar2")], rich_table=True - ) + ui.table([("foo", "bar"), ("foo1", "bar1"), ("foo2", "bar2")], rich_table=True) captured = capsys.readouterr() - assert [ - row.strip() for row in captured.out.splitlines() if row.strip() - ] == ["foo bar", "foo1 bar1", "foo2 bar2"] + assert [row.strip() for row in captured.out.splitlines() if row.strip()] == [ + "foo bar", + "foo1 bar1", + "foo2 bar2", + ] def test_rich_border(capsys): @@ -103,9 +106,7 @@ def test_rich_border(capsys): borders="simple", ) captured = capsys.readouterr() - assert [ - row.strip() for row in captured.out.splitlines() if row.strip() - ] == [ + assert [row.strip() for row in captured.out.splitlines() if row.strip()] == [ "first second", "────────────────", "foo bar", @@ -127,13 +128,16 @@ def test_rich_styles(capsys, extra_opts): [("foo", "bar"), ("foo1", "bar1"), ("foo2", "bar2")], headers=["first", "second"], rich_table=True, - **extra_opts + **extra_opts, ) # not able to test the actual style for now captured = capsys.readouterr() - assert [ - row.strip() for row in captured.out.splitlines() if row.strip() - ] == ["first second", "foo bar", "foo1 bar1", "foo2 bar2"] + assert [row.strip() for row in captured.out.splitlines() if row.strip()] == [ + "first second", + "foo bar", + "foo1 bar1", + "foo2 bar2", + ] def test_rich_pager(mocker): @@ -146,9 +150,12 @@ def test_rich_pager(mocker): pager=True, ) received_text = pager_mock.call_args[0][0] - assert [ - row.strip() for row in received_text.splitlines() if row.strip() - ] == ["first second", "foo bar", "foo1 bar1", "foo2 bar2"] + assert [row.strip() for row in received_text.splitlines() if row.strip()] == [ + "first second", + "foo bar", + "foo1 bar1", + "foo2 bar2", + ] @pytest.mark.parametrize("rich_table", [True, False]) diff --git a/tests/unit/utils/test_fs.py b/tests/unit/utils/test_fs.py index 1429d1550..032d86836 100644 --- a/tests/unit/utils/test_fs.py +++ b/tests/unit/utils/test_fs.py @@ -33,9 +33,7 @@ def test_should_return_false_on_path_eq_to_base_path(mocker): def test_should_return_false_on_no_more_dirs_below_path(mocker): mocker.patch.object(system, "is_symlink", return_value=False) - dirname_patch = mocker.patch.object( - os.path, "dirname", side_effect=lambda arg: arg - ) + dirname_patch = mocker.patch.object(os.path, "dirname", side_effect=lambda arg: arg) assert not contains_symlink_up_to(os.path.join("foo", "path"), "foo") dirname_patch.assert_called_once() @@ -71,9 +69,7 @@ def test_should_call_recursive_on_no_condition_matched(mocker): contains_symlink_spy = mocker.spy(dvc.utils.fs, "contains_symlink_up_to") # call from full path to match contains_symlink_spy patch path - assert not dvc.utils.fs.contains_symlink_up_to( - os.path.join("foo", "path"), "foo" - ) + assert not dvc.utils.fs.contains_symlink_up_to(os.path.join("foo", "path"), "foo") assert contains_symlink_spy.mock.call_count == 2 diff --git a/tests/unit/utils/test_humanize.py b/tests/unit/utils/test_humanize.py index 0efe11513..67674018a 100644 --- a/tests/unit/utils/test_humanize.py +++ b/tests/unit/utils/test_humanize.py @@ -16,12 +16,9 @@ def test_get_summary(): ] ) - assert get_summary(stats.items()) == ( - "3 files fetched, " - "3 files added, " - "2 files deleted " - "and " - "2 files modified" + assert ( + get_summary(stats.items()) + == "3 files fetched, 3 files added, 2 files deleted and 2 files modified" ) del stats["fetched"] diff --git a/tests/unit/utils/test_utils.py b/tests/unit/utils/test_utils.py index 7745a8758..0e159d86e 100644 --- a/tests/unit/utils/test_utils.py +++ b/tests/unit/utils/test_utils.py @@ -2,13 +2,7 @@ import os import pytest -from dvc.utils import ( - dict_sha256, - fix_env, - parse_target, - relpath, - resolve_output, -) +from dvc.utils import dict_sha256, fix_env, parse_target, relpath, resolve_output @pytest.mark.skipif(os.name == "nt", reason="pyenv-win is not supported") @@ -16,10 +10,7 @@ from dvc.utils import ( "path, orig", [ ( - ( - "/pyenv/bin:/pyenv/libexec:/pyenv/plugins/plugin:" - "/orig/path1:/orig/path2" - ), + "/pyenv/bin:/pyenv/libexec:/pyenv/plugins/plugin:/orig/path1:/orig/path2", "/orig/path1:/orig/path2", ), ( @@ -27,10 +18,7 @@ from dvc.utils import ( "/orig/path1:/orig/path2", ), ( - ( - "/pyenv/bin:/some/libexec:/pyenv/plugins/plugin:" - "/orig/path1:/orig/path2" - ), + "/pyenv/bin:/some/libexec:/pyenv/plugins/plugin:/orig/path1:/orig/path2", "/orig/path1:/orig/path2", ), ("/orig/path1:/orig/path2", "/orig/path1:/orig/path2"), diff --git a/tests/utils/plots.py b/tests/utils/plots.py index aa8aa7842..cf9155154 100644 --- a/tests/utils/plots.py +++ b/tests/utils/plots.py @@ -5,7 +5,5 @@ dpath.options.ALLOW_EMPTY_STRING_KEYS = True def get_plot(plots_data, revision, typ="sources", file=None, endkey="data"): if file is not None: - return dpath.util.get( - plots_data, [revision, typ, "data", file, endkey] - ) + return dpath.util.get(plots_data, [revision, typ, "data", file, endkey]) return dpath.util.get(plots_data, [revision, typ, endkey])