diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c7d39eebf..b74d89df9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,7 +43,7 @@ repos: hooks: - id: isort - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: 'v0.0.235' + rev: 'v0.0.236' hooks: - id: ruff - repo: https://github.com/codespell-project/codespell diff --git a/dvc/annotations.py b/dvc/annotations.py index 05b02fb89..663a75fb2 100644 --- a/dvc/annotations.py +++ b/dvc/annotations.py @@ -12,7 +12,7 @@ class Annotation: PARAM_META: ClassVar[str] = "meta" desc: Optional[str] = None - type: Optional[str] = None + type: Optional[str] = None # noqa: A003 labels: List[str] = field(default_factory=list) meta: Dict[str, Any] = field(default_factory=dict) diff --git a/dvc/cli/__init__.py b/dvc/cli/__init__.py index b85a0dd9f..f3bd3e685 100644 --- a/dvc/cli/__init__.py +++ b/dvc/cli/__init__.py @@ -57,8 +57,8 @@ def _log_exceptions(exc: Exception) -> Optional[int]: if exc.errno == errno.EMFILE: logger.exception( "too many open files, please visit " - "{} to see how to handle this " - "problem".format(error_link("many-files")), + "%s to see how to handle this problem", + error_link("many-files"), extra={"tb_only": True}, ) else: @@ -91,8 +91,11 @@ def _log_exceptions(exc: Exception) -> Optional[int]: hint = f"\nPlease report this bug to {link}. Thank you!" logger.exception( - f"URL '{exc.url}' is supported but requires these missing " - f"dependencies: {exc.missing_deps}. {hint}", + "URL '%s' is supported but requires these missing " + "dependencies: %s. %s", + exc.url, + exc.missing_deps, + hint, extra={"tb_only": True}, ) return None @@ -101,7 +104,9 @@ def _log_exceptions(exc: Exception) -> Optional[int]: link = format_link("https://man.dvc.org/remote/modify") logger.exception("configuration error") logger.exception( - f"{exc!s}\nLearn more about configuration settings at {link}.", + "%s\nLearn more about configuration settings at %s.", + exc, + link, extra={"tb_only": True}, ) return 251 @@ -113,9 +118,12 @@ def _log_exceptions(exc: Exception) -> Optional[int]: directory = relpath(exc.directory) logger.exception( - f"Could not open pickled '{exc.type}' cache.\n" - f"Remove the '{directory}' directory and then retry this command." - f"\nSee {error_link('pickle')} for more information.", + "Could not open pickled '%s' cache.\n" + "Remove the '%s' directory and then retry this command." + "\nSee %s for more information.", + exc.type, + directory, + error_link("pickle"), extra={"tb_only": True}, ) return None diff --git a/dvc/cli/actions.py b/dvc/cli/actions.py index 12430bea9..299974d39 100644 --- a/dvc/cli/actions.py +++ b/dvc/cli/actions.py @@ -2,7 +2,9 @@ from argparse import _AppendAction class CommaSeparatedArgs(_AppendAction): # pylint: disable=protected-access - def __call__(self, parser, namespace, values, option_string=None): + def __call__( + self, parser, namespace, values, option_string=None # noqa: ARG002 + ): from funcy import ldistinct items = getattr(namespace, self.dest) or [] @@ -11,7 +13,9 @@ class CommaSeparatedArgs(_AppendAction): # pylint: disable=protected-access class KeyValueArgs(_AppendAction): - def __call__(self, parser, namespace, values, option_string=None): + def __call__( + self, parser, namespace, values, option_string=None # noqa: ARG002 + ): items = getattr(namespace, self.dest) or {} for value in filter(bool, values): key, _, value = value.partition("=") diff --git a/dvc/commands/commit.py b/dvc/commands/commit.py index d1ccfe5f8..ba155a29b 100644 --- a/dvc/commands/commit.py +++ b/dvc/commands/commit.py @@ -25,9 +25,7 @@ class CmdCommit(CmdBase): ) except DvcException: logger.exception( - "failed to commit{}".format( - (" " + target) if target else "" - ) + "failed to commit%s", (" " + target) if target else "" ) return 1 return 0 diff --git a/dvc/commands/get.py b/dvc/commands/get.py index 0f4ee6a4d..7b59d965f 100644 --- a/dvc/commands/get.py +++ b/dvc/commands/get.py @@ -45,13 +45,11 @@ class CmdGet(CmdBaseNoRepo): ) return 0 except CloneError: - logger.exception(f"failed to get '{self.args.path}'") + logger.exception("failed to get '%s'", self.args.path) return 1 except DvcException: logger.exception( - "failed to get '{}' from '{}'".format( - self.args.path, self.args.url - ) + "failed to get '%s' from '%s'", self.args.path, self.args.url ) return 1 diff --git a/dvc/commands/imp.py b/dvc/commands/imp.py index c6a6d9122..862729ecf 100644 --- a/dvc/commands/imp.py +++ b/dvc/commands/imp.py @@ -28,13 +28,13 @@ class CmdImport(CmdBase): jobs=self.args.jobs, ) except CloneError: - logger.exception(f"failed to import '{self.args.path}'") + logger.exception("failed to import '%s'", self.args.path) return 1 except DvcException: logger.exception( - "failed to import '{}' from '{}'.".format( - self.args.path, self.args.url - ) + "failed to import '%s' from '%s'.", + self.args.path, + self.args.url, ) return 1 return 0 diff --git a/dvc/commands/imp_url.py b/dvc/commands/imp_url.py index 0e00df68a..8e5d570ef 100644 --- a/dvc/commands/imp_url.py +++ b/dvc/commands/imp_url.py @@ -29,10 +29,9 @@ class CmdImportUrl(CmdBase): ) except DvcException: logger.exception( - "failed to import {}. You could also try downloading " - "it manually, and adding it with `dvc add`.".format( - self.args.url - ) + "failed to import %s. You could also try downloading " + "it manually, and adding it with `dvc add`.", + self.args.url, ) return 1 return 0 diff --git a/dvc/commands/ls/ls_colors.py b/dvc/commands/ls/ls_colors.py index 92e6f7a10..08254d63b 100644 --- a/dvc/commands/ls/ls_colors.py +++ b/dvc/commands/ls/ls_colors.py @@ -20,7 +20,7 @@ class LsColors: else: self._codes[code] = color - def format(self, entry): + def format(self, entry): # noqa: A003 text = entry["path"] if entry.get("isout", False) and "out" in self._codes: diff --git a/dvc/config.py b/dvc/config.py index 0e41f02a1..007d1b6aa 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -9,12 +9,12 @@ from typing import TYPE_CHECKING, Dict, Optional from funcy import compact, memoize, re_find from dvc.exceptions import DvcException, NotDvcRepoError -from dvc.types import DictStrAny, StrPath from .utils.objects import cached_property if TYPE_CHECKING: from dvc.fs import FileSystem + from dvc.types import DictStrAny, StrPath logger = logging.getLogger(__name__) @@ -92,10 +92,10 @@ class Config(dict): def __init__( self, - dvc_dir: Optional[StrPath] = None, + dvc_dir: Optional["StrPath"] = None, validate: bool = True, fs: Optional["FileSystem"] = None, - config: Optional[DictStrAny] = None, + config: Optional["DictStrAny"] = None, ): # pylint: disable=super-init-not-called from dvc.fs import LocalFileSystem @@ -153,7 +153,9 @@ class Config(dict): with open(config_file, "w+", encoding="utf-8"): return Config(dvc_dir) - def load(self, validate: bool = True, config: Optional[DictStrAny] = None): + def load( + self, validate: bool = True, config: Optional["DictStrAny"] = None + ): """Loads config from all the config files. Raises: diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index 12bc376b8..d89bce521 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -112,7 +112,8 @@ class DataCloud: ) logger.warning( "Some of the cache files do not exist neither locally " - f"nor on remote. Missing cache files:\n{missing_desc}" + "nor on remote. Missing cache files:\n%s", + missing_desc, ) def transfer( diff --git a/dvc/dependency/repo.py b/dvc/dependency/repo.py index ed7cfe800..7238b1bd4 100644 --- a/dvc/dependency/repo.py +++ b/dvc/dependency/repo.py @@ -1,28 +1,22 @@ import os from collections import defaultdict from copy import copy -from typing import ( - TYPE_CHECKING, - ContextManager, - Dict, - Optional, - Set, - Tuple, - Union, -) +from typing import TYPE_CHECKING, Dict, Optional, Set, Tuple, Union from voluptuous import Required from dvc.prompt import confirm -from dvc_data.hashfile.meta import Meta from .base import Dependency if TYPE_CHECKING: + from typing import ContextManager + from dvc.output import Output from dvc.repo import Repo from dvc.stage import Stage from dvc_data.hashfile.hash_info import HashInfo + from dvc_data.hashfile.meta import Meta from dvc_data.hashfile.obj import HashFile from dvc_objects.db import ObjectDB @@ -46,10 +40,10 @@ class RepoDependency(Dependency): ): self.def_repo = def_repo self._objs: Dict[str, "HashFile"] = {} - self._meta: Dict[str, Meta] = {} + self._meta: Dict[str, "Meta"] = {} super().__init__(stage, *args, **kwargs) - def _parse_path(self, fs, fs_path): + def _parse_path(self, fs, fs_path): # noqa: ARG002 return None @property @@ -114,7 +108,9 @@ class RepoDependency(Dependency): def _get_used_and_obj( self, obj_only: bool = False, **kwargs - ) -> Tuple[Dict[Optional["ObjectDB"], Set["HashInfo"]], Meta, "HashFile"]: + ) -> Tuple[ + Dict[Optional["ObjectDB"], Set["HashInfo"]], "Meta", "HashFile" + ]: from dvc.config import NoRemoteError from dvc.exceptions import NoOutputOrStageError, PathMissingError from dvc.utils import as_posix diff --git a/dvc/dvcfile.py b/dvc/dvcfile.py index 49eaf5100..da177020b 100644 --- a/dvc/dvcfile.py +++ b/dvc/dvcfile.py @@ -21,7 +21,6 @@ from dvc.stage.exceptions import ( StageFileDoesNotExistError, StageFileIsNotDvcFileError, ) -from dvc.types import StrOrBytesPath from dvc.utils import relpath from dvc.utils.collections import apply_diff from dvc.utils.objects import cached_property @@ -29,6 +28,7 @@ from dvc.utils.serialize import dump_yaml, modify_yaml if TYPE_CHECKING: from dvc.repo import Repo + from dvc.types import StrOrBytesPath from .parsing import DataResolver from .stage import Stage @@ -179,7 +179,8 @@ class FileMixin: **kwargs, ) - def remove(self, force=False): # pylint: disable=unused-argument + # pylint: disable-next=unused-argument + def remove(self, force=False): # noqa: ARG002 with contextlib.suppress(FileNotFoundError): os.unlink(self.path) @@ -219,7 +220,8 @@ class SingleStageFile(FileMixin): dump_yaml(self.path, serialize.to_single_stage_file(stage, **kwargs)) self.repo.scm_context.track_file(self.relpath) - def remove_stage(self, stage): # pylint: disable=unused-argument + # pylint: disable-next=unused-argument + def remove_stage(self, stage): # noqa: ARG002 self.remove() def merge(self, ancestor, other, allowed=None): @@ -465,7 +467,7 @@ class Lockfile(FileMixin): def load_file( - repo: "Repo", path: StrOrBytesPath, **kwargs: Any + repo: "Repo", path: "StrOrBytesPath", **kwargs: Any ) -> Union[ProjectFile, SingleStageFile]: _, ext = os.path.splitext(path) if ext in (".yaml", ".yml"): diff --git a/dvc/external_repo.py b/dvc/external_repo.py index de235ec4f..bf71eba57 100644 --- a/dvc/external_repo.py +++ b/dvc/external_repo.py @@ -17,12 +17,13 @@ from dvc.exceptions import ( ) from dvc.repo import Repo from dvc.scm import CloneError, map_scm_exception -from dvc.types import StrPath from dvc.utils import relpath if TYPE_CHECKING: from scmrepo import Git + from dvc.types import StrPath + logger = logging.getLogger(__name__) @@ -32,7 +33,7 @@ def external_repo( url, rev: Optional[str] = None, for_write: bool = False, - cache_dir: Optional[StrPath] = None, + cache_dir: Optional["StrPath"] = None, cache_types: Optional[Collection[str]] = None, **kwargs, ) -> Iterator["Repo"]: diff --git a/dvc/fs/callbacks.py b/dvc/fs/callbacks.py index 68a8c5cdb..25555b7bf 100644 --- a/dvc/fs/callbacks.py +++ b/dvc/fs/callbacks.py @@ -70,7 +70,7 @@ class RichCallback(Callback): self.progress.clear_task(self.task) self._stack.close() - def call(self, hook_name=None, **kwargs): + def call(self, hook_name=None, **kwargs): # noqa: ARG002 self.progress.update( self.task, completed=self.value, diff --git a/dvc/fs/dvc.py b/dvc/fs/dvc.py index b9e2af672..b5df81054 100644 --- a/dvc/fs/dvc.py +++ b/dvc/fs/dvc.py @@ -20,7 +20,6 @@ from typing import ( from fsspec.spec import AbstractFileSystem from funcy import wrap_with -from dvc.types import StrPath from dvc_objects.fs.base import FileSystem from dvc_objects.fs.path import Path @@ -28,6 +27,7 @@ from .data import DataFileSystem if TYPE_CHECKING: from dvc.repo import Repo + from dvc.types import StrPath logger = logging.getLogger(__name__) @@ -166,7 +166,7 @@ class _DVCFileSystem(AbstractFileSystem): # pylint:disable=abstract-method if hasattr(repo, "dvc_dir"): self._datafss[key] = DataFileSystem(index=repo.index.data["repo"]) - def _get_key(self, path: StrPath) -> Key: + def _get_key(self, path: "StrPath") -> Key: parts = self.repo.fs.path.relparts(path, self.repo.root_dir) if parts == (os.curdir,): return () diff --git a/dvc/ignore.py b/dvc/ignore.py index 3c6108acf..e2b63dae6 100644 --- a/dvc/ignore.py +++ b/dvc/ignore.py @@ -3,15 +3,18 @@ import os import re from collections import namedtuple from itertools import chain, groupby, takewhile -from typing import List, Optional +from typing import TYPE_CHECKING, List, Optional from pathspec.patterns import GitWildMatchPattern from pathspec.util import normalize_file from pygtrie import Trie -from dvc.fs import AnyFSPath, FileSystem, Schemes, localfs +from dvc.fs import Schemes, localfs from dvc.pathspec_math import PatternInfo, merge_patterns +if TYPE_CHECKING: + from dvc.fs import AnyFSPath, FileSystem + logger = logging.getLogger(__name__) @@ -306,7 +309,7 @@ class DvcIgnoreFilter: return [fs_dict[name] for name in chain(dirs, nondirs)] - def walk(self, fs: FileSystem, path: AnyFSPath, **kwargs): + def walk(self, fs: "FileSystem", path: "AnyFSPath", **kwargs): detail = kwargs.get("detail", False) ignore_subrepos = kwargs.pop("ignore_subrepos", True) if fs.protocol == Schemes.LOCAL: @@ -330,7 +333,7 @@ class DvcIgnoreFilter: else: yield from fs.walk(path, **kwargs) - def find(self, fs: FileSystem, path: AnyFSPath, **kwargs): + def find(self, fs: "FileSystem", path: "AnyFSPath", **kwargs): if fs.protocol == Schemes.LOCAL: for root, _, files in self.walk(fs, path, **kwargs): for file in files: @@ -420,7 +423,7 @@ class DvcIgnoreFilter: return _no_match(target) def is_ignored( - self, fs: FileSystem, path: str, ignore_subrepos: bool = True + self, fs: "FileSystem", path: str, ignore_subrepos: bool = True ) -> bool: # NOTE: can't use self.check_ignore(path).match for now, see # https://github.com/iterative/dvc/issues/4555 diff --git a/dvc/logger.py b/dvc/logger.py index f43475b1c..5a1366121 100644 --- a/dvc/logger.py +++ b/dvc/logger.py @@ -68,7 +68,7 @@ class LoggingException(Exception): def excludeFilter(level): class ExcludeLevelFilter(logging.Filter): - def filter(self, record): + def filter(self, record): # noqa: A003 return record.levelno < level return ExcludeLevelFilter @@ -94,7 +94,7 @@ class ColorFormatter(logging.Formatter): "CRITICAL": colorama.Fore.RED, } - def format(self, record): + def format(self, record): # noqa: A003 record.message = record.getMessage() msg = self.formatMessage(record) diff --git a/dvc/machine/__init__.py b/dvc/machine/__init__.py index 4fe54e854..8c575402d 100644 --- a/dvc/machine/__init__.py +++ b/dvc/machine/__init__.py @@ -13,10 +13,10 @@ from typing import ( ) from dvc.exceptions import DvcException -from dvc.types import StrPath if TYPE_CHECKING: from dvc.repo import Repo + from dvc.types import StrPath from .backend.base import BaseMachineBackend @@ -76,7 +76,7 @@ class MachineBackends(Mapping): def __init__( self, selected: Optional[Iterable[str]], - tmp_dir: StrPath, + tmp_dir: "StrPath", **kwargs, ) -> None: selected = selected or list(self.DEFAULT) diff --git a/dvc/output.py b/dvc/output.py index 59d09cca2..945904932 100644 --- a/dvc/output.py +++ b/dvc/output.py @@ -32,7 +32,7 @@ from dvc_objects.errors import ObjectFormatError from .annotations import ANNOTATION_FIELDS, ANNOTATION_SCHEMA, Annotation from .fs import LocalFileSystem, RemoteMissingDepsError, Schemes, get_cloud_fs -from .fs.callbacks import DEFAULT_CALLBACK, Callback +from .fs.callbacks import DEFAULT_CALLBACK from .utils import relpath from .utils.fs import path_isin @@ -41,6 +41,7 @@ if TYPE_CHECKING: from dvc_data.index import DataIndexKey from dvc_objects.db import ObjectDB + from .fs.callbacks import Callback from .ignore import DvcIgnoreFilter logger = logging.getLogger(__name__) diff --git a/dvc/parsing/__init__.py b/dvc/parsing/__init__.py index 056c72f3c..b0dbf4e47 100644 --- a/dvc/parsing/__init__.py +++ b/dvc/parsing/__init__.py @@ -26,7 +26,6 @@ from .context import ( KeyNotInContext, MergeError, Node, - SeqOrMap, VarsAlreadyLoaded, ) from .interpolate import ( @@ -40,8 +39,10 @@ if TYPE_CHECKING: from typing import NoReturn from dvc.repo import Repo + from dvc.types import DictStrAny + + from .context import SeqOrMap -from dvc.types import DictStrAny logger = logging.getLogger(__name__) @@ -92,7 +93,7 @@ def _reraise_err( def check_syntax_errors( - definition: DictStrAny, name: str, path: str, where: str = "stages" + definition: "DictStrAny", name: str, path: str, where: str = "stages" ): for key, d in definition.items(): try: @@ -111,8 +112,8 @@ def split_foreach_name(name: str) -> Tuple[str, Optional[str]]: return group, first(keys) -def check_interpolations(data: DictStrAny, where: str, path: str): - def func(s: DictStrAny) -> None: +def check_interpolations(data: "DictStrAny", where: str, path: str): + def func(s: "DictStrAny") -> None: if is_interpolated_string(s): raise ResolveError( _format_preamble(f"'{where}'", path) @@ -126,7 +127,7 @@ Definition = Union["ForeachDefinition", "EntryDefinition"] def make_definition( - resolver: "DataResolver", name: str, definition: DictStrAny, **kwargs + resolver: "DataResolver", name: str, definition: "DictStrAny", **kwargs ) -> Definition: args = resolver, resolver.context, name, definition if FOREACH_KWD in definition: @@ -225,7 +226,7 @@ class EntryDefinition: resolver: DataResolver, context: Context, name: str, - definition: DictStrAny, + definition: "DictStrAny", where: str = STAGES_KWD, ): self.resolver = resolver @@ -254,7 +255,7 @@ class EntryDefinition: except ContextError as exc: format_and_raise(exc, f"stage '{self.name}'", self.relpath) - def resolve_stage(self, skip_checks: bool = False) -> DictStrAny: + def resolve_stage(self, skip_checks: bool = False) -> "DictStrAny": context = self.context name = self.name if not skip_checks: @@ -302,7 +303,7 @@ class EntryDefinition: def _resolve( self, context: "Context", value: Any, key: str, skip_checks: bool - ) -> DictStrAny: + ) -> "DictStrAny": try: return context.resolve( value, skip_interpolation_checks=skip_checks, key=key @@ -324,7 +325,7 @@ class ForeachDefinition: resolver: DataResolver, context: Context, name: str, - definition: DictStrAny, + definition: "DictStrAny", where: str = STAGES_KWD, ): self.resolver = resolver @@ -349,7 +350,7 @@ class ForeachDefinition: def resolved_iterable(self): return self._resolve_foreach_data() - def _resolve_foreach_data(self) -> SeqOrMap: + def _resolve_foreach_data(self) -> "SeqOrMap": try: iterable = self.context.resolve(self.foreach_data, unwrap=False) except (ContextError, ParseError) as exc: @@ -417,13 +418,13 @@ class ForeachDefinition: def _generate_name(self, key: str) -> str: return f"{self.name}{JOIN}{key}" - def resolve_all(self) -> DictStrAny: + def resolve_all(self) -> "DictStrAny": return join(map(self.resolve_one, self.normalized_iterable)) - def resolve_one(self, key: str) -> DictStrAny: + def resolve_one(self, key: str) -> "DictStrAny": return self._each_iter(key) - def _each_iter(self, key: str) -> DictStrAny: + def _each_iter(self, key: str) -> "DictStrAny": err_message = f"Could not find '{key}' in foreach group '{self.name}'" with reraise(KeyError, EntryNotFound(err_message)): value = self.normalized_iterable[key] diff --git a/dvc/proc/manager.py b/dvc/proc/manager.py index e01dd6422..bc52fb563 100644 --- a/dvc/proc/manager.py +++ b/dvc/proc/manager.py @@ -92,7 +92,7 @@ class ProcessManager: def handle_closed_process(): logging.warning( - f"Process {name} had already aborted unexpectedly." + "Process %s had already aborted unexpectedly.", name ) process_info.returncode = -1 self[name] = process_info diff --git a/dvc/prompt.py b/dvc/prompt.py index 132a1b060..d6664d3ef 100644 --- a/dvc/prompt.py +++ b/dvc/prompt.py @@ -25,8 +25,7 @@ def ask(prompt: str, limited_to: Optional[Collection[str]] = None): return answer logger.info( - "Your response must be one of: {options}. " - "Please try again.".format(options=limited_to) + "Your response must be one of: %s. Please try again.", limited_to ) diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index ffebc06b4..27baa6110 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -275,7 +275,7 @@ class Repo: new.check_graph() @staticmethod - def open(url, *args, **kwargs): + def open(url, *args, **kwargs): # noqa: A003 if url is None: url = os.getcwd() diff --git a/dvc/repo/experiments/__init__.py b/dvc/repo/experiments/__init__.py index 93c3c8b28..d87534bed 100644 --- a/dvc/repo/experiments/__init__.py +++ b/dvc/repo/experiments/__init__.py @@ -2,7 +2,7 @@ import logging import os import re import time -from typing import Dict, Iterable, Optional +from typing import TYPE_CHECKING, Dict, Iterable, Optional from funcy import chain, first @@ -18,7 +18,6 @@ from .exceptions import ( MultipleBranchError, ) from .executor.base import BaseExecutor -from .queue.base import BaseStashQueue, QueueEntry from .queue.celery import LocalCeleryQueue from .queue.tempdir import TempDirQueue from .queue.workspace import WorkspaceQueue @@ -32,9 +31,12 @@ from .refs import ( WORKSPACE_STASH, ExpRefInfo, ) -from .stash import ExpStashEntry from .utils import check_ref_format, exp_refs_by_rev, unlocked_repo +if TYPE_CHECKING: + from .queue.base import BaseStashQueue, QueueEntry + from .stash import ExpStashEntry + logger = logging.getLogger(__name__) @@ -85,7 +87,7 @@ class Experiments: return LocalCeleryQueue(self.repo, CELERY_STASH, CELERY_FAILED_STASH) @property - def stash_revs(self) -> Dict[str, ExpStashEntry]: + def stash_revs(self) -> Dict[str, "ExpStashEntry"]: revs = {} for queue in (self.workspace_queue, self.celery_queue): revs.update(queue.stash.stash_revs) @@ -107,7 +109,7 @@ class Experiments: ) self.scm.reset() - exp_queue: BaseStashQueue = ( + exp_queue: "BaseStashQueue" = ( self.tempdir_queue if tmp_dir else self.workspace_queue ) self.queue_one(exp_queue, **kwargs) @@ -119,11 +121,11 @@ class Experiments: def queue_one( self, - queue: BaseStashQueue, + queue: "BaseStashQueue", checkpoint_resume: Optional[str] = None, reset: bool = False, **kwargs, - ) -> QueueEntry: + ) -> "QueueEntry": """Queue a single experiment.""" if reset: self.reset_checkpoints() @@ -162,7 +164,7 @@ class Experiments: return None def reproduce_celery( # noqa: C901 - self, entries: Optional[Iterable[QueueEntry]] = None, **kwargs + self, entries: Optional[Iterable["QueueEntry"]] = None, **kwargs ) -> Dict[str, str]: results: Dict[str, str] = {} if entries is None: @@ -238,11 +240,11 @@ class Experiments: def new( self, - queue: BaseStashQueue, + queue: "BaseStashQueue", *args, checkpoint_resume: Optional[str] = None, **kwargs, - ) -> QueueEntry: + ) -> "QueueEntry": """Create and enqueue a new experiment. Experiment will be derived from the current workspace. @@ -266,11 +268,11 @@ class Experiments: def _resume_checkpoint( self, - queue: BaseStashQueue, + queue: "BaseStashQueue", *args, resume_rev: Optional[str] = None, **kwargs, - ) -> QueueEntry: + ) -> "QueueEntry": """Create and queue a resumed checkpoint experiment.""" assert resume_rev @@ -340,7 +342,7 @@ class Experiments: @unlocked_repo def _reproduce_queue( - self, queue: BaseStashQueue, **kwargs + self, queue: "BaseStashQueue", **kwargs ) -> Dict[str, str]: """Reproduce queued experiments. diff --git a/dvc/repo/experiments/executor/local.py b/dvc/repo/experiments/executor/local.py index e369aaca4..36bf71a47 100644 --- a/dvc/repo/experiments/executor/local.py +++ b/dvc/repo/experiments/executor/local.py @@ -18,7 +18,6 @@ from dvc.repo.experiments.refs import ( EXEC_MERGE, EXEC_NAMESPACE, EXPS_TEMP, - ExpRefInfo, ) from dvc.repo.experiments.utils import EXEC_TMP_DIR, get_exp_rwlock from dvc.scm import SCM, GitMergeError @@ -31,6 +30,7 @@ if TYPE_CHECKING: from scmrepo.git import Git from dvc.repo import Repo + from dvc.repo.experiments.refs import ExpRefInfo from dvc.repo.experiments.stash import ExpStashEntry from dvc.scm import NoSCM @@ -150,7 +150,9 @@ class TempDirExecutor(BaseLocalExecutor): with open(local_config, "w", encoding="utf-8") as fobj: fobj.write(f"[cache]\n dir = {cache_dir}") - def init_cache(self, repo: "Repo", rev: str, run_cache: bool = True): + def init_cache( + self, repo: "Repo", rev: str, run_cache: bool = True # noqa: ARG002 + ): """Initialize DVC cache.""" self._config(repo.odb.repo.path) diff --git a/dvc/repo/experiments/executor/ssh.py b/dvc/repo/experiments/executor/ssh.py index 292b469bc..2f247dc8a 100644 --- a/dvc/repo/experiments/executor/ssh.py +++ b/dvc/repo/experiments/executor/ssh.py @@ -121,7 +121,7 @@ class SSHExecutor(BaseExecutor): def init_git( self, - repo: "Repo", + repo: "Repo", # noqa: ARG002 scm: "Git", stash_rev: str, entry: "ExpStashEntry", diff --git a/dvc/repo/experiments/init.py b/dvc/repo/experiments/init.py index 4aceba92d..d8c078b5b 100644 --- a/dvc/repo/experiments/init.py +++ b/dvc/repo/experiments/init.py @@ -19,7 +19,6 @@ from funcy import compact, lremove, lsplit from dvc.exceptions import DvcException from dvc.stage import PipelineStage -from dvc.types import OptStr if TYPE_CHECKING: from dvc.dvcfile import ProjectFile, SingleStageFile @@ -47,7 +46,7 @@ def _prompts( ] = None, allow_omission: bool = True, stream: Optional[TextIO] = None, -) -> Dict[str, OptStr]: +) -> Dict[str, Optional[str]]: from dvc.ui.prompt import Prompt defaults = defaults or {} diff --git a/dvc/repo/experiments/ls.py b/dvc/repo/experiments/ls.py index e209f49d9..b3f0a6fb4 100644 --- a/dvc/repo/experiments/ls.py +++ b/dvc/repo/experiments/ls.py @@ -1,10 +1,10 @@ import logging from collections import defaultdict +from typing import Optional from dvc.repo import locked from dvc.repo.scm_context import scm_context from dvc.scm import iter_revs -from dvc.types import Optional from .utils import exp_refs_by_baseline diff --git a/dvc/repo/experiments/queue/base.py b/dvc/repo/experiments/queue/base.py index 5c14338d4..666853edc 100644 --- a/dvc/repo/experiments/queue/base.py +++ b/dvc/repo/experiments/queue/base.py @@ -27,7 +27,7 @@ from dvc.repo.experiments.exceptions import ( CheckpointExistsError, ExperimentExistsError, ) -from dvc.repo.experiments.executor.base import BaseExecutor, ExecutorResult +from dvc.repo.experiments.executor.base import BaseExecutor from dvc.repo.experiments.executor.local import WorkspaceExecutor from dvc.repo.experiments.refs import ExpRefInfo from dvc.repo.experiments.stash import ExpStash, ExpStashEntry @@ -48,6 +48,7 @@ if TYPE_CHECKING: from dvc.repo import Repo from dvc.repo.experiments import Experiments + from dvc.repo.experiments.executor.base import ExecutorResult logger = logging.getLogger(__name__) @@ -87,7 +88,7 @@ class QueueGetResult(NamedTuple): class QueueDoneResult(NamedTuple): entry: QueueEntry - result: Optional[ExecutorResult] + result: Optional["ExecutorResult"] class ExpRefAndQueueEntry(NamedTuple): @@ -200,7 +201,7 @@ class BaseStashQueue(ABC): def _format_entry( entry: QueueEntry, - exp_result: Optional[ExecutorResult] = None, + exp_result: Optional["ExecutorResult"] = None, status: str = "Unknown", ) -> Dict[str, Any]: name = entry.name @@ -258,7 +259,7 @@ class BaseStashQueue(ABC): """Reproduce queued experiments sequentially.""" @abstractmethod - def get_result(self, entry: QueueEntry) -> Optional[ExecutorResult]: + def get_result(self, entry: QueueEntry) -> Optional["ExecutorResult"]: """Return result of the specified item. This method blocks until the specified item has been collected. @@ -632,7 +633,7 @@ class BaseStashQueue(ABC): def collect_git( exp: "Experiments", executor: BaseExecutor, - exec_result: ExecutorResult, + exec_result: "ExecutorResult", ) -> Dict[str, str]: results = {} @@ -664,7 +665,7 @@ class BaseStashQueue(ABC): cls, exp: "Experiments", executor: BaseExecutor, - exec_result: ExecutorResult, + exec_result: "ExecutorResult", ) -> Dict[str, str]: results = cls.collect_git(exp, executor, exec_result) diff --git a/dvc/repo/experiments/queue/celery.py b/dvc/repo/experiments/queue/celery.py index ba668e786..e03fa01ae 100644 --- a/dvc/repo/experiments/queue/celery.py +++ b/dvc/repo/experiments/queue/celery.py @@ -17,7 +17,6 @@ from typing import ( from celery.result import AsyncResult from funcy import first -from kombu.message import Message from dvc.daemon import daemonize from dvc.exceptions import DvcException @@ -25,7 +24,7 @@ from dvc.repo.experiments.exceptions import ( UnresolvedQueueExpNamesError, UnresolvedRunningExpNamesError, ) -from dvc.repo.experiments.executor.base import ExecutorInfo, ExecutorResult +from dvc.repo.experiments.executor.base import ExecutorInfo from dvc.repo.experiments.refs import CELERY_STASH from dvc.repo.experiments.utils import EXEC_TMP_DIR, get_exp_rwlock from dvc.ui import ui @@ -36,23 +35,27 @@ from .base import ( ExpRefAndQueueEntry, QueueDoneResult, QueueEntry, - QueueGetResult, ) from .exceptions import CannotKillTasksError from .tasks import run_exp from .utils import fetch_running_exp_from_temp_dir if TYPE_CHECKING: + from kombu.message import Message + + from dvc.repo.experiments.executor.base import ExecutorResult from dvc.repo.experiments.refs import ExpRefInfo from dvc_task.app import FSApp from dvc_task.proc.manager import ProcessManager from dvc_task.worker import TemporaryWorker + from .base import QueueGetResult + logger = logging.getLogger(__name__) class _MessageEntry(NamedTuple): - msg: Message + msg: "Message" entry: QueueEntry @@ -142,7 +145,7 @@ class LocalCeleryQueue(BaseStashQueue): cmd = ["exp", "queue-worker", node_name] name = f"dvc-exp-worker-{num}" - logger.debug(f"start a new worker: {name}, node: {node_name}") + logger.debug("start a new worker: %s, node: %s", name, node_name) if os.name == "nt": daemonize(cmd) else: @@ -158,7 +161,7 @@ class LocalCeleryQueue(BaseStashQueue): newly spawned worker number. """ - logger.debug(f"Spawning {count} exp queue workers") + logger.debug("Spawning %s exp queue workers", count) active_worker: Dict = self.worker_status() started = 0 @@ -168,7 +171,7 @@ class LocalCeleryQueue(BaseStashQueue): ] node_name = f"dvc-exp-{wdir_hash}-{num}@localhost" if node_name in active_worker: - logger.debug(f"Exp queue worker {node_name} already exist") + logger.debug("Exp queue worker %s already exist", node_name) continue self._spawn_worker(num) started += 1 @@ -184,7 +187,7 @@ class LocalCeleryQueue(BaseStashQueue): # NOTE: Queue consumption should not be done directly. Celery worker(s) # will automatically consume available experiments. - def get(self) -> QueueGetResult: + def get(self) -> "QueueGetResult": raise NotImplementedError def iter_queued(self) -> Generator[QueueEntry, None, None]: @@ -330,7 +333,7 @@ class LocalCeleryQueue(BaseStashQueue): ) -> None: remained_revs: List[str] = [] running_ids = self._get_running_task_ids() - logger.debug(f"Current running tasks ids: {running_ids}.") + logger.debug("Current running tasks ids: %s.", running_ids) for msg, entry in self._iter_processed(): if entry not in remained_entries: continue @@ -341,8 +344,9 @@ class LocalCeleryQueue(BaseStashQueue): result: AsyncResult = AsyncResult(task_id) if not result.ready(): logger.debug( - f"Task id {task_id} rev {remained_entries[entry]} " - "marked as failure." + "Task id %s rev %s marked as failure.", + task_id, + remained_entries[entry], ) self.celery.backend.mark_as_failure(task_id, None) @@ -441,8 +445,7 @@ class LocalCeleryQueue(BaseStashQueue): def worker_status(self) -> Dict[str, List[Dict]]: """Return the current active celery worker""" status = self.celery.control.inspect().active() or {} - logger.debug(f"Worker status: {status}") - + logger.debug("Worker status: %s", status) return status def clear(self, *args, **kwargs): diff --git a/dvc/repo/experiments/queue/tasks.py b/dvc/repo/experiments/queue/tasks.py index 5dd6ac58b..9f8af5675 100644 --- a/dvc/repo/experiments/queue/tasks.py +++ b/dvc/repo/experiments/queue/tasks.py @@ -74,7 +74,7 @@ def collect_exp( else: logger.debug("Experiment failed (Exec result was None)") celery_queue.stash_failed(entry) - except Exception: # noqa: BLE001, pylint: disable=broad-except + except Exception: # pylint: disable=broad-except # Log exceptions but do not re-raise so that task chain execution # continues logger.exception("Failed to collect experiment") diff --git a/dvc/repo/experiments/queue/tempdir.py b/dvc/repo/experiments/queue/tempdir.py index 4f534b88c..15b23881e 100644 --- a/dvc/repo/experiments/queue/tempdir.py +++ b/dvc/repo/experiments/queue/tempdir.py @@ -7,12 +7,7 @@ from funcy import first from dvc.exceptions import DvcException from dvc.repo.experiments.exceptions import ExpQueueEmptyError -from dvc.repo.experiments.executor.base import ( - BaseExecutor, - ExecutorInfo, - ExecutorResult, - TaskStatus, -) +from dvc.repo.experiments.executor.base import ExecutorInfo, TaskStatus from dvc.repo.experiments.executor.local import TempDirExecutor from dvc.repo.experiments.utils import EXEC_PID_DIR, EXEC_TMP_DIR from dvc.utils.objects import cached_property @@ -23,6 +18,7 @@ from .workspace import WorkspaceQueue if TYPE_CHECKING: from dvc.repo.experiments import Experiments + from dvc.repo.experiments.executor.base import BaseExecutor, ExecutorResult from dvc_task.proc.manager import ProcessManager logger = logging.getLogger(__name__) @@ -96,7 +92,7 @@ class TempDirQueue(WorkspaceQueue): ) def _reproduce_entry( - self, entry: QueueEntry, executor: BaseExecutor + self, entry: QueueEntry, executor: "BaseExecutor" ) -> Dict[str, Dict[str, str]]: from dvc.stage.monitor import CheckpointKilledError @@ -143,8 +139,8 @@ class TempDirQueue(WorkspaceQueue): @staticmethod def collect_executor( exp: "Experiments", - executor: BaseExecutor, - exec_result: ExecutorResult, + executor: "BaseExecutor", + exec_result: "ExecutorResult", ) -> Dict[str, str]: return BaseStashQueue.collect_executor(exp, executor, exec_result) diff --git a/dvc/repo/experiments/queue/workspace.py b/dvc/repo/experiments/queue/workspace.py index af52ea485..ba4afb5ac 100644 --- a/dvc/repo/experiments/queue/workspace.py +++ b/dvc/repo/experiments/queue/workspace.py @@ -10,12 +10,7 @@ from voluptuous import Invalid from dvc.exceptions import DvcException from dvc.lock import make_lock from dvc.repo.experiments.exceptions import ExpQueueEmptyError -from dvc.repo.experiments.executor.base import ( - BaseExecutor, - ExecutorInfo, - ExecutorResult, - TaskStatus, -) +from dvc.repo.experiments.executor.base import ExecutorInfo, TaskStatus from dvc.repo.experiments.executor.local import WorkspaceExecutor from dvc.repo.experiments.refs import EXEC_BRANCH, WORKSPACE_STASH from dvc.repo.experiments.utils import get_exp_rwlock, to_studio_params @@ -23,10 +18,13 @@ from dvc.rwlock import RWLOCK_FILE, RWLOCK_LOCK, SCHEMA from dvc.utils import relpath from dvc.utils.fs import remove -from .base import BaseStashQueue, QueueDoneResult, QueueEntry, QueueGetResult +from .base import BaseStashQueue, QueueEntry, QueueGetResult if TYPE_CHECKING: from dvc.repo.experiments import Experiments + from dvc.repo.experiments.executor.base import BaseExecutor, ExecutorResult + + from .base import QueueDoneResult logger = logging.getLogger(__name__) @@ -74,13 +72,13 @@ class WorkspaceQueue(BaseStashQueue): # need to be handled via the queue raise NotImplementedError - def iter_done(self) -> Generator[QueueDoneResult, None, None]: + def iter_done(self) -> Generator["QueueDoneResult", None, None]: raise NotImplementedError - def iter_failed(self) -> Generator[QueueDoneResult, None, None]: + def iter_failed(self) -> Generator["QueueDoneResult", None, None]: raise NotImplementedError - def iter_success(self) -> Generator[QueueDoneResult, None, None]: + def iter_success(self) -> Generator["QueueDoneResult", None, None]: raise NotImplementedError def reproduce(self) -> Dict[str, Dict[str, str]]: @@ -94,7 +92,7 @@ class WorkspaceQueue(BaseStashQueue): return results def _reproduce_entry( - self, entry: QueueEntry, executor: BaseExecutor + self, entry: QueueEntry, executor: "BaseExecutor" ) -> Dict[str, Dict[str, str]]: from dvc_studio_client.post_live_metrics import post_live_metrics @@ -152,8 +150,8 @@ class WorkspaceQueue(BaseStashQueue): @staticmethod def collect_executor( # pylint: disable=unused-argument exp: "Experiments", - executor: BaseExecutor, # noqa: ARG004 - exec_result: ExecutorResult, + executor: "BaseExecutor", # noqa: ARG004 + exec_result: "ExecutorResult", ) -> Dict[str, str]: results: Dict[str, str] = {} exp_rev = exp.scm.get_ref(EXEC_BRANCH) @@ -164,7 +162,7 @@ class WorkspaceQueue(BaseStashQueue): return results - def get_result(self, entry: QueueEntry) -> Optional[ExecutorResult]: + def get_result(self, entry: QueueEntry) -> Optional["ExecutorResult"]: raise NotImplementedError def kill(self, revs: Collection[str]) -> None: @@ -239,7 +237,10 @@ class WorkspaceQueue(BaseStashQueue): remove(path) return False - def get_running_exps(self, fetch_refs: bool = True) -> Dict[str, Dict]: + def get_running_exps( + self, + fetch_refs: bool = True, # noqa: ARG002 + ) -> Dict[str, Dict]: from dvc.utils.serialize import load_json assert self._EXEC_NAME diff --git a/dvc/repo/experiments/remove.py b/dvc/repo/experiments/remove.py index d0a9148db..46068df99 100644 --- a/dvc/repo/experiments/remove.py +++ b/dvc/repo/experiments/remove.py @@ -6,8 +6,6 @@ from dvc.repo.scm_context import scm_context from dvc.scm import iter_revs from .exceptions import UnresolvedExpNamesError -from .queue.base import QueueEntry -from .refs import ExpRefInfo from .utils import exp_refs, exp_refs_by_baseline, push_refspec if TYPE_CHECKING: @@ -15,7 +13,8 @@ if TYPE_CHECKING: from dvc.repo.experiments.queue.celery import LocalCeleryQueue from dvc.scm import Git - from .queue.base import ExpRefAndQueueEntry + from .queue.base import ExpRefAndQueueEntry, QueueEntry + from .refs import ExpRefInfo logger = logging.getLogger(__name__) @@ -99,7 +98,7 @@ def _resolve_exp_by_baseline( def _remove_commited_exps( - scm: "Git", exp_refs_list: Iterable[ExpRefInfo], remote: Optional[str] + scm: "Git", exp_refs_list: Iterable["ExpRefInfo"], remote: Optional[str] ) -> List[str]: if remote: from dvc.scm import TqdmGit diff --git a/dvc/repo/metrics/show.py b/dvc/repo/metrics/show.py index 99c6dc7d7..638ff89d5 100644 --- a/dvc/repo/metrics/show.py +++ b/dvc/repo/metrics/show.py @@ -1,11 +1,10 @@ import logging import os -from typing import List +from typing import TYPE_CHECKING, List from scmrepo.exceptions import SCMError from dvc.fs.dvc import DVCFileSystem -from dvc.output import Output from dvc.repo import locked from dvc.repo.collect import collect from dvc.scm import NoSCMError @@ -18,14 +17,17 @@ from dvc.utils import ( from dvc.utils.collections import ensure_list from dvc.utils.serialize import load_path +if TYPE_CHECKING: + from dvc.output import Output + logger = logging.getLogger(__name__) -def _is_metric(out: Output) -> bool: +def _is_metric(out: "Output") -> bool: return bool(out.metric) -def _to_fs_paths(metrics: List[Output]) -> List["str"]: +def _to_fs_paths(metrics: List["Output"]) -> List["str"]: result = [] for out in metrics: if out.metric: diff --git a/dvc/repo/plots/__init__.py b/dvc/repo/plots/__init__.py index ad89af4c7..eb196d45c 100644 --- a/dvc/repo/plots/__init__.py +++ b/dvc/repo/plots/__init__.py @@ -32,7 +32,7 @@ if TYPE_CHECKING: from dvc.fs import FileSystem from dvc.output import Output from dvc.repo import Repo - from dvc.types import DictAny, DictStrAny, StrPath + from dvc.types import DictStrAny, StrPath dpath.options.ALLOW_EMPTY_STRING_KEYS = True @@ -424,7 +424,7 @@ def _adjust_sources(fs, plot_props, config_dir): def _resolve_definitions( fs: "FileSystem", targets: List[str], - props: "DictAny", + props: Dict[str, Any], config_path: "StrPath", definitions: "DictStrAny", onerror: Optional[Callable[[Any], Any]] = None, diff --git a/dvc/repo/scm_context.py b/dvc/repo/scm_context.py index d8468cdde..888fc38c5 100644 --- a/dvc/repo/scm_context.py +++ b/dvc/repo/scm_context.py @@ -133,7 +133,7 @@ class SCMContext: and logger.isEnabledFor(logging.INFO) ): add_cmd = self._make_git_add_cmd(self.files_to_track) - logger.info(f"\nTo track the changes with git, run:\n\n{add_cmd}") + logger.info("\nTo track the changes with git, run:\n\n%s", add_cmd) logger.info( "\nTo enable auto staging, run:\n\n" "\tdvc config core.autostage true" diff --git a/dvc/repo/stage.py b/dvc/repo/stage.py index 8fe7f63d8..0f44ae76d 100644 --- a/dvc/repo/stage.py +++ b/dvc/repo/stage.py @@ -18,7 +18,6 @@ if typing.TYPE_CHECKING: from dvc.repo import Repo from dvc.stage import PipelineStage, Stage from dvc.stage.loader import StageLoader - from dvc.types import OptStr PROJECT_FILE = "dvc.yaml" @@ -61,7 +60,7 @@ def _collect_specific_target( target: str, with_deps: bool, recursive: bool, -) -> Tuple[StageIter, "OptStr", "OptStr"]: +) -> Tuple[StageIter, Optional[str], Optional[str]]: from dvc.dvcfile import is_valid_filename # Optimization: do not collect the graph for a specific target diff --git a/dvc/stage/loader.py b/dvc/stage/loader.py index 4b8fd66ab..0fc5a3743 100644 --- a/dvc/stage/loader.py +++ b/dvc/stage/loader.py @@ -60,7 +60,7 @@ class StageLoader(Mapping): from dvc.output import merge_file_meta_from_cloud assert isinstance(lock_data, dict) - items: Iterable[Tuple[str, Output]] = chain( + items: Iterable[Tuple[str, "Output"]] = chain( ((StageParams.PARAM_DEPS, dep) for dep in stage.deps), ((StageParams.PARAM_OUTS, out) for out in stage.outs), ) diff --git a/dvc/stage/monitor.py b/dvc/stage/monitor.py index 8dc49c167..b0869ba2b 100644 --- a/dvc/stage/monitor.py +++ b/dvc/stage/monitor.py @@ -110,7 +110,7 @@ class Monitor: try: task.execute() task.updated.set() - except Exception: # noqa: BLE001, pylint: disable=W0703 + except Exception: # pylint: disable=broad-except logger.exception( "Error running '%s' task, '%s' will be aborted", task.name, diff --git a/dvc/types.py b/dvc/types.py index 0b6966975..b061af6e4 100644 --- a/dvc/types.py +++ b/dvc/types.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, AnyStr, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, AnyStr, Dict, List, Union if TYPE_CHECKING: from os import PathLike @@ -8,7 +8,5 @@ BytesPath = Union[bytes, "PathLike[bytes]"] GenericPath = Union[AnyStr, "PathLike[AnyStr]"] StrOrBytesPath = Union[str, bytes, "PathLike[str]", "PathLike[bytes]"] -OptStr = Optional[str] TargetType = Union[List[str], str] DictStrAny = Dict[str, Any] -DictAny = Dict[Any, Any] diff --git a/dvc/ui/__init__.py b/dvc/ui/__init__.py index 0e343c657..e86a4226c 100644 --- a/dvc/ui/__init__.py +++ b/dvc/ui/__init__.py @@ -38,7 +38,7 @@ class Formatter: } self.theme = defaultdict(lambda: defaults or {}, theme) - def format( + def format( # noqa: A003 self, message: str, style: Optional[str] = None, **kwargs ) -> str: from dvc.utils import colorize diff --git a/pyproject.toml b/pyproject.toml index 461bcc2f3..9670cf96a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -161,26 +161,31 @@ line_length = 79 line-length = 79 target-version = "py38" show-source = true -ignore = ["S101", "SIM105", "SIM108", "SIM110", "SIM111", "SIM117", "A003", "ARG002", "PLR2004", "RUF005"] -select = ["F", "E", "W", "C90", "UP", "YTT", "S", "BLE", "B", "A", "C4", "T10", "ISC", "ICN", "T20", "PT003", "PT008", "PT009", "PT010", "PT015", "PT016", "PT017", "PT020", "PT021", "PT024", "PT025", "PT026", "Q", "RET501", "RET504", "SIM", "ARG", "TID", "PGH", "PLC", "PLE", "PLR", "PLW", "PIE", "INP", "TRY004", "TRY201", "RUF"] +ignore = ["S101", "SIM105", "SIM108", "SIM110", "SIM111", "SIM117", "PLR2004", "RUF005"] +select = ["F", "E", "W", "C90", "UP", "YTT", "S", "BLE", "B", "A", "C4", "T10", "ISC", "ICN", "G", "INP", "PIE", "T20", "PT003", "PT008", "PT009", "PT010", "PT015", "PT016", "PT017", "PT020", "PT021", "PT024", "PT025", "PT026", "Q", "RET501", "RET504", "SIM", "TID", "TCH", "ARG", "PGH", "PLC", "PLE", "PLR", "PLW", "TRY002", "TRY004", "TRY201", "TRY400", "RUF"] # external flake8 codes that should be preserved external = ["B301", "B024", "C901", "E302", "F401", "W601"] [tool.ruff.per-file-ignores] -"scripts/**/*.py" = ["T201", "INP001"] +"dvc/testing/*.py" = ["ARG002"] +"scripts/**/*.py" = ["T201", "INP001", "TRY002"] [tool.ruff.flake8-pytest-style] fixture-parentheses = false mark-parentheses = false parametrize-names-type = "csv" -[tool.ruff.flake8-unused-arguments] -ignore-variadic-names = true - [tool.ruff.flake8-tidy-imports] [tool.ruff.flake8-tidy-imports.banned-api] "funcy.cached_property".msg = "use `from dvc.utils.objects import cached_property` instead." +[tool.ruff.flake8-type-checking] +exempt-modules = ["typing"] +strict = true + +[tool.ruff.flake8-unused-arguments] +ignore-variadic-names = true + [tool.pytest.ini_options] log_level = "debug" addopts = "-ra --cov-config pyproject.toml" @@ -264,11 +269,9 @@ ignore_missing_imports = true [tool.pylint.message_control] disable = [ - "format", "refactoring", "spelling", "design", - "invalid-name", "duplicate-code", "fixme", - "unused-wildcard-import", "cyclic-import", "wrong-import-order", + "format", "refactoring", "spelling", "design", "invalid-name", "fixme", + "duplicate-code", "unused-wildcard-import", "cyclic-import", "wrong-import-order", "wrong-import-position", "ungrouped-imports", "multiple-imports", - "logging-format-interpolation", "logging-fstring-interpolation", "missing-function-docstring", "missing-module-docstring", "missing-class-docstring", "raise-missing-from", "import-outside-toplevel", ] diff --git a/tests/func/test_ignore.py b/tests/func/test_ignore.py index 1842800b6..804e27059 100644 --- a/tests/func/test_ignore.py +++ b/tests/func/test_ignore.py @@ -1,6 +1,7 @@ import os import shutil from pathlib import Path +from typing import List import pytest @@ -9,7 +10,6 @@ from dvc.output import OutputIsIgnoredError from dvc.pathspec_math import PatternInfo, merge_patterns from dvc.repo import Repo from dvc.testing.tmp_dir import TmpDir -from dvc.types import List from dvc_data.hashfile.build import IgnoreInCollectedDirError from dvc_data.hashfile.utils import get_mtime_and_size diff --git a/tests/ruff.toml b/tests/ruff.toml index d7cbe2ee6..36d358bd7 100644 --- a/tests/ruff.toml +++ b/tests/ruff.toml @@ -1,3 +1,3 @@ extend = "../pyproject.toml" extend-select = ["RUF005"] -extend-ignore = ["S", "RUF001", "ARG001", "ARG002"] +extend-ignore = ["S", "RUF001", "ARG001", "ARG002", "TRY002"] diff --git a/tests/unit/repo/experiments/conftest.py b/tests/unit/repo/experiments/conftest.py index 94679e83e..e016cea5c 100644 --- a/tests/unit/repo/experiments/conftest.py +++ b/tests/unit/repo/experiments/conftest.py @@ -3,8 +3,6 @@ from textwrap import dedent import pytest -from dvc.repo.experiments.queue.celery import LocalCeleryQueue - DEFAULT_ITERATIONS = 2 CHECKPOINT_SCRIPT_FORMAT = dedent( """\ @@ -156,7 +154,7 @@ def _thread_worker(app, **kwargs): @pytest.fixture -def test_queue(tmp_dir, dvc, scm, mocker) -> LocalCeleryQueue: +def test_queue(tmp_dir, dvc, scm, mocker): """Patches experiments celery queue for pytest testing. Test queue worker runs for the duration of the test in separate thread(s). diff --git a/tests/unit/repo/experiments/test_show.py b/tests/unit/repo/experiments/test_show.py index 8cc33a7ca..2293e043b 100644 --- a/tests/unit/repo/experiments/test_show.py +++ b/tests/unit/repo/experiments/test_show.py @@ -1,6 +1,5 @@ import pytest from funcy import first -from scmrepo.git import Git from dvc.repo.experiments.show import ( ExpStatus, @@ -10,7 +9,7 @@ from dvc.repo.experiments.show import ( ) -def test_get_show_branch(tmp_dir, scm: "Git", dvc, exp_stage): +def test_get_show_branch(tmp_dir, scm, dvc, exp_stage): new_branch = "new" baseline = scm.get_rev() diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py index 67f06b259..f7a6324bb 100644 --- a/tests/unit/test_logger.py +++ b/tests/unit/test_logger.py @@ -65,7 +65,7 @@ class TestColorFormatter: with caplog.at_level(logging.INFO, logger="dvc"): try: raise ValueError - except Exception: # noqa: BLE001 + except Exception: logger.exception("message") expected = "{red}ERROR{nc}: message".format(**colors) @@ -76,7 +76,7 @@ class TestColorFormatter: with caplog.at_level(logging.INFO, logger="dvc"): try: raise Exception("description") - except Exception: # noqa: BLE001 + except Exception: logger.exception("") expected = "{red}ERROR{nc}: description".format(**colors) @@ -87,7 +87,7 @@ class TestColorFormatter: with caplog.at_level(logging.INFO, logger="dvc"): try: raise Exception("description") - except Exception: # noqa: BLE001 + except Exception: logger.exception("message") expected = "{red}ERROR{nc}: message - description".format(**colors) @@ -98,7 +98,7 @@ class TestColorFormatter: with caplog.at_level(logging.DEBUG, logger="dvc"): try: raise Exception("description") - except Exception: # noqa: BLE001 + except Exception: stack_trace = traceback.format_exc() logger.exception("") @@ -144,7 +144,7 @@ class TestColorFormatter: with caplog.at_level(logging.DEBUG, logger="dvc"): try: raise Exception("description") - except Exception: # noqa: BLE001 + except Exception: stack_trace = traceback.format_exc() logger.exception("something", extra={"tb_only": True}) diff --git a/tests/unit/ui/test_console.py b/tests/unit/ui/test_console.py index 7f72e6712..eebdf5859 100644 --- a/tests/unit/ui/test_console.py +++ b/tests/unit/ui/test_console.py @@ -2,13 +2,11 @@ import datetime import textwrap import pytest -from pytest import CaptureFixture -from pytest_mock import MockerFixture from dvc.ui import Console -def test_write(capsys: CaptureFixture[str]): +def test_write(capsys): """Test that ui.write works.""" console = Console(enable=True) message = "hello world" @@ -44,9 +42,7 @@ def test_write(capsys: CaptureFixture[str]): ), ], ) -def test_write_json( - capsys: CaptureFixture[str], mocker: MockerFixture, isatty, expected_output -): +def test_write_json(capsys, mocker, isatty, expected_output): """Test that ui.write_json works.""" console = Console(enable=True) @@ -57,7 +53,7 @@ def test_write_json( assert captured.out == expected_output -def test_capsys_works(capsys: CaptureFixture[str]): +def test_capsys_works(capsys): """Sanity check that capsys can capture outputs from a global ui.""" from dvc.ui import ui diff --git a/tests/unit/ui/test_table.py b/tests/unit/ui/test_table.py index 71126d855..da528cd62 100644 --- a/tests/unit/ui/test_table.py +++ b/tests/unit/ui/test_table.py @@ -1,14 +1,12 @@ import textwrap import pytest -from pytest import CaptureFixture -from pytest_mock import MockerFixture from rich.style import Style from dvc.ui import ui -def test_plain(capsys: CaptureFixture[str]): +def test_plain(capsys): ui.table( [("foo", "bar"), ("foo1", "bar1"), ("foo2", "bar2")], headers=["first", "second"], @@ -24,7 +22,7 @@ def test_plain(capsys: CaptureFixture[str]): ) -def test_plain_md(capsys: CaptureFixture[str]): +def test_plain_md(capsys): ui.table( [("foo", "bar"), ("foo1", "bar1"), ("foo2", "bar2")], headers=["first", "second"], @@ -42,7 +40,7 @@ def test_plain_md(capsys: CaptureFixture[str]): ) -def test_plain_pager(mocker: MockerFixture): +def test_plain_pager(mocker): pager_mock = mocker.patch("dvc.ui.pager.pager") ui.table( [("foo", "bar"), ("foo1", "bar1"), ("foo2", "bar2")], @@ -62,7 +60,7 @@ def test_plain_pager(mocker: MockerFixture): ) -def test_plain_headerless(capsys: CaptureFixture[str]): +def test_plain_headerless(capsys): ui.table([("foo", "bar"), ("foo1", "bar1"), ("foo2", "bar2")]) captured = capsys.readouterr() assert captured.out == textwrap.dedent( @@ -74,7 +72,7 @@ def test_plain_headerless(capsys: CaptureFixture[str]): ) -def test_rich_simple(capsys: CaptureFixture[str]): +def test_rich_simple(capsys): ui.table( [("foo", "bar"), ("foo1", "bar1"), ("foo2", "bar2")], headers=["first", "second"], @@ -87,7 +85,7 @@ def test_rich_simple(capsys: CaptureFixture[str]): ] == ["first second", "foo bar", "foo1 bar1", "foo2 bar2"] -def test_rich_headerless(capsys: CaptureFixture[str]): +def test_rich_headerless(capsys): ui.table( [("foo", "bar"), ("foo1", "bar1"), ("foo2", "bar2")], rich_table=True ) @@ -97,7 +95,7 @@ def test_rich_headerless(capsys: CaptureFixture[str]): ] == ["foo bar", "foo1 bar1", "foo2 bar2"] -def test_rich_border(capsys: CaptureFixture[str]): +def test_rich_border(capsys): ui.table( [("foo", "bar"), ("foo1", "bar1"), ("foo2", "bar2")], headers=["first", "second"], @@ -124,7 +122,7 @@ def test_rich_border(capsys: CaptureFixture[str]): {"row_styles": [{"style": Style(bold=True)}]}, ], ) -def test_rich_styles(capsys: CaptureFixture[str], extra_opts): +def test_rich_styles(capsys, extra_opts): ui.table( [("foo", "bar"), ("foo1", "bar1"), ("foo2", "bar2")], headers=["first", "second"], @@ -138,7 +136,7 @@ def test_rich_styles(capsys: CaptureFixture[str], extra_opts): ] == ["first second", "foo bar", "foo1 bar1", "foo2 bar2"] -def test_rich_pager(mocker: MockerFixture): +def test_rich_pager(mocker): pager_mock = mocker.patch("dvc.ui.pager.pager") ui.table( @@ -154,13 +152,13 @@ def test_rich_pager(mocker: MockerFixture): @pytest.mark.parametrize("rich_table", [True, False]) -def test_empty(capsys: CaptureFixture[str], rich_table: str): +def test_empty(capsys, rich_table): ui.table([], rich_table=rich_table) out, err = capsys.readouterr() assert (out, err) == ("", "") -def test_empty_markdown(capsys: CaptureFixture[str]): +def test_empty_markdown(capsys): ui.table([], headers=["Col1", "Col2"], markdown=True) out, err = capsys.readouterr() assert (out, err) == ("| Col1 | Col2 |\n|--------|--------|\n\n", "") diff --git a/tests/utils/asserts.py b/tests/utils/asserts.py index 579cf3124..ae589fe93 100644 --- a/tests/utils/asserts.py +++ b/tests/utils/asserts.py @@ -1,5 +1,8 @@ -from typing import Any, Dict -from unittest.mock import ANY, Mock +from typing import TYPE_CHECKING, Any, Dict +from unittest.mock import ANY + +if TYPE_CHECKING: + from unittest.mock import Mock def issubset(subset: Dict, superset: Dict) -> bool: @@ -7,7 +10,7 @@ def issubset(subset: Dict, superset: Dict) -> bool: return True -def called_once_with_subset(m: Mock, *args: Any, **kwargs: Any) -> bool: +def called_once_with_subset(m: "Mock", *args: Any, **kwargs: Any) -> bool: m.assert_called_once() m_args, m_kwargs = m.call_args