Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

update ruff and enable flake8-logging-format and flake8-type-checking (#8894)

Browse Source 
* update ruff and enable flake8-logging-format and flake8-type-checking check

* Update dvc/external_repo.py

* Update dvc/fs/dvc.py

* Update dvc/repo/experiments/__init__.py

* Update dvc/repo/experiments/queue/base.py

* Update pyproject.toml
This commit is contained in:
skshetry
2023-01-27 19:14:19 +05:45
committed by GitHub
parent 6f0e8a8db5
commit a137b2a6bc
52 changed files with 222 additions and 210 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.pre-commit-config.yaml
+1 -1
View File
@@ -43,7 +43,7 @@ repos:
hooks:
- id: isort
- repo: https://github.com/charliermarsh/ruff-pre-commit
rev: 'v0.0.235'
rev: 'v0.0.236'
hooks:
- id: ruff
- repo: https://github.com/codespell-project/codespell
dvc/annotations.py
+1 -1
View File
@@ -12,7 +12,7 @@ class Annotation:
PARAM_META: ClassVar[str] = "meta"
desc: Optional[str] = None
type: Optional[str] = None
type: Optional[str] = None # noqa: A003
labels: List[str] = field(default_factory=list)
meta: Dict[str, Any] = field(default_factory=dict)
dvc/cli/__init__.py
+16 -8
View File
@@ -57,8 +57,8 @@ def _log_exceptions(exc: Exception) -> Optional[int]:
if exc.errno == errno.EMFILE:
logger.exception(
"too many open files, please visit "
"{} to see how to handle this "
"problem".format(error_link("many-files")),
"%s to see how to handle this problem",
error_link("many-files"),
extra={"tb_only": True},
)
else:
@@ -91,8 +91,11 @@ def _log_exceptions(exc: Exception) -> Optional[int]:
hint = f"\nPlease report this bug to {link}. Thank you!"
logger.exception(
f"URL '{exc.url}' is supported but requires these missing "
f"dependencies: {exc.missing_deps}. {hint}",
"URL '%s' is supported but requires these missing "
"dependencies: %s. %s",
exc.url,
exc.missing_deps,
hint,
extra={"tb_only": True},
)
return None
@@ -101,7 +104,9 @@ def _log_exceptions(exc: Exception) -> Optional[int]:
link = format_link("https://man.dvc.org/remote/modify")
logger.exception("configuration error")
logger.exception(
f"{exc!s}\nLearn more about configuration settings at {link}.",
"%s\nLearn more about configuration settings at %s.",
exc,
link,
extra={"tb_only": True},
)
return 251
@@ -113,9 +118,12 @@ def _log_exceptions(exc: Exception) -> Optional[int]:
directory = relpath(exc.directory)
logger.exception(
f"Could not open pickled '{exc.type}' cache.\n"
f"Remove the '{directory}' directory and then retry this command."
f"\nSee {error_link('pickle')} for more information.",
"Could not open pickled '%s' cache.\n"
"Remove the '%s' directory and then retry this command."
"\nSee %s for more information.",
exc.type,
directory,
error_link("pickle"),
extra={"tb_only": True},
)
return None
dvc/cli/actions.py
+6 -2
View File
@@ -2,7 +2,9 @@ from argparse import _AppendAction
class CommaSeparatedArgs(_AppendAction): # pylint: disable=protected-access
def __call__(self, parser, namespace, values, option_string=None):
def __call__(
self, parser, namespace, values, option_string=None # noqa: ARG002
):
from funcy import ldistinct
items = getattr(namespace, self.dest) or []
@@ -11,7 +13,9 @@ class CommaSeparatedArgs(_AppendAction): # pylint: disable=protected-access
class KeyValueArgs(_AppendAction):
def __call__(self, parser, namespace, values, option_string=None):
def __call__(
self, parser, namespace, values, option_string=None # noqa: ARG002
):
items = getattr(namespace, self.dest) or {}
for value in filter(bool, values):
key, _, value = value.partition("=")
dvc/commands/commit.py
+1 -3
View File
@@ -25,9 +25,7 @@ class CmdCommit(CmdBase):
)
except DvcException:
logger.exception(
"failed to commit{}".format(
(" " + target) if target else ""
)
"failed to commit%s", (" " + target) if target else ""
)
return 1
return 0
dvc/commands/get.py
+2 -4
View File
@@ -45,13 +45,11 @@ class CmdGet(CmdBaseNoRepo):
)
return 0
except CloneError:
logger.exception(f"failed to get '{self.args.path}'")
logger.exception("failed to get '%s'", self.args.path)
return 1
except DvcException:
logger.exception(
"failed to get '{}' from '{}'".format(
self.args.path, self.args.url
)
"failed to get '%s' from '%s'", self.args.path, self.args.url
)
return 1
dvc/commands/imp.py
+4 -4
View File
@@ -28,13 +28,13 @@ class CmdImport(CmdBase):
jobs=self.args.jobs,
)
except CloneError:
logger.exception(f"failed to import '{self.args.path}'")
logger.exception("failed to import '%s'", self.args.path)
return 1
except DvcException:
logger.exception(
"failed to import '{}' from '{}'.".format(
self.args.path, self.args.url
)
"failed to import '%s' from '%s'.",
self.args.path,
self.args.url,
)
return 1
return 0
dvc/commands/imp_url.py
+3 -4
View File
@@ -29,10 +29,9 @@ class CmdImportUrl(CmdBase):
)
except DvcException:
logger.exception(
"failed to import {}. You could also try downloading "
"it manually, and adding it with `dvc add`.".format(
self.args.url
)
"failed to import %s. You could also try downloading "
"it manually, and adding it with `dvc add`.",
self.args.url,
)
return 1
return 0
dvc/commands/ls/ls_colors.py
+1 -1
View File
@@ -20,7 +20,7 @@ class LsColors:
else:
self._codes[code] = color
def format(self, entry):
def format(self, entry): # noqa: A003
text = entry["path"]
if entry.get("isout", False) and "out" in self._codes:
dvc/config.py
+6 -4
View File
@@ -9,12 +9,12 @@ from typing import TYPE_CHECKING, Dict, Optional
from funcy import compact, memoize, re_find
from dvc.exceptions import DvcException, NotDvcRepoError
from dvc.types import DictStrAny, StrPath
from .utils.objects import cached_property
if TYPE_CHECKING:
from dvc.fs import FileSystem
from dvc.types import DictStrAny, StrPath
logger = logging.getLogger(__name__)
@@ -92,10 +92,10 @@ class Config(dict):
def __init__(
self,
dvc_dir: Optional[StrPath] = None,
dvc_dir: Optional["StrPath"] = None,
validate: bool = True,
fs: Optional["FileSystem"] = None,
config: Optional[DictStrAny] = None,
config: Optional["DictStrAny"] = None,
): # pylint: disable=super-init-not-called
from dvc.fs import LocalFileSystem
@@ -153,7 +153,9 @@ class Config(dict):
with open(config_file, "w+", encoding="utf-8"):
return Config(dvc_dir)
def load(self, validate: bool = True, config: Optional[DictStrAny] = None):
def load(
self, validate: bool = True, config: Optional["DictStrAny"] = None
):
"""Loads config from all the config files.
Raises:
dvc/data_cloud.py
+2 -1
View File
@@ -112,7 +112,8 @@ class DataCloud:
)
logger.warning(
"Some of the cache files do not exist neither locally "
f"nor on remote. Missing cache files:\n{missing_desc}"
"nor on remote. Missing cache files:\n%s",
missing_desc,
)
def transfer(
dvc/dependency/repo.py
+9 -13
View File
@@ -1,28 +1,22 @@
import os
from collections import defaultdict
from copy import copy
from typing import (
TYPE_CHECKING,
ContextManager,
Dict,
Optional,
Set,
Tuple,
Union,
)
from typing import TYPE_CHECKING, Dict, Optional, Set, Tuple, Union
from voluptuous import Required
from dvc.prompt import confirm
from dvc_data.hashfile.meta import Meta
from .base import Dependency
if TYPE_CHECKING:
from typing import ContextManager
from dvc.output import Output
from dvc.repo import Repo
from dvc.stage import Stage
from dvc_data.hashfile.hash_info import HashInfo
from dvc_data.hashfile.meta import Meta
from dvc_data.hashfile.obj import HashFile
from dvc_objects.db import ObjectDB
@@ -46,10 +40,10 @@ class RepoDependency(Dependency):
):
self.def_repo = def_repo
self._objs: Dict[str, "HashFile"] = {}
self._meta: Dict[str, Meta] = {}
self._meta: Dict[str, "Meta"] = {}
super().__init__(stage, *args, **kwargs)
def _parse_path(self, fs, fs_path):
def _parse_path(self, fs, fs_path): # noqa: ARG002
return None
@property
@@ -114,7 +108,9 @@ class RepoDependency(Dependency):
def _get_used_and_obj(
self, obj_only: bool = False, **kwargs
) -> Tuple[Dict[Optional["ObjectDB"], Set["HashInfo"]], Meta, "HashFile"]:
) -> Tuple[
Dict[Optional["ObjectDB"], Set["HashInfo"]], "Meta", "HashFile"
]:
from dvc.config import NoRemoteError
from dvc.exceptions import NoOutputOrStageError, PathMissingError
from dvc.utils import as_posix
dvc/dvcfile.py
+6 -4
View File
@@ -21,7 +21,6 @@ from dvc.stage.exceptions import (
StageFileDoesNotExistError,
StageFileIsNotDvcFileError,
)
from dvc.types import StrOrBytesPath
from dvc.utils import relpath
from dvc.utils.collections import apply_diff
from dvc.utils.objects import cached_property
@@ -29,6 +28,7 @@ from dvc.utils.serialize import dump_yaml, modify_yaml
if TYPE_CHECKING:
from dvc.repo import Repo
from dvc.types import StrOrBytesPath
from .parsing import DataResolver
from .stage import Stage
@@ -179,7 +179,8 @@ class FileMixin:
**kwargs,
)
def remove(self, force=False): # pylint: disable=unused-argument
# pylint: disable-next=unused-argument
def remove(self, force=False): # noqa: ARG002
with contextlib.suppress(FileNotFoundError):
os.unlink(self.path)
@@ -219,7 +220,8 @@ class SingleStageFile(FileMixin):
dump_yaml(self.path, serialize.to_single_stage_file(stage, **kwargs))
self.repo.scm_context.track_file(self.relpath)
def remove_stage(self, stage): # pylint: disable=unused-argument
# pylint: disable-next=unused-argument
def remove_stage(self, stage): # noqa: ARG002
self.remove()
def merge(self, ancestor, other, allowed=None):
@@ -465,7 +467,7 @@ class Lockfile(FileMixin):
def load_file(
repo: "Repo", path: StrOrBytesPath, **kwargs: Any
repo: "Repo", path: "StrOrBytesPath", **kwargs: Any
) -> Union[ProjectFile, SingleStageFile]:
_, ext = os.path.splitext(path)
if ext in (".yaml", ".yml"):
dvc/external_repo.py
+3 -2
View File
@@ -17,12 +17,13 @@ from dvc.exceptions import (
)
from dvc.repo import Repo
from dvc.scm import CloneError, map_scm_exception
from dvc.types import StrPath
from dvc.utils import relpath
if TYPE_CHECKING:
from scmrepo import Git
from dvc.types import StrPath
logger = logging.getLogger(__name__)
@@ -32,7 +33,7 @@ def external_repo(
url,
rev: Optional[str] = None,
for_write: bool = False,
cache_dir: Optional[StrPath] = None,
cache_dir: Optional["StrPath"] = None,
cache_types: Optional[Collection[str]] = None,
**kwargs,
) -> Iterator["Repo"]:
dvc/fs/callbacks.py
+1 -1
View File
@@ -70,7 +70,7 @@ class RichCallback(Callback):
self.progress.clear_task(self.task)
self._stack.close()
def call(self, hook_name=None, **kwargs):
def call(self, hook_name=None, **kwargs): # noqa: ARG002
self.progress.update(
self.task,
completed=self.value,
dvc/fs/dvc.py
+2 -2
View File
@@ -20,7 +20,6 @@ from typing import (
from fsspec.spec import AbstractFileSystem
from funcy import wrap_with
from dvc.types import StrPath
from dvc_objects.fs.base import FileSystem
from dvc_objects.fs.path import Path
@@ -28,6 +27,7 @@ from .data import DataFileSystem
if TYPE_CHECKING:
from dvc.repo import Repo
from dvc.types import StrPath
logger = logging.getLogger(__name__)
@@ -166,7 +166,7 @@ class _DVCFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
if hasattr(repo, "dvc_dir"):
self._datafss[key] = DataFileSystem(index=repo.index.data["repo"])
def _get_key(self, path: StrPath) -> Key:
def _get_key(self, path: "StrPath") -> Key:
parts = self.repo.fs.path.relparts(path, self.repo.root_dir)
if parts == (os.curdir,):
return ()
dvc/ignore.py
+8 -5
View File
@@ -3,15 +3,18 @@ import os
import re
from collections import namedtuple
from itertools import chain, groupby, takewhile
from typing import List, Optional
from typing import TYPE_CHECKING, List, Optional
from pathspec.patterns import GitWildMatchPattern
from pathspec.util import normalize_file
from pygtrie import Trie
from dvc.fs import AnyFSPath, FileSystem, Schemes, localfs
from dvc.fs import Schemes, localfs
from dvc.pathspec_math import PatternInfo, merge_patterns
if TYPE_CHECKING:
from dvc.fs import AnyFSPath, FileSystem
logger = logging.getLogger(__name__)
@@ -306,7 +309,7 @@ class DvcIgnoreFilter:
return [fs_dict[name] for name in chain(dirs, nondirs)]
def walk(self, fs: FileSystem, path: AnyFSPath, **kwargs):
def walk(self, fs: "FileSystem", path: "AnyFSPath", **kwargs):
detail = kwargs.get("detail", False)
ignore_subrepos = kwargs.pop("ignore_subrepos", True)
if fs.protocol == Schemes.LOCAL:
@@ -330,7 +333,7 @@ class DvcIgnoreFilter:
else:
yield from fs.walk(path, **kwargs)
def find(self, fs: FileSystem, path: AnyFSPath, **kwargs):
def find(self, fs: "FileSystem", path: "AnyFSPath", **kwargs):
if fs.protocol == Schemes.LOCAL:
for root, _, files in self.walk(fs, path, **kwargs):
for file in files:
@@ -420,7 +423,7 @@ class DvcIgnoreFilter:
return _no_match(target)
def is_ignored(
self, fs: FileSystem, path: str, ignore_subrepos: bool = True
self, fs: "FileSystem", path: str, ignore_subrepos: bool = True
) -> bool:
# NOTE: can't use self.check_ignore(path).match for now, see
# https://github.com/iterative/dvc/issues/4555
dvc/logger.py
+2 -2
View File
@@ -68,7 +68,7 @@ class LoggingException(Exception):
def excludeFilter(level):
class ExcludeLevelFilter(logging.Filter):
def filter(self, record):
def filter(self, record): # noqa: A003
return record.levelno < level
return ExcludeLevelFilter
@@ -94,7 +94,7 @@ class ColorFormatter(logging.Formatter):
"CRITICAL": colorama.Fore.RED,
}
def format(self, record):
def format(self, record): # noqa: A003
record.message = record.getMessage()
msg = self.formatMessage(record)
dvc/machine/__init__.py
+2 -2
View File
@@ -13,10 +13,10 @@ from typing import (
)
from dvc.exceptions import DvcException
from dvc.types import StrPath
if TYPE_CHECKING:
from dvc.repo import Repo
from dvc.types import StrPath
from .backend.base import BaseMachineBackend
@@ -76,7 +76,7 @@ class MachineBackends(Mapping):
def __init__(
self,
selected: Optional[Iterable[str]],
tmp_dir: StrPath,
tmp_dir: "StrPath",
**kwargs,
) -> None:
selected = selected or list(self.DEFAULT)
dvc/output.py
+2 -1
View File
@@ -32,7 +32,7 @@ from dvc_objects.errors import ObjectFormatError
from .annotations import ANNOTATION_FIELDS, ANNOTATION_SCHEMA, Annotation
from .fs import LocalFileSystem, RemoteMissingDepsError, Schemes, get_cloud_fs
from .fs.callbacks import DEFAULT_CALLBACK, Callback
from .fs.callbacks import DEFAULT_CALLBACK
from .utils import relpath
from .utils.fs import path_isin
@@ -41,6 +41,7 @@ if TYPE_CHECKING:
from dvc_data.index import DataIndexKey
from dvc_objects.db import ObjectDB
from .fs.callbacks import Callback
from .ignore import DvcIgnoreFilter
logger = logging.getLogger(__name__)
dvc/parsing/__init__.py
+15 -14
View File
@@ -26,7 +26,6 @@ from .context import (
KeyNotInContext,
MergeError,
Node,
SeqOrMap,
VarsAlreadyLoaded,
)
from .interpolate import (
@@ -40,8 +39,10 @@ if TYPE_CHECKING:
from typing import NoReturn
from dvc.repo import Repo
from dvc.types import DictStrAny
from .context import SeqOrMap
from dvc.types import DictStrAny
logger = logging.getLogger(__name__)
@@ -92,7 +93,7 @@ def _reraise_err(
def check_syntax_errors(
definition: DictStrAny, name: str, path: str, where: str = "stages"
definition: "DictStrAny", name: str, path: str, where: str = "stages"
):
for key, d in definition.items():
try:
@@ -111,8 +112,8 @@ def split_foreach_name(name: str) -> Tuple[str, Optional[str]]:
return group, first(keys)
def check_interpolations(data: DictStrAny, where: str, path: str):
def func(s: DictStrAny) -> None:
def check_interpolations(data: "DictStrAny", where: str, path: str):
def func(s: "DictStrAny") -> None:
if is_interpolated_string(s):
raise ResolveError(
_format_preamble(f"'{where}'", path)
@@ -126,7 +127,7 @@ Definition = Union["ForeachDefinition", "EntryDefinition"]
def make_definition(
resolver: "DataResolver", name: str, definition: DictStrAny, **kwargs
resolver: "DataResolver", name: str, definition: "DictStrAny", **kwargs
) -> Definition:
args = resolver, resolver.context, name, definition
if FOREACH_KWD in definition:
@@ -225,7 +226,7 @@ class EntryDefinition:
resolver: DataResolver,
context: Context,
name: str,
definition: DictStrAny,
definition: "DictStrAny",
where: str = STAGES_KWD,
):
self.resolver = resolver
@@ -254,7 +255,7 @@ class EntryDefinition:
except ContextError as exc:
format_and_raise(exc, f"stage '{self.name}'", self.relpath)
def resolve_stage(self, skip_checks: bool = False) -> DictStrAny:
def resolve_stage(self, skip_checks: bool = False) -> "DictStrAny":
context = self.context
name = self.name
if not skip_checks:
@@ -302,7 +303,7 @@ class EntryDefinition:
def _resolve(
self, context: "Context", value: Any, key: str, skip_checks: bool
) -> DictStrAny:
) -> "DictStrAny":
try:
return context.resolve(
value, skip_interpolation_checks=skip_checks, key=key
@@ -324,7 +325,7 @@ class ForeachDefinition:
resolver: DataResolver,
context: Context,
name: str,
definition: DictStrAny,
definition: "DictStrAny",
where: str = STAGES_KWD,
):
self.resolver = resolver
@@ -349,7 +350,7 @@ class ForeachDefinition:
def resolved_iterable(self):
return self._resolve_foreach_data()
def _resolve_foreach_data(self) -> SeqOrMap:
def _resolve_foreach_data(self) -> "SeqOrMap":
try:
iterable = self.context.resolve(self.foreach_data, unwrap=False)
except (ContextError, ParseError) as exc:
@@ -417,13 +418,13 @@ class ForeachDefinition:
def _generate_name(self, key: str) -> str:
return f"{self.name}{JOIN}{key}"
def resolve_all(self) -> DictStrAny:
def resolve_all(self) -> "DictStrAny":
return join(map(self.resolve_one, self.normalized_iterable))
def resolve_one(self, key: str) -> DictStrAny:
def resolve_one(self, key: str) -> "DictStrAny":
return self._each_iter(key)
def _each_iter(self, key: str) -> DictStrAny:
def _each_iter(self, key: str) -> "DictStrAny":
err_message = f"Could not find '{key}' in foreach group '{self.name}'"
with reraise(KeyError, EntryNotFound(err_message)):
value = self.normalized_iterable[key]
dvc/proc/manager.py
+1 -1
View File
@@ -92,7 +92,7 @@ class ProcessManager:
def handle_closed_process():
logging.warning(
f"Process {name} had already aborted unexpectedly."
"Process %s had already aborted unexpectedly.", name
)
process_info.returncode = -1
self[name] = process_info
dvc/prompt.py
+1 -2
View File
@@ -25,8 +25,7 @@ def ask(prompt: str, limited_to: Optional[Collection[str]] = None):
return answer
logger.info(
"Your response must be one of: {options}. "
"Please try again.".format(options=limited_to)
"Your response must be one of: %s. Please try again.", limited_to
)
dvc/repo/__init__.py
+1 -1
View File
@@ -275,7 +275,7 @@ class Repo:
new.check_graph()
@staticmethod
def open(url, *args, **kwargs):
def open(url, *args, **kwargs): # noqa: A003
if url is None:
url = os.getcwd()
dvc/repo/experiments/__init__.py
+15 -13
View File
@@ -2,7 +2,7 @@ import logging
import os
import re
import time
from typing import Dict, Iterable, Optional
from typing import TYPE_CHECKING, Dict, Iterable, Optional
from funcy import chain, first
@@ -18,7 +18,6 @@ from .exceptions import (
MultipleBranchError,
)
from .executor.base import BaseExecutor
from .queue.base import BaseStashQueue, QueueEntry
from .queue.celery import LocalCeleryQueue
from .queue.tempdir import TempDirQueue
from .queue.workspace import WorkspaceQueue
@@ -32,9 +31,12 @@ from .refs import (
WORKSPACE_STASH,
ExpRefInfo,
)
from .stash import ExpStashEntry
from .utils import check_ref_format, exp_refs_by_rev, unlocked_repo
if TYPE_CHECKING:
from .queue.base import BaseStashQueue, QueueEntry
from .stash import ExpStashEntry
logger = logging.getLogger(__name__)
@@ -85,7 +87,7 @@ class Experiments:
return LocalCeleryQueue(self.repo, CELERY_STASH, CELERY_FAILED_STASH)
@property
def stash_revs(self) -> Dict[str, ExpStashEntry]:
def stash_revs(self) -> Dict[str, "ExpStashEntry"]:
revs = {}
for queue in (self.workspace_queue, self.celery_queue):
revs.update(queue.stash.stash_revs)
@@ -107,7 +109,7 @@ class Experiments:
)
self.scm.reset()
exp_queue: BaseStashQueue = (
exp_queue: "BaseStashQueue" = (
self.tempdir_queue if tmp_dir else self.workspace_queue
)
self.queue_one(exp_queue, **kwargs)
@@ -119,11 +121,11 @@ class Experiments:
def queue_one(
self,
queue: BaseStashQueue,
queue: "BaseStashQueue",
checkpoint_resume: Optional[str] = None,
reset: bool = False,
**kwargs,
) -> QueueEntry:
) -> "QueueEntry":
"""Queue a single experiment."""
if reset:
self.reset_checkpoints()
@@ -162,7 +164,7 @@ class Experiments:
return None
def reproduce_celery( # noqa: C901
self, entries: Optional[Iterable[QueueEntry]] = None, **kwargs
self, entries: Optional[Iterable["QueueEntry"]] = None, **kwargs
) -> Dict[str, str]:
results: Dict[str, str] = {}
if entries is None:
@@ -238,11 +240,11 @@ class Experiments:
def new(
self,
queue: BaseStashQueue,
queue: "BaseStashQueue",
*args,
checkpoint_resume: Optional[str] = None,
**kwargs,
) -> QueueEntry:
) -> "QueueEntry":
"""Create and enqueue a new experiment.
Experiment will be derived from the current workspace.
@@ -266,11 +268,11 @@ class Experiments:
def _resume_checkpoint(
self,
queue: BaseStashQueue,
queue: "BaseStashQueue",
*args,
resume_rev: Optional[str] = None,
**kwargs,
) -> QueueEntry:
) -> "QueueEntry":
"""Create and queue a resumed checkpoint experiment."""
assert resume_rev
@@ -340,7 +342,7 @@ class Experiments:
@unlocked_repo
def _reproduce_queue(
self, queue: BaseStashQueue, **kwargs
self, queue: "BaseStashQueue", **kwargs
) -> Dict[str, str]:
"""Reproduce queued experiments.
dvc/repo/experiments/executor/local.py
+4 -2
View File
@@ -18,7 +18,6 @@ from dvc.repo.experiments.refs import (
EXEC_MERGE,
EXEC_NAMESPACE,
EXPS_TEMP,
ExpRefInfo,
)
from dvc.repo.experiments.utils import EXEC_TMP_DIR, get_exp_rwlock
from dvc.scm import SCM, GitMergeError
@@ -31,6 +30,7 @@ if TYPE_CHECKING:
from scmrepo.git import Git
from dvc.repo import Repo
from dvc.repo.experiments.refs import ExpRefInfo
from dvc.repo.experiments.stash import ExpStashEntry
from dvc.scm import NoSCM
@@ -150,7 +150,9 @@ class TempDirExecutor(BaseLocalExecutor):
with open(local_config, "w", encoding="utf-8") as fobj:
fobj.write(f"[cache]\n dir = {cache_dir}")
def init_cache(self, repo: "Repo", rev: str, run_cache: bool = True):
def init_cache(
self, repo: "Repo", rev: str, run_cache: bool = True # noqa: ARG002
):
"""Initialize DVC cache."""
self._config(repo.odb.repo.path)
dvc/repo/experiments/executor/ssh.py
+1 -1
View File
@@ -121,7 +121,7 @@ class SSHExecutor(BaseExecutor):
def init_git(
self,
repo: "Repo",
repo: "Repo", # noqa: ARG002
scm: "Git",
stash_rev: str,
entry: "ExpStashEntry",
dvc/repo/experiments/init.py
+1 -2
View File
@@ -19,7 +19,6 @@ from funcy import compact, lremove, lsplit
from dvc.exceptions import DvcException
from dvc.stage import PipelineStage
from dvc.types import OptStr
if TYPE_CHECKING:
from dvc.dvcfile import ProjectFile, SingleStageFile
@@ -47,7 +46,7 @@ def _prompts(
] = None,
allow_omission: bool = True,
stream: Optional[TextIO] = None,
) -> Dict[str, OptStr]:
) -> Dict[str, Optional[str]]:
from dvc.ui.prompt import Prompt
defaults = defaults or {}
dvc/repo/experiments/ls.py
+1 -1
View File
@@ -1,10 +1,10 @@
import logging
from collections import defaultdict
from typing import Optional
from dvc.repo import locked
from dvc.repo.scm_context import scm_context
from dvc.scm import iter_revs
from dvc.types import Optional
from .utils import exp_refs_by_baseline
dvc/repo/experiments/queue/base.py
+7 -6
View File
@@ -27,7 +27,7 @@ from dvc.repo.experiments.exceptions import (
CheckpointExistsError,
ExperimentExistsError,
)
from dvc.repo.experiments.executor.base import BaseExecutor, ExecutorResult
from dvc.repo.experiments.executor.base import BaseExecutor
from dvc.repo.experiments.executor.local import WorkspaceExecutor
from dvc.repo.experiments.refs import ExpRefInfo
from dvc.repo.experiments.stash import ExpStash, ExpStashEntry
@@ -48,6 +48,7 @@ if TYPE_CHECKING:
from dvc.repo import Repo
from dvc.repo.experiments import Experiments
from dvc.repo.experiments.executor.base import ExecutorResult
logger = logging.getLogger(__name__)
@@ -87,7 +88,7 @@ class QueueGetResult(NamedTuple):
class QueueDoneResult(NamedTuple):
entry: QueueEntry
result: Optional[ExecutorResult]
result: Optional["ExecutorResult"]
class ExpRefAndQueueEntry(NamedTuple):
@@ -200,7 +201,7 @@ class BaseStashQueue(ABC):
def _format_entry(
entry: QueueEntry,
exp_result: Optional[ExecutorResult] = None,
exp_result: Optional["ExecutorResult"] = None,
status: str = "Unknown",
) -> Dict[str, Any]:
name = entry.name
@@ -258,7 +259,7 @@ class BaseStashQueue(ABC):
"""Reproduce queued experiments sequentially."""
@abstractmethod
def get_result(self, entry: QueueEntry) -> Optional[ExecutorResult]:
def get_result(self, entry: QueueEntry) -> Optional["ExecutorResult"]:
"""Return result of the specified item.
This method blocks until the specified item has been collected.
@@ -632,7 +633,7 @@ class BaseStashQueue(ABC):
def collect_git(
exp: "Experiments",
executor: BaseExecutor,
exec_result: ExecutorResult,
exec_result: "ExecutorResult",
) -> Dict[str, str]:
results = {}
@@ -664,7 +665,7 @@ class BaseStashQueue(ABC):
cls,
exp: "Experiments",
executor: BaseExecutor,
exec_result: ExecutorResult,
exec_result: "ExecutorResult",
) -> Dict[str, str]:
results = cls.collect_git(exp, executor, exec_result)
dvc/repo/experiments/queue/celery.py
+16 -13
View File
@@ -17,7 +17,6 @@ from typing import (
from celery.result import AsyncResult
from funcy import first
from kombu.message import Message
from dvc.daemon import daemonize
from dvc.exceptions import DvcException
@@ -25,7 +24,7 @@ from dvc.repo.experiments.exceptions import (
UnresolvedQueueExpNamesError,
UnresolvedRunningExpNamesError,
)
from dvc.repo.experiments.executor.base import ExecutorInfo, ExecutorResult
from dvc.repo.experiments.executor.base import ExecutorInfo
from dvc.repo.experiments.refs import CELERY_STASH
from dvc.repo.experiments.utils import EXEC_TMP_DIR, get_exp_rwlock
from dvc.ui import ui
@@ -36,23 +35,27 @@ from .base import (
ExpRefAndQueueEntry,
QueueDoneResult,
QueueEntry,
QueueGetResult,
)
from .exceptions import CannotKillTasksError
from .tasks import run_exp
from .utils import fetch_running_exp_from_temp_dir
if TYPE_CHECKING:
from kombu.message import Message
from dvc.repo.experiments.executor.base import ExecutorResult
from dvc.repo.experiments.refs import ExpRefInfo
from dvc_task.app import FSApp
from dvc_task.proc.manager import ProcessManager
from dvc_task.worker import TemporaryWorker
from .base import QueueGetResult
logger = logging.getLogger(__name__)
class _MessageEntry(NamedTuple):
msg: Message
msg: "Message"
entry: QueueEntry
@@ -142,7 +145,7 @@ class LocalCeleryQueue(BaseStashQueue):
cmd = ["exp", "queue-worker", node_name]
name = f"dvc-exp-worker-{num}"
logger.debug(f"start a new worker: {name}, node: {node_name}")
logger.debug("start a new worker: %s, node: %s", name, node_name)
if os.name == "nt":
daemonize(cmd)
else:
@@ -158,7 +161,7 @@ class LocalCeleryQueue(BaseStashQueue):
newly spawned worker number.
"""
logger.debug(f"Spawning {count} exp queue workers")
logger.debug("Spawning %s exp queue workers", count)
active_worker: Dict = self.worker_status()
started = 0
@@ -168,7 +171,7 @@ class LocalCeleryQueue(BaseStashQueue):
]
node_name = f"dvc-exp-{wdir_hash}-{num}@localhost"
if node_name in active_worker:
logger.debug(f"Exp queue worker {node_name} already exist")
logger.debug("Exp queue worker %s already exist", node_name)
continue
self._spawn_worker(num)
started += 1
@@ -184,7 +187,7 @@ class LocalCeleryQueue(BaseStashQueue):
# NOTE: Queue consumption should not be done directly. Celery worker(s)
# will automatically consume available experiments.
def get(self) -> QueueGetResult:
def get(self) -> "QueueGetResult":
raise NotImplementedError
def iter_queued(self) -> Generator[QueueEntry, None, None]:
@@ -330,7 +333,7 @@ class LocalCeleryQueue(BaseStashQueue):
) -> None:
remained_revs: List[str] = []
running_ids = self._get_running_task_ids()
logger.debug(f"Current running tasks ids: {running_ids}.")
logger.debug("Current running tasks ids: %s.", running_ids)
for msg, entry in self._iter_processed():
if entry not in remained_entries:
continue
@@ -341,8 +344,9 @@ class LocalCeleryQueue(BaseStashQueue):
result: AsyncResult = AsyncResult(task_id)
if not result.ready():
logger.debug(
f"Task id {task_id} rev {remained_entries[entry]} "
"marked as failure."
"Task id %s rev %s marked as failure.",
task_id,
remained_entries[entry],
)
self.celery.backend.mark_as_failure(task_id, None)
@@ -441,8 +445,7 @@ class LocalCeleryQueue(BaseStashQueue):
def worker_status(self) -> Dict[str, List[Dict]]:
"""Return the current active celery worker"""
status = self.celery.control.inspect().active() or {}
logger.debug(f"Worker status: {status}")
logger.debug("Worker status: %s", status)
return status
def clear(self, *args, **kwargs):
dvc/repo/experiments/queue/tasks.py
+1 -1
View File
@@ -74,7 +74,7 @@ def collect_exp(
else:
logger.debug("Experiment failed (Exec result was None)")
celery_queue.stash_failed(entry)
except Exception: # noqa: BLE001, pylint: disable=broad-except
except Exception: # pylint: disable=broad-except
# Log exceptions but do not re-raise so that task chain execution
# continues
logger.exception("Failed to collect experiment")
dvc/repo/experiments/queue/tempdir.py
+5 -9
View File
@@ -7,12 +7,7 @@ from funcy import first
from dvc.exceptions import DvcException
from dvc.repo.experiments.exceptions import ExpQueueEmptyError
from dvc.repo.experiments.executor.base import (
BaseExecutor,
ExecutorInfo,
ExecutorResult,
TaskStatus,
)
from dvc.repo.experiments.executor.base import ExecutorInfo, TaskStatus
from dvc.repo.experiments.executor.local import TempDirExecutor
from dvc.repo.experiments.utils import EXEC_PID_DIR, EXEC_TMP_DIR
from dvc.utils.objects import cached_property
@@ -23,6 +18,7 @@ from .workspace import WorkspaceQueue
if TYPE_CHECKING:
from dvc.repo.experiments import Experiments
from dvc.repo.experiments.executor.base import BaseExecutor, ExecutorResult
from dvc_task.proc.manager import ProcessManager
logger = logging.getLogger(__name__)
@@ -96,7 +92,7 @@ class TempDirQueue(WorkspaceQueue):
)
def _reproduce_entry(
self, entry: QueueEntry, executor: BaseExecutor
self, entry: QueueEntry, executor: "BaseExecutor"
) -> Dict[str, Dict[str, str]]:
from dvc.stage.monitor import CheckpointKilledError
@@ -143,8 +139,8 @@ class TempDirQueue(WorkspaceQueue):
@staticmethod
def collect_executor(
exp: "Experiments",
executor: BaseExecutor,
exec_result: ExecutorResult,
executor: "BaseExecutor",
exec_result: "ExecutorResult",
) -> Dict[str, str]:
return BaseStashQueue.collect_executor(exp, executor, exec_result)
dvc/repo/experiments/queue/workspace.py
+16 -15
View File
@@ -10,12 +10,7 @@ from voluptuous import Invalid
from dvc.exceptions import DvcException
from dvc.lock import make_lock
from dvc.repo.experiments.exceptions import ExpQueueEmptyError
from dvc.repo.experiments.executor.base import (
BaseExecutor,
ExecutorInfo,
ExecutorResult,
TaskStatus,
)
from dvc.repo.experiments.executor.base import ExecutorInfo, TaskStatus
from dvc.repo.experiments.executor.local import WorkspaceExecutor
from dvc.repo.experiments.refs import EXEC_BRANCH, WORKSPACE_STASH
from dvc.repo.experiments.utils import get_exp_rwlock, to_studio_params
@@ -23,10 +18,13 @@ from dvc.rwlock import RWLOCK_FILE, RWLOCK_LOCK, SCHEMA
from dvc.utils import relpath
from dvc.utils.fs import remove
from .base import BaseStashQueue, QueueDoneResult, QueueEntry, QueueGetResult
from .base import BaseStashQueue, QueueEntry, QueueGetResult
if TYPE_CHECKING:
from dvc.repo.experiments import Experiments
from dvc.repo.experiments.executor.base import BaseExecutor, ExecutorResult
from .base import QueueDoneResult
logger = logging.getLogger(__name__)
@@ -74,13 +72,13 @@ class WorkspaceQueue(BaseStashQueue):
# need to be handled via the queue
raise NotImplementedError
def iter_done(self) -> Generator[QueueDoneResult, None, None]:
def iter_done(self) -> Generator["QueueDoneResult", None, None]:
raise NotImplementedError
def iter_failed(self) -> Generator[QueueDoneResult, None, None]:
def iter_failed(self) -> Generator["QueueDoneResult", None, None]:
raise NotImplementedError
def iter_success(self) -> Generator[QueueDoneResult, None, None]:
def iter_success(self) -> Generator["QueueDoneResult", None, None]:
raise NotImplementedError
def reproduce(self) -> Dict[str, Dict[str, str]]:
@@ -94,7 +92,7 @@ class WorkspaceQueue(BaseStashQueue):
return results
def _reproduce_entry(
self, entry: QueueEntry, executor: BaseExecutor
self, entry: QueueEntry, executor: "BaseExecutor"
) -> Dict[str, Dict[str, str]]:
from dvc_studio_client.post_live_metrics import post_live_metrics
@@ -152,8 +150,8 @@ class WorkspaceQueue(BaseStashQueue):
@staticmethod
def collect_executor( # pylint: disable=unused-argument
exp: "Experiments",
executor: BaseExecutor, # noqa: ARG004
exec_result: ExecutorResult,
executor: "BaseExecutor", # noqa: ARG004
exec_result: "ExecutorResult",
) -> Dict[str, str]:
results: Dict[str, str] = {}
exp_rev = exp.scm.get_ref(EXEC_BRANCH)
@@ -164,7 +162,7 @@ class WorkspaceQueue(BaseStashQueue):
return results
def get_result(self, entry: QueueEntry) -> Optional[ExecutorResult]:
def get_result(self, entry: QueueEntry) -> Optional["ExecutorResult"]:
raise NotImplementedError
def kill(self, revs: Collection[str]) -> None:
@@ -239,7 +237,10 @@ class WorkspaceQueue(BaseStashQueue):
remove(path)
return False
def get_running_exps(self, fetch_refs: bool = True) -> Dict[str, Dict]:
def get_running_exps(
self,
fetch_refs: bool = True, # noqa: ARG002
) -> Dict[str, Dict]:
from dvc.utils.serialize import load_json
assert self._EXEC_NAME
dvc/repo/experiments/remove.py
+3 -4
View File
@@ -6,8 +6,6 @@ from dvc.repo.scm_context import scm_context
from dvc.scm import iter_revs
from .exceptions import UnresolvedExpNamesError
from .queue.base import QueueEntry
from .refs import ExpRefInfo
from .utils import exp_refs, exp_refs_by_baseline, push_refspec
if TYPE_CHECKING:
@@ -15,7 +13,8 @@ if TYPE_CHECKING:
from dvc.repo.experiments.queue.celery import LocalCeleryQueue
from dvc.scm import Git
from .queue.base import ExpRefAndQueueEntry
from .queue.base import ExpRefAndQueueEntry, QueueEntry
from .refs import ExpRefInfo
logger = logging.getLogger(__name__)
@@ -99,7 +98,7 @@ def _resolve_exp_by_baseline(
def _remove_commited_exps(
scm: "Git", exp_refs_list: Iterable[ExpRefInfo], remote: Optional[str]
scm: "Git", exp_refs_list: Iterable["ExpRefInfo"], remote: Optional[str]
) -> List[str]:
if remote:
from dvc.scm import TqdmGit
dvc/repo/metrics/show.py
+6 -4
View File
@@ -1,11 +1,10 @@
import logging
import os
from typing import List
from typing import TYPE_CHECKING, List
from scmrepo.exceptions import SCMError
from dvc.fs.dvc import DVCFileSystem
from dvc.output import Output
from dvc.repo import locked
from dvc.repo.collect import collect
from dvc.scm import NoSCMError
@@ -18,14 +17,17 @@ from dvc.utils import (
from dvc.utils.collections import ensure_list
from dvc.utils.serialize import load_path
if TYPE_CHECKING:
from dvc.output import Output
logger = logging.getLogger(__name__)
def _is_metric(out: Output) -> bool:
def _is_metric(out: "Output") -> bool:
return bool(out.metric)
def _to_fs_paths(metrics: List[Output]) -> List["str"]:
def _to_fs_paths(metrics: List["Output"]) -> List["str"]:
result = []
for out in metrics:
if out.metric:
dvc/repo/plots/__init__.py
+2 -2
View File
@@ -32,7 +32,7 @@ if TYPE_CHECKING:
from dvc.fs import FileSystem
from dvc.output import Output
from dvc.repo import Repo
from dvc.types import DictAny, DictStrAny, StrPath
from dvc.types import DictStrAny, StrPath
dpath.options.ALLOW_EMPTY_STRING_KEYS = True
@@ -424,7 +424,7 @@ def _adjust_sources(fs, plot_props, config_dir):
def _resolve_definitions(
fs: "FileSystem",
targets: List[str],
props: "DictAny",
props: Dict[str, Any],
config_path: "StrPath",
definitions: "DictStrAny",
onerror: Optional[Callable[[Any], Any]] = None,
dvc/repo/scm_context.py
+1 -1
View File
@@ -133,7 +133,7 @@ class SCMContext:
and logger.isEnabledFor(logging.INFO)
):
add_cmd = self._make_git_add_cmd(self.files_to_track)
logger.info(f"\nTo track the changes with git, run:\n\n{add_cmd}")
logger.info("\nTo track the changes with git, run:\n\n%s", add_cmd)
logger.info(
"\nTo enable auto staging, run:\n\n"
"\tdvc config core.autostage true"
dvc/repo/stage.py
+1 -2
View File
@@ -18,7 +18,6 @@ if typing.TYPE_CHECKING:
from dvc.repo import Repo
from dvc.stage import PipelineStage, Stage
from dvc.stage.loader import StageLoader
from dvc.types import OptStr
PROJECT_FILE = "dvc.yaml"
@@ -61,7 +60,7 @@ def _collect_specific_target(
target: str,
with_deps: bool,
recursive: bool,
) -> Tuple[StageIter, "OptStr", "OptStr"]:
) -> Tuple[StageIter, Optional[str], Optional[str]]:
from dvc.dvcfile import is_valid_filename
# Optimization: do not collect the graph for a specific target
dvc/stage/loader.py
+1 -1
View File
@@ -60,7 +60,7 @@ class StageLoader(Mapping):
from dvc.output import merge_file_meta_from_cloud
assert isinstance(lock_data, dict)
items: Iterable[Tuple[str, Output]] = chain(
items: Iterable[Tuple[str, "Output"]] = chain(
((StageParams.PARAM_DEPS, dep) for dep in stage.deps),
((StageParams.PARAM_OUTS, out) for out in stage.outs),
)
dvc/stage/monitor.py
+1 -1
View File
@@ -110,7 +110,7 @@ class Monitor:
try:
task.execute()
task.updated.set()
except Exception: # noqa: BLE001, pylint: disable=W0703
except Exception: # pylint: disable=broad-except
logger.exception(
"Error running '%s' task, '%s' will be aborted",
task.name,
dvc/types.py
+1 -3
View File
@@ -1,4 +1,4 @@
from typing import TYPE_CHECKING, Any, AnyStr, Dict, List, Optional, Union
from typing import TYPE_CHECKING, Any, AnyStr, Dict, List, Union
if TYPE_CHECKING:
from os import PathLike
@@ -8,7 +8,5 @@ BytesPath = Union[bytes, "PathLike[bytes]"]
GenericPath = Union[AnyStr, "PathLike[AnyStr]"]
StrOrBytesPath = Union[str, bytes, "PathLike[str]", "PathLike[bytes]"]
OptStr = Optional[str]
TargetType = Union[List[str], str]
DictStrAny = Dict[str, Any]
DictAny = Dict[Any, Any]
dvc/ui/__init__.py
+1 -1
View File
@@ -38,7 +38,7 @@ class Formatter:
}
self.theme = defaultdict(lambda: defaults or {}, theme)
def format(
def format( # noqa: A003
self, message: str, style: Optional[str] = None, **kwargs
) -> str:
from dvc.utils import colorize
pyproject.toml
+13 -10
View File
@@ -161,26 +161,31 @@ line_length = 79
line-length = 79
target-version = "py38"
show-source = true
ignore = ["S101", "SIM105", "SIM108", "SIM110", "SIM111", "SIM117", "A003", "ARG002", "PLR2004", "RUF005"]
select = ["F", "E", "W", "C90", "UP", "YTT", "S", "BLE", "B", "A", "C4", "T10", "ISC", "ICN", "T20", "PT003", "PT008", "PT009", "PT010", "PT015", "PT016", "PT017", "PT020", "PT021", "PT024", "PT025", "PT026", "Q", "RET501", "RET504", "SIM", "ARG", "TID", "PGH", "PLC", "PLE", "PLR", "PLW", "PIE", "INP", "TRY004", "TRY201", "RUF"]
ignore = ["S101", "SIM105", "SIM108", "SIM110", "SIM111", "SIM117", "PLR2004", "RUF005"]
select = ["F", "E", "W", "C90", "UP", "YTT", "S", "BLE", "B", "A", "C4", "T10", "ISC", "ICN", "G", "INP", "PIE", "T20", "PT003", "PT008", "PT009", "PT010", "PT015", "PT016", "PT017", "PT020", "PT021", "PT024", "PT025", "PT026", "Q", "RET501", "RET504", "SIM", "TID", "TCH", "ARG", "PGH", "PLC", "PLE", "PLR", "PLW", "TRY002", "TRY004", "TRY201", "TRY400", "RUF"]
# external flake8 codes that should be preserved
external = ["B301", "B024", "C901", "E302", "F401", "W601"]
[tool.ruff.per-file-ignores]
"scripts/**/*.py" = ["T201", "INP001"]
"dvc/testing/*.py" = ["ARG002"]
"scripts/**/*.py" = ["T201", "INP001", "TRY002"]
[tool.ruff.flake8-pytest-style]
fixture-parentheses = false
mark-parentheses = false
parametrize-names-type = "csv"
[tool.ruff.flake8-unused-arguments]
ignore-variadic-names = true
[tool.ruff.flake8-tidy-imports]
[tool.ruff.flake8-tidy-imports.banned-api]
"funcy.cached_property".msg = "use `from dvc.utils.objects import cached_property` instead."
[tool.ruff.flake8-type-checking]
exempt-modules = ["typing"]
strict = true
[tool.ruff.flake8-unused-arguments]
ignore-variadic-names = true
[tool.pytest.ini_options]
log_level = "debug"
addopts = "-ra --cov-config pyproject.toml"
@@ -264,11 +269,9 @@ ignore_missing_imports = true
[tool.pylint.message_control]
disable = [
"format", "refactoring", "spelling", "design",
"invalid-name", "duplicate-code", "fixme",
"unused-wildcard-import", "cyclic-import", "wrong-import-order",
"format", "refactoring", "spelling", "design", "invalid-name", "fixme",
"duplicate-code", "unused-wildcard-import", "cyclic-import", "wrong-import-order",
"wrong-import-position", "ungrouped-imports", "multiple-imports",
"logging-format-interpolation", "logging-fstring-interpolation",
"missing-function-docstring", "missing-module-docstring",
"missing-class-docstring", "raise-missing-from", "import-outside-toplevel",
]
tests/func/test_ignore.py
+1 -1
View File
@@ -1,6 +1,7 @@
import os
import shutil
from pathlib import Path
from typing import List
import pytest
@@ -9,7 +10,6 @@ from dvc.output import OutputIsIgnoredError
from dvc.pathspec_math import PatternInfo, merge_patterns
from dvc.repo import Repo
from dvc.testing.tmp_dir import TmpDir
from dvc.types import List
from dvc_data.hashfile.build import IgnoreInCollectedDirError
from dvc_data.hashfile.utils import get_mtime_and_size
tests/ruff.toml
+1 -1
View File
@@ -1,3 +1,3 @@
extend = "../pyproject.toml"
extend-select = ["RUF005"]
extend-ignore = ["S", "RUF001", "ARG001", "ARG002"]
extend-ignore = ["S", "RUF001", "ARG001", "ARG002", "TRY002"]
tests/unit/repo/experiments/conftest.py
+1 -3
View File
@@ -3,8 +3,6 @@ from textwrap import dedent
import pytest
from dvc.repo.experiments.queue.celery import LocalCeleryQueue
DEFAULT_ITERATIONS = 2
CHECKPOINT_SCRIPT_FORMAT = dedent(
"""\
@@ -156,7 +154,7 @@ def _thread_worker(app, **kwargs):
@pytest.fixture
def test_queue(tmp_dir, dvc, scm, mocker) -> LocalCeleryQueue:
def test_queue(tmp_dir, dvc, scm, mocker):
"""Patches experiments celery queue for pytest testing.
Test queue worker runs for the duration of the test in separate thread(s).
tests/unit/repo/experiments/test_show.py
+1 -2
View File
@@ -1,6 +1,5 @@
import pytest
from funcy import first
from scmrepo.git import Git
from dvc.repo.experiments.show import (
ExpStatus,
@@ -10,7 +9,7 @@ from dvc.repo.experiments.show import (
)
def test_get_show_branch(tmp_dir, scm: "Git", dvc, exp_stage):
def test_get_show_branch(tmp_dir, scm, dvc, exp_stage):
new_branch = "new"
baseline = scm.get_rev()
tests/unit/test_logger.py
+5 -5
View File
@@ -65,7 +65,7 @@ class TestColorFormatter:
with caplog.at_level(logging.INFO, logger="dvc"):
try:
raise ValueError
except Exception: # noqa: BLE001
except Exception:
logger.exception("message")
expected = "{red}ERROR{nc}: message".format(**colors)
@@ -76,7 +76,7 @@ class TestColorFormatter:
with caplog.at_level(logging.INFO, logger="dvc"):
try:
raise Exception("description")
except Exception: # noqa: BLE001
except Exception:
logger.exception("")
expected = "{red}ERROR{nc}: description".format(**colors)
@@ -87,7 +87,7 @@ class TestColorFormatter:
with caplog.at_level(logging.INFO, logger="dvc"):
try:
raise Exception("description")
except Exception: # noqa: BLE001
except Exception:
logger.exception("message")
expected = "{red}ERROR{nc}: message - description".format(**colors)
@@ -98,7 +98,7 @@ class TestColorFormatter:
with caplog.at_level(logging.DEBUG, logger="dvc"):
try:
raise Exception("description")
except Exception: # noqa: BLE001
except Exception:
stack_trace = traceback.format_exc()
logger.exception("")
@@ -144,7 +144,7 @@ class TestColorFormatter:
with caplog.at_level(logging.DEBUG, logger="dvc"):
try:
raise Exception("description")
except Exception: # noqa: BLE001
except Exception:
stack_trace = traceback.format_exc()
logger.exception("something", extra={"tb_only": True})
tests/unit/ui/test_console.py
+3 -7
View File
@@ -2,13 +2,11 @@ import datetime
import textwrap
import pytest
from pytest import CaptureFixture
from pytest_mock import MockerFixture
from dvc.ui import Console
def test_write(capsys: CaptureFixture[str]):
def test_write(capsys):
"""Test that ui.write works."""
console = Console(enable=True)
message = "hello world"
@@ -44,9 +42,7 @@ def test_write(capsys: CaptureFixture[str]):
),
],
)
def test_write_json(
capsys: CaptureFixture[str], mocker: MockerFixture, isatty, expected_output
):
def test_write_json(capsys, mocker, isatty, expected_output):
"""Test that ui.write_json works."""
console = Console(enable=True)
@@ -57,7 +53,7 @@ def test_write_json(
assert captured.out == expected_output
def test_capsys_works(capsys: CaptureFixture[str]):
def test_capsys_works(capsys):
"""Sanity check that capsys can capture outputs from a global ui."""
from dvc.ui import ui
tests/unit/ui/test_table.py
+11 -13
View File
@@ -1,14 +1,12 @@
import textwrap
import pytest
from pytest import CaptureFixture
from pytest_mock import MockerFixture
from rich.style import Style
from dvc.ui import ui
def test_plain(capsys: CaptureFixture[str]):
def test_plain(capsys):
ui.table(
[("foo", "bar"), ("foo1", "bar1"), ("foo2", "bar2")],
headers=["first", "second"],
@@ -24,7 +22,7 @@ def test_plain(capsys: CaptureFixture[str]):
)
def test_plain_md(capsys: CaptureFixture[str]):
def test_plain_md(capsys):
ui.table(
[("foo", "bar"), ("foo1", "bar1"), ("foo2", "bar2")],
headers=["first", "second"],
@@ -42,7 +40,7 @@ def test_plain_md(capsys: CaptureFixture[str]):
)
def test_plain_pager(mocker: MockerFixture):
def test_plain_pager(mocker):
pager_mock = mocker.patch("dvc.ui.pager.pager")
ui.table(
[("foo", "bar"), ("foo1", "bar1"), ("foo2", "bar2")],
@@ -62,7 +60,7 @@ def test_plain_pager(mocker: MockerFixture):
)
def test_plain_headerless(capsys: CaptureFixture[str]):
def test_plain_headerless(capsys):
ui.table([("foo", "bar"), ("foo1", "bar1"), ("foo2", "bar2")])
captured = capsys.readouterr()
assert captured.out == textwrap.dedent(
@@ -74,7 +72,7 @@ def test_plain_headerless(capsys: CaptureFixture[str]):
)
def test_rich_simple(capsys: CaptureFixture[str]):
def test_rich_simple(capsys):
ui.table(
[("foo", "bar"), ("foo1", "bar1"), ("foo2", "bar2")],
headers=["first", "second"],
@@ -87,7 +85,7 @@ def test_rich_simple(capsys: CaptureFixture[str]):
] == ["first second", "foo bar", "foo1 bar1", "foo2 bar2"]
def test_rich_headerless(capsys: CaptureFixture[str]):
def test_rich_headerless(capsys):
ui.table(
[("foo", "bar"), ("foo1", "bar1"), ("foo2", "bar2")], rich_table=True
)
@@ -97,7 +95,7 @@ def test_rich_headerless(capsys: CaptureFixture[str]):
] == ["foo bar", "foo1 bar1", "foo2 bar2"]
def test_rich_border(capsys: CaptureFixture[str]):
def test_rich_border(capsys):
ui.table(
[("foo", "bar"), ("foo1", "bar1"), ("foo2", "bar2")],
headers=["first", "second"],
@@ -124,7 +122,7 @@ def test_rich_border(capsys: CaptureFixture[str]):
{"row_styles": [{"style": Style(bold=True)}]},
],
)
def test_rich_styles(capsys: CaptureFixture[str], extra_opts):
def test_rich_styles(capsys, extra_opts):
ui.table(
[("foo", "bar"), ("foo1", "bar1"), ("foo2", "bar2")],
headers=["first", "second"],
@@ -138,7 +136,7 @@ def test_rich_styles(capsys: CaptureFixture[str], extra_opts):
] == ["first second", "foo bar", "foo1 bar1", "foo2 bar2"]
def test_rich_pager(mocker: MockerFixture):
def test_rich_pager(mocker):
pager_mock = mocker.patch("dvc.ui.pager.pager")
ui.table(
@@ -154,13 +152,13 @@ def test_rich_pager(mocker: MockerFixture):
@pytest.mark.parametrize("rich_table", [True, False])
def test_empty(capsys: CaptureFixture[str], rich_table: str):
def test_empty(capsys, rich_table):
ui.table([], rich_table=rich_table)
out, err = capsys.readouterr()
assert (out, err) == ("", "")
def test_empty_markdown(capsys: CaptureFixture[str]):
def test_empty_markdown(capsys):
ui.table([], headers=["Col1", "Col2"], markdown=True)
out, err = capsys.readouterr()
assert (out, err) == ("| Col1 | Col2 |\n|--------|--------|\n\n", "")
tests/utils/asserts.py
+6 -3
View File
@@ -1,5 +1,8 @@
from typing import Any, Dict
from unittest.mock import ANY, Mock
from typing import TYPE_CHECKING, Any, Dict
from unittest.mock import ANY
if TYPE_CHECKING:
from unittest.mock import Mock
def issubset(subset: Dict, superset: Dict) -> bool:
@@ -7,7 +10,7 @@ def issubset(subset: Dict, superset: Dict) -> bool:
return True
def called_once_with_subset(m: Mock, *args: Any, **kwargs: Any) -> bool:
def called_once_with_subset(m: "Mock", *args: Any, **kwargs: Any) -> bool:
m.assert_called_once()
m_args, m_kwargs = m.call_args