diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 35b26e2d4..7fb762fb4 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -25,7 +25,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.12 - run: python -m pip install --upgrade pip setuptools_scm build twine diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 8e8a9a3cc..6b52e7c8d 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -23,7 +23,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-20.04, windows-latest, macos-latest] - pyv: ["3.8", "3.9", "3.10", "3.11", "3.12"] + pyv: ["3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 diff --git a/dvc/annotations.py b/dvc/annotations.py index a46b0cca6..e443c802f 100644 --- a/dvc/annotations.py +++ b/dvc/annotations.py @@ -1,5 +1,5 @@ from dataclasses import asdict, dataclass, field, fields -from typing import Any, ClassVar, Dict, List, Optional +from typing import Any, ClassVar, Optional from funcy import compact from voluptuous import Required @@ -14,10 +14,10 @@ class Annotation: desc: Optional[str] = None type: Optional[str] = None - labels: List[str] = field(default_factory=list) - meta: Dict[str, Any] = field(default_factory=dict) + labels: list[str] = field(default_factory=list) + meta: dict[str, Any] = field(default_factory=dict) - def to_dict(self) -> Dict[str, str]: + def to_dict(self) -> dict[str, str]: return compact(asdict(self)) @@ -32,10 +32,10 @@ class Artifact: path: str desc: Optional[str] = None type: Optional[str] = None - labels: List[str] = field(default_factory=list) - meta: Dict[str, Any] = field(default_factory=dict) + labels: list[str] = field(default_factory=list) + meta: dict[str, Any] = field(default_factory=dict) - def to_dict(self) -> Dict[str, str]: + def to_dict(self) -> dict[str, str]: return compact(asdict(self)) @@ -46,7 +46,7 @@ ANNOTATION_SCHEMA = { Annotation.PARAM_LABELS: [str], Annotation.PARAM_META: object, } -ARTIFACT_SCHEMA: Dict[Any, Any] = { +ARTIFACT_SCHEMA: dict[Any, Any] = { Required(Artifact.PARAM_PATH): str, **ANNOTATION_SCHEMA, # type: ignore[arg-type] } diff --git a/dvc/api/artifacts.py b/dvc/api/artifacts.py index 33971bc60..687fc86a8 100644 --- a/dvc/api/artifacts.py +++ b/dvc/api/artifacts.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional +from typing import Any, Optional from dvc.repo import Repo @@ -8,7 +8,7 @@ def artifacts_show( version: Optional[str] = None, stage: Optional[str] = None, repo: Optional[str] = None, -) -> Dict[str, str]: +) -> dict[str, str]: """ Return path and Git revision for an artifact in a DVC project. @@ -36,7 +36,7 @@ def artifacts_show( if version and stage: raise ValueError("Artifact version and stage are mutually exclusive.") - repo_kwargs: Dict[str, Any] = { + repo_kwargs: dict[str, Any] = { "subrepos": True, "uninitialized": True, } diff --git a/dvc/api/data.py b/dvc/api/data.py index e40f9bbc8..d1c38b4fe 100644 --- a/dvc/api/data.py +++ b/dvc/api/data.py @@ -1,6 +1,6 @@ from contextlib import _GeneratorContextManager as GCM from contextlib import contextmanager -from typing import Any, Dict, Optional +from typing import Any, Optional from funcy import reraise @@ -39,7 +39,7 @@ def get_url(path, repo=None, rev=None, remote=None): from dvc.config import NoRemoteError from dvc_data.index import StorageKeyError - repo_kwargs: Dict[str, Any] = {} + repo_kwargs: dict[str, Any] = {} if remote: repo_kwargs["config"] = {"core": {"remote": remote}} with Repo.open( @@ -74,8 +74,8 @@ def open( # noqa: A001 remote: Optional[str] = None, mode: str = "r", encoding: Optional[str] = None, - config: Optional[Dict[str, Any]] = None, - remote_config: Optional[Dict[str, Any]] = None, + config: Optional[dict[str, Any]] = None, + remote_config: Optional[dict[str, Any]] = None, ): """ Opens a file tracked in a DVC project. @@ -233,7 +233,7 @@ def _open( config=None, remote_config=None, ): - repo_kwargs: Dict[str, Any] = { + repo_kwargs: dict[str, Any] = { "subrepos": True, "uninitialized": True, "remote": remote, diff --git a/dvc/api/experiments.py b/dvc/api/experiments.py index 4a1ead96e..ec99f832e 100644 --- a/dvc/api/experiments.py +++ b/dvc/api/experiments.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Optional, Union +from typing import Optional, Union from rich.text import Text @@ -9,7 +9,7 @@ from dvc.repo.experiments.show import tabulate def exp_save( name: Optional[str] = None, force: bool = False, - include_untracked: Optional[List[str]] = None, + include_untracked: Optional[list[str]] = None, ): """ Create a new DVC experiment using `exp save`. @@ -61,12 +61,12 @@ def _postprocess(exp_rows): def exp_show( repo: Optional[str] = None, - revs: Optional[Union[str, List[str]]] = None, + revs: Optional[Union[str, list[str]]] = None, num: int = 1, param_deps: bool = False, force: bool = False, - config: Optional[Dict] = None, -) -> List[Dict]: + config: Optional[dict] = None, +) -> list[dict]: """Get DVC experiments tracked in `repo`. Without arguments, this function will retrieve all experiments derived from diff --git a/dvc/api/scm.py b/dvc/api/scm.py index b3656bd7c..0c900e724 100644 --- a/dvc/api/scm.py +++ b/dvc/api/scm.py @@ -1,9 +1,9 @@ -from typing import List, Optional +from typing import Optional from dvc.repo import Repo -def all_branches(repo: Optional[str] = None) -> List[str]: +def all_branches(repo: Optional[str] = None) -> list[str]: """Get all Git branches in a DVC repository. Args: @@ -20,7 +20,7 @@ def all_branches(repo: Optional[str] = None) -> List[str]: return _repo.scm.list_branches() -def all_commits(repo: Optional[str] = None) -> List[str]: +def all_commits(repo: Optional[str] = None) -> list[str]: """Get all Git commits in a DVC repository. Args: @@ -37,7 +37,7 @@ def all_commits(repo: Optional[str] = None) -> List[str]: return _repo.scm.list_all_commits() -def all_tags(repo: Optional[str] = None) -> List[str]: +def all_tags(repo: Optional[str] = None) -> list[str]: """Get all Git tags in a DVC repository. Args: diff --git a/dvc/api/show.py b/dvc/api/show.py index 3323fff6b..bbf92a27b 100644 --- a/dvc/api/show.py +++ b/dvc/api/show.py @@ -1,6 +1,7 @@ import typing from collections import Counter -from typing import Dict, Iterable, Optional, Union +from collections.abc import Iterable +from typing import Optional, Union from funcy import first @@ -8,7 +9,7 @@ from dvc.repo import Repo def _postprocess(results): - processed: Dict[str, Dict] = {} + processed: dict[str, dict] = {} for rev, rev_data in results.items(): if not rev_data: continue @@ -36,8 +37,8 @@ def metrics_show( *targets: str, repo: Optional[str] = None, rev: Optional[str] = None, - config: Optional[Dict] = None, -) -> Dict: + config: Optional[dict] = None, +) -> dict: """Get metrics tracked in `repo`. Without arguments, this function will retrieve all metrics from all tracked @@ -159,8 +160,8 @@ def params_show( stages: Optional[Union[str, Iterable[str]]] = None, rev: Optional[str] = None, deps: bool = False, - config: Optional[Dict] = None, -) -> Dict: + config: Optional[dict] = None, +) -> dict: """Get parameters tracked in `repo`. Without arguments, this function will retrieve all params from all tracked diff --git a/dvc/cachemgr.py b/dvc/cachemgr.py index bbb7a02a6..f6a41b9eb 100644 --- a/dvc/cachemgr.py +++ b/dvc/cachemgr.py @@ -1,5 +1,5 @@ import os -from typing import TYPE_CHECKING, Optional, Tuple +from typing import TYPE_CHECKING, Optional from dvc.fs import GitFileSystem, Schemes from dvc_data.hashfile.db import get_odb @@ -15,7 +15,7 @@ def _get_odb( repo, settings, fs=None, - prefix: Optional[Tuple[str, ...]] = None, + prefix: Optional[tuple[str, ...]] = None, hash_name: Optional[str] = None, **kwargs, ): diff --git a/dvc/commands/data.py b/dvc/commands/data.py index 4e89b6e05..a9ea6e038 100644 --- a/dvc/commands/data.py +++ b/dvc/commands/data.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, ClassVar, Dict, Tuple +from typing import TYPE_CHECKING, ClassVar from funcy import chunks, compact, log_durations @@ -17,14 +17,14 @@ logger = logger.getChild(__name__) class CmdDataStatus(CmdBase): - COLORS: ClassVar[Dict[str, str]] = { + COLORS: ClassVar[dict[str, str]] = { "not_in_remote": "red", "not_in_cache": "red", "committed": "green", "uncommitted": "yellow", "untracked": "cyan", } - LABELS: ClassVar[Dict[str, str]] = { + LABELS: ClassVar[dict[str, str]] = { "not_in_remote": "Not in remote", "not_in_cache": "Not in cache", "committed": "DVC committed changes", @@ -32,7 +32,7 @@ class CmdDataStatus(CmdBase): "untracked": "Untracked files", "unchanged": "DVC unchanged files", } - HINTS: ClassVar[Dict[str, Tuple[str, ...]]] = { + HINTS: ClassVar[dict[str, tuple[str, ...]]] = { "not_in_remote": ('use "dvc push ..." to upload files',), "not_in_cache": ('use "dvc fetch ..." to download files',), "committed": ("git commit the corresponding dvc files to update the repo",), diff --git a/dvc/commands/experiments/push.py b/dvc/commands/experiments/push.py index 4b1ec750b..31a2faa84 100644 --- a/dvc/commands/experiments/push.py +++ b/dvc/commands/experiments/push.py @@ -1,4 +1,4 @@ -from typing import Any, Dict +from typing import Any from dvc.cli import completion, formatter from dvc.cli.command import CmdBase @@ -11,7 +11,7 @@ logger = logger.getChild(__name__) class CmdExperimentsPush(CmdBase): @staticmethod - def log_result(result: Dict[str, Any], remote: str): + def log_result(result: dict[str, Any], remote: str): from dvc.utils import humanize def join_exps(exps): diff --git a/dvc/commands/experiments/show.py b/dvc/commands/experiments/show.py index 8c4e7cd23..8f3370835 100644 --- a/dvc/commands/experiments/show.py +++ b/dvc/commands/experiments/show.py @@ -1,7 +1,8 @@ import argparse import re +from collections.abc import Iterable from datetime import date, datetime -from typing import TYPE_CHECKING, Dict, Iterable +from typing import TYPE_CHECKING from funcy import lmap @@ -57,7 +58,7 @@ def baseline_styler(typ): def show_experiments( td: "TabularData", - headers: Dict[str, Iterable[str]], + headers: dict[str, Iterable[str]], keep=None, drop=None, pager=True, diff --git a/dvc/commands/plots.py b/dvc/commands/plots.py index 596c22723..0639952a1 100644 --- a/dvc/commands/plots.py +++ b/dvc/commands/plots.py @@ -1,6 +1,6 @@ import argparse import os -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING, Optional from funcy import compact, first, get_in @@ -20,14 +20,14 @@ logger = logger.getChild(__name__) def _show_json( - renderers_with_errors: List["RendererWithErrors"], + renderers_with_errors: list["RendererWithErrors"], split=False, - errors: Optional[Dict[str, Exception]] = None, + errors: Optional[dict[str, Exception]] = None, ): from dvc.render.convert import to_json from dvc.utils.serialize import encode_exception - all_errors: List[Dict] = [] + all_errors: list[dict] = [] data = {} for renderer, src_errors, def_errors in renderers_with_errors: diff --git a/dvc/commands/stage.py b/dvc/commands/stage.py index f6c2a8bd0..ea93b6d26 100644 --- a/dvc/commands/stage.py +++ b/dvc/commands/stage.py @@ -1,8 +1,9 @@ import argparse import logging +from collections.abc import Iterable from contextlib import contextmanager from itertools import chain, filterfalse -from typing import TYPE_CHECKING, Dict, Iterable, List +from typing import TYPE_CHECKING from dvc.cli import completion, formatter from dvc.cli.command import CmdBase @@ -33,7 +34,7 @@ def generate_description(stage: "Stage") -> str: def is_plot_or_metric(out: "Output"): return bool(out.plot) or bool(out.metric) - desc: List[str] = [] + desc: list[str] = [] outs = list(filterfalse(is_plot_or_metric, stage.outs)) if outs: @@ -55,7 +56,7 @@ def prepare_stages_data( stages: Iterable["Stage"], description: bool = True, max_length: int = MAX_TEXT_LENGTH, -) -> Dict[str, str]: +) -> dict[str, str]: return { stage.addressing: ( prepare_description(stage, max_length=max_length) if description else "" @@ -67,7 +68,7 @@ def prepare_stages_data( class CmdStageList(CmdBase): def _get_stages(self) -> Iterable["Stage"]: if self.args.all: - stages: List["Stage"] = self.repo.index.stages + stages: list["Stage"] = self.repo.index.stages logger.trace("%d no. of stages found", len(stages)) return stages @@ -96,7 +97,7 @@ class CmdStageList(CmdBase): return 0 -def parse_cmd(commands: List[str]) -> str: +def parse_cmd(commands: list[str]) -> str: """ We need to take into account two cases: diff --git a/dvc/compare.py b/dvc/compare.py index ad57c3268..5b55d9ac9 100644 --- a/dvc/compare.py +++ b/dvc/compare.py @@ -1,20 +1,18 @@ from collections import abc +from collections.abc import ( + ItemsView, + Iterable, + Iterator, + Mapping, + MutableSequence, + Sequence, +) from itertools import chain, repeat, zip_longest from operator import itemgetter from typing import ( TYPE_CHECKING, Any, - Dict, - ItemsView, - Iterable, - Iterator, - List, - Mapping, - MutableSequence, Optional, - Sequence, - Set, - Tuple, Union, overload, ) @@ -25,7 +23,7 @@ if TYPE_CHECKING: from dvc.ui.table import CellT -class Column(List["CellT"]): +class Column(list["CellT"]): pass @@ -35,13 +33,13 @@ def with_value(value, default): class TabularData(MutableSequence[Sequence["CellT"]]): def __init__(self, columns: Sequence[str], fill_value: Optional[str] = ""): - self._columns: Dict[str, Column] = {name: Column() for name in columns} - self._keys: List[str] = list(columns) + self._columns: dict[str, Column] = {name: Column() for name in columns} + self._keys: list[str] = list(columns) self._fill_value = fill_value - self._protected: Set[str] = set() + self._protected: set[str] = set() @property - def columns(self) -> List[Column]: + def columns(self) -> list[Column]: return list(map(self.column, self.keys())) def is_protected(self, col_name) -> bool: @@ -60,10 +58,10 @@ class TabularData(MutableSequence[Sequence["CellT"]]): projection = {k: self.column(k) for k in self.keys()} return projection.items() - def keys(self) -> List[str]: + def keys(self) -> list[str]: return self._keys - def _iter_col_row(self, row: Sequence["CellT"]) -> Iterator[Tuple["CellT", Column]]: + def _iter_col_row(self, row: Sequence["CellT"]) -> Iterator[tuple["CellT", Column]]: for val, col in zip_longest(row, self.columns): if col is None: break @@ -81,7 +79,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]): for val, col in self._iter_col_row(value): col.insert(index, val) - def __iter__(self) -> Iterator[List["CellT"]]: + def __iter__(self) -> Iterator[list["CellT"]]: return map(list, zip(*self.columns)) def __getattr__(self, item: str) -> Column: @@ -127,7 +125,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]): return len(self.columns[0]) @property - def shape(self) -> Tuple[int, int]: + def shape(self) -> tuple[int, int]: return len(self.columns), len(self) def drop(self, *col_names: str) -> None: @@ -170,7 +168,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]): if key not in keys: self.add_column(key) - row: List["CellT"] = [ + row: list["CellT"] = [ with_value(d.get(key), self._fill_value) for key in self.keys() ] self.append(row) @@ -185,7 +183,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]): def as_dict( self, cols: Optional[Iterable[str]] = None - ) -> Iterable[Dict[str, "CellT"]]: + ) -> Iterable[dict[str, "CellT"]]: keys = self.keys() if cols is None else set(cols) return [{k: self._columns[k][i] for k in keys} for i in range(len(self))] @@ -202,7 +200,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]): if how not in ["any", "all"]: raise ValueError(f"Invalid 'how' value {how}. Choose one of ['any', 'all']") - match_line: Set = set() + match_line: set = set() match_any = True if how == "all": match_any = False @@ -245,7 +243,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]): ) if axis == "cols": - cols_to_drop: List[str] = [] + cols_to_drop: list[str] = [] for n_col, col in enumerate(self.columns): if subset and self.keys()[n_col] not in subset: continue @@ -259,7 +257,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]): elif axis == "rows": unique_rows = [] - rows_to_drop: List[int] = [] + rows_to_drop: list[int] = [] for n_row, row in enumerate(self): if subset: row = [ @@ -319,7 +317,7 @@ def diff_table( ) -> TabularData: a_rev = a_rev or "HEAD" b_rev = b_rev or "workspace" - headers: List[str] = ["Path", title, a_rev, b_rev, "Change"] + headers: list[str] = ["Path", title, a_rev, b_rev, "Change"] fill_value = "-" td = TabularData(headers, fill_value=fill_value) @@ -394,7 +392,7 @@ def metrics_table( for branch, val in metrics.items(): for fname, metric in val.get("data", {}).items(): - row_data: Dict[str, str] = {"Revision": branch, "Path": fname} + row_data: dict[str, str] = {"Revision": branch, "Path": fname} metric = metric.get("data", {}) flattened = ( flatten(format_dict(metric)) diff --git a/dvc/config.py b/dvc/config.py index 94d019801..3522404ea 100644 --- a/dvc/config.py +++ b/dvc/config.py @@ -3,7 +3,7 @@ import os import re from contextlib import contextmanager from functools import partial -from typing import TYPE_CHECKING, Dict, Optional +from typing import TYPE_CHECKING, Optional from funcy import compact, memoize, re_find @@ -138,7 +138,7 @@ class Config(dict): return system_config_dir() @cached_property - def files(self) -> Dict[str, str]: + def files(self) -> dict[str, str]: files = { level: os.path.join(self.get_dir(level), self.CONFIG) for level in ("system", "global") @@ -344,7 +344,7 @@ class Config(dict): return Schema(dirs_schema, extra=ALLOW_EXTRA)(conf) def load_config_to_level(self, level=None): - merged_conf: Dict = {} + merged_conf: dict = {} for merge_level in self.LEVELS: if merge_level == level: break @@ -390,7 +390,7 @@ class Config(dict): def _parse_named(conf): - result: Dict[str, Dict] = {"remote": {}, "machine": {}, "db": {}} + result: dict[str, dict] = {"remote": {}, "machine": {}, "db": {}} for section, val in conf.items(): match = re_find(r'^\s*(remote|machine|db)\s*"(.*)"\s*$', section) diff --git a/dvc/daemon.py b/dvc/daemon.py index c88ae4801..cb89de810 100644 --- a/dvc/daemon.py +++ b/dvc/daemon.py @@ -5,13 +5,14 @@ import logging import os import subprocess import sys +from collections.abc import Mapping, Sequence from contextlib import nullcontext -from typing import TYPE_CHECKING, Any, List, Mapping, Optional, Sequence, Union +from typing import TYPE_CHECKING, Any, Optional, Union from dvc.log import logger if TYPE_CHECKING: - from typing import ContextManager + from contextlib import AbstractContextManager from dvc.env import DVC_DAEMON, DVC_DAEMON_LOGFILE from dvc.utils import fix_env, is_binary @@ -55,7 +56,7 @@ def _win_detached_subprocess(args: Sequence[str], **kwargs) -> int: return popen.pid -def _get_dvc_args() -> List[str]: +def _get_dvc_args() -> list[str]: args = [sys.executable] if not is_binary(): root_dir = os.path.abspath(os.path.dirname(__file__)) @@ -136,7 +137,7 @@ def _map_log_level_to_flag() -> Optional[str]: return flags.get(logger.getEffectiveLevel()) -def daemon(args: List[str]) -> None: +def daemon(args: list[str]) -> None: """Launch a `dvc daemon` command in a detached process. Args: @@ -148,12 +149,12 @@ def daemon(args: List[str]) -> None: def _spawn( - args: List[str], - executable: Optional[Union[str, List[str]]] = None, + args: list[str], + executable: Optional[Union[str, list[str]]] = None, env: Optional[Mapping[str, str]] = None, output_file: Optional[str] = None, ) -> int: - file: "ContextManager[Any]" = nullcontext() + file: "AbstractContextManager[Any]" = nullcontext() kwargs = {} if output_file: file = open(output_file, "ab") # noqa: SIM115 @@ -168,7 +169,7 @@ def _spawn( return _detached_subprocess(executable + args, env=env, **kwargs) -def daemonize(args: List[str], executable: Union[None, str, List[str]] = None) -> None: +def daemonize(args: list[str], executable: Union[None, str, list[str]] = None) -> None: if os.name not in ("posix", "nt"): return diff --git a/dvc/data_cloud.py b/dvc/data_cloud.py index 2722b0c92..12d40317a 100644 --- a/dvc/data_cloud.py +++ b/dvc/data_cloud.py @@ -1,6 +1,7 @@ """Manages dvc remotes that user can use with push/pull/status commands.""" -from typing import TYPE_CHECKING, Iterable, Optional, Set, Tuple +from collections.abc import Iterable +from typing import TYPE_CHECKING, Optional from dvc.config import NoRemoteError, RemoteConfigError from dvc.log import logger @@ -50,7 +51,7 @@ class Remote: def _split_legacy_hash_infos( hash_infos: Iterable["HashInfo"], -) -> Tuple[Set["HashInfo"], Set["HashInfo"]]: +) -> tuple[set["HashInfo"], set["HashInfo"]]: from dvc.cachemgr import LEGACY_HASH_NAMES legacy = set() diff --git a/dvc/database.py b/dvc/database.py index a87d289e5..4289779b3 100644 --- a/dvc/database.py +++ b/dvc/database.py @@ -1,8 +1,9 @@ import os +from collections.abc import Iterator from contextlib import contextmanager from dataclasses import dataclass from tempfile import NamedTemporaryFile -from typing import TYPE_CHECKING, Any, Callable, Dict, Iterator, Optional, Union +from typing import TYPE_CHECKING, Any, Callable, Optional, Union from sqlalchemy import create_engine from sqlalchemy.engine import make_url as _make_url @@ -30,7 +31,7 @@ def make_url(url: Union["URL", str], **kwargs: Any) -> "URL": return _make_url(url).set(**kwargs) -def url_from_config(config: Union[str, "URL", Dict[str, str]]) -> "URL": +def url_from_config(config: Union[str, "URL", dict[str, str]]) -> "URL": if isinstance(config, dict): return make_url(**config) return make_url(config) @@ -118,7 +119,7 @@ def handle_error(url: "URL"): @contextmanager def client( - url_or_config: Union[str, "URL", Dict[str, str]], **engine_kwargs: Any + url_or_config: Union[str, "URL", dict[str, str]], **engine_kwargs: Any ) -> Iterator[Client]: url = url_from_config(url_or_config) echo = env2bool(env.DVC_SQLALCHEMY_ECHO, False) diff --git a/dvc/dependency/__init__.py b/dvc/dependency/__init__.py index e3def3c9b..9e7c474cd 100644 --- a/dvc/dependency/__init__.py +++ b/dvc/dependency/__init__.py @@ -1,5 +1,6 @@ from collections import defaultdict -from typing import Any, Dict, List, Mapping, Set +from collections.abc import Mapping +from typing import Any from dvc.output import ARTIFACT_SCHEMA, DIR_FILES_SCHEMA, Output @@ -66,12 +67,12 @@ def loads_from(stage, s_list, erepo=None, fs_config=None, db=None): ] -def _merge_params(s_list) -> Dict[str, List[str]]: +def _merge_params(s_list) -> dict[str, list[str]]: d = defaultdict(list) default_file = ParamsDependency.DEFAULT_PARAMS_FILE # figure out completely tracked params file, and ignore specific keys - wholly_tracked: Set[str] = set() + wholly_tracked: set[str] = set() for key in s_list: if not isinstance(key, dict): continue diff --git a/dvc/dependency/base.py b/dvc/dependency/base.py index 649805fe0..239c1bcb1 100644 --- a/dvc/dependency/base.py +++ b/dvc/dependency/base.py @@ -1,5 +1,3 @@ -from typing import Dict, Type - from dvc.exceptions import DvcException from dvc.fs import download as fs_download from dvc.output import Output @@ -25,11 +23,11 @@ class DependencyIsStageFileError(DvcException): class Dependency(Output): IS_DEPENDENCY = True - DoesNotExistError: Type[DvcException] = DependencyDoesNotExistError - IsNotFileOrDirError: Type[DvcException] = DependencyIsNotFileOrDirError - IsStageFileError: Type[DvcException] = DependencyIsStageFileError + DoesNotExistError: type[DvcException] = DependencyDoesNotExistError + IsNotFileOrDirError: type[DvcException] = DependencyIsNotFileOrDirError + IsStageFileError: type[DvcException] = DependencyIsStageFileError - def workspace_status(self) -> Dict[str, str]: + def workspace_status(self) -> dict[str, str]: if self.fs.version_aware: old_fs_path = self.fs_path try: diff --git a/dvc/dependency/db.py b/dvc/dependency/db.py index 99ebb0923..2d663150c 100644 --- a/dvc/dependency/db.py +++ b/dvc/dependency/db.py @@ -1,5 +1,6 @@ +from collections.abc import Iterator from contextlib import contextmanager -from typing import TYPE_CHECKING, Any, Callable, ClassVar, Dict, Iterator, Optional +from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional from funcy import compact, log_durations @@ -31,7 +32,7 @@ def download_progress(to: "Output") -> Iterator[Callable[[int], Any]]: class AbstractDependency(Dependency): """Dependency without workspace/fs/fs_path""" - def __init__(self, stage: "Stage", info: Dict[str, Any], *args, **kwargs): + def __init__(self, stage: "Stage", info: dict[str, Any], *args, **kwargs): self.repo = stage.repo self.stage = stage self.fs = None @@ -46,7 +47,7 @@ class DbDependency(AbstractDependency): PARAM_QUERY = "query" PARAM_TABLE = "table" PARAM_FILE_FORMAT = "file_format" - DB_SCHEMA: ClassVar[Dict] = { + DB_SCHEMA: ClassVar[dict] = { PARAM_DB: { PARAM_QUERY: str, PARAM_CONNECTION: str, @@ -57,7 +58,7 @@ class DbDependency(AbstractDependency): def __init__(self, stage: "Stage", info, *args, **kwargs): super().__init__(stage, info, *args, **kwargs) - self.db_info: Dict[str, str] = self.info.get(self.PARAM_DB, {}) + self.db_info: dict[str, str] = self.info.get(self.PARAM_DB, {}) self.connection = self.db_info.get(self.PARAM_CONNECTION) @property diff --git a/dvc/dependency/param.py b/dvc/dependency/param.py index 311998ff6..19a8d02b5 100644 --- a/dvc/dependency/param.py +++ b/dvc/dependency/param.py @@ -1,7 +1,7 @@ import os import typing from collections import defaultdict -from typing import TYPE_CHECKING, Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Optional import dpath @@ -37,7 +37,7 @@ class BadParamFileError(DvcException): def read_param_file( fs: "FileSystem", path: str, - key_paths: Optional[List[str]] = None, + key_paths: Optional[list[str]] = None, flatten: bool = False, **load_kwargs, ) -> Any: @@ -110,7 +110,7 @@ class ParamsDependency(Dependency): def read_params( self, flatten: bool = True, **kwargs: typing.Any - ) -> Dict[str, typing.Any]: + ) -> dict[str, typing.Any]: try: self.validate_filepath() except MissingParamsFile: @@ -134,7 +134,7 @@ class ParamsDependency(Dependency): from funcy import ldistinct - status: Dict[str, Any] = defaultdict(dict) + status: dict[str, Any] = defaultdict(dict) info = self.hash_info.value if self.hash_info else {} assert isinstance(info, dict) actual = self.read_params() diff --git a/dvc/dependency/repo.py b/dvc/dependency/repo.py index 42071c5ed..eaf776a2e 100644 --- a/dvc/dependency/repo.py +++ b/dvc/dependency/repo.py @@ -1,5 +1,5 @@ from copy import deepcopy -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union +from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union import voluptuous as vol @@ -20,7 +20,7 @@ class RepoDependency(Dependency): PARAM_CONFIG = "config" PARAM_REMOTE = "remote" - REPO_SCHEMA: ClassVar[Dict] = { + REPO_SCHEMA: ClassVar[dict] = { PARAM_REPO: { vol.Required(PARAM_URL): str, PARAM_REV: str, @@ -30,7 +30,7 @@ class RepoDependency(Dependency): } } - def __init__(self, def_repo: Dict[str, Any], stage: "Stage", *args, **kwargs): + def __init__(self, def_repo: dict[str, Any], stage: "Stage", *args, **kwargs): self.def_repo = def_repo super().__init__(stage, *args, **kwargs) @@ -65,7 +65,7 @@ class RepoDependency(Dependency): self.def_repo[self.PARAM_REV_LOCK] = rev @classmethod - def _dump_def_repo(cls, def_repo) -> Dict[str, str]: + def _dump_def_repo(cls, def_repo) -> dict[str, str]: repo = {cls.PARAM_URL: def_repo[cls.PARAM_URL]} rev = def_repo.get(cls.PARAM_REV) @@ -85,7 +85,7 @@ class RepoDependency(Dependency): repo[cls.PARAM_REMOTE] = remote return repo - def dumpd(self, **kwargs) -> Dict[str, Union[str, Dict[str, str]]]: + def dumpd(self, **kwargs) -> dict[str, Union[str, dict[str, str]]]: return { self.PARAM_PATH: self.def_path, self.PARAM_REPO: self._dump_def_repo(self.def_repo), diff --git a/dvc/dvcfile.py b/dvc/dvcfile.py index 06718ca7f..e568b843d 100644 --- a/dvc/dvcfile.py +++ b/dvc/dvcfile.py @@ -5,10 +5,7 @@ from typing import ( Any, Callable, ClassVar, - Dict, - List, Optional, - Tuple, TypeVar, Union, ) @@ -134,7 +131,7 @@ class FileMixin: d, _ = self._load(**kwargs) return d - def _load(self, **kwargs: Any) -> Tuple[Any, str]: + def _load(self, **kwargs: Any) -> tuple[Any, str]: # it raises the proper exceptions by priority: # 1. when the file doesn't exists # 2. filename is not a DVC file @@ -157,7 +154,7 @@ class FileMixin: return validate(d, cls.SCHEMA, path=fname) # type: ignore[arg-type] - def _load_yaml(self, **kwargs: Any) -> Tuple[Any, str]: + def _load_yaml(self, **kwargs: Any) -> tuple[Any, str]: from dvc.utils import strictyaml return strictyaml.load( @@ -182,10 +179,10 @@ class SingleStageFile(FileMixin): from dvc.schema import COMPILED_SINGLE_STAGE_SCHEMA as SCHEMA from dvc.stage.loader import SingleStageLoader as LOADER # noqa: N814 - metrics: ClassVar[List[str]] = [] + metrics: ClassVar[list[str]] = [] plots: ClassVar[Any] = {} - params: ClassVar[List[str]] = [] - artifacts: ClassVar[Dict[str, Optional[Dict[str, Any]]]] = {} + params: ClassVar[list[str]] = [] + artifacts: ClassVar[dict[str, Optional[dict[str, Any]]]] = {} @property def stage(self) -> "Stage": @@ -284,11 +281,11 @@ class ProjectFile(FileMixin): raise DvcException("ProjectFile has multiple stages. Please specify it's name.") @cached_property - def contents(self) -> Dict[str, Any]: + def contents(self) -> dict[str, Any]: return self._load()[0] @cached_property - def lockfile_contents(self) -> Dict[str, Any]: + def lockfile_contents(self) -> dict[str, Any]: return self._lockfile.load() @cached_property @@ -303,7 +300,7 @@ class ProjectFile(FileMixin): return self.LOADER(self, self.contents, self.lockfile_contents) @property - def metrics(self) -> List[str]: + def metrics(self) -> list[str]: return self.contents.get("metrics", []) @property @@ -311,11 +308,11 @@ class ProjectFile(FileMixin): return self.contents.get("plots", {}) @property - def params(self) -> List[str]: + def params(self) -> list[str]: return self.contents.get("params", []) @property - def artifacts(self) -> Dict[str, Optional[Dict[str, Any]]]: + def artifacts(self) -> dict[str, Optional[dict[str, Any]]]: return self.contents.get("artifacts", {}) def remove(self, force=False): diff --git a/dvc/exceptions.py b/dvc/exceptions.py index c79ee09b4..bf1f418e8 100644 --- a/dvc/exceptions.py +++ b/dvc/exceptions.py @@ -1,6 +1,6 @@ """Exceptions raised by the dvc.""" import errno -from typing import TYPE_CHECKING, Dict, List, Optional, Set +from typing import TYPE_CHECKING, Optional from dvc.utils import format_link @@ -34,7 +34,7 @@ class OutputDuplicationError(DvcException): stages (list): list of paths to stages. """ - def __init__(self, output: str, stages: Set["Stage"]): + def __init__(self, output: str, stages: set["Stage"]): from funcy import first assert isinstance(output, str) @@ -241,7 +241,7 @@ class UploadError(FileTransferError): class CheckoutError(DvcException): - def __init__(self, target_infos: List[str], stats: Dict[str, List[str]]): + def __init__(self, target_infos: list[str], stats: dict[str, list[str]]): from dvc.utils import error_link self.target_infos = target_infos diff --git a/dvc/fs/callbacks.py b/dvc/fs/callbacks.py index ae977ad00..aa6f7aec9 100644 --- a/dvc/fs/callbacks.py +++ b/dvc/fs/callbacks.py @@ -1,5 +1,5 @@ from contextlib import ExitStack -from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optional, Union +from typing import TYPE_CHECKING, Any, BinaryIO, Optional, Union from dvc.progress import Tqdm from dvc.utils.objects import cached_property @@ -55,7 +55,7 @@ class TqdmCallback(Callback): self, path_1: "Union[str, BinaryIO]", path_2: str, - kwargs: Dict[str, Any], + kwargs: dict[str, Any], child: Optional[Callback] = None, ): child = child or TqdmCallback( @@ -130,7 +130,7 @@ class RichCallback(Callback): self, path_1: Union[str, BinaryIO], path_2: str, - kwargs: Dict[str, Any], + kwargs: dict[str, Any], child: Optional["Callback"] = None, ): child = child or RichCallback( diff --git a/dvc/fs/dvc.py b/dvc/fs/dvc.py index c75a86cec..6a99f08f5 100644 --- a/dvc/fs/dvc.py +++ b/dvc/fs/dvc.py @@ -6,7 +6,7 @@ import posixpath import threading from collections import deque from contextlib import ExitStack, suppress -from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple, Type, Union +from typing import TYPE_CHECKING, Any, Callable, Optional, Union from fsspec.spec import AbstractFileSystem from funcy import wrap_with @@ -22,8 +22,8 @@ if TYPE_CHECKING: logger = logger.getChild(__name__) -RepoFactory = Union[Callable[..., "Repo"], Type["Repo"]] -Key = Tuple[str, ...] +RepoFactory = Union[Callable[..., "Repo"], type["Repo"]] +Key = tuple[str, ...] def as_posix(path: str) -> str: @@ -85,7 +85,7 @@ class _DVCFileSystem(AbstractFileSystem): subrepos: bool = False, repo_factory: Optional[RepoFactory] = None, fo: Optional[str] = None, - target_options: Optional[Dict[str, Any]] = None, # noqa: ARG002 + target_options: Optional[dict[str, Any]] = None, # noqa: ARG002 target_protocol: Optional[str] = None, # noqa: ARG002 config: Optional["DictStrAny"] = None, remote: Optional[str] = None, @@ -145,7 +145,7 @@ class _DVCFileSystem(AbstractFileSystem): } def getcwd(self): - relparts: Tuple[str, ...] = () + relparts: tuple[str, ...] = () assert self.repo is not None if self.repo.fs.isin(self.repo.fs.getcwd(), self.repo.root_dir): relparts = self.repo.fs.relparts(self.repo.fs.getcwd(), self.repo.root_dir) @@ -156,7 +156,7 @@ class _DVCFileSystem(AbstractFileSystem): return posixpath.join(*parts) @classmethod - def parts(cls, path: str) -> Tuple[str, ...]: + def parts(cls, path: str) -> tuple[str, ...]: ret = [] while True: path, part = posixpath.split(path) @@ -187,7 +187,7 @@ class _DVCFileSystem(AbstractFileSystem): start = "." return posixpath.relpath(self.abspath(path), start=self.abspath(start)) - def relparts(self, path: str, start: Optional[str] = None) -> Tuple[str, ...]: + def relparts(self, path: str, start: Optional[str] = None) -> tuple[str, ...]: return self.parts(self.relpath(path, start=start)) @functools.cached_property @@ -321,7 +321,7 @@ class _DVCFileSystem(AbstractFileSystem): def _get_subrepo_info( self, key: Key - ) -> Tuple["Repo", Optional[DataFileSystem], Key]: + ) -> tuple["Repo", Optional[DataFileSystem], Key]: """ Returns information about the subrepo the key is part of. """ @@ -525,7 +525,7 @@ class DVCFileSystem(FileSystem): protocol = "local" PARAM_CHECKSUM = "md5" - def _prepare_credentials(self, **config) -> Dict[str, Any]: + def _prepare_credentials(self, **config) -> dict[str, Any]: return config @functools.cached_property diff --git a/dvc/ignore.py b/dvc/ignore.py index a39c72b16..a622fb1fd 100644 --- a/dvc/ignore.py +++ b/dvc/ignore.py @@ -2,7 +2,7 @@ import os import re from collections import namedtuple from itertools import chain, groupby, takewhile -from typing import TYPE_CHECKING, List, Optional +from typing import TYPE_CHECKING, Optional from pathspec.patterns import GitWildMatchPattern from pathspec.util import normalize_file @@ -62,7 +62,7 @@ class DvcIgnorePatterns(DvcIgnore): return cls(path_spec_lines, dirname, fs.sep) - def __call__(self, root: List[str], dirs: List[str], files: List[str]): + def __call__(self, root: list[str], dirs: list[str], files: list[str]): files = [f for f in files if not self.matches(root, f)] dirs = [d for d in dirs if not self.matches(root, d, True)] @@ -226,7 +226,7 @@ class DvcIgnoreFilter: self, dirname: str, ignore_trie: Trie, - dnames: Optional["List"], + dnames: Optional["list"], ignore_subrepos: bool, ) -> None: self._update_trie(dirname, ignore_trie) @@ -331,7 +331,7 @@ class DvcIgnoreFilter: yield from fs.find(path) def _get_trie_pattern( - self, dirname, dnames: Optional["List"] = None, ignore_subrepos=True + self, dirname, dnames: Optional["list"] = None, ignore_subrepos=True ) -> Optional["DvcIgnorePatterns"]: if ignore_subrepos: ignores_trie = self.ignores_trie_fs diff --git a/dvc/logger.py b/dvc/logger.py index 7256fe5c2..274555d11 100644 --- a/dvc/logger.py +++ b/dvc/logger.py @@ -5,7 +5,7 @@ import logging.config import logging.handlers import os import sys -from typing import ClassVar, Dict +from typing import ClassVar import colorama @@ -79,7 +79,7 @@ class ColorFormatter(logging.Formatter): """ reset = colorama.Fore.RESET - color_codes: ClassVar[Dict[str, str]] = { + color_codes: ClassVar[dict[str, str]] = { "TRACE": colorama.Fore.GREEN, "DEBUG": colorama.Fore.BLUE, "WARNING": colorama.Fore.YELLOW, diff --git a/dvc/output.py b/dvc/output.py index 2f6fa42ff..56dd5fb0a 100644 --- a/dvc/output.py +++ b/dvc/output.py @@ -4,7 +4,7 @@ import posixpath from collections import defaultdict from contextlib import suppress from operator import itemgetter -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Type, Union +from typing import TYPE_CHECKING, Any, Optional, Union from urllib.parse import urlparse import voluptuous as vol @@ -148,7 +148,7 @@ def _split_dict(d, keys): def _merge_data(s_list): - d: Dict[str, Dict] = defaultdict(dict) + d: dict[str, dict] = defaultdict(dict) for key in s_list: if isinstance(key, str): d[key].update({}) @@ -207,7 +207,7 @@ def load_from_pipeline(stage, data, typ="outs"): ) -def split_file_meta_from_cloud(entry: Dict) -> Dict: +def split_file_meta_from_cloud(entry: dict) -> dict: if remote_name := entry.pop(Meta.PARAM_REMOTE, None): remote_meta = {} for key in ( @@ -223,7 +223,7 @@ def split_file_meta_from_cloud(entry: Dict) -> Dict: return entry -def merge_file_meta_from_cloud(entry: Dict) -> Dict: +def merge_file_meta_from_cloud(entry: dict) -> dict: cloud_meta = entry.pop(Output.PARAM_CLOUD, {}) if remote_name := first(cloud_meta): entry.update(cloud_meta[remote_name]) @@ -231,7 +231,7 @@ def merge_file_meta_from_cloud(entry: Dict) -> Dict: return entry -def _serialize_tree_obj_to_files(obj: Tree) -> List[Dict[str, Any]]: +def _serialize_tree_obj_to_files(obj: Tree) -> list[dict[str, Any]]: key = obj.PARAM_RELPATH return sorted( ( @@ -246,7 +246,7 @@ def _serialize_tree_obj_to_files(obj: Tree) -> List[Dict[str, Any]]: ) -def _serialize_hi_to_dict(hash_info: Optional[HashInfo]) -> Dict[str, Any]: +def _serialize_hi_to_dict(hash_info: Optional[HashInfo]) -> dict[str, Any]: if hash_info: if hash_info.name == "md5-dos2unix": return {"md5": hash_info.value} @@ -316,10 +316,10 @@ class Output: PARAM_HASH = "hash" PARAM_FS_CONFIG = "fs_config" - DoesNotExistError: Type[DvcException] = OutputDoesNotExistError - IsNotFileOrDirError: Type[DvcException] = OutputIsNotFileOrDirError - IsStageFileError: Type[DvcException] = OutputIsStageFileError - IsIgnoredError: Type[DvcException] = OutputIsIgnoredError + DoesNotExistError: type[DvcException] = OutputDoesNotExistError + IsNotFileOrDirError: type[DvcException] = OutputIsNotFileOrDirError + IsStageFileError: type[DvcException] = OutputIsStageFileError + IsIgnoredError: type[DvcException] = OutputIsIgnoredError def __init__( # noqa: PLR0913 self, @@ -337,7 +337,7 @@ class Output: remote=None, repo=None, fs_config=None, - files: Optional[List[Dict[str, Any]]] = None, + files: Optional[list[dict[str, Any]]] = None, push: bool = True, hash_name: Optional[str] = DEFAULT_ALGORITHM, ): @@ -420,7 +420,7 @@ class Output: def _compute_hash_info_from_meta( self, hash_name: Optional[str] - ) -> Tuple[str, HashInfo]: + ) -> tuple[str, HashInfo]: if self.is_in_repo: if hash_name is None: # Legacy 2.x output, use "md5-dos2unix" but read "md5" from @@ -555,7 +555,7 @@ class Output: def _build( self, *args, no_progress_bar=False, **kwargs - ) -> Tuple["HashFileDB", "Meta", "HashFile"]: + ) -> tuple["HashFileDB", "Meta", "HashFile"]: from dvc.ui import ui with ui.progress( @@ -601,7 +601,7 @@ class Output: return self.fs.exists(self.fs_path) @cached_property - def index_key(self) -> Tuple[str, "DataIndexKey"]: + def index_key(self) -> tuple[str, "DataIndexKey"]: if self.is_in_repo: workspace = "repo" key = self.repo.fs.relparts(self.fs_path, self.repo.root_dir) @@ -633,7 +633,7 @@ class Output: return self.meta.version_id != self.get_meta().version_id return False - def workspace_status(self) -> Dict[str, str]: + def workspace_status(self) -> dict[str, str]: if not self.exists: return {str(self): "deleted"} @@ -645,7 +645,7 @@ class Output: return {} - def status(self) -> Dict[str, str]: + def status(self) -> dict[str, str]: if self.hash_info and self.use_cache and self.changed_cache(): return {str(self): "not in cache"} @@ -840,7 +840,7 @@ class Output: def dumpd(self, **kwargs): # noqa: C901, PLR0912 from dvc.cachemgr import LEGACY_HASH_NAMES - ret: Dict[str, Any] = {} + ret: dict[str, Any] = {} with_files = ( (not self.IS_DEPENDENCY or self.stage.is_import) and self.hash_info.isdir @@ -962,7 +962,7 @@ class Output: filter_info: Optional[str] = None, allow_missing: bool = False, **kwargs, - ) -> Optional[Tuple[bool, Optional[bool]]]: + ) -> Optional[tuple[bool, Optional[bool]]]: # callback passed act as a aggregate callback. # do not let checkout to call set_size and change progressbar. class CallbackProxy(Callback): @@ -1157,7 +1157,7 @@ class Output: def get_used_objs( # noqa: PLR0911 self, **kwargs - ) -> Dict[Optional["HashFileDB"], Set["HashInfo"]]: + ) -> dict[Optional["HashFileDB"], set["HashInfo"]]: """Return filtered set of used object IDs for this out.""" from dvc.cachemgr import LEGACY_HASH_NAMES @@ -1291,7 +1291,7 @@ class Output: nfiles=len(merged), ) - def unstage(self, path: str) -> Tuple["Meta", "Tree"]: + def unstage(self, path: str) -> tuple["Meta", "Tree"]: from pygtrie import Trie rel_key = tuple(self.fs.parts(self.fs.relpath(path, self.fs_path))) @@ -1324,7 +1324,7 @@ class Output: path: str, obj: Union["Tree", "HashFile"], meta: "Meta", - ) -> Tuple["Meta", "Tree"]: + ) -> tuple["Meta", "Tree"]: from pygtrie import Trie append_only = True @@ -1514,7 +1514,7 @@ META_SCHEMA = { CLOUD_SCHEMA = vol.All({str: {**META_SCHEMA, **CHECKSUMS_SCHEMA}}, vol.Length(max=1)) -ARTIFACT_SCHEMA: Dict[Any, Any] = { +ARTIFACT_SCHEMA: dict[Any, Any] = { **CHECKSUMS_SCHEMA, **META_SCHEMA, Output.PARAM_PATH: str, @@ -1523,7 +1523,7 @@ ARTIFACT_SCHEMA: Dict[Any, Any] = { Output.PARAM_HASH: str, } -DIR_FILES_SCHEMA: Dict[Any, Any] = { +DIR_FILES_SCHEMA: dict[Any, Any] = { **CHECKSUMS_SCHEMA, **META_SCHEMA, vol.Required(Tree.PARAM_RELPATH): str, diff --git a/dvc/parsing/__init__.py b/dvc/parsing/__init__.py index a318bc7b2..bb64c5843 100644 --- a/dvc/parsing/__init__.py +++ b/dvc/parsing/__init__.py @@ -6,12 +6,8 @@ from itertools import product from typing import ( TYPE_CHECKING, Any, - Dict, - List, NamedTuple, Optional, - Tuple, - Type, Union, ) @@ -85,7 +81,7 @@ def format_and_raise(exc: Exception, msg: str, path: str) -> "NoReturn": def _reraise_err( - exc_cls: Type[Exception], *args, from_exc: Optional[Exception] = None + exc_cls: type[Exception], *args, from_exc: Optional[Exception] = None ) -> "NoReturn": err = exc_cls(*args) if from_exc and logger.isEnabledFor(logging.DEBUG): @@ -108,7 +104,7 @@ def is_map_or_seq(data: Any) -> bool: return not isinstance(data, str) and _is_map_or_seq(data) -def split_group_name(name: str) -> Tuple[str, Optional[str]]: +def split_group_name(name: str) -> tuple[str, Optional[str]]: group, *keys = name.rsplit(JOIN, maxsplit=1) return group, first(keys) @@ -161,14 +157,14 @@ class DataResolver: # we use `tracked_vars` to keep a dictionary of used variables # by the interpolated entries. - self.tracked_vars: Dict[str, Mapping] = {} + self.tracked_vars: dict[str, Mapping] = {} stages_data = d.get(STAGES_KWD, {}) # we wrap the definitions into: # ForeachDefinition, MatrixDefinition, and EntryDefinition # that helps us to optimize, cache and selectively load each one of # them as we need, and simplify all of this DSL/parsing logic. - self.definitions: Dict[str, Definition] = { + self.definitions: dict[str, Definition] = { name: make_definition(self, name, definition) for name, definition in stages_data.items() } @@ -376,7 +372,7 @@ class ForeachDefinition: f" in '{self.relpath}': expected list/dictionary, got " + typ ) - def _warn_if_overwriting(self, keys: List[str]): + def _warn_if_overwriting(self, keys: list[str]): warn_for = [k for k in keys if k in self.context] if warn_for: linking_verb = "is" if len(warn_for) == 1 else "are" @@ -390,7 +386,7 @@ class ForeachDefinition: self.name, ) - def _inserted_keys(self, iterable) -> List[str]: + def _inserted_keys(self, iterable) -> list[str]: keys = [self.pair.value] if isinstance(iterable, Mapping): keys.append(self.pair.key) @@ -490,10 +486,10 @@ class MatrixDefinition: return self._template @cached_property - def resolved_iterable(self) -> Dict[str, List]: + def resolved_iterable(self) -> dict[str, list]: return self._resolve_matrix_data() - def _resolve_matrix_data(self) -> Dict[str, List]: + def _resolve_matrix_data(self) -> dict[str, list]: try: iterable = self.context.resolve(self.matrix_data, unwrap=False) except (ContextError, ParseError) as exc: @@ -504,7 +500,7 @@ class MatrixDefinition: self._warn_if_overwriting([self.pair.key, self.pair.value]) return iterable - def _warn_if_overwriting(self, keys: List[str]): + def _warn_if_overwriting(self, keys: list[str]): warn_for = [k for k in keys if k in self.context] if warn_for: linking_verb = "is" if len(warn_for) == 1 else "are" @@ -519,16 +515,16 @@ class MatrixDefinition: ) @cached_property - def normalized_iterable(self) -> Dict[str, "DictStrAny"]: + def normalized_iterable(self) -> dict[str, "DictStrAny"]: """Convert sequence to Mapping with keys normalized.""" iterable = self.resolved_iterable assert isinstance(iterable, Mapping) - ret: Dict[str, "DictStrAny"] = {} + ret: dict[str, "DictStrAny"] = {} matrix = {key: enumerate(v) for key, v in iterable.items()} for combination in product(*matrix.values()): d: "DictStrAny" = {} - fragments: List[str] = [] + fragments: list[str] = [] for k, (i, v) in zip(matrix.keys(), combination): d[k] = v fragments.append(f"{k}{i}" if is_map_or_seq(v) else to_str(v)) @@ -540,7 +536,7 @@ class MatrixDefinition: def has_member(self, key: str) -> bool: return key in self.normalized_iterable - def get_generated_names(self) -> List[str]: + def get_generated_names(self) -> list[str]: return list(map(self._generate_name, self.normalized_iterable)) def _generate_name(self, key: str) -> str: diff --git a/dvc/parsing/context.py b/dvc/parsing/context.py index 7416a156d..3d7beeec6 100644 --- a/dvc/parsing/context.py +++ b/dvc/parsing/context.py @@ -4,7 +4,7 @@ from collections.abc import Mapping, MutableMapping, MutableSequence, Sequence from contextlib import contextmanager from copy import deepcopy from dataclasses import dataclass, field, replace -from typing import Any, Dict, List, Optional, Union +from typing import Any, Optional, Union from funcy import identity, lfilter, nullcontext, select @@ -22,7 +22,7 @@ from dvc.parsing.interpolate import ( logger = logger.getChild(__name__) SeqOrMap = Union[Sequence, Mapping] -DictStr = Dict[str, Any] +DictStr = dict[str, Any] class ContextError(DvcException): @@ -100,7 +100,7 @@ def recurse_not_a_node(data: dict): @dataclass class Meta: source: Optional[str] = None - dpaths: List[str] = field(default_factory=list) + dpaths: list[str] = field(default_factory=list) local: bool = True @staticmethod @@ -297,7 +297,7 @@ class Context(CtxDict): """ super().__init__(*args, **kwargs) self._track = False - self._tracked_data: Dict[str, Dict] = defaultdict(dict) + self._tracked_data: dict[str, dict] = defaultdict(dict) self.imports = {} self._reserved_keys = {} @@ -348,7 +348,7 @@ class Context(CtxDict): @classmethod def load_from( - cls, fs, path: str, select_keys: Optional[List[str]] = None + cls, fs, path: str, select_keys: Optional[list[str]] = None ) -> "Context": from dvc.utils.serialize import load_path @@ -423,7 +423,7 @@ class Context(CtxDict): def load_from_vars( self, fs, - vars_: List, + vars_: list, wdir: str, stage_name: Optional[str] = None, default: Optional[str] = None, diff --git a/dvc/parsing/interpolate.py b/dvc/parsing/interpolate.py index e5462775e..8948d045a 100644 --- a/dvc/parsing/interpolate.py +++ b/dvc/parsing/interpolate.py @@ -10,7 +10,8 @@ from dvc.exceptions import DvcException from dvc.utils.flatten import flatten if typing.TYPE_CHECKING: - from typing import List, Match, NoReturn + from re import Match + from typing import NoReturn from pyparsing import ParseException @@ -205,7 +206,7 @@ def validate_value(value, key): def str_interpolate( template: str, - matches: "List[Match]", + matches: "list[Match]", context: "Context", skip_checks: bool = False, key=None, @@ -225,5 +226,5 @@ def str_interpolate( return buf.replace(r"\${", BRACE_OPEN) -def is_exact_string(src: str, matches: "List[Match]"): +def is_exact_string(src: str, matches: "list[Match]"): return len(matches) == 1 and src == matches[0].group(0) diff --git a/dvc/progress.py b/dvc/progress.py index 67d7c8514..81b7a9f7b 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -2,7 +2,7 @@ import logging import sys from threading import RLock -from typing import TYPE_CHECKING, Any, ClassVar, Dict +from typing import TYPE_CHECKING, Any, ClassVar from tqdm import tqdm @@ -33,7 +33,7 @@ class Tqdm(tqdm): " [{elapsed}<{remaining}, {rate_fmt:>11}]" ) BAR_FMT_NOTOTAL = "{desc}{bar:b}|{postfix[info]}{n_fmt} [{elapsed}, {rate_fmt:>11}]" - BYTES_DEFAULTS: ClassVar[Dict[str, Any]] = { + BYTES_DEFAULTS: ClassVar[dict[str, Any]] = { "unit": "B", "unit_scale": True, "unit_divisor": 1024, diff --git a/dvc/prompt.py b/dvc/prompt.py index 8cacc876c..2f7c0e0c0 100644 --- a/dvc/prompt.py +++ b/dvc/prompt.py @@ -1,7 +1,8 @@ """Manages user prompts.""" +from collections.abc import Collection from getpass import getpass -from typing import Collection, Optional +from typing import Optional from dvc.log import logger diff --git a/dvc/render/convert.py b/dvc/render/convert.py index bfbc6fa40..11e98114e 100644 --- a/dvc/render/convert.py +++ b/dvc/render/convert.py @@ -1,4 +1,4 @@ -from typing import Dict, List, Union +from typing import Union from dvc.render import REVISION, REVISIONS, SRC, TYPE_KEY from dvc.render.converter.image import ImageConverter @@ -18,7 +18,7 @@ def _get_converter( raise ValueError(f"Invalid renderer class {renderer_class}") -def to_json(renderer, split: bool = False) -> List[Dict]: +def to_json(renderer, split: bool = False) -> list[dict]: if renderer.TYPE == "vega": if not renderer.datapoints: return [] diff --git a/dvc/render/converter/__init__.py b/dvc/render/converter/__init__.py index 77e790495..38dec5552 100644 --- a/dvc/render/converter/__init__.py +++ b/dvc/render/converter/__init__.py @@ -1,19 +1,19 @@ -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Optional class Converter: def __init__( self, plot_id: str, - data: Optional[Dict[str, Any]] = None, - properties: Optional[Dict] = None, + data: Optional[dict[str, Any]] = None, + properties: Optional[dict] = None, ): self.plot_id = plot_id self.properties = properties or {} self.data = data or {} - def convert(self) -> Tuple[List[Tuple[str, str, Any]], Dict]: + def convert(self) -> tuple[list[tuple[str, str, Any]], dict]: raise NotImplementedError - def flat_datapoints(self, revision: str) -> Tuple[List[Dict], Dict]: + def flat_datapoints(self, revision: str) -> tuple[list[dict], dict]: raise NotImplementedError diff --git a/dvc/render/converter/image.py b/dvc/render/converter/image.py index fed3a02f4..076ddd864 100644 --- a/dvc/render/converter/image.py +++ b/dvc/render/converter/image.py @@ -1,6 +1,6 @@ import base64 import os -from typing import TYPE_CHECKING, Any, Dict, List, Tuple +from typing import TYPE_CHECKING, Any from dvc.render import FILENAME, REVISION, SRC @@ -34,13 +34,13 @@ class ImageConverter(Converter): base64_str = base64.b64encode(image_data).decode() return f"data:image;base64,{base64_str}" - def convert(self) -> Tuple[List[Tuple[str, str, Any]], Dict]: + def convert(self) -> tuple[list[tuple[str, str, Any]], dict]: datas = [] for filename, image_data in self.data.items(): datas.append((filename, "", image_data)) return datas, self.properties - def flat_datapoints(self, revision: str) -> Tuple[List[Dict], Dict]: + def flat_datapoints(self, revision: str) -> tuple[list[dict], dict]: """ Convert the DVC Plots content to DVC Render datapoints. Return both generated datapoints and updated properties. diff --git a/dvc/render/converter/vega.py b/dvc/render/converter/vega.py index fd97f6c2e..0a87743d0 100644 --- a/dvc/render/converter/vega.py +++ b/dvc/render/converter/vega.py @@ -1,5 +1,6 @@ import os -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union +from collections.abc import Iterable +from typing import Any, Optional, Union from funcy import first, last @@ -18,7 +19,7 @@ class FieldNotFoundError(DvcException): ) -def _lists(blob: Union[Dict, List]) -> Iterable[List]: +def _lists(blob: Union[dict, list]) -> Iterable[list]: if isinstance(blob, list): yield blob else: @@ -43,7 +44,7 @@ def _file_field(*args): def _find( filename: str, field: str, - data_series: List[Tuple[str, str, Any]], + data_series: list[tuple[str, str, Any]], ): for data_file, data_field, data in data_series: if data_file == filename and data_field == field: @@ -51,14 +52,14 @@ def _find( return None -def _verify_field(file2datapoints: Dict[str, List], filename: str, field: str): +def _verify_field(file2datapoints: dict[str, list], filename: str, field: str): if filename in file2datapoints: datapoint = first(file2datapoints[filename]) if field not in datapoint: raise FieldNotFoundError(field, datapoint.keys()) -def _get_xs(properties: Dict, file2datapoints: Dict[str, List[Dict]]): +def _get_xs(properties: dict, file2datapoints: dict[str, list[dict]]): x = properties.get("x") if x is not None and isinstance(x, dict): for filename, field in _file_field(x): @@ -66,7 +67,7 @@ def _get_xs(properties: Dict, file2datapoints: Dict[str, List[Dict]]): yield filename, field -def _get_ys(properties, file2datapoints: Dict[str, List[Dict]]): +def _get_ys(properties, file2datapoints: dict[str, list[dict]]): y = properties.get("y", None) if y is not None: for filename, field in _file_field(y): @@ -74,7 +75,7 @@ def _get_ys(properties, file2datapoints: Dict[str, List[Dict]]): yield filename, field -def _is_datapoints(lst: List[Dict]): +def _is_datapoints(lst: list[dict]): """ check if dict keys match, datapoints with different keys mgiht lead to unexpected behavior @@ -85,8 +86,8 @@ def _is_datapoints(lst: List[Dict]): } -def get_datapoints(file_content: Dict): - result: List[Dict[str, Any]] = [] +def get_datapoints(file_content: dict): + result: list[dict[str, Any]] = [] for lst in _lists(file_content): if _is_datapoints(lst): for index, datapoint in enumerate(lst): @@ -107,8 +108,8 @@ class VegaConverter(Converter): def __init__( self, plot_id: str, - data: Optional[Dict] = None, - properties: Optional[Dict] = None, + data: Optional[dict] = None, + properties: Optional[dict] = None, ): super().__init__(plot_id, data, properties) self.plot_id = plot_id @@ -126,7 +127,7 @@ class VegaConverter(Converter): x = self.properties.get("x", None) y = self.properties.get("y", None) - inferred_properties: Dict = {} + inferred_properties: dict = {} # Infer x. if isinstance(x, str): @@ -195,7 +196,7 @@ class VegaConverter(Converter): def flat_datapoints(self, revision): # noqa: C901, PLR0912 file2datapoints, properties = self.convert() - props_update: Dict[str, Union[str, List[Dict[str, str]]]] = {} + props_update: dict[str, Union[str, list[dict[str, str]]]] = {} xs = list(_get_xs(properties, file2datapoints)) @@ -322,9 +323,9 @@ def _get_short_y_file(y_file, common_prefix_len): def _update_from_field( - target_datapoints: List[Dict], + target_datapoints: list[dict], field: str, - source_datapoints: Optional[List[Dict]] = None, + source_datapoints: Optional[list[dict]] = None, source_field: Optional[str] = None, ): if source_datapoints is None: @@ -341,11 +342,11 @@ def _update_from_field( datapoint[field] = source_datapoint[source_field] -def _update_from_index(datapoints: List[Dict], new_field: str): +def _update_from_index(datapoints: list[dict], new_field: str): for index, datapoint in enumerate(datapoints): datapoint[new_field] = index -def _update_all(datapoints: List[Dict], update_dict: Dict): +def _update_all(datapoints: list[dict], update_dict: dict): for datapoint in datapoints: datapoint.update(update_dict) diff --git a/dvc/render/match.py b/dvc/render/match.py index 0a861cb4f..dbb710b2c 100644 --- a/dvc/render/match.py +++ b/dvc/render/match.py @@ -1,6 +1,6 @@ import os from collections import defaultdict -from typing import TYPE_CHECKING, DefaultDict, Dict, List, NamedTuple, Optional +from typing import TYPE_CHECKING, NamedTuple, Optional import dpath import dpath.options @@ -21,16 +21,16 @@ dpath.options.ALLOW_EMPTY_STRING_KEYS = True logger = logger.getChild(__name__) -def _squash_plots_properties(data: List) -> Dict: +def _squash_plots_properties(data: list) -> dict: configs = [last(group) for group in data] - resolved: Dict = {} + resolved: dict = {} for config in reversed(configs): resolved = {**resolved, **config} return resolved class PlotsData: - def __init__(self, data: Dict): + def __init__(self, data: dict): self.data = data def group_definitions(self): @@ -62,15 +62,15 @@ class PlotsData: class RendererWithErrors(NamedTuple): renderer: "Renderer" - source_errors: Dict[str, Dict[str, Exception]] - definition_errors: Dict[str, Exception] + source_errors: dict[str, dict[str, Exception]] + definition_errors: dict[str, Exception] def match_defs_renderers( # noqa: C901, PLR0912 data, out=None, templates_dir: Optional["StrPath"] = None, -) -> List[RendererWithErrors]: +) -> list[RendererWithErrors]: from dvc_render import ImageRenderer, VegaRenderer plots_data = PlotsData(data) @@ -78,12 +78,12 @@ def match_defs_renderers( # noqa: C901, PLR0912 renderer_cls = None for plot_id, group in plots_data.group_definitions().items(): - plot_datapoints: List[Dict] = [] + plot_datapoints: list[dict] = [] props = _squash_plots_properties(group) - first_props: Dict = {} + first_props: dict = {} - def_errors: Dict[str, Exception] = {} - src_errors: DefaultDict[str, Dict[str, Exception]] = defaultdict(dict) + def_errors: dict[str, Exception] = {} + src_errors: defaultdict[str, dict[str, Exception]] = defaultdict(dict) if out is not None: props["out"] = out diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 606571c6a..57b7a2813 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -1,14 +1,12 @@ import os from collections import defaultdict -from contextlib import contextmanager +from collections.abc import Iterable +from contextlib import AbstractContextManager, contextmanager from functools import wraps from typing import ( TYPE_CHECKING, Callable, - ContextManager, - Iterable, Optional, - Tuple, Union, ) @@ -111,7 +109,7 @@ class Repo: fs: Optional["FileSystem"] = None, uninitialized: bool = False, scm: Optional[Union["Git", "NoSCM"]] = None, - ) -> Tuple[str, Optional[str]]: + ) -> tuple[str, Optional[str]]: from dvc.fs import localfs from dvc.scm import SCM, SCMError @@ -268,7 +266,7 @@ class Repo: if not isinstance(self.fs, GitFileSystem): return None - relparts: Tuple[str, ...] = () + relparts: tuple[str, ...] = () if self.root_dir != "/": # subrepo relparts = self.fs.relparts(self.root_dir, "/") @@ -392,7 +390,7 @@ class Repo: self, path: str, workspace: str = "repo", - ) -> Tuple["DataIndex", "DataIndexEntry"]: + ) -> tuple["DataIndex", "DataIndexEntry"]: if self.subrepos: fs_path = self.dvcfs.from_os_path(path) fs = self.dvcfs.fs @@ -476,7 +474,7 @@ class Repo: return brancher(self, *args, **kwargs) - def switch(self, rev: str) -> ContextManager[str]: + def switch(self, rev: str) -> AbstractContextManager[str]: from dvc.repo.brancher import switch return switch(self, rev) @@ -626,7 +624,7 @@ class Repo: cache_dir = self.config["core"].get("site_cache_dir") or site_cache_dir() if isinstance(self.fs, GitFileSystem): - relparts: Tuple[str, ...] = () + relparts: tuple[str, ...] = () if self.root_dir != "/": # subrepo relparts = self.fs.relparts(self.root_dir, "/") diff --git a/dvc/repo/add.py b/dvc/repo/add.py index be1977c76..91dc04c44 100644 --- a/dvc/repo/add.py +++ b/dvc/repo/add.py @@ -1,13 +1,10 @@ import os +from collections.abc import Iterator from contextlib import contextmanager from typing import ( TYPE_CHECKING, - Dict, - Iterator, - List, NamedTuple, Optional, - Tuple, Union, ) @@ -37,7 +34,7 @@ class StageInfo(NamedTuple): def find_targets( targets: Union["StrOrBytesPath", Iterator["StrOrBytesPath"]], glob: bool = False -) -> List[str]: +) -> list[str]: if isinstance(targets, (str, bytes, os.PathLike)): targets_list = [os.fsdecode(targets)] else: @@ -99,7 +96,7 @@ OVERLAPPING_PARENT_FMT = ( @contextmanager -def translate_graph_error(stages: List["Stage"]) -> Iterator[None]: +def translate_graph_error(stages: list["Stage"]) -> Iterator[None]: try: yield except OverlappingOutputPathsError as exc: @@ -124,7 +121,7 @@ def translate_graph_error(stages: List["Stage"]) -> Iterator[None]: ) -def progress_iter(stages: Dict[str, StageInfo]) -> Iterator[Tuple[str, StageInfo]]: +def progress_iter(stages: dict[str, StageInfo]) -> Iterator[tuple[str, StageInfo]]: total = len(stages) desc = "Adding..." with ui.progress( @@ -151,8 +148,8 @@ LINK_FAILURE_MESSAGE = ( @contextmanager -def warn_link_failures() -> Iterator[List[str]]: - link_failures: List[str] = [] +def warn_link_failures() -> Iterator[list[str]]: + link_failures: list[str] = [] try: yield link_failures finally: @@ -202,7 +199,7 @@ def add( to_remote: bool = False, remote_jobs: Optional[int] = None, force: bool = False, -) -> List["Stage"]: +) -> list["Stage"]: add_targets = find_targets(targets, glob=glob) if not add_targets: return [] diff --git a/dvc/repo/artifacts.py b/dvc/repo/artifacts.py index e2056e82e..be65a1e03 100644 --- a/dvc/repo/artifacts.py +++ b/dvc/repo/artifacts.py @@ -1,7 +1,7 @@ import os import posixpath from pathlib import Path -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Optional, Union from dvc.annotations import Artifact from dvc.dvcfile import PROJECT_FILE @@ -95,9 +95,9 @@ class Artifacts: return self.repo.scm return None - def read(self) -> Dict[str, Dict[str, Artifact]]: + def read(self) -> dict[str, dict[str, Artifact]]: """Read artifacts from dvc.yaml.""" - artifacts: Dict[str, Dict[str, Artifact]] = {} + artifacts: dict[str, dict[str, Artifact]] = {} for ( dvcfile, dvcfile_artifacts, @@ -140,14 +140,14 @@ class Artifacts: assert not (version and stage) name = _reformat_name(name) - tags: List["GitTag"] = find( + tags: list["GitTag"] = find( name=name, version=version, stage=stage, scm=self.scm ) if not tags: raise ArtifactNotFoundError(name, version=version, stage=stage) if version or stage: return tags[-1].target - gto_tags: List["GTOTag"] = sort_versions(parse_tag(tag) for tag in tags) + gto_tags: list["GTOTag"] = sort_versions(parse_tag(tag) for tag in tags) return gto_tags[0].tag.target def get_path(self, name: str): @@ -177,7 +177,7 @@ class Artifacts: out: Optional[str] = None, force: bool = False, jobs: Optional[int] = None, - ) -> Tuple[int, str]: + ) -> tuple[int, str]: """Download the specified artifact.""" from dvc.fs import download as fs_download @@ -205,9 +205,9 @@ class Artifacts: out: Optional[str] = None, force: bool = False, jobs: Optional[int] = None, - dvc_studio_config: Optional[Dict[str, Any]] = None, + dvc_studio_config: Optional[dict[str, Any]] = None, **kwargs, - ) -> Tuple[int, str]: + ) -> tuple[int, str]: from dvc_studio_client.model_registry import get_download_uris from dvc.fs import HTTPFileSystem, generic, localfs @@ -215,8 +215,8 @@ class Artifacts: logger.debug("Trying to download artifact '%s' via studio", name) out = out or os.getcwd() - to_infos: List[str] = [] - from_infos: List[str] = [] + to_infos: list[str] = [] + from_infos: list[str] = [] if dvc_studio_config is None: dvc_studio_config = {} dvc_studio_config["repo_url"] = repo_url @@ -263,9 +263,9 @@ class Artifacts: name: str, version: Optional[str] = None, stage: Optional[str] = None, - config: Optional[Union[str, Dict[str, Any]]] = None, + config: Optional[Union[str, dict[str, Any]]] = None, remote: Optional[str] = None, - remote_config: Optional[Union[str, Dict[str, Any]]] = None, + remote_config: Optional[Union[str, dict[str, Any]]] = None, out: Optional[str] = None, force: bool = False, jobs: Optional[int] = None, diff --git a/dvc/repo/brancher.py b/dvc/repo/brancher.py index 549c531bf..a3b90870e 100644 --- a/dvc/repo/brancher.py +++ b/dvc/repo/brancher.py @@ -1,5 +1,6 @@ +from collections.abc import Iterator from contextlib import contextmanager -from typing import TYPE_CHECKING, Iterator, Optional, Tuple +from typing import TYPE_CHECKING, Optional from scmrepo.git import Git @@ -59,11 +60,11 @@ def brancher( from dvc.fs import LocalFileSystem - repo_root_parts: Tuple[str, ...] = () + repo_root_parts: tuple[str, ...] = () if self.fs.isin(self.root_dir, self.scm.root_dir): repo_root_parts = self.fs.relparts(self.root_dir, self.scm.root_dir) - cwd_parts: Tuple[str, ...] = () + cwd_parts: tuple[str, ...] = () if self.fs.isin(self.fs.getcwd(), self.scm.root_dir): cwd_parts = self.fs.relparts(self.fs.getcwd(), self.scm.root_dir) @@ -112,8 +113,8 @@ def brancher( def _switch_fs( repo: "Repo", rev: str, - repo_root_parts: Tuple[str, ...], - cwd_parts: Tuple[str, ...], + repo_root_parts: tuple[str, ...], + cwd_parts: tuple[str, ...], ): from dvc.fs import GitFileSystem, LocalFileSystem @@ -147,11 +148,11 @@ def switch(repo: "Repo", rev: str) -> Iterator[str]: if rev != "workspace": rev = resolve_rev(repo.scm, rev) - repo_root_parts: Tuple[str, ...] = () + repo_root_parts: tuple[str, ...] = () if repo.fs.isin(repo.root_dir, repo.scm.root_dir): repo_root_parts = repo.fs.relparts(repo.root_dir, repo.scm.root_dir) - cwd_parts: Tuple[str, ...] = () + cwd_parts: tuple[str, ...] = () if repo.fs.isin(repo.fs.getcwd(), repo.scm.root_dir): cwd_parts = repo.fs.relparts(repo.fs.getcwd(), repo.scm.root_dir) diff --git a/dvc/repo/checkout.py b/dvc/repo/checkout.py index 8f14c8c1c..6a6bff33a 100644 --- a/dvc/repo/checkout.py +++ b/dvc/repo/checkout.py @@ -1,5 +1,5 @@ import os -from typing import TYPE_CHECKING, Dict, List +from typing import TYPE_CHECKING from dvc.exceptions import ( CheckoutError, @@ -66,7 +66,7 @@ def _build_out_changes(index, changes): def _check_can_delete( - entries: List["DataIndexEntry"], + entries: list["DataIndexEntry"], index: "BaseDataIndex", path: str, fs: "FileSystem", @@ -107,7 +107,7 @@ def checkout( # noqa: C901 from dvc.stage.exceptions import StageFileBadNameError, StageFileDoesNotExistError from dvc_data.index.checkout import ADD, DELETE, MODIFY, apply, compare - stats: Dict[str, List[str]] = { + stats: dict[str, list[str]] = { "added": [], "deleted": [], "modified": [], diff --git a/dvc/repo/collect.py b/dvc/repo/collect.py index 69bdb7d4f..31805e950 100644 --- a/dvc/repo/collect.py +++ b/dvc/repo/collect.py @@ -1,4 +1,5 @@ -from typing import TYPE_CHECKING, Callable, Iterable, List, Optional, Tuple +from collections.abc import Iterable +from typing import TYPE_CHECKING, Callable, Optional from dvc.log import logger @@ -10,8 +11,8 @@ logger = logger.getChild(__name__) FilterFn = Callable[["Output"], bool] -Outputs = List["Output"] -StrPaths = List[str] +Outputs = list["Output"] +StrPaths = list[str] def _collect_outs( @@ -43,7 +44,7 @@ def _collect_paths( def _filter_outs( outs: Outputs, fs_paths: StrPaths, duplicates=False -) -> Tuple[Outputs, StrPaths]: +) -> tuple[Outputs, StrPaths]: res_outs: Outputs = [] fs_res_paths = fs_paths @@ -65,7 +66,7 @@ def collect( output_filter: Optional[FilterFn] = None, recursive: bool = False, duplicates: bool = False, -) -> Tuple[Outputs, StrPaths]: +) -> tuple[Outputs, StrPaths]: assert targets or output_filter outs: Outputs = _collect_outs(repo, output_filter=output_filter, deps=deps) diff --git a/dvc/repo/commit.py b/dvc/repo/commit.py index a324737d2..cea727f20 100644 --- a/dvc/repo/commit.py +++ b/dvc/repo/commit.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Set +from typing import TYPE_CHECKING from dvc import prompt @@ -122,7 +122,7 @@ def commit_2_to_3(repo: "Repo", dry: bool = False): stage.dump(update_pipeline=False) -def _migrateable_dvcfiles(view: "IndexView") -> Set[str]: +def _migrateable_dvcfiles(view: "IndexView") -> set[str]: from dvc.dvcfile import ProjectFile migrated = set() diff --git a/dvc/repo/data.py b/dvc/repo/data.py index ce6344fdc..4be4ccb51 100644 --- a/dvc/repo/data.py +++ b/dvc/repo/data.py @@ -1,6 +1,7 @@ import os import posixpath -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, TypedDict, Union +from collections.abc import Iterable +from typing import TYPE_CHECKING, Any, TypedDict, Union from dvc.fs.callbacks import DEFAULT_CALLBACK from dvc.ui import ui @@ -53,11 +54,11 @@ def _diff( not_in_remote: bool = False, remote_refresh: bool = False, callback: "Callback" = DEFAULT_CALLBACK, -) -> Dict[str, List[str]]: +) -> dict[str, list[str]]: from dvc_data.index import StorageError from dvc_data.index.diff import UNCHANGED, UNKNOWN, diff - ret: Dict[str, List[str]] = {} + ret: dict[str, list[str]] = {} def _add_change(typ, change): typ = _adapt_typ(typ) @@ -115,9 +116,9 @@ def _diff( class GitInfo(TypedDict, total=False): - staged: Dict[str, List[str]] - unstaged: Dict[str, List[str]] - untracked: List[str] + staged: dict[str, list[str]] + unstaged: dict[str, list[str]] + untracked: list[str] is_empty: bool is_dirty: bool @@ -150,7 +151,7 @@ def _git_info(scm: Union["Git", "NoSCM"], untracked_files: str = "all") -> GitIn ) -def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> Dict[str, List[str]]: +def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> dict[str, list[str]]: from .index import build_data_index with ui.progress( @@ -180,7 +181,7 @@ def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> Dict[str, List[str]]: def _diff_head_to_index( repo: "Repo", head: str = "HEAD", **kwargs: Any -) -> Dict[str, List[str]]: +) -> dict[str, list[str]]: index = repo.index.data["repo"] with repo.switch(head): @@ -194,16 +195,16 @@ def _diff_head_to_index( class Status(TypedDict): - not_in_cache: List[str] - not_in_remote: List[str] - committed: Dict[str, List[str]] - uncommitted: Dict[str, List[str]] - untracked: List[str] - unchanged: List[str] + not_in_cache: list[str] + not_in_remote: list[str] + committed: dict[str, list[str]] + uncommitted: dict[str, list[str]] + untracked: list[str] + unchanged: list[str] git: GitInfo -def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> List[str]: +def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> list[str]: """Transform files rel. to Git root to DVC root, and drop outside files.""" rel = repo.fs.relpath(repo.root_dir, repo.scm.root_dir).rstrip("/") diff --git a/dvc/repo/diff.py b/dvc/repo/diff.py index d29f96d93..0406dcad1 100644 --- a/dvc/repo/diff.py +++ b/dvc/repo/diff.py @@ -1,7 +1,7 @@ import errno import os from collections import defaultdict -from typing import Dict, List, Optional +from typing import Optional from dvc.log import logger from dvc.repo import locked @@ -26,7 +26,7 @@ def _diff(old, new, data_keys, with_missing=False): from dvc_data.index.diff import ADD, DELETE, MODIFY, RENAME from dvc_data.index.diff import diff as idiff - ret: "Dict[str, List[Dict]]" = { + ret: "dict[str, list[dict]]" = { "added": [], "deleted": [], "modified": [], @@ -107,7 +107,7 @@ def diff( self, a_rev: str = "HEAD", b_rev: Optional[str] = None, - targets: Optional[List[str]] = None, + targets: Optional[list[str]] = None, recursive: bool = False, ): """ diff --git a/dvc/repo/du.py b/dvc/repo/du.py index f51148088..99eaf6394 100644 --- a/dvc/repo/du.py +++ b/dvc/repo/du.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Union +from typing import Any, Optional, Union def du( @@ -6,7 +6,7 @@ def du( path: Optional[str] = None, rev: Optional[str] = None, summarize: bool = False, - config: Union[None, Dict[str, Any], str] = None, + config: Union[None, dict[str, Any], str] = None, remote: Optional[str] = None, remote_config: Optional[dict] = None, ): diff --git a/dvc/repo/experiments/__init__.py b/dvc/repo/experiments/__init__.py index c346672a6..86a0a971b 100644 --- a/dvc/repo/experiments/__init__.py +++ b/dvc/repo/experiments/__init__.py @@ -1,6 +1,7 @@ import os import re -from typing import TYPE_CHECKING, Dict, Iterable, List, Optional +from collections.abc import Iterable +from typing import TYPE_CHECKING, Optional from funcy import chain, first @@ -104,7 +105,7 @@ class Experiments: return ExpCache(self.repo) @property - def stash_revs(self) -> Dict[str, "ExpStashEntry"]: + def stash_revs(self) -> dict[str, "ExpStashEntry"]: revs = {} for queue in (self.workspace_queue, self.celery_queue): revs.update(queue.stash.stash_revs) @@ -113,7 +114,7 @@ class Experiments: def reproduce_one( self, tmp_dir: bool = False, - copy_paths: Optional[List[str]] = None, + copy_paths: Optional[list[str]] = None, message: Optional[str] = None, **kwargs, ): @@ -143,8 +144,8 @@ class Experiments: def reproduce_celery( self, entries: Optional[Iterable["QueueEntry"]] = None, **kwargs - ) -> Dict[str, str]: - results: Dict[str, str] = {} + ) -> dict[str, str]: + results: dict[str, str] = {} if entries is None: entries = list( chain( @@ -250,10 +251,10 @@ class Experiments: def _reproduce_queue( self, queue: "BaseStashQueue", - copy_paths: Optional[List[str]] = None, + copy_paths: Optional[list[str]] = None, message: Optional[str] = None, **kwargs, - ) -> Dict[str, str]: + ) -> dict[str, str]: """Reproduce queued experiments. Arguments: @@ -265,7 +266,7 @@ class Experiments: """ exec_results = queue.reproduce(copy_paths=copy_paths, message=message) - results: Dict[str, str] = {} + results: dict[str, str] = {} for _, exp_result in exec_results.items(): results.update(exp_result) return results @@ -319,12 +320,12 @@ class Experiments: raise MultipleBranchError(rev, ref_infos) return str(ref_infos[0]) - def get_exact_name(self, revs: Iterable[str]) -> Dict[str, Optional[str]]: + def get_exact_name(self, revs: Iterable[str]) -> dict[str, Optional[str]]: """Returns preferred name for the specified revision. Prefers tags, branches (heads), experiments in that order. """ - result: Dict[str, Optional[str]] = {} + result: dict[str, Optional[str]] = {} exclude = f"{EXEC_NAMESPACE}/*" ref_dict = self.scm.describe(revs, base=EXPS_NAMESPACE, exclude=exclude) for rev in revs: diff --git a/dvc/repo/experiments/brancher.py b/dvc/repo/experiments/brancher.py index 5a46ffede..cc1665cbe 100644 --- a/dvc/repo/experiments/brancher.py +++ b/dvc/repo/experiments/brancher.py @@ -1,5 +1,6 @@ +from collections.abc import Iterator from contextlib import ExitStack, contextmanager -from typing import TYPE_CHECKING, Iterator, Tuple +from typing import TYPE_CHECKING from dvc.repo.experiments.exceptions import InvalidExpRevError from dvc.scm import RevError @@ -12,7 +13,7 @@ if TYPE_CHECKING: def switch_repo( repo: "Repo", rev: str, -) -> Iterator[Tuple["Repo", str]]: +) -> Iterator[tuple["Repo", str]]: """Return a repo instance (brancher) switched to rev. If rev is the name of a running experiment, the returned instance will be diff --git a/dvc/repo/experiments/collect.py b/dvc/repo/experiments/collect.py index b68b57af3..7b5e64981 100644 --- a/dvc/repo/experiments/collect.py +++ b/dvc/repo/experiments/collect.py @@ -1,16 +1,11 @@ import itertools import os +from collections.abc import Collection, Iterable, Iterator from dataclasses import fields from datetime import datetime from typing import ( TYPE_CHECKING, - Collection, - Dict, - Iterable, - Iterator, - List, Optional, - Tuple, Union, ) @@ -142,7 +137,7 @@ def collect_queued( repo: "Repo", baseline_revs: Collection[str], **kwargs, -) -> Dict[str, List["ExpRange"]]: +) -> dict[str, list["ExpRange"]]: """Collect queued experiments derived from the specified revisions. Args: @@ -177,7 +172,7 @@ def collect_active( repo: "Repo", baseline_revs: Collection[str], **kwargs, -) -> Dict[str, List["ExpRange"]]: +) -> dict[str, list["ExpRange"]]: """Collect active (running) experiments derived from the specified revisions. Args: @@ -189,7 +184,7 @@ def collect_active( """ if not baseline_revs: return {} - result: Dict[str, List["ExpRange"]] = {} + result: dict[str, list["ExpRange"]] = {} exps = repo.experiments for queue in (exps.workspace_queue, exps.tempdir_queue, exps.celery_queue): for baseline, active_exps in queue.collect_active_data( @@ -206,7 +201,7 @@ def collect_failed( repo: "Repo", baseline_revs: Collection[str], **kwargs, -) -> Dict[str, List["ExpRange"]]: +) -> dict[str, list["ExpRange"]]: """Collect failed experiments derived from the specified revisions. Args: @@ -225,7 +220,7 @@ def collect_successful( repo: "Repo", baseline_revs: Collection[str], **kwargs, -) -> Dict[str, List["ExpRange"]]: +) -> dict[str, list["ExpRange"]]: """Collect successful experiments derived from the specified revisions. Args: @@ -235,7 +230,7 @@ def collect_successful( Returns: Dict mapping baseline revision to successful experiments. """ - result: Dict[str, List["ExpRange"]] = {} + result: dict[str, list["ExpRange"]] = {} for baseline_rev in baseline_revs: result[baseline_rev] = list(_collect_baseline(repo, baseline_rev, **kwargs)) return result @@ -282,7 +277,7 @@ def _collect_baseline( def collect( repo: "Repo", - revs: Union[List[str], str, None] = None, + revs: Union[list[str], str, None] = None, all_branches: bool = False, all_tags: bool = False, all_commits: bool = False, @@ -291,7 +286,7 @@ def collect( hide_failed: bool = False, sha_only: bool = False, **kwargs, -) -> List["ExpState"]: +) -> list["ExpState"]: """Collect baseline revisions and derived experiments.""" assert isinstance(repo.scm, Git) if repo.scm.no_commits: @@ -312,14 +307,14 @@ def collect( ) ) if sha_only: - baseline_names: Dict[str, Optional[str]] = {} + baseline_names: dict[str, Optional[str]] = {} else: baseline_names = describe( repo.scm, baseline_revs, refs=cached_refs, logger=logger ) workspace_data = collect_rev(repo, "workspace", **kwargs) - result: List["ExpState"] = [workspace_data] + result: list["ExpState"] = [workspace_data] queued = collect_queued(repo, baseline_revs, **kwargs) if not hide_queued else {} active = collect_active(repo, baseline_revs, **kwargs) failed = collect_failed(repo, baseline_revs, **kwargs) if not hide_failed else {} @@ -345,10 +340,10 @@ def collect( return result -def _sorted_ranges(exp_ranges: Iterable["ExpRange"]) -> List["ExpRange"]: +def _sorted_ranges(exp_ranges: Iterable["ExpRange"]) -> list["ExpRange"]: """Return list of ExpRange sorted by (timestamp, rev).""" - def _head_timestamp(exp_range: "ExpRange") -> Tuple[datetime, str]: + def _head_timestamp(exp_range: "ExpRange") -> tuple[datetime, str]: head_exp = first(exp_range.revs) if head_exp and head_exp.data and head_exp.data.timestamp: return head_exp.data.timestamp, head_exp.rev diff --git a/dvc/repo/experiments/exceptions.py b/dvc/repo/experiments/exceptions.py index eac56d55f..56c55ef2f 100644 --- a/dvc/repo/experiments/exceptions.py +++ b/dvc/repo/experiments/exceptions.py @@ -1,4 +1,5 @@ -from typing import TYPE_CHECKING, Collection, Iterable, Optional +from collections.abc import Collection, Iterable +from typing import TYPE_CHECKING, Optional from dvc.exceptions import DvcException, InvalidArgumentError diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py index 9d2a1299c..2d749b369 100644 --- a/dvc/repo/experiments/executor/base.py +++ b/dvc/repo/experiments/executor/base.py @@ -3,6 +3,7 @@ import os import pickle import shutil from abc import ABC, abstractmethod +from collections.abc import Iterable, Iterator from contextlib import contextmanager from dataclasses import asdict, dataclass from enum import IntEnum @@ -11,15 +12,8 @@ from typing import ( TYPE_CHECKING, Any, Callable, - Dict, - Iterable, - Iterator, - List, NamedTuple, Optional, - Tuple, - Type, - TypeVar, Union, ) @@ -41,6 +35,8 @@ from dvc.utils.studio import env_to_config if TYPE_CHECKING: from queue import Queue + from typing_extensions import Self + from dvc.repo import Repo from dvc.repo.experiments.stash import ExpStashEntry from dvc.scm import Git @@ -112,9 +108,6 @@ class ExecutorInfo: return cls.from_dict(load_json(filename)) -_T = TypeVar("_T", bound="BaseExecutor") - - class BaseExecutor(ABC): """Base class for executing experiments in parallel. @@ -185,7 +178,7 @@ class BaseExecutor(ABC): @property def info(self) -> "ExecutorInfo": if self.result is not None: - result_dict: Dict[str, Any] = { + result_dict: dict[str, Any] = { "result_hash": self.result.exp_hash, "result_ref": ( str(self.result.ref_info) if self.result.ref_info else None @@ -207,7 +200,7 @@ class BaseExecutor(ABC): ) @classmethod - def from_info(cls: Type[_T], info: "ExecutorInfo") -> _T: + def from_info(cls, info: "ExecutorInfo") -> "Self": if info.result_hash: result: Optional["ExecutorResult"] = ExecutorResult( info.result_hash, @@ -229,21 +222,21 @@ class BaseExecutor(ABC): @classmethod @abstractmethod def from_stash_entry( - cls: Type[_T], + cls, repo: "Repo", entry: "ExpStashEntry", **kwargs, - ) -> _T: + ) -> "Self": pass @classmethod def _from_stash_entry( - cls: Type[_T], + cls, repo: "Repo", entry: "ExpStashEntry", root_dir: str, **kwargs, - ) -> _T: + ) -> "Self": return cls( root_dir=root_dir, dvc_dir=relpath(repo.dvc_dir, repo.scm.root_dir), @@ -255,7 +248,7 @@ class BaseExecutor(ABC): ) @classmethod - def _get_top_level_paths(cls, repo: "Repo") -> List["str"]: + def _get_top_level_paths(cls, repo: "Repo") -> list["str"]: return list( chain( _collect_top_level_metrics(repo), @@ -271,7 +264,7 @@ class BaseExecutor(ABC): targets: Optional[Iterable[str]] = None, recursive: bool = False, force: bool = False, - include_untracked: Optional[List[str]] = None, + include_untracked: Optional[list[str]] = None, message: Optional[str] = None, ) -> ExecutorResult: from dvc.dvcfile import LOCK_FILE @@ -382,7 +375,7 @@ class BaseExecutor(ABC): def fetch_exps( self, dest_scm: "Git", - refs: List[str], + refs: list[str], force: bool = False, on_diverged: Optional[Callable[[str], None]] = None, **kwargs, @@ -457,7 +450,7 @@ class BaseExecutor(ABC): infofile: Optional[str] = None, log_errors: bool = True, log_level: Optional[int] = None, - copy_paths: Optional[List[str]] = None, + copy_paths: Optional[list[str]] = None, message: Optional[str] = None, **kwargs, ) -> "ExecutorResult": @@ -565,7 +558,7 @@ class BaseExecutor(ABC): git_remote, repro_force, message: Optional[str] = None, - ) -> Tuple[Optional[str], Optional["ExpRefInfo"], bool]: + ) -> tuple[Optional[str], Optional["ExpRefInfo"], bool]: cls.commit( dvc.scm, exp_hash, @@ -598,7 +591,7 @@ class BaseExecutor(ABC): info: "ExecutorInfo", infofile: Optional[str] = None, log_errors: bool = True, - copy_paths: Optional[List[str]] = None, + copy_paths: Optional[list[str]] = None, message: Optional[str] = None, **kwargs, ) -> Iterator["Repo"]: @@ -790,7 +783,7 @@ class BaseExecutor(ABC): raise DvcException(f"Unable to copy '{src}' to '{dst}'.") from exc @contextmanager - def set_temp_refs(self, scm: "Git", temp_dict: Dict[str, str]): + def set_temp_refs(self, scm: "Git", temp_dict: dict[str, str]): try: for ref, rev in temp_dict.items(): scm.set_ref(ref, rev) diff --git a/dvc/repo/experiments/ls.py b/dvc/repo/experiments/ls.py index 6d2a23ff2..9e3654d08 100644 --- a/dvc/repo/experiments/ls.py +++ b/dvc/repo/experiments/ls.py @@ -1,5 +1,5 @@ from collections import defaultdict -from typing import Dict, List, Optional, Tuple, Union +from typing import Optional, Union from dvc.log import logger from dvc.repo import locked @@ -15,11 +15,11 @@ logger = logger.getChild(__name__) @scm_context def ls( repo, - rev: Optional[Union[List[str], str]] = None, + rev: Optional[Union[list[str], str]] = None, all_commits: bool = False, num: int = 1, git_remote: Optional[str] = None, -) -> Dict[str, List[Tuple[str, Optional[str]]]]: +) -> dict[str, list[tuple[str, Optional[str]]]]: """List experiments. Returns a dict mapping baseline revs to a list of (exp_name, exp_sha) tuples. diff --git a/dvc/repo/experiments/pull.py b/dvc/repo/experiments/pull.py index 699a54a42..81ed1cc4a 100644 --- a/dvc/repo/experiments/pull.py +++ b/dvc/repo/experiments/pull.py @@ -1,4 +1,5 @@ -from typing import Iterable, List, Mapping, Optional, Set, Union +from collections.abc import Iterable, Mapping +from typing import Optional, Union from funcy import group_by from scmrepo.git.backend.base import SyncStatus @@ -23,13 +24,13 @@ def pull( # noqa: C901 git_remote: str, exp_names: Optional[Union[Iterable[str], str]] = None, all_commits=False, - rev: Optional[Union[List[str], str]] = None, + rev: Optional[Union[list[str], str]] = None, num=1, force: bool = False, pull_cache: bool = False, **kwargs, ) -> Iterable[str]: - exp_ref_set: Set["ExpRefInfo"] = set() + exp_ref_set: set["ExpRefInfo"] = set() if all_commits: exp_ref_set.update(exp_refs(repo.scm, git_remote)) elif exp_names: @@ -81,7 +82,7 @@ def _pull( git_remote: str, refs: Iterable["ExpRefInfo"], force: bool, -) -> Mapping[SyncStatus, List["ExpRefInfo"]]: +) -> Mapping[SyncStatus, list["ExpRefInfo"]]: refspec_list = [f"{exp_ref}:{exp_ref}" for exp_ref in refs] logger.debug("git pull experiment '%s' -> '%s'", git_remote, refspec_list) @@ -96,7 +97,7 @@ def _pull( def group_result(refspec): return results[str(refspec)] - pull_result: Mapping[SyncStatus, List["ExpRefInfo"]] = group_by(group_result, refs) + pull_result: Mapping[SyncStatus, list["ExpRefInfo"]] = group_by(group_result, refs) return pull_result diff --git a/dvc/repo/experiments/push.py b/dvc/repo/experiments/push.py index 184bf2140..df87155d9 100644 --- a/dvc/repo/experiments/push.py +++ b/dvc/repo/experiments/push.py @@ -1,12 +1,8 @@ +from collections.abc import Iterable, Mapping from typing import ( TYPE_CHECKING, Any, - Dict, - Iterable, - List, - Mapping, Optional, - Set, Union, ) @@ -39,7 +35,7 @@ class UploadError(DvcException): def notify_refs_to_studio( - repo: "Repo", git_remote: str, **refs: List[str] + repo: "Repo", git_remote: str, **refs: list[str] ) -> Optional[str]: import os @@ -67,7 +63,7 @@ def notify_refs_to_studio( return d.get("url") -def exp_refs_from_names(scm: "Git", exp_names: List[str]) -> Set["ExpRefInfo"]: +def exp_refs_from_names(scm: "Git", exp_names: list[str]) -> set["ExpRefInfo"]: exp_ref_set = set() exp_ref_dict = resolve_name(scm, exp_names) unresolved_exp_names = [] @@ -82,7 +78,7 @@ def exp_refs_from_names(scm: "Git", exp_names: List[str]) -> Set["ExpRefInfo"]: return exp_ref_set -def exp_refs_from_rev(scm: "Git", rev: List[str], num: int = 1) -> Set["ExpRefInfo"]: +def exp_refs_from_rev(scm: "Git", rev: list[str], num: int = 1) -> set["ExpRefInfo"]: exp_ref_set = set() rev_dict = iter_revs(scm, rev, num) rev_set = set(rev_dict.keys()) @@ -97,15 +93,15 @@ def exp_refs_from_rev(scm: "Git", rev: List[str], num: int = 1) -> Set["ExpRefIn def push( repo: "Repo", git_remote: str, - exp_names: Optional[Union[List[str], str]] = None, + exp_names: Optional[Union[list[str], str]] = None, all_commits: bool = False, - rev: Optional[Union[List[str], str]] = None, + rev: Optional[Union[list[str], str]] = None, num: int = 1, force: bool = False, push_cache: bool = False, **kwargs: Any, -) -> Dict[str, Any]: - exp_ref_set: Set["ExpRefInfo"] = set() +) -> dict[str, Any]: + exp_ref_set: set["ExpRefInfo"] = set() assert isinstance(repo.scm, Git) if all_commits: exp_ref_set.update(exp_refs(repo.scm)) @@ -123,7 +119,7 @@ def push( status.name.lower(): [ref.name for ref in ref_list] for status, ref_list in push_result.items() } - result: Dict[str, Any] = {**refs, "uploaded": 0} + result: dict[str, Any] = {**refs, "uploaded": 0} pushed_refs_info = ( push_result[SyncStatus.UP_TO_DATE] + push_result[SyncStatus.SUCCESS] @@ -149,7 +145,7 @@ def _push( git_remote: str, refs: Iterable["ExpRefInfo"], force: bool, -) -> Mapping[SyncStatus, List["ExpRefInfo"]]: +) -> Mapping[SyncStatus, list["ExpRefInfo"]]: from scmrepo.exceptions import AuthError from dvc.scm import GitAuthError @@ -171,7 +167,7 @@ def _push( def group_result(refspec): return results[str(refspec)] - pull_result: Mapping[SyncStatus, List["ExpRefInfo"]] = group_by(group_result, refs) + pull_result: Mapping[SyncStatus, list["ExpRefInfo"]] = group_by(group_result, refs) return pull_result diff --git a/dvc/repo/experiments/queue/base.py b/dvc/repo/experiments/queue/base.py index d04056ce8..b18935c9f 100644 --- a/dvc/repo/experiments/queue/base.py +++ b/dvc/repo/experiments/queue/base.py @@ -1,18 +1,12 @@ import os from abc import ABC, abstractmethod +from collections.abc import Collection, Generator, Iterable, Mapping from dataclasses import asdict, dataclass from typing import ( TYPE_CHECKING, Any, - Collection, - Dict, - Generator, - Iterable, - List, - Mapping, NamedTuple, Optional, - Type, Union, ) @@ -69,11 +63,11 @@ class QueueEntry: and self.stash_rev == other.stash_rev ) - def asdict(self) -> Dict[str, Any]: + def asdict(self) -> dict[str, Any]: return asdict(self) @classmethod - def from_dict(cls, d: Dict[str, Any]) -> "QueueEntry": + def from_dict(cls, d: dict[str, Any]) -> "QueueEntry": return cls(**d) @@ -150,7 +144,7 @@ class BaseStashQueue(ABC): all_: bool = False, queued: bool = False, **kwargs, - ) -> List[str]: + ) -> list[str]: """Remove the specified entries from the queue. Arguments: @@ -165,8 +159,8 @@ class BaseStashQueue(ABC): if all_ or queued: return self.clear() - name_to_remove: List[str] = [] - entry_to_remove: List[ExpStashEntry] = [] + name_to_remove: list[str] = [] + entry_to_remove: list[ExpStashEntry] = [] queue_entries = self.match_queue_entry_by_name(revs, self.iter_queued()) for name, entry in queue_entries.items(): if entry: @@ -176,7 +170,7 @@ class BaseStashQueue(ABC): self.stash.remove_revs(entry_to_remove) return name_to_remove - def clear(self, **kwargs) -> List[str]: + def clear(self, **kwargs) -> list[str]: """Remove all entries from the queue.""" stash_revs = self.stash.stash_revs name_to_remove = list(stash_revs) @@ -184,11 +178,11 @@ class BaseStashQueue(ABC): return name_to_remove - def status(self) -> List[Dict[str, Any]]: + def status(self) -> list[dict[str, Any]]: """Show the status of exp tasks in queue""" from datetime import datetime - result: List[Dict[str, Optional[str]]] = [] + result: list[dict[str, Optional[str]]] = [] def _get_timestamp(rev: str) -> datetime: commit = self.scm.resolve_commit(rev) @@ -198,7 +192,7 @@ class BaseStashQueue(ABC): entry: QueueEntry, exp_result: Optional["ExecutorResult"] = None, status: str = "Unknown", - ) -> Dict[str, Any]: + ) -> dict[str, Any]: name = entry.name if not name and exp_result and exp_result.ref_info: name = exp_result.ref_info.name @@ -251,7 +245,7 @@ class BaseStashQueue(ABC): @abstractmethod def reproduce( - self, copy_paths: Optional[List[str]] = None, message: Optional[str] = None + self, copy_paths: Optional[list[str]] = None, message: Optional[str] = None ) -> Mapping[str, Mapping[str, str]]: """Reproduce queued experiments sequentially.""" @@ -300,7 +294,7 @@ class BaseStashQueue(ABC): def _stash_exp( self, *args, - params: Optional[Dict[str, List[str]]] = None, + params: Optional[dict[str, list[str]]] = None, baseline_rev: Optional[str] = None, branch: Optional[str] = None, name: Optional[str] = None, @@ -460,7 +454,7 @@ class BaseStashQueue(ABC): f"from '{config_path}': {param_list}" ) - def _update_params(self, params: Dict[str, List[str]]): + def _update_params(self, params: dict[str, list[str]]): """Update param files with the provided `Hydra Override`_ patterns. Args: @@ -528,7 +522,7 @@ class BaseStashQueue(ABC): cls, exp: "Experiments", queue_entry: QueueEntry, - executor_cls: Type[BaseExecutor] = WorkspaceExecutor, + executor_cls: type[BaseExecutor] = WorkspaceExecutor, **kwargs, ) -> BaseExecutor: stash_entry = cls.get_stash_entry(exp, queue_entry) @@ -563,7 +557,7 @@ class BaseStashQueue(ABC): exp: "Experiments", executor: BaseExecutor, exec_result: "ExecutorResult", - ) -> Dict[str, str]: + ) -> dict[str, str]: results = {} def on_diverged(ref: str): @@ -593,7 +587,7 @@ class BaseStashQueue(ABC): exp: "Experiments", executor: BaseExecutor, exec_result: "ExecutorResult", - ) -> Dict[str, str]: + ) -> dict[str, str]: results = cls.collect_git(exp, executor, exec_result) if exec_result.ref_info is not None: @@ -605,11 +599,11 @@ class BaseStashQueue(ABC): self, exp_names: Collection[str], *entries: Iterable[Union[QueueEntry, QueueDoneResult]], - ) -> Dict[str, Optional[QueueEntry]]: + ) -> dict[str, Optional[QueueEntry]]: from funcy import concat - entry_name_dict: Dict[str, QueueEntry] = {} - entry_rev_dict: Dict[str, QueueEntry] = {} + entry_name_dict: dict[str, QueueEntry] = {} + entry_rev_dict: dict[str, QueueEntry] = {} for entry in concat(*entries): if isinstance(entry, QueueDoneResult): queue_entry: QueueEntry = entry.entry @@ -624,7 +618,7 @@ class BaseStashQueue(ABC): entry_name_dict[name] = queue_entry entry_rev_dict[queue_entry.stash_rev] = queue_entry - result: Dict[str, Optional[QueueEntry]] = {} + result: dict[str, Optional[QueueEntry]] = {} for exp_name in exp_names: result[exp_name] = None if exp_name in entry_name_dict: @@ -666,7 +660,7 @@ class BaseStashQueue(ABC): baseline_revs: Optional[Collection[str]], fetch_refs: bool = False, **kwargs, - ) -> Dict[str, List["ExpRange"]]: + ) -> dict[str, list["ExpRange"]]: """Collect data for active (running) experiments. Args: @@ -685,7 +679,7 @@ class BaseStashQueue(ABC): self, baseline_revs: Optional[Collection[str]], **kwargs, - ) -> Dict[str, List["ExpRange"]]: + ) -> dict[str, list["ExpRange"]]: """Collect data for queued experiments. Args: @@ -702,7 +696,7 @@ class BaseStashQueue(ABC): self, baseline_revs: Optional[Collection[str]], **kwargs, - ) -> Dict[str, List["ExpRange"]]: + ) -> dict[str, list["ExpRange"]]: """Collect data for failed experiments. Args: diff --git a/dvc/repo/experiments/queue/celery.py b/dvc/repo/experiments/queue/celery.py index 1274df334..345a26b8c 100644 --- a/dvc/repo/experiments/queue/celery.py +++ b/dvc/repo/experiments/queue/celery.py @@ -3,16 +3,11 @@ import locale import logging import os from collections import defaultdict +from collections.abc import Collection, Generator, Mapping from typing import ( TYPE_CHECKING, - Collection, - Dict, - Generator, - List, - Mapping, NamedTuple, Optional, - Set, Union, ) @@ -166,7 +161,7 @@ class LocalCeleryQueue(BaseStashQueue): """ logger.debug("Spawning %s exp queue workers", count) - active_worker: Dict = self.worker_status() + active_worker: dict = self.worker_status() started = 0 for num in range(1, 1 + count): @@ -183,7 +178,7 @@ class LocalCeleryQueue(BaseStashQueue): def put( self, *args, - copy_paths: Optional[List[str]] = None, + copy_paths: Optional[list[str]] = None, message: Optional[str] = None, **kwargs, ) -> QueueEntry: @@ -265,7 +260,7 @@ class LocalCeleryQueue(BaseStashQueue): yield QueueDoneResult(queue_entry, exp_result) def reproduce( - self, copy_paths: Optional[List[str]] = None, message: Optional[str] = None + self, copy_paths: Optional[list[str]] = None, message: Optional[str] = None ) -> Mapping[str, Mapping[str, str]]: raise NotImplementedError @@ -333,8 +328,8 @@ class LocalCeleryQueue(BaseStashQueue): while not self.proc.get(entry.stash_rev): time.sleep(sleep_interval) - def _get_running_task_ids(self) -> Set[str]: - running_task_ids: Set[str] = set() + def _get_running_task_ids(self) -> set[str]: + running_task_ids: set[str] = set() active_workers = self.worker_status() for _, tasks in active_workers.items(): task = first(tasks) @@ -343,9 +338,9 @@ class LocalCeleryQueue(BaseStashQueue): return running_task_ids def _try_to_kill_tasks( - self, to_kill: Dict[QueueEntry, str], force: bool - ) -> Dict[QueueEntry, str]: - fail_to_kill_entries: Dict[QueueEntry, str] = {} + self, to_kill: dict[QueueEntry, str], force: bool + ) -> dict[QueueEntry, str]: + fail_to_kill_entries: dict[QueueEntry, str] = {} for queue_entry, rev in to_kill.items(): try: if force: @@ -358,9 +353,9 @@ class LocalCeleryQueue(BaseStashQueue): return fail_to_kill_entries def _mark_inactive_tasks_failure( - self, remained_entries: Dict[QueueEntry, str] + self, remained_entries: dict[QueueEntry, str] ) -> None: - remained_revs: List[str] = [] + remained_revs: list[str] = [] running_ids = self._get_running_task_ids() logger.debug("Current running tasks ids: %s.", running_ids) for msg, entry in self._iter_processed(): @@ -383,12 +378,12 @@ class LocalCeleryQueue(BaseStashQueue): if remained_revs: raise CannotKillTasksError(remained_revs) - def _kill_entries(self, entries: Dict[QueueEntry, str], force: bool) -> None: + def _kill_entries(self, entries: dict[QueueEntry, str], force: bool) -> None: logger.debug( "Found active tasks: '%s' to kill", list(entries.values()), ) - inactive_entries: Dict[QueueEntry, str] = self._try_to_kill_tasks( + inactive_entries: dict[QueueEntry, str] = self._try_to_kill_tasks( entries, force ) @@ -396,12 +391,12 @@ class LocalCeleryQueue(BaseStashQueue): self._mark_inactive_tasks_failure(inactive_entries) def kill(self, revs: Collection[str], force: bool = False) -> None: - name_dict: Dict[str, Optional[QueueEntry]] = self.match_queue_entry_by_name( + name_dict: dict[str, Optional[QueueEntry]] = self.match_queue_entry_by_name( set(revs), self.iter_active() ) - missing_revs: List[str] = [] - to_kill: Dict[QueueEntry, str] = {} + missing_revs: list[str] = [] + to_kill: dict[QueueEntry, str] = {} for rev, queue_entry in name_dict.items(): if queue_entry is None: missing_revs.append(rev) @@ -417,7 +412,7 @@ class LocalCeleryQueue(BaseStashQueue): def shutdown(self, kill: bool = False): self.celery.control.shutdown() if kill: - to_kill: Dict[QueueEntry, str] = {} + to_kill: dict[QueueEntry, str] = {} for entry in self.iter_active(): to_kill[entry] = entry.name or entry.stash_rev if to_kill: @@ -468,7 +463,7 @@ class LocalCeleryQueue(BaseStashQueue): ) as fobj: ui.write(fobj.read()) - def worker_status(self) -> Dict[str, List[Dict]]: + def worker_status(self) -> dict[str, list[dict]]: """Return the current active celery worker""" status = self.celery.control.inspect().active() or {} logger.debug("Worker status: %s", status) @@ -486,21 +481,21 @@ class LocalCeleryQueue(BaseStashQueue): def get_ref_and_entry_by_names( self, - exp_names: Union[str, List[str]], + exp_names: Union[str, list[str]], git_remote: Optional[str] = None, - ) -> Dict[str, ExpRefAndQueueEntry]: + ) -> dict[str, ExpRefAndQueueEntry]: """Find finished ExpRefInfo or queued or failed QueueEntry by name""" from dvc.repo.experiments.utils import resolve_name if isinstance(exp_names, str): exp_names = [exp_names] - results: Dict[str, ExpRefAndQueueEntry] = {} + results: dict[str, ExpRefAndQueueEntry] = {} - exp_ref_match: Dict[str, Optional["ExpRefInfo"]] = resolve_name( + exp_ref_match: dict[str, Optional["ExpRefInfo"]] = resolve_name( self.scm, exp_names, git_remote ) if not git_remote: - queue_entry_match: Dict[ + queue_entry_match: dict[ str, Optional["QueueEntry"] ] = self.match_queue_entry_by_name( exp_names, self.iter_queued(), self.iter_done() @@ -517,7 +512,7 @@ class LocalCeleryQueue(BaseStashQueue): baseline_revs: Optional[Collection[str]], fetch_refs: bool = False, **kwargs, - ) -> Dict[str, List["ExpRange"]]: + ) -> dict[str, list["ExpRange"]]: from dvc.repo import Repo from dvc.repo.experiments.collect import collect_exec_branch from dvc.repo.experiments.serialize import ( @@ -526,7 +521,7 @@ class LocalCeleryQueue(BaseStashQueue): LocalExpExecutor, ) - result: Dict[str, List[ExpRange]] = defaultdict(list) + result: dict[str, list[ExpRange]] = defaultdict(list) for entry in self.iter_active(): if baseline_revs and entry.baseline_rev not in baseline_revs: continue @@ -568,7 +563,7 @@ class LocalCeleryQueue(BaseStashQueue): self, baseline_revs: Optional[Collection[str]], **kwargs, - ) -> Dict[str, List["ExpRange"]]: + ) -> dict[str, list["ExpRange"]]: from dvc.repo.experiments.collect import collect_rev from dvc.repo.experiments.serialize import ( ExpExecutor, @@ -576,7 +571,7 @@ class LocalCeleryQueue(BaseStashQueue): LocalExpExecutor, ) - result: Dict[str, List[ExpRange]] = defaultdict(list) + result: dict[str, list[ExpRange]] = defaultdict(list) for entry in self.iter_queued(): if baseline_revs and entry.baseline_rev not in baseline_revs: continue @@ -598,7 +593,7 @@ class LocalCeleryQueue(BaseStashQueue): self, baseline_revs: Optional[Collection[str]], **kwargs, - ) -> Dict[str, List["ExpRange"]]: + ) -> dict[str, list["ExpRange"]]: from dvc.repo.experiments.collect import collect_rev from dvc.repo.experiments.serialize import ( ExpExecutor, @@ -607,7 +602,7 @@ class LocalCeleryQueue(BaseStashQueue): SerializableError, ) - result: Dict[str, List[ExpRange]] = defaultdict(list) + result: dict[str, list[ExpRange]] = defaultdict(list) for entry, _ in self.iter_failed(): if baseline_revs and entry.baseline_rev not in baseline_revs: continue @@ -637,11 +632,11 @@ class LocalCeleryQueue(BaseStashQueue): self, baseline_revs: Optional[Collection[str]], **kwargs, - ) -> Dict[str, "ExpExecutor"]: + ) -> dict[str, "ExpExecutor"]: """Map exp refs to any available successful executors.""" from dvc.repo.experiments.serialize import ExpExecutor, LocalExpExecutor - result: Dict[str, "ExpExecutor"] = {} + result: dict[str, "ExpExecutor"] = {} for entry, exec_result in self.iter_success(): if baseline_revs and entry.baseline_rev not in baseline_revs: continue diff --git a/dvc/repo/experiments/queue/exceptions.py b/dvc/repo/experiments/queue/exceptions.py index 7bc29ad4a..0e098cede 100644 --- a/dvc/repo/experiments/queue/exceptions.py +++ b/dvc/repo/experiments/queue/exceptions.py @@ -1,4 +1,4 @@ -from typing import Collection +from collections.abc import Collection from dvc.exceptions import DvcException diff --git a/dvc/repo/experiments/queue/remove.py b/dvc/repo/experiments/queue/remove.py index 76d7f0bd9..9880b8b8c 100644 --- a/dvc/repo/experiments/queue/remove.py +++ b/dvc/repo/experiments/queue/remove.py @@ -1,4 +1,5 @@ -from typing import TYPE_CHECKING, Collection, Dict, Iterable, List, Set, Union +from collections.abc import Collection, Iterable +from typing import TYPE_CHECKING, Union from dvc.repo.experiments.exceptions import UnresolvedExpNamesError from dvc.repo.experiments.queue.base import QueueDoneResult @@ -20,11 +21,11 @@ def remove_tasks( # noqa: C901, PLR0912 """ from celery.result import AsyncResult - stash_revs: Dict[str, "ExpStashEntry"] = {} - failed_stash_revs: List["ExpStashEntry"] = [] - done_entry_set: Set["QueueEntry"] = set() + stash_revs: dict[str, "ExpStashEntry"] = {} + failed_stash_revs: list["ExpStashEntry"] = [] + done_entry_set: set["QueueEntry"] = set() stash_rev_all = celery_queue.stash.stash_revs - failed_rev_all: Dict[str, "ExpStashEntry"] = {} + failed_rev_all: dict[str, "ExpStashEntry"] = {} if celery_queue.failed_stash: failed_rev_all = celery_queue.failed_stash.stash_revs for entry in queue_entries: @@ -64,7 +65,7 @@ def remove_tasks( # noqa: C901, PLR0912 def _get_names(entries: Iterable[Union["QueueEntry", "QueueDoneResult"]]): - names: List[str] = [] + names: list[str] = [] for entry in entries: if isinstance(entry, QueueDoneResult): if entry.result and entry.result.ref_info: @@ -82,7 +83,7 @@ def celery_clear( queued: bool = False, failed: bool = False, success: bool = False, -) -> List[str]: +) -> list[str]: """Remove entries from the queue. Arguments: @@ -94,18 +95,18 @@ def celery_clear( Revisions which were removed. """ - removed: List[str] = [] - entry_list: List["QueueEntry"] = [] + removed: list[str] = [] + entry_list: list["QueueEntry"] = [] if queued: - queue_entries: List["QueueEntry"] = list(self.iter_queued()) + queue_entries: list["QueueEntry"] = list(self.iter_queued()) entry_list.extend(queue_entries) removed.extend(_get_names(queue_entries)) if failed: - failed_tasks: List["QueueDoneResult"] = list(self.iter_failed()) + failed_tasks: list["QueueDoneResult"] = list(self.iter_failed()) entry_list.extend([result.entry for result in failed_tasks]) removed.extend(_get_names(failed_tasks)) if success: - success_tasks: List["QueueDoneResult"] = list(self.iter_success()) + success_tasks: list["QueueDoneResult"] = list(self.iter_success()) entry_list.extend([result.entry for result in success_tasks]) removed.extend(_get_names(success_tasks)) @@ -117,7 +118,7 @@ def celery_clear( def celery_remove( self: "LocalCeleryQueue", revs: Collection[str], -) -> List[str]: +) -> list[str]: """Remove the specified entries from the queue. Arguments: @@ -131,9 +132,9 @@ def celery_remove( revs, self.iter_queued(), self.iter_done() ) - remained: List[str] = [] - removed: List[str] = [] - entry_to_remove: List["QueueEntry"] = [] + remained: list[str] = [] + removed: list[str] = [] + entry_to_remove: list["QueueEntry"] = [] for name, entry in match_results.items(): if entry: entry_to_remove.append(entry) diff --git a/dvc/repo/experiments/queue/tasks.py b/dvc/repo/experiments/queue/tasks.py index 566c6dbb7..deb250f82 100644 --- a/dvc/repo/experiments/queue/tasks.py +++ b/dvc/repo/experiments/queue/tasks.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Optional from celery import shared_task from celery.utils.log import get_task_logger @@ -16,7 +16,7 @@ logger = get_task_logger(__name__) @shared_task -def setup_exp(entry_dict: Dict[str, Any]) -> "BaseExecutor": +def setup_exp(entry_dict: dict[str, Any]) -> "BaseExecutor": """Setup an experiment. Arguments: @@ -44,8 +44,8 @@ def setup_exp(entry_dict: Dict[str, Any]) -> "BaseExecutor": @shared_task def collect_exp( - proc_dict: Dict[str, Any], # noqa: ARG001 - entry_dict: Dict[str, Any], + proc_dict: dict[str, Any], # noqa: ARG001 + entry_dict: dict[str, Any], ) -> str: """Collect results for an experiment. @@ -92,8 +92,8 @@ def cleanup_exp(executor: TempDirExecutor, infofile: str) -> None: @shared_task def run_exp( - entry_dict: Dict[str, Any], - copy_paths: Optional[List[str]] = None, + entry_dict: dict[str, Any], + copy_paths: Optional[list[str]] = None, message: Optional[str] = None, ) -> None: """Run a full experiment. diff --git a/dvc/repo/experiments/queue/tempdir.py b/dvc/repo/experiments/queue/tempdir.py index fef0dc9b7..d6490bbd4 100644 --- a/dvc/repo/experiments/queue/tempdir.py +++ b/dvc/repo/experiments/queue/tempdir.py @@ -1,6 +1,7 @@ import os from collections import defaultdict -from typing import TYPE_CHECKING, Collection, Dict, Generator, List, Optional +from collections.abc import Collection, Generator +from typing import TYPE_CHECKING, Optional from funcy import first @@ -96,11 +97,11 @@ class TempDirQueue(WorkspaceQueue): self, entry: QueueEntry, executor: "BaseExecutor", - copy_paths: Optional[List[str]] = None, + copy_paths: Optional[list[str]] = None, message: Optional[str] = None, **kwargs, - ) -> Dict[str, Dict[str, str]]: - results: Dict[str, Dict[str, str]] = defaultdict(dict) + ) -> dict[str, dict[str, str]]: + results: dict[str, dict[str, str]] = defaultdict(dict) exec_name = self._EXEC_NAME or entry.stash_rev infofile = self.get_infofile_path(exec_name) try: @@ -135,7 +136,7 @@ class TempDirQueue(WorkspaceQueue): exp: "Experiments", executor: "BaseExecutor", exec_result: "ExecutorResult", - ) -> Dict[str, str]: + ) -> dict[str, str]: return BaseStashQueue.collect_executor(exp, executor, exec_result) def collect_active_data( @@ -143,7 +144,7 @@ class TempDirQueue(WorkspaceQueue): baseline_revs: Optional[Collection[str]], fetch_refs: bool = False, **kwargs, - ) -> Dict[str, List["ExpRange"]]: + ) -> dict[str, list["ExpRange"]]: from dvc.repo import Repo from dvc.repo.experiments.collect import collect_exec_branch from dvc.repo.experiments.serialize import ( @@ -152,7 +153,7 @@ class TempDirQueue(WorkspaceQueue): LocalExpExecutor, ) - result: Dict[str, List[ExpRange]] = defaultdict(list) + result: dict[str, list[ExpRange]] = defaultdict(list) for entry in self.iter_active(): if baseline_revs and entry.baseline_rev not in baseline_revs: continue diff --git a/dvc/repo/experiments/queue/utils.py b/dvc/repo/experiments/queue/utils.py index 26524a5d0..cfde2ed38 100644 --- a/dvc/repo/experiments/queue/utils.py +++ b/dvc/repo/experiments/queue/utils.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Dict, List +from typing import TYPE_CHECKING from scmrepo.exceptions import SCMError @@ -16,7 +16,7 @@ if TYPE_CHECKING: from .base import BaseStashQueue -def get_remote_executor_refs(scm: "Git", remote_url: str) -> List[str]: +def get_remote_executor_refs(scm: "Git", remote_url: str) -> list[str]: """Get result list refs from a remote repository Args: @@ -35,7 +35,7 @@ def get_remote_executor_refs(scm: "Git", remote_url: str) -> List[str]: def fetch_running_exp_from_temp_dir( queue: "BaseStashQueue", rev: str, fetch_refs: bool -) -> Dict[str, Dict]: +) -> dict[str, dict]: """Fetch status of running exps out of current working directory Args: @@ -50,7 +50,7 @@ def fetch_running_exp_from_temp_dir( from dvc.scm import InvalidRemoteSCMRepo from dvc.utils.serialize import load_json - result: Dict[str, Dict] = {} + result: dict[str, dict] = {} infofile = queue.get_infofile_path(rev) try: info = ExecutorInfo.from_dict(load_json(infofile)) diff --git a/dvc/repo/experiments/queue/workspace.py b/dvc/repo/experiments/queue/workspace.py index ed67d4a01..1d72f082a 100644 --- a/dvc/repo/experiments/queue/workspace.py +++ b/dvc/repo/experiments/queue/workspace.py @@ -1,7 +1,8 @@ import json import os from collections import defaultdict -from typing import TYPE_CHECKING, Collection, Dict, Generator, List, Optional +from collections.abc import Collection, Generator +from typing import TYPE_CHECKING, Optional import psutil from funcy import first @@ -82,9 +83,9 @@ class WorkspaceQueue(BaseStashQueue): raise NotImplementedError def reproduce( - self, copy_paths: Optional[List[str]] = None, message: Optional[str] = None - ) -> Dict[str, Dict[str, str]]: - results: Dict[str, Dict[str, str]] = defaultdict(dict) + self, copy_paths: Optional[list[str]] = None, message: Optional[str] = None + ) -> dict[str, dict[str, str]]: + results: dict[str, dict[str, str]] = defaultdict(dict) try: while True: entry, executor = self.get() @@ -99,11 +100,11 @@ class WorkspaceQueue(BaseStashQueue): def _reproduce_entry( self, entry: QueueEntry, executor: "BaseExecutor", **kwargs - ) -> Dict[str, Dict[str, str]]: + ) -> dict[str, dict[str, str]]: kwargs.pop("copy_paths", None) from dvc_task.proc.process import ProcessInfo - results: Dict[str, Dict[str, str]] = defaultdict(dict) + results: dict[str, dict[str, str]] = defaultdict(dict) exec_name = self._EXEC_NAME or entry.stash_rev proc_info = ProcessInfo(os.getpid(), None, None, None, None) proc_info_path = self._proc_info_path(exec_name) @@ -162,8 +163,8 @@ class WorkspaceQueue(BaseStashQueue): exp: "Experiments", executor: "BaseExecutor", # noqa: ARG004 exec_result: "ExecutorResult", - ) -> Dict[str, str]: - results: Dict[str, str] = {} + ) -> dict[str, str]: + results: dict[str, str] = {} exp_rev = exp.scm.get_ref(EXEC_BRANCH) if exp_rev: assert exec_result.exp_hash @@ -207,7 +208,7 @@ class WorkspaceQueue(BaseStashQueue): baseline_revs: Optional[Collection[str]], fetch_refs: bool = False, # noqa: ARG002 **kwargs, - ) -> Dict[str, List["ExpRange"]]: + ) -> dict[str, list["ExpRange"]]: from dvc.repo.experiments.collect import collect_exec_branch from dvc.repo.experiments.serialize import ( ExpExecutor, @@ -215,7 +216,7 @@ class WorkspaceQueue(BaseStashQueue): LocalExpExecutor, ) - result: Dict[str, List[ExpRange]] = defaultdict(list) + result: dict[str, list[ExpRange]] = defaultdict(list) pid = self._active_pid if pid is None: return result @@ -248,12 +249,12 @@ class WorkspaceQueue(BaseStashQueue): self, baseline_revs: Optional[Collection[str]], **kwargs, - ) -> Dict[str, List["ExpRange"]]: + ) -> dict[str, list["ExpRange"]]: raise NotImplementedError def collect_failed_data( self, baseline_revs: Optional[Collection[str]], **kwargs, - ) -> Dict[str, List["ExpRange"]]: + ) -> dict[str, list["ExpRange"]]: raise NotImplementedError diff --git a/dvc/repo/experiments/remove.py b/dvc/repo/experiments/remove.py index 437befa05..2fc7373ec 100644 --- a/dvc/repo/experiments/remove.py +++ b/dvc/repo/experiments/remove.py @@ -1,4 +1,5 @@ -from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Union +from collections.abc import Iterable +from typing import TYPE_CHECKING, Optional, Union from dvc.log import logger from dvc.repo import locked @@ -23,14 +24,14 @@ logger = logger.getChild(__name__) @scm_context def remove( # noqa: C901, PLR0912 repo: "Repo", - exp_names: Union[None, str, List[str]] = None, - rev: Optional[Union[List[str], str]] = None, + exp_names: Union[None, str, list[str]] = None, + rev: Optional[Union[list[str], str]] = None, all_commits: bool = False, num: int = 1, queue: bool = False, git_remote: Optional[str] = None, -) -> List[str]: - removed: List[str] = [] +) -> list[str]: + removed: list[str] = [] if not any([exp_names, queue, all_commits, rev]): return removed celery_queue: "LocalCeleryQueue" = repo.experiments.celery_queue @@ -40,13 +41,13 @@ def remove( # noqa: C901, PLR0912 assert isinstance(repo.scm, Git) - exp_ref_list: List["ExpRefInfo"] = [] - queue_entry_list: List["QueueEntry"] = [] + exp_ref_list: list["ExpRefInfo"] = [] + queue_entry_list: list["QueueEntry"] = [] if exp_names: - results: Dict[ + results: dict[ str, "ExpRefAndQueueEntry" ] = celery_queue.get_ref_and_entry_by_names(exp_names, git_remote) - remained: List[str] = [] + remained: list[str] = [] for name, result in results.items(): if not result.exp_ref_info and not result.queue_entry: remained.append(name) @@ -87,13 +88,13 @@ def remove( # noqa: C901, PLR0912 def _resolve_exp_by_baseline( repo: "Repo", - rev: List[str], + rev: list[str], num: int, git_remote: Optional[str] = None, -) -> Dict[str, "ExpRefInfo"]: +) -> dict[str, "ExpRefInfo"]: assert isinstance(repo.scm, Git) - commit_ref_dict: Dict[str, "ExpRefInfo"] = {} + commit_ref_dict: dict[str, "ExpRefInfo"] = {} rev_dict = iter_revs(repo.scm, rev, num) rev_set = set(rev_dict.keys()) ref_info_dict = exp_refs_by_baseline(repo.scm, rev_set, git_remote) @@ -105,7 +106,7 @@ def _resolve_exp_by_baseline( def _remove_commited_exps( scm: "Git", exp_refs_list: Iterable["ExpRefInfo"], remote: Optional[str] -) -> List[str]: +) -> list[str]: if remote: from dvc.scm import TqdmGit diff --git a/dvc/repo/experiments/rename.py b/dvc/repo/experiments/rename.py index 747be3d7b..4b0b374f9 100644 --- a/dvc/repo/experiments/rename.py +++ b/dvc/repo/experiments/rename.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Optional, Union from dvc.log import logger from dvc.repo.experiments.exceptions import ( @@ -22,16 +22,16 @@ def rename( exp_name: Union[str, None] = None, git_remote: Optional[str] = None, force: bool = False, -) -> Union[List[str], None]: - renamed: List[str] = [] - remained: List[str] = [] +) -> Union[list[str], None]: + renamed: list[str] = [] + remained: list[str] = [] assert isinstance(repo.scm, Git) if exp_name == new_name: return None if exp_name: - results: Dict[str, Union[ExpRefInfo, None]] = resolve_name( + results: dict[str, Union[ExpRefInfo, None]] = resolve_name( scm=repo.scm, exp_names=exp_name, git_remote=git_remote ) for name, result in results.items(): diff --git a/dvc/repo/experiments/run.py b/dvc/repo/experiments/run.py index 298ea1a3b..6de5f1680 100644 --- a/dvc/repo/experiments/run.py +++ b/dvc/repo/experiments/run.py @@ -1,4 +1,5 @@ -from typing import Dict, Iterable, Optional +from collections.abc import Iterable +from typing import Optional from dvc.dependency.param import ParamsDependency from dvc.exceptions import InvalidArgumentError @@ -22,7 +23,7 @@ def run( # noqa: C901, PLR0912 copy_paths: Optional[Iterable[str]] = None, message: Optional[str] = None, **kwargs, -) -> Dict[str, str]: +) -> dict[str, str]: """Reproduce the specified targets as an experiment. Accepts the same additional kwargs as Repo.reproduce. diff --git a/dvc/repo/experiments/save.py b/dvc/repo/experiments/save.py index 1ae4e1594..64e07ab4f 100644 --- a/dvc/repo/experiments/save.py +++ b/dvc/repo/experiments/save.py @@ -1,5 +1,6 @@ import os -from typing import TYPE_CHECKING, Iterable, List, Optional +from collections.abc import Iterable +from typing import TYPE_CHECKING, Optional from funcy import first @@ -18,7 +19,7 @@ def save( name: Optional[str] = None, recursive: bool = False, force: bool = False, - include_untracked: Optional[List[str]] = None, + include_untracked: Optional[list[str]] = None, message: Optional[str] = None, ) -> Optional[str]: """Save the current workspace status as an experiment. diff --git a/dvc/repo/experiments/serialize.py b/dvc/repo/experiments/serialize.py index 556a1834f..e4d2b96c2 100644 --- a/dvc/repo/experiments/serialize.py +++ b/dvc/repo/experiments/serialize.py @@ -1,7 +1,8 @@ import json +from collections.abc import Iterator from dataclasses import asdict, dataclass, field from datetime import datetime -from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Literal, Optional +from typing import TYPE_CHECKING, Any, Literal, Optional from dvc.exceptions import DvcException from dvc.repo.metrics.show import _gather_metrics @@ -30,11 +31,11 @@ class SerializableExp: rev: str timestamp: Optional[datetime] = None - params: Dict[str, "FileResult"] = field(default_factory=dict) - metrics: Dict[str, "FileResult"] = field(default_factory=dict) - deps: Dict[str, "ExpDep"] = field(default_factory=dict) - outs: Dict[str, "ExpOut"] = field(default_factory=dict) - meta: Dict[str, Any] = field(default_factory=dict) + params: dict[str, "FileResult"] = field(default_factory=dict) + metrics: dict[str, "FileResult"] = field(default_factory=dict) + deps: dict[str, "ExpDep"] = field(default_factory=dict) + outs: dict[str, "ExpOut"] = field(default_factory=dict) + meta: dict[str, Any] = field(default_factory=dict) @classmethod def from_repo( @@ -83,7 +84,7 @@ class SerializableExp: **kwargs, ) - def dumpd(self) -> Dict[str, Any]: + def dumpd(self) -> dict[str, Any]: return asdict(self) def as_bytes(self) -> bytes: @@ -131,7 +132,7 @@ class SerializableError: msg: str type: str = "" - def dumpd(self) -> Dict[str, Any]: + def dumpd(self) -> dict[str, Any]: return asdict(self) def as_bytes(self) -> bytes: @@ -154,15 +155,15 @@ class ExpState: name: Optional[str] = None data: Optional[SerializableExp] = None error: Optional[SerializableError] = None - experiments: Optional[List["ExpRange"]] = None + experiments: Optional[list["ExpRange"]] = None - def dumpd(self) -> Dict[str, Any]: + def dumpd(self) -> dict[str, Any]: return asdict(self) @dataclass class ExpRange: - revs: List["ExpState"] + revs: list["ExpState"] executor: Optional["ExpExecutor"] = None name: Optional[str] = None @@ -175,7 +176,7 @@ class ExpRange: def __getitem__(self, index: int) -> "ExpState": return self.revs[index] - def dumpd(self) -> Dict[str, Any]: + def dumpd(self) -> dict[str, Any]: return asdict(self) diff --git a/dvc/repo/experiments/show.py b/dvc/repo/experiments/show.py index d80271640..2d4096417 100644 --- a/dvc/repo/experiments/show.py +++ b/dvc/repo/experiments/show.py @@ -1,18 +1,12 @@ from collections import Counter, defaultdict +from collections.abc import Iterable, Iterator, Mapping from datetime import date, datetime from typing import ( TYPE_CHECKING, Any, - Dict, - Iterable, - Iterator, - List, Literal, - Mapping, NamedTuple, Optional, - Set, - Tuple, Union, ) @@ -36,7 +30,7 @@ logger = logger.getChild(__name__) def show( repo: "Repo", - revs: Union[List[str], str, None] = None, + revs: Union[list[str], str, None] = None, all_branches: bool = False, all_tags: bool = False, all_commits: bool = False, @@ -45,7 +39,7 @@ def show( hide_failed: bool = False, sha_only: bool = False, **kwargs, -) -> List["ExpState"]: +) -> list["ExpState"]: return collect( repo, revs=revs, @@ -65,7 +59,7 @@ def tabulate( fill_value: Optional[str] = "-", error_value: str = "!", **kwargs, -) -> Tuple["TabularData", Dict[str, Iterable[str]]]: +) -> tuple["TabularData", dict[str, Iterable[str]]]: """Return table data for experiments. Returns: @@ -112,7 +106,7 @@ def tabulate( **kwargs, ) ) - data_headers: Dict[str, Iterable[str]] = { + data_headers: dict[str, Iterable[str]] = { "metrics": metrics_headers, "params": params_headers, "deps": deps_names, @@ -128,9 +122,9 @@ def _build_rows( sort_by: Optional[str] = None, sort_order: Optional[Literal["asc", "desc"]] = None, **kwargs, -) -> Iterator[Tuple["CellT", ...]]: +) -> Iterator[tuple["CellT", ...]]: for baseline in baseline_states: - row: Dict[str, "CellT"] = dict.fromkeys(all_headers, fill_value) + row: dict[str, "CellT"] = dict.fromkeys(all_headers, fill_value) row["Experiment"] = "" if baseline.name: row["rev"] = baseline.name @@ -177,10 +171,10 @@ def _sort_column( # noqa: C901 sort_by: str, metric_names: Mapping[str, Iterable[str]], param_names: Mapping[str, Iterable[str]], -) -> Tuple[str, str, str]: +) -> tuple[str, str, str]: sep = ":" parts = sort_by.split(sep) - matches: Set[Tuple[str, str, str]] = set() + matches: set[tuple[str, str, str]] = set() for split_num in range(len(parts)): path = sep.join(parts[:split_num]) @@ -217,7 +211,7 @@ def _sort_exp( sort_name: str, typ: str, reverse: bool, -) -> List["ExpRange"]: +) -> list["ExpRange"]: from funcy import first def _sort(exp_range: "ExpRange"): @@ -238,14 +232,14 @@ def _exp_range_rows( fill_value: Optional[str], is_base: bool = False, **kwargs, -) -> Iterator[Tuple["CellT", ...]]: +) -> Iterator[tuple["CellT", ...]]: from funcy import first if len(exp_range.revs) > 1: logger.debug("Returning tip commit for legacy checkpoint exp") exp = first(exp_range.revs) if exp: - row: Dict[str, "CellT"] = dict.fromkeys(all_headers, fill_value) + row: dict[str, "CellT"] = dict.fromkeys(all_headers, fill_value) row["Experiment"] = exp.name or "" row["rev"] = exp.rev[:7] if Git.is_sha(exp.rev) else exp.rev row["typ"] = "branch_base" if is_base else "branch_commit" @@ -274,12 +268,12 @@ def _data_cells( error_value: str = "!", precision: Optional[int] = None, **kwargs, -) -> Iterator[Tuple[str, "CellT"]]: +) -> Iterator[tuple[str, "CellT"]]: def _d_cells( d: Mapping[str, Any], names: Mapping[str, Iterable[str]], headers: Iterable[str], - ) -> Iterator[Tuple[str, "CellT"]]: + ) -> Iterator[tuple[str, "CellT"]]: from dvc.compare import _format_field, with_value for fname, data in d.items(): @@ -329,9 +323,9 @@ def format_time( class _DataNames(NamedTuple): # NOTE: we use nested dict instead of set for metrics/params names to # preserve key ordering - metrics: Dict[str, Dict[str, Any]] - params: Dict[str, Dict[str, Any]] - deps: Set[str] + metrics: dict[str, dict[str, Any]] + params: dict[str, dict[str, Any]] + deps: set[str] @property def sorted_deps(self): @@ -339,7 +333,7 @@ class _DataNames(NamedTuple): def update(self, other: "_DataNames"): def _update_d( - d: Dict[str, Dict[str, Any]], other_d: Mapping[str, Mapping[str, Any]] + d: dict[str, dict[str, Any]], other_d: Mapping[str, Mapping[str, Any]] ): for k, v in other_d.items(): if k in d: @@ -355,7 +349,7 @@ class _DataNames(NamedTuple): def _collect_names(exp_states: Iterable["ExpState"]) -> _DataNames: result = _DataNames(defaultdict(dict), defaultdict(dict), set()) - def _collect_d(result_d: Dict[str, Dict[str, Any]], data_d: Dict[str, Any]): + def _collect_d(result_d: dict[str, dict[str, Any]], data_d: dict[str, Any]): for path, item in data_d.items(): item = item.get("data", {}) if isinstance(item, dict): @@ -376,7 +370,7 @@ def _collect_names(exp_states: Iterable["ExpState"]) -> _DataNames: def _normalize_headers( names: Mapping[str, Mapping[str, Any]], count: Mapping[str, int] -) -> List[str]: +) -> list[str]: return [ name if count[name] == 1 else f"{path}:{name}" for path in names diff --git a/dvc/repo/experiments/stash.py b/dvc/repo/experiments/stash.py index 607fef5c7..8fbb2c801 100644 --- a/dvc/repo/experiments/stash.py +++ b/dvc/repo/experiments/stash.py @@ -1,6 +1,7 @@ import re +from collections.abc import Iterable, Iterator from contextlib import contextmanager -from typing import Dict, Iterable, Iterator, NamedTuple, Optional +from typing import NamedTuple, Optional from scmrepo.git import Stash @@ -42,7 +43,7 @@ class ExpStash(Stash): ) @property - def stash_revs(self) -> Dict[str, ExpStashEntry]: + def stash_revs(self) -> dict[str, ExpStashEntry]: revs = {} for i, entry in enumerate(self): msg = entry.message.decode("utf-8").strip() @@ -110,7 +111,7 @@ class ApplyStash(Stash): ) @property - def stash_revs(self) -> Dict[str, ApplyStashEntry]: + def stash_revs(self) -> dict[str, ApplyStashEntry]: revs = {} for i, entry in enumerate(self): msg = entry.message.decode("utf-8").strip() diff --git a/dvc/repo/experiments/utils.py b/dvc/repo/experiments/utils.py index 6c6fbe35f..b5ffb1486 100644 --- a/dvc/repo/experiments/utils.py +++ b/dvc/repo/experiments/utils.py @@ -2,18 +2,12 @@ import os import random import sys from collections import defaultdict +from collections.abc import Generator, Iterable, Mapping from functools import wraps from typing import ( TYPE_CHECKING, Callable, - Dict, - Generator, - Iterable, - List, - Mapping, Optional, - Set, - Tuple, Union, ) @@ -43,8 +37,8 @@ EXEC_PID_DIR = "run" def get_exp_rwlock( repo: "Repo", - reads: Optional[List[str]] = None, - writes: Optional[List[str]] = None, + reads: Optional[list[str]] = None, + writes: Optional[list[str]] = None, ): reads = reads or [] writes = writes or [] @@ -109,9 +103,9 @@ def exp_refs_by_rev(scm: "Git", rev: str) -> Generator[ExpRefInfo, None, None]: def exp_refs_by_baseline( scm: "Git", - revs: Optional[Set[str]] = None, + revs: Optional[set[str]] = None, url: Optional[str] = None, -) -> Mapping[str, List[ExpRefInfo]]: +) -> Mapping[str, list[ExpRefInfo]]: """Iterate over all experiment refs with the specified baseline.""" all_exp_refs = exp_refs(scm, url) result = defaultdict(list) @@ -137,7 +131,7 @@ def iter_remote_refs(scm: "Git", url: str, base: Optional[str] = None, **kwargs) def push_refspec( scm: "Git", url: str, - push_list=List[Tuple[Optional[str], str]], + push_list=list[tuple[Optional[str], str]], force: bool = False, on_diverged: Optional[Callable[[str, str], bool]] = None, **kwargs, @@ -183,8 +177,8 @@ def remote_exp_refs(scm: "Git", url: str) -> Generator[ExpRefInfo, None, None]: def exp_refs_by_names( - scm: "Git", names: Set[str], url: Optional[str] = None -) -> Dict[str, List[ExpRefInfo]]: + scm: "Git", names: set[str], url: Optional[str] = None +) -> dict[str, list[ExpRefInfo]]: """Iterate over all experiment refs matching the specified names.""" resolve_results = defaultdict(list) ref_info_gen = exp_refs(scm, url) @@ -210,7 +204,7 @@ def exp_commits( scm: "Git", ref_infos: Optional[Iterable[ExpRefInfo]] = None ) -> Iterable[str]: """Iterate over all experiment commits.""" - shas: Set["str"] = set() + shas: set["str"] = set() refs = ref_infos if ref_infos else exp_refs(scm) for ref_info in refs: shas.update(scm.branch_revs(str(ref_info), ref_info.baseline_sha)) @@ -242,7 +236,7 @@ def resolve_name( scm: "Git", exp_names: Union[Iterable[str], str], git_remote: Optional[str] = None, -) -> Dict[str, Optional[ExpRefInfo]]: +) -> dict[str, Optional[ExpRefInfo]]: """find the ref_info of specified names.""" if isinstance(exp_names, str): exp_names = [exp_names] @@ -338,7 +332,7 @@ def to_studio_params(dvc_params): "params.yaml": {"foo": 1} } """ - result: Dict = {} + result: dict = {} if not dvc_params: return result for rev_data in dvc_params.values(): @@ -353,7 +347,7 @@ def describe( revs: Iterable[str], logger, refs: Optional[Iterable[str]] = None, -) -> Dict[str, Optional[str]]: +) -> dict[str, Optional[str]]: """Describe revisions using a tag, branch. The first matching name will be returned for each rev. Names are preferred in this @@ -390,7 +384,7 @@ def describe( if is_branch and rev not in branches: branches[rev] = ref[len("refs/heads/") :] - names: Dict[str, Optional[str]] = {} + names: dict[str, Optional[str]] = {} for rev in revs: if rev == head_rev and head_branch: names[rev] = head_branch diff --git a/dvc/repo/fetch.py b/dvc/repo/fetch.py index 1967725a0..1a0839caa 100644 --- a/dvc/repo/fetch.py +++ b/dvc/repo/fetch.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, List, Tuple +from typing import TYPE_CHECKING from dvc.exceptions import DownloadError from dvc.log import logger @@ -205,9 +205,9 @@ def fetch( # noqa: PLR0913 return transferred_count -def _log_unversioned(data: List["DataIndex"]) -> Tuple[List["DataIndex"], int]: - ret: List["DataIndex"] = [] - unversioned: List[str] = [] +def _log_unversioned(data: list["DataIndex"]) -> tuple[list["DataIndex"], int]: + ret: list["DataIndex"] = [] + unversioned: list[str] = [] for fs_index in data: remote = fs_index.storage_map[()].remote if not isinstance(remote, FileStorage) or not remote.fs.version_aware: diff --git a/dvc/repo/gc.py b/dvc/repo/gc.py index 682d05a3f..eb660c4ce 100644 --- a/dvc/repo/gc.py +++ b/dvc/repo/gc.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, List, Optional +from typing import TYPE_CHECKING, Optional from dvc.exceptions import InvalidArgumentError from dvc.log import logger @@ -61,7 +61,7 @@ def gc( # noqa: PLR0913, C901 all_experiments: bool = False, force: bool = False, jobs: Optional[int] = None, - repos: Optional[List[str]] = None, + repos: Optional[list[str]] = None, workspace: bool = False, commit_date: Optional[str] = None, rev: Optional[str] = None, diff --git a/dvc/repo/graph.py b/dvc/repo/graph.py index 24b62ffd0..6534d2300 100644 --- a/dvc/repo/graph.py +++ b/dvc/repo/graph.py @@ -1,4 +1,5 @@ -from typing import TYPE_CHECKING, Any, Iterator, List, Optional, Set, TypeVar +from collections.abc import Iterator +from typing import TYPE_CHECKING, Any, Optional, TypeVar from dvc.fs import localfs from dvc.utils.fs import path_isin @@ -21,7 +22,7 @@ def check_acyclic(graph: "DiGraph") -> None: except nx.NetworkXNoCycle: return - stages: Set["Stage"] = set() + stages: set["Stage"] = set() for from_node, to_node, _ in edges: stages.add(from_node) stages.add(to_node) @@ -45,7 +46,7 @@ def get_pipelines(graph: "DiGraph"): def get_subgraph_of_nodes( - graph: "DiGraph", sources: Optional[List[Any]] = None, downstream: bool = False + graph: "DiGraph", sources: Optional[list[Any]] = None, downstream: bool = False ) -> "DiGraph": from networkx import dfs_postorder_nodes, reverse_view @@ -69,7 +70,7 @@ def collect_pipeline(stage: "Stage", graph: "DiGraph") -> Iterator["Stage"]: return nx.dfs_postorder_nodes(pipeline, stage) -def collect_inside_path(path: str, graph: "DiGraph") -> List["Stage"]: +def collect_inside_path(path: str, graph: "DiGraph") -> list["Stage"]: import networkx as nx stages = nx.dfs_postorder_nodes(graph) diff --git a/dvc/repo/imp_db.py b/dvc/repo/imp_db.py index 0366dd941..dc5aa2912 100644 --- a/dvc/repo/imp_db.py +++ b/dvc/repo/imp_db.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Dict, Optional +from typing import TYPE_CHECKING, Optional from funcy import compact @@ -33,7 +33,7 @@ def imp_db( assert sql or table assert output_format in ("csv", "json") - db: Dict[str, str] = compact( + db: dict[str, str] = compact( { "connection": connection, "file_format": output_format, diff --git a/dvc/repo/imports.py b/dvc/repo/imports.py index c373d7ace..516bb129b 100644 --- a/dvc/repo/imports.py +++ b/dvc/repo/imports.py @@ -1,7 +1,7 @@ import os from functools import partial from tempfile import TemporaryDirectory -from typing import TYPE_CHECKING, List, Set, Tuple, Union +from typing import TYPE_CHECKING, Union from dvc.log import logger @@ -19,7 +19,7 @@ logger = logger.getChild(__name__) def unfetched_view( index: "Index", targets: "TargetType", unpartial: bool = False, **kwargs -) -> Tuple["IndexView", "IndexView", List["Dependency"]]: +) -> tuple["IndexView", "IndexView", list["Dependency"]]: """Return index view of imports which have not been fetched. Returns: @@ -28,7 +28,7 @@ def unfetched_view( """ from dvc.cachemgr import LEGACY_HASH_NAMES - changed_deps: List["Dependency"] = [] + changed_deps: list["Dependency"] = [] def need_fetch(stage: "Stage", legacy: bool = False) -> bool: if not stage.is_import or (stage.is_partial_import and not unpartial): @@ -92,7 +92,7 @@ def unpartial_imports(index: Union["Index", "IndexView"]) -> int: def save_imports( repo: "Repo", targets: "TargetType", unpartial: bool = False, **kwargs -) -> Set["HashInfo"]: +) -> set["HashInfo"]: """Save (download) imports from their original source location. Imports which are already cached will not be downloaded. @@ -105,7 +105,7 @@ def save_imports( from dvc_data.index import md5, save from dvc_data.index.checkout import apply, compare - downloaded: Set["HashInfo"] = set() + downloaded: set["HashInfo"] = set() legacy_unfetched, unfetched, changed = unfetched_view( repo.index, targets, unpartial=unpartial, **kwargs diff --git a/dvc/repo/index.py b/dvc/repo/index.py index fe9d29ca8..c32e537c4 100644 --- a/dvc/repo/index.py +++ b/dvc/repo/index.py @@ -1,20 +1,15 @@ import logging import time from collections import defaultdict +from collections.abc import Iterable, Iterator from functools import partial from itertools import chain from typing import ( TYPE_CHECKING, Any, Callable, - Dict, - Iterable, - Iterator, - List, NamedTuple, Optional, - Set, - Tuple, Union, ) @@ -44,7 +39,7 @@ if TYPE_CHECKING: logger = logger.getChild(__name__) -ObjectContainer = Dict[Optional["HashFileDB"], Set["HashInfo"]] +ObjectContainer = dict[Optional["HashFileDB"], set["HashInfo"]] def log_walk(seq): @@ -74,7 +69,7 @@ def collect_files( scm = repo.scm fs = repo.fs sep = fs.sep - outs: Set[str] = set() + outs: set[str] = set() is_local_fs = isinstance(fs, LocalFileSystem) @@ -288,11 +283,11 @@ class Index: def __init__( self, repo: "Repo", - stages: Optional[List["Stage"]] = None, - metrics: Optional[Dict[str, List[str]]] = None, - plots: Optional[Dict[str, List[str]]] = None, - params: Optional[Dict[str, Any]] = None, - artifacts: Optional[Dict[str, Any]] = None, + stages: Optional[list["Stage"]] = None, + metrics: Optional[dict[str, list[str]]] = None, + plots: Optional[dict[str, list[str]]] = None, + params: Optional[dict[str, Any]] = None, + artifacts: Optional[dict[str, Any]] = None, ) -> None: self.repo = repo self.stages = stages or [] @@ -300,7 +295,7 @@ class Index: self._plots = plots or {} self._params = params or {} self._artifacts = artifacts or {} - self._collected_targets: Dict[int, List["StageInfo"]] = {} + self._collected_targets: dict[int, list["StageInfo"]] = {} @cached_property def rev(self) -> Optional[str]: @@ -404,8 +399,8 @@ class Index: yield from stage.outs @cached_property - def out_data_keys(self) -> Dict[str, Set["DataIndexKey"]]: - by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set) + def out_data_keys(self) -> dict[str, set["DataIndexKey"]]: + by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set) by_workspace["repo"] = set() by_workspace["local"] = set() @@ -443,10 +438,10 @@ class Index: yield from stage.deps @cached_property - def _plot_sources(self) -> List[str]: + def _plot_sources(self) -> list[str]: from dvc.repo.plots import _collect_pipeline_files - sources: List[str] = [] + sources: list[str] = [] for data in _collect_pipeline_files(self.repo, [], {}).values(): for plot_id, props in data.get("data", {}).items(): if isinstance(props.get("y"), dict): @@ -458,8 +453,8 @@ class Index: return sources @cached_property - def data_keys(self) -> Dict[str, Set["DataIndexKey"]]: - by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set) + def data_keys(self) -> dict[str, set["DataIndexKey"]]: + by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set) by_workspace["repo"] = set() by_workspace["local"] = set() @@ -474,10 +469,10 @@ class Index: return dict(by_workspace) @cached_property - def metric_keys(self) -> Dict[str, Set["DataIndexKey"]]: + def metric_keys(self) -> dict[str, set["DataIndexKey"]]: from .metrics.show import _collect_top_level_metrics - by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set) + by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set) by_workspace["repo"] = set() @@ -495,10 +490,10 @@ class Index: return dict(by_workspace) @cached_property - def param_keys(self) -> Dict[str, Set["DataIndexKey"]]: + def param_keys(self) -> dict[str, set["DataIndexKey"]]: from .params.show import _collect_top_level_params - by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set) + by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set) by_workspace["repo"] = set() param_paths = _collect_top_level_params(self.repo) @@ -513,8 +508,8 @@ class Index: return dict(by_workspace) @cached_property - def plot_keys(self) -> Dict[str, Set["DataIndexKey"]]: - by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set) + def plot_keys(self) -> dict[str, set["DataIndexKey"]]: + by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set) by_workspace["repo"] = set() @@ -536,7 +531,7 @@ class Index: return _build_tree_from_outs(self.outs) @cached_property - def data(self) -> "Dict[str, DataIndex]": + def data(self) -> "dict[str, DataIndex]": prefix: "DataIndexKey" loaded = False @@ -584,7 +579,7 @@ class Index: def collect_targets( self, targets: Optional["TargetType"], *, onerror=None, **kwargs: Any - ) -> List["StageInfo"]: + ) -> list["StageInfo"]: from dvc.exceptions import DvcException from dvc.repo.stage import StageInfo from dvc.utils.collections import ensure_list @@ -658,7 +653,7 @@ class Index: stage_filter: Optional[Callable[["Stage"], bool]] = None, outs_filter: Optional[Callable[["Output"], bool]] = None, max_size: Optional[int] = None, - types: Optional[List[str]] = None, + types: Optional[list[str]] = None, **kwargs: Any, ) -> "IndexView": """Return read-only view of index for the specified targets. @@ -698,8 +693,8 @@ class Index: class _DataPrefixes(NamedTuple): - explicit: Set["DataIndexKey"] - recursive: Set["DataIndexKey"] + explicit: set["DataIndexKey"] + recursive: set["DataIndexKey"] class IndexView: @@ -728,7 +723,7 @@ class IndexView: yield from stage.deps @property - def _filtered_outs(self) -> Iterator[Tuple["Output", Optional[str]]]: + def _filtered_outs(self) -> Iterator[tuple["Output", Optional[str]]]: for stage, filter_info in self._stage_infos: for out in stage.filter_outs(filter_info): if not self._outs_filter or self._outs_filter(out): @@ -739,8 +734,8 @@ class IndexView: yield from {out for (out, _) in self._filtered_outs} @cached_property - def out_data_keys(self) -> Dict[str, Set["DataIndexKey"]]: - by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set) + def out_data_keys(self) -> dict[str, set["DataIndexKey"]]: + by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set) by_workspace["repo"] = set() by_workspace["local"] = set() @@ -755,8 +750,8 @@ class IndexView: return dict(by_workspace) @cached_property - def _data_prefixes(self) -> Dict[str, "_DataPrefixes"]: - prefixes: Dict[str, "_DataPrefixes"] = defaultdict( + def _data_prefixes(self) -> dict[str, "_DataPrefixes"]: + prefixes: dict[str, "_DataPrefixes"] = defaultdict( lambda: _DataPrefixes(set(), set()) ) for out, filter_info in self._filtered_outs: @@ -772,8 +767,8 @@ class IndexView: return prefixes @cached_property - def data_keys(self) -> Dict[str, Set["DataIndexKey"]]: - ret: Dict[str, Set["DataIndexKey"]] = defaultdict(set) + def data_keys(self) -> dict[str, set["DataIndexKey"]]: + ret: dict[str, set["DataIndexKey"]] = defaultdict(set) for out, filter_info in self._filtered_outs: if not out.use_cache: @@ -791,7 +786,7 @@ class IndexView: return _build_tree_from_outs(self.outs) @cached_property - def data(self) -> Dict[str, Union["DataIndex", "DataIndexView"]]: + def data(self) -> dict[str, Union["DataIndex", "DataIndexView"]]: from dvc_data.index import DataIndex, view def key_filter(workspace: str, key: "DataIndexKey"): @@ -803,7 +798,7 @@ class IndexView: except KeyError: return False - data: Dict[str, Union["DataIndex", "DataIndexView"]] = {} + data: dict[str, Union["DataIndex", "DataIndexView"]] = {} for workspace, data_index in self._index.data.items(): if self.stages: data[workspace] = view(data_index, partial(key_filter, workspace)) diff --git a/dvc/repo/ls.py b/dvc/repo/ls.py index 19d4d7b82..0dd09c95f 100644 --- a/dvc/repo/ls.py +++ b/dvc/repo/ls.py @@ -1,5 +1,5 @@ import os -from typing import TYPE_CHECKING, Any, Dict, Optional, Union +from typing import TYPE_CHECKING, Any, Optional, Union if TYPE_CHECKING: from dvc.fs.dvc import DVCFileSystem @@ -13,7 +13,7 @@ def ls( rev: Optional[str] = None, recursive: Optional[bool] = None, dvc_only: bool = False, - config: Union[None, Dict[str, Any], str] = None, + config: Union[None, dict[str, Any], str] = None, remote: Optional[str] = None, remote_config: Optional[dict] = None, ): diff --git a/dvc/repo/metrics/diff.py b/dvc/repo/metrics/diff.py index 5fc6a35bb..99077407b 100644 --- a/dvc/repo/metrics/diff.py +++ b/dvc/repo/metrics/diff.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Dict, TypedDict, Union +from typing import TYPE_CHECKING, TypedDict, Union from funcy import compact @@ -12,12 +12,12 @@ if TYPE_CHECKING: class DiffResult(TypedDict, total=False): - errors: Dict[str, Union[Exception, Dict[str, Exception]]] - diff: Dict[str, Dict[str, Dict]] + errors: dict[str, Union[Exception, dict[str, Exception]]] + diff: dict[str, dict[str, dict]] def _diff( - result: Dict[str, "Result"], + result: dict[str, "Result"], old_rev: str, new_rev: str, **kwargs, diff --git a/dvc/repo/metrics/show.py b/dvc/repo/metrics/show.py index 3f2ff7f37..ba3728140 100644 --- a/dvc/repo/metrics/show.py +++ b/dvc/repo/metrics/show.py @@ -1,14 +1,10 @@ import os +from collections.abc import Iterable, Iterator from itertools import chain from typing import ( TYPE_CHECKING, Any, - Dict, - Iterable, - Iterator, - List, Optional, - Tuple, TypedDict, Union, ) @@ -71,7 +67,7 @@ def _read_metric(fs: "FileSystem", path: str, **load_kwargs) -> Any: def _read_metrics( fs: "FileSystem", metrics: Iterable[str], **load_kwargs -) -> Iterator[Tuple[str, Union[Exception, Any]]]: +) -> Iterator[tuple[str, Union[Exception, Any]]]: for metric in metrics: try: yield metric, _read_metric(fs, metric, **load_kwargs) @@ -80,7 +76,7 @@ def _read_metrics( yield metric, exc -def metrics_from_target(repo: "Repo", targets: List[str]) -> Iterator["Output"]: +def metrics_from_target(repo: "Repo", targets: list[str]) -> Iterator["Output"]: stages = chain.from_iterable(repo.stage.collect(target) for target in targets) for stage in stages: yield from stage.metrics @@ -88,11 +84,11 @@ def metrics_from_target(repo: "Repo", targets: List[str]) -> Iterator["Output"]: def _collect_metrics( repo: "Repo", - targets: Optional[List[str]] = None, - stages: Optional[List[str]] = None, + targets: Optional[list[str]] = None, + stages: Optional[list[str]] = None, outs_only: bool = False, -) -> List[str]: - metrics: List[str] = [] +) -> list[str]: + metrics: list[str] = [] if targets: # target is a repo-relative path @@ -122,7 +118,7 @@ class FileResult(TypedDict, total=False): class Result(TypedDict, total=False): - data: Dict[str, FileResult] + data: dict[str, FileResult] error: Exception @@ -139,11 +135,11 @@ def to_relpath(fs: "FileSystem", root_dir: str, d: Result) -> Result: def _gather_metrics( repo: "Repo", - targets: Optional[List[str]] = None, + targets: Optional[list[str]] = None, outs_only: bool = False, - stages: Optional[List[str]] = None, + stages: Optional[list[str]] = None, on_error: str = "return", -) -> Dict[str, FileResult]: +) -> dict[str, FileResult]: assert on_error in ("raise", "return", "ignore") # `files` is a repo-relative posixpath that can be passed to DVCFileSystem @@ -168,8 +164,8 @@ def _gather_metrics( def _hide_workspace( - scm: Union["Git", "NoSCM"], res: Dict[str, Result] -) -> Dict[str, Result]: + scm: Union["Git", "NoSCM"], res: dict[str, Result] +) -> dict[str, Result]: # Hide workspace params if they are the same as in the active branch try: active_branch = scm.active_branch() @@ -186,16 +182,16 @@ def _hide_workspace( def show( repo: "Repo", - targets: Optional[List[str]] = None, - stages: Optional[List[str]] = None, + targets: Optional[list[str]] = None, + stages: Optional[list[str]] = None, outs_only: bool = False, all_branches: bool = False, all_tags: bool = False, - revs: Optional[List[str]] = None, + revs: Optional[list[str]] = None, all_commits: bool = False, hide_workspace: bool = True, on_error: str = "return", -) -> Dict[str, Result]: +) -> dict[str, Result]: assert on_error in ("raise", "return", "ignore") targets = [os.path.abspath(target) for target in ensure_list(targets)] diff --git a/dvc/repo/open_repo.py b/dvc/repo/open_repo.py index 8cc08b187..2788b56e8 100644 --- a/dvc/repo/open_repo.py +++ b/dvc/repo/open_repo.py @@ -1,7 +1,7 @@ import os import tempfile import threading -from typing import TYPE_CHECKING, Dict, Optional, Tuple +from typing import TYPE_CHECKING, Optional from funcy import retry, wrap_with @@ -78,8 +78,8 @@ def erepo_factory(url, root_dir, cache_config): return make_repo -CLONES: Dict[str, Tuple[str, bool]] = {} -CACHE_DIRS: Dict[str, str] = {} +CLONES: dict[str, tuple[str, bool]] = {} +CACHE_DIRS: dict[str, str] = {} @wrap_with(threading.Lock()) diff --git a/dvc/repo/params/show.py b/dvc/repo/params/show.py index 8873e6032..036a39b84 100644 --- a/dvc/repo/params/show.py +++ b/dvc/repo/params/show.py @@ -1,7 +1,8 @@ import os from collections import defaultdict +from collections.abc import Iterator from itertools import chain -from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Optional, Union from dvc.dependency.param import ParamsDependency, read_param_file from dvc.log import logger @@ -27,7 +28,7 @@ def _collect_top_level_params(repo: "Repo") -> Iterator[str]: def params_from_target( - repo: "Repo", targets: List[str] + repo: "Repo", targets: list[str] ) -> Iterator["ParamsDependency"]: stages = chain.from_iterable(repo.stage.collect(target) for target in targets) for stage in stages: @@ -36,17 +37,17 @@ def params_from_target( def _collect_params( repo: "Repo", - targets: Union[List[str], Dict[str, List[str]], None] = None, - stages: Optional[List[str]] = None, + targets: Union[list[str], dict[str, list[str]], None] = None, + stages: Optional[list[str]] = None, deps_only: bool = False, default_file: Optional[str] = None, -) -> Dict[str, List[str]]: +) -> dict[str, list[str]]: from dvc.dependency import _merge_params if isinstance(targets, list): targets = {target: [] for target in targets} - params: List[Dict[str, List[str]]] = [] + params: list[dict[str, list[str]]] = [] if targets: # target is a repo-relative path @@ -80,8 +81,8 @@ def _collect_params( return ret -def _collect_vars(repo, params, stages=None) -> Dict: - vars_params: Dict[str, Dict] = defaultdict(dict) +def _collect_vars(repo, params, stages=None) -> dict: + vars_params: dict[str, dict] = defaultdict(dict) for stage in repo.index.stages: if isinstance(stage, PipelineStage) and stage.tracked_vars: @@ -102,8 +103,8 @@ def _collect_vars(repo, params, stages=None) -> Dict: def _read_params( - fs: "FileSystem", params: Dict[str, List[str]], **load_kwargs -) -> Iterator[Tuple[str, Union[Exception, Any]]]: + fs: "FileSystem", params: dict[str, list[str]], **load_kwargs +) -> Iterator[tuple[str, Union[Exception, Any]]]: for file_path, key_paths in params.items(): try: yield file_path, read_param_file(fs, file_path, key_paths, **load_kwargs) @@ -114,9 +115,9 @@ def _read_params( def _gather_params( repo: "Repo", - targets: Union[List[str], Dict[str, List[str]], None] = None, + targets: Union[list[str], dict[str, list[str]], None] = None, deps_only: bool = False, - stages: Optional[List[str]] = None, + stages: Optional[list[str]] = None, on_error: str = "return", ): assert on_error in ("raise", "return", "ignore") @@ -132,7 +133,7 @@ def _gather_params( default_file=ParamsDependency.DEFAULT_PARAMS_FILE, ) - data: Dict[str, FileResult] = {} + data: dict[str, FileResult] = {} fs = repo.dvcfs for fs_path, result in _read_params(fs, files_keypaths, cache=True): @@ -159,16 +160,16 @@ def _gather_params( def show( repo: "Repo", - targets: Optional[List[str]] = None, - stages: Optional[List[str]] = None, + targets: Optional[list[str]] = None, + stages: Optional[list[str]] = None, deps_only: bool = False, all_branches: bool = False, all_tags: bool = False, - revs: Optional[List[str]] = None, + revs: Optional[list[str]] = None, all_commits: bool = False, hide_workspace: bool = True, on_error: str = "return", -) -> Dict[str, Result]: +) -> dict[str, Result]: assert on_error in ("raise", "return", "ignore") res = {} diff --git a/dvc/repo/plots/__init__.py b/dvc/repo/plots/__init__.py index 22b0898f9..83b56e73d 100644 --- a/dvc/repo/plots/__init__.py +++ b/dvc/repo/plots/__init__.py @@ -2,6 +2,7 @@ import csv import io import os from collections import defaultdict +from collections.abc import Iterator from copy import deepcopy from functools import partial from multiprocessing import cpu_count @@ -9,11 +10,7 @@ from typing import ( TYPE_CHECKING, Any, Callable, - Dict, - Iterator, - List, Optional, - Set, Union, ) @@ -40,7 +37,7 @@ dpath.options.ALLOW_EMPTY_STRING_KEYS = True logger = logger.getChild(__name__) -def onerror_collect(result: Dict, exception: Exception, *args, **kwargs): +def onerror_collect(result: dict, exception: Exception, *args, **kwargs): logger.debug("", exc_info=True) result["error"] = exception @@ -82,12 +79,12 @@ class Plots: def collect( self, - targets: Optional[List[str]] = None, - revs: Optional[List[str]] = None, + targets: Optional[list[str]] = None, + revs: Optional[list[str]] = None, recursive: bool = False, onerror: Optional[Callable] = None, - props: Optional[Dict] = None, - ) -> Iterator[Dict]: + props: Optional[dict] = None, + ) -> Iterator[dict]: """Collects plots definitions and data sources. Generator yielding a structure like: @@ -142,7 +139,7 @@ class Plots: revs = ["workspace", *revs] for rev in revs: with switch_repo(self.repo, rev) as (repo, _): - res: Dict = {} + res: dict = {} definitions = _collect_definitions( repo, targets=targets, @@ -168,9 +165,9 @@ class Plots: def _collect_data_sources( self, repo: "Repo", - targets: Optional[List[str]] = None, + targets: Optional[list[str]] = None, recursive: bool = False, - props: Optional[Dict] = None, + props: Optional[dict] = None, onerror: Optional[Callable] = None, ): fs = repo.dvcfs @@ -178,7 +175,7 @@ class Plots: props = props or {} plots = _collect_plots(repo, targets, recursive) - res: Dict[str, Any] = {} + res: dict[str, Any] = {} for fs_path, rev_props in plots.items(): joined_props = {**rev_props, **props} res[fs_path] = {"props": joined_props} @@ -197,7 +194,7 @@ class Plots: def show( self, - targets: Optional[List[str]] = None, + targets: Optional[list[str]] = None, revs=None, props=None, recursive=False, @@ -206,7 +203,7 @@ class Plots: if onerror is None: onerror = onerror_collect - result: Dict[str, Dict] = {} + result: dict[str, dict] = {} for data in self.collect( targets, revs, @@ -286,7 +283,7 @@ def _is_plot(out: "Output") -> bool: return bool(out.plot) -def _resolve_data_sources(plots_data: Dict, rev: str, cache: bool = False): +def _resolve_data_sources(plots_data: dict, rev: str, cache: bool = False): from dvc.progress import Tqdm values = list(plots_data.values()) @@ -325,9 +322,9 @@ def _resolve_data_sources(plots_data: Dict, rev: str, cache: bool = False): def _collect_plots( repo: "Repo", - targets: Optional[List[str]] = None, + targets: Optional[list[str]] = None, recursive: bool = False, -) -> Dict[str, Dict]: +) -> dict[str, dict]: from dvc.repo.collect import collect plots, fs_paths = collect( @@ -344,8 +341,8 @@ def _collect_plots( return result -def _get_data_targets(definitions: Dict): - result: Set = set() +def _get_data_targets(definitions: dict): + result: set = set() if "data" in definitions: for content in definitions["data"].values(): if "data" in content: @@ -404,7 +401,7 @@ def _relpath(fs, path): def _collect_output_plots(repo, targets, props, onerror: Optional[Callable] = None): fs = repo.dvcfs - result: Dict[str, Dict] = {} + result: dict[str, dict] = {} for plot in repo.index.plots: plot_props = _plot_props(plot) dvcfile = plot.stage.dvcfile @@ -445,15 +442,15 @@ def _adjust_sources(fs, plot_props, config_dir): def _resolve_definitions( fs: "FileSystem", - targets: List[str], - props: Dict[str, Any], + targets: list[str], + props: dict[str, Any], config_path: "StrPath", definitions: "DictStrAny", onerror: Optional[Callable[[Any], Any]] = None, ): config_path = os.fspath(config_path) config_dir = fs.dirname(config_path) - result: Dict[str, Dict] = {} + result: dict[str, dict] = {} for plot_id, plot_props in definitions.items(): if plot_props is None: plot_props = {} @@ -477,12 +474,12 @@ def _resolve_definitions( return result -def _collect_pipeline_files(repo, targets: List[str], props, onerror=None): - result: Dict[str, Dict] = {} +def _collect_pipeline_files(repo, targets: list[str], props, onerror=None): + result: dict[str, dict] = {} top_plots = repo.index._plots for dvcfile, plots_def in top_plots.items(): dvcfile_path = _relpath(repo.dvcfs, dvcfile) - dvcfile_defs_dict: Dict[str, Union[Dict, None]] = {} + dvcfile_defs_dict: dict[str, Union[dict, None]] = {} for elem in plots_def: if isinstance(elem, str): dvcfile_defs_dict[elem] = None @@ -510,11 +507,11 @@ def _collect_pipeline_files(repo, targets: List[str], props, onerror=None): def _collect_definitions( repo: "Repo", targets=None, - props: Optional[Dict] = None, + props: Optional[dict] = None, onerror: Optional[Callable] = None, **kwargs, -) -> Dict: - result: Dict = defaultdict(dict) +) -> dict: + result: dict = defaultdict(dict) props = props or {} fs = repo.dvcfs @@ -536,8 +533,8 @@ def _collect_definitions( return dict(result) -def unpack_if_dir(fs, path, props: Dict[str, str], onerror: Optional[Callable] = None): - result: Dict[str, Dict] = defaultdict(dict) +def unpack_if_dir(fs, path, props: dict[str, str], onerror: Optional[Callable] = None): + result: dict[str, dict] = defaultdict(dict) if fs.isdir(path): unpacked = _unpack_dir_files(fs, path, onerror=onerror) else: @@ -574,7 +571,7 @@ def parse(fs, path, props=None, **fs_kwargs): return PARSERS[extension](contents, path) -def _plot_props(out: "Output") -> Dict: +def _plot_props(out: "Output") -> dict: from dvc.schema import PLOT_PROPS if not (out.plot): diff --git a/dvc/repo/reproduce.py b/dvc/repo/reproduce.py index 14e7f9385..d66408cc2 100644 --- a/dvc/repo/reproduce.py +++ b/dvc/repo/reproduce.py @@ -1,13 +1,9 @@ +from collections.abc import Iterable from typing import ( TYPE_CHECKING, Callable, - Dict, - Iterable, - List, NoReturn, Optional, - Set, - Tuple, TypeVar, Union, cast, @@ -40,8 +36,8 @@ def collect_stages( targets: Iterable[str], recursive: bool = False, glob: bool = False, -) -> List["Stage"]: - stages: List["Stage"] = [] +) -> list["Stage"]: + stages: list["Stage"] = [] for target in targets: stages.extend(repo.stage.collect(target, recursive=recursive, glob=glob)) return ldistinct(stages) @@ -49,7 +45,7 @@ def collect_stages( def get_subgraph( graph: "DiGraph", - nodes: Optional[List] = None, + nodes: Optional[list] = None, pipeline: bool = False, downstream: bool = False, ) -> "DiGraph": @@ -76,10 +72,10 @@ def get_active_graph(graph: "DiGraph") -> "DiGraph": def plan_repro( graph: "DiGraph", - stages: Optional[List["T"]] = None, + stages: Optional[list["T"]] = None, pipeline: bool = False, downstream: bool = False, -) -> List["T"]: +) -> list["T"]: r"""Derive the evaluation of the given node for the given graph. When you _reproduce a stage_, you want to _evaluate the descendants_ @@ -134,7 +130,7 @@ def _reproduce_stage(stage: "Stage", **kwargs) -> Optional["Stage"]: def _get_upstream_downstream_nodes( graph: Optional["DiGraph"], node: T -) -> Tuple[List[T], List[T]]: +) -> tuple[list[T], list[T]]: succ = list(graph.successors(node)) if graph else [] pre = list(graph.predecessors(node)) if graph else [] return succ, pre @@ -146,7 +142,7 @@ def _repr(stages: Iterable["Stage"]) -> str: def handle_error( graph: Optional["DiGraph"], on_error: str, exc: Exception, stage: "Stage" -) -> Set["Stage"]: +) -> set["Stage"]: import networkx as nx logger.warning("%s%s", exc, " (ignored)" if on_error == "ignore" else "") @@ -168,19 +164,19 @@ def _raise_error(exc: Optional[Exception], *stages: "Stage") -> NoReturn: def _reproduce( - stages: List["Stage"], + stages: list["Stage"], graph: Optional["DiGraph"] = None, force_downstream: bool = False, on_error: str = "fail", force: bool = False, repro_fn: Callable = _reproduce_stage, **kwargs, -) -> List["Stage"]: +) -> list["Stage"]: assert on_error in ("fail", "keep-going", "ignore") - result: List["Stage"] = [] - failed: List["Stage"] = [] - to_skip: Dict["Stage", "Stage"] = {} + result: list["Stage"] = [] + failed: list["Stage"] = [] + to_skip: dict["Stage", "Stage"] = {} ret: Optional["Stage"] = None force_state = dict.fromkeys(stages, force) @@ -240,7 +236,7 @@ def reproduce( if not kwargs.get("interactive", False): kwargs["interactive"] = self.config["core"].get("interactive", False) - stages: List["Stage"] = [] + stages: list["Stage"] = [] if not all_pipelines: targets_list = ensure_list(targets or PROJECT_FILE) stages = collect_stages(self, targets_list, recursive=recursive, glob=glob) diff --git a/dvc/repo/scm_context.py b/dvc/repo/scm_context.py index c8bb6865a..d8e110959 100644 --- a/dvc/repo/scm_context.py +++ b/dvc/repo/scm_context.py @@ -1,16 +1,12 @@ import logging import shlex +from collections.abc import Iterable, Iterator from contextlib import contextmanager from functools import wraps from typing import ( TYPE_CHECKING, Any, - Dict, - Iterable, - Iterator, - List, Optional, - Set, Union, ) @@ -27,15 +23,15 @@ logger = logger.getChild(__name__) class SCMContext: - def __init__(self, scm: "Base", config: Optional[Dict[str, Any]] = None) -> None: + def __init__(self, scm: "Base", config: Optional[dict[str, Any]] = None) -> None: from funcy import get_in self.scm: "Base" = scm self.autostage: bool = get_in( config or {}, ["core", "autostage"], default=False ) - self.ignored_paths: List[str] = [] - self.files_to_track: Set[str] = set() + self.ignored_paths: list[str] = [] + self.files_to_track: set[str] = set() self.quiet: bool = False def track_file(self, paths: Union[str, Iterable[str], None] = None) -> None: diff --git a/dvc/repo/stage.py b/dvc/repo/stage.py index 6824ce51b..7c5221cb0 100644 --- a/dvc/repo/stage.py +++ b/dvc/repo/stage.py @@ -1,8 +1,9 @@ import fnmatch import typing +from collections.abc import Iterable from contextlib import suppress from functools import wraps -from typing import Iterable, List, NamedTuple, Optional, Set, Tuple, Union +from typing import NamedTuple, Optional, Union from dvc.exceptions import ( NoOutputOrStageError, @@ -31,9 +32,9 @@ class StageInfo(NamedTuple): filter_info: Optional[str] = None -StageList = List["Stage"] +StageList = list["Stage"] StageIter = Iterable["Stage"] -StageSet = Set["Stage"] +StageSet = set["Stage"] def _collect_with_deps(stages: StageList, graph: "DiGraph") -> StageSet: @@ -72,7 +73,7 @@ def _collect_specific_target( target: str, with_deps: bool, recursive: bool, -) -> Tuple[StageIter, Optional[str], Optional[str]]: +) -> tuple[StageIter, Optional[str], Optional[str]]: from dvc.dvcfile import is_valid_filename # Optimization: do not collect the graph for a specific target @@ -363,7 +364,7 @@ class StageLoad: with_deps: bool = False, recursive: bool = False, graph: Optional["DiGraph"] = None, - ) -> List[StageInfo]: + ) -> list[StageInfo]: """Collects a list of (stage, filter_info) from the given target. Priority is in the order of following in case of ambiguity: diff --git a/dvc/repo/update.py b/dvc/repo/update.py index ce34cddcf..e623efb2d 100644 --- a/dvc/repo/update.py +++ b/dvc/repo/update.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING from dvc.exceptions import InvalidArgumentError @@ -34,7 +34,7 @@ def update( # noqa: C901 raise InvalidArgumentError("--remote can't be used without --to-remote") import_stages = set() - other_stage_infos: List["StageInfo"] = [] + other_stage_infos: list["StageInfo"] = [] for stage_info in self.index.collect_targets(targets, recursive=recursive): if stage_info.stage.is_import: diff --git a/dvc/repo/worktree.py b/dvc/repo/worktree.py index f3d94ba3e..e8bbc006f 100644 --- a/dvc/repo/worktree.py +++ b/dvc/repo/worktree.py @@ -1,5 +1,6 @@ +from collections.abc import Iterable from functools import partial -from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Optional, Union from funcy import first @@ -35,7 +36,7 @@ def worktree_view_by_remotes( targets: Optional["TargetType"] = None, push: bool = False, **kwargs: Any, -) -> Iterable[Tuple[Optional[str], "IndexView"]]: +) -> Iterable[tuple[Optional[str], "IndexView"]]: from dvc.repo.index import IndexView def outs_filter(view: "IndexView", remote: Optional[str]): @@ -176,7 +177,7 @@ def update_worktree_stages( outs_filter=outs_filter, ) local_index = view.data["repo"] - remote_indexes: Dict[str, Tuple["Remote", "DataIndex"]] = {} + remote_indexes: dict[str, tuple["Remote", "DataIndex"]] = {} for stage in view.stages: for out in stage.outs: _update_worktree_out(repo, out, local_index, remote_indexes) @@ -187,7 +188,7 @@ def _update_worktree_out( repo: "Repo", out: "Output", local_index: Union["DataIndex", "DataIndexView"], - remote_indexes: Dict[str, Tuple["Remote", "DataIndex"]], + remote_indexes: dict[str, tuple["Remote", "DataIndex"]], ): from dvc_data.index import build @@ -283,7 +284,7 @@ def _get_diff_indexes( out: "Output", local_index: Union["DataIndex", "DataIndexView"], remote_index: Union["DataIndex", "DataIndexView"], -) -> Tuple["DataIndex", "DataIndex"]: +) -> tuple["DataIndex", "DataIndex"]: from dvc_data.index import DataIndex _, key = out.index_key diff --git a/dvc/schema.py b/dvc/schema.py index b2d70de2e..ca110566c 100644 --- a/dvc/schema.py +++ b/dvc/schema.py @@ -1,5 +1,5 @@ from collections.abc import Mapping -from typing import Any, Dict +from typing import Any import voluptuous as vol @@ -28,7 +28,7 @@ SINGLE_STAGE_SCHEMA = { StageParams.PARAM_DESC: str, } -DATA_SCHEMA: Dict[Any, Any] = { +DATA_SCHEMA: dict[Any, Any] = { **CHECKSUMS_SCHEMA, **META_SCHEMA, vol.Required("path"): str, diff --git a/dvc/scm.py b/dvc/scm.py index 435b34a51..bd5f29c7b 100644 --- a/dvc/scm.py +++ b/dvc/scm.py @@ -1,13 +1,11 @@ """Manages source control systems (e.g. Git).""" import os +from collections.abc import Iterator, Mapping from contextlib import contextmanager from functools import partial from typing import ( TYPE_CHECKING, - Iterator, - List, Literal, - Mapping, Optional, Union, overload, @@ -189,7 +187,7 @@ def resolve_rev(scm: Union["Git", "NoSCM"], rev: str) -> str: raise RevError(str(exc)) # noqa: B904 -def _get_n_commits(scm: "Git", revs: List[str], num: int) -> List[str]: +def _get_n_commits(scm: "Git", revs: list[str], num: int) -> list[str]: results = [] for rev in revs: if num == 0: @@ -210,14 +208,14 @@ def _get_n_commits(scm: "Git", revs: List[str], num: int) -> List[str]: def iter_revs( scm: "Git", - revs: Optional[List[str]] = None, + revs: Optional[list[str]] = None, num: int = 1, all_branches: bool = False, all_tags: bool = False, all_commits: bool = False, all_experiments: bool = False, commit_date: Optional[str] = None, -) -> Mapping[str, List[str]]: +) -> Mapping[str, list[str]]: from scmrepo.exceptions import SCMError as _SCMError from dvc.repo.experiments.utils import exp_commits @@ -235,7 +233,7 @@ def iter_revs( return {} revs = revs or [] - results: List[str] = _get_n_commits(scm, revs, num) + results: list[str] = _get_n_commits(scm, revs, num) if all_commits: results.extend(scm.list_all_commits()) @@ -266,7 +264,7 @@ def iter_revs( return group_by(rev_resolver, results) -def lfs_prefetch(fs: "FileSystem", paths: List[str]): +def lfs_prefetch(fs: "FileSystem", paths: list[str]): from scmrepo.git.lfs import fetch as _lfs_fetch from dvc.fs.dvc import DVCFileSystem diff --git a/dvc/stage/__init__.py b/dvc/stage/__init__.py index 5f96cfef2..f595aca18 100644 --- a/dvc/stage/__init__.py +++ b/dvc/stage/__init__.py @@ -1,18 +1,13 @@ import os import string from collections import defaultdict +from collections.abc import Iterable from contextlib import suppress from dataclasses import dataclass from typing import ( TYPE_CHECKING, Any, - Dict, - Iterable, - List, Optional, - Set, - Tuple, - Type, TypeVar, Union, ) @@ -55,14 +50,14 @@ if TYPE_CHECKING: logger = logger.getChild(__name__) # Disallow all punctuation characters except hyphen and underscore INVALID_STAGENAME_CHARS = set(string.punctuation) - {"_", "-"} -Env = Dict[str, str] -ChangedEntries = Tuple[List[str], List[str], Optional[str]] +Env = dict[str, str] +ChangedEntries = tuple[list[str], list[str], Optional[str]] _T = TypeVar("_T") def loads_from( - cls: Type[_T], repo: "Repo", path: str, wdir: str, data: Dict[str, Any] + cls: type[_T], repo: "Repo", path: str, wdir: str, data: dict[str, Any] ) -> _T: kw = { "repo": repo, @@ -91,7 +86,7 @@ class RawData: generated_from: Optional[str] = None -def create_stage(cls: Type[_T], repo, path, **kwargs) -> _T: +def create_stage(cls: type[_T], repo, path, **kwargs) -> _T: from dvc.dvcfile import check_dvcfile_path wdir = os.path.abspath(kwargs.get("wdir", None) or os.curdir) @@ -205,13 +200,13 @@ class Stage(params.StageParams): self._dvcfile = dvcfile @property - def params(self) -> List["ParamsDependency"]: + def params(self) -> list["ParamsDependency"]: from dvc.dependency import ParamsDependency return [dep for dep in self.deps if isinstance(dep, ParamsDependency)] @property - def metrics(self) -> List["Output"]: + def metrics(self) -> list["Output"]: return [out for out in self.outs if out.metric] def __repr__(self): @@ -310,7 +305,7 @@ class Stage(params.StageParams): return desc def changed_deps( - self, allow_missing: bool = False, upstream: Optional[List] = None + self, allow_missing: bool = False, upstream: Optional[list] = None ) -> bool: if self.frozen: return False @@ -322,7 +317,7 @@ class Stage(params.StageParams): @rwlocked(read=["deps"]) def _changed_deps( - self, allow_missing: bool = False, upstream: Optional[List] = None + self, allow_missing: bool = False, upstream: Optional[list] = None ) -> bool: for dep in self.deps: status = dep.status() @@ -370,7 +365,7 @@ class Stage(params.StageParams): @rwlocked(read=["deps", "outs"]) def changed( - self, allow_missing: bool = False, upstream: Optional[List] = None + self, allow_missing: bool = False, upstream: Optional[list] = None ) -> bool: is_changed = ( # Short-circuit order: stage md5 is fast, @@ -477,7 +472,7 @@ class Stage(params.StageParams): def reload(self) -> "Stage": return self.dvcfile.stage - def dumpd(self, **kwargs) -> Dict[str, Any]: + def dumpd(self, **kwargs) -> dict[str, Any]: return get_dump(self, **kwargs) def compute_md5(self) -> Optional[str]: @@ -509,7 +504,7 @@ class Stage(params.StageParams): if not allow_missing: raise - def get_versioned_outs(self) -> Dict[str, "Output"]: + def get_versioned_outs(self) -> dict[str, "Output"]: from .exceptions import StageFileDoesNotExistError, StageNotFound try: @@ -543,7 +538,7 @@ class Stage(params.StageParams): out.ignore() @staticmethod - def _changed_entries(entries) -> List[str]: + def _changed_entries(entries) -> list[str]: return [str(entry) for entry in entries if entry.workspace_status()] def _changed_stage_entry(self) -> str: @@ -663,8 +658,8 @@ class Stage(params.StageParams): @rwlocked(write=["outs"]) def checkout( self, allow_missing: bool = False, **kwargs - ) -> Dict[str, List["StrPath"]]: - stats: Dict[str, List["StrPath"]] = defaultdict(list) + ) -> dict[str, list["StrPath"]]: + stats: dict[str, list["StrPath"]] = defaultdict(list) if self.is_partial_import: return stats @@ -679,7 +674,7 @@ class Stage(params.StageParams): return stats @staticmethod - def _checkout(out, **kwargs) -> Tuple[Optional[str], List[str]]: + def _checkout(out, **kwargs) -> tuple[Optional[str], list[str]]: try: result = out.checkout(**kwargs) added, modified = result or (None, None) @@ -692,8 +687,8 @@ class Stage(params.StageParams): @rwlocked(read=["deps", "outs"]) def status( self, check_updates: bool = False, filter_info: Optional[bool] = None - ) -> Dict[str, List[Union[str, Dict[str, str]]]]: - ret: List[Union[str, Dict[str, str]]] = [] + ) -> dict[str, list[Union[str, dict[str, str]]]]: + ret: list[Union[str, dict[str, str]]] = [] show_import = ( self.is_repo_import or self.is_versioned_import ) and check_updates @@ -706,7 +701,7 @@ class Stage(params.StageParams): return {self.addressing: ret} if ret else {} @staticmethod - def _status(entries: Iterable["Output"]) -> Dict[str, str]: + def _status(entries: Iterable["Output"]) -> dict[str, str]: ret = {} for entry in entries: @@ -747,7 +742,7 @@ class Stage(params.StageParams): def get_used_objs( self, *args, **kwargs - ) -> Dict[Optional["HashFileDB"], Set["HashInfo"]]: + ) -> dict[Optional["HashFileDB"], set["HashInfo"]]: """Return set of object IDs used by this stage.""" if self.is_partial_import and not self.is_repo_import: return {} @@ -802,7 +797,7 @@ class PipelineStage(Stage): super().__init__(*args, **kwargs) self.name = name self.cmd_changed = False - self.tracked_vars: Dict[str, Dict[str, Dict[str, str]]] = {} + self.tracked_vars: dict[str, dict[str, dict[str, str]]] = {} def __eq__(self, other): return super().__eq__(other) and self.name == other.name diff --git a/dvc/stage/cache.py b/dvc/stage/cache.py index 7d3d1078c..7b6df7cb7 100644 --- a/dvc/stage/cache.py +++ b/dvc/stage/cache.py @@ -1,6 +1,6 @@ import os from contextlib import contextmanager -from typing import TYPE_CHECKING, List, Optional, Tuple +from typing import TYPE_CHECKING, Optional from funcy import first @@ -247,7 +247,7 @@ class StageCache: message = f"run-cache is not supported for http filesystem: {path}" raise RunCacheNotSupported(message) - ret: List[Tuple[str, str]] = [] + ret: list[tuple[str, str]] = [] if not from_fs.exists(runs): return ret diff --git a/dvc/stage/loader.py b/dvc/stage/loader.py index 75fa95def..a63f9152b 100644 --- a/dvc/stage/loader.py +++ b/dvc/stage/loader.py @@ -1,7 +1,7 @@ from collections.abc import Mapping from copy import deepcopy from itertools import chain -from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Tuple +from typing import TYPE_CHECKING, Any, Optional from funcy import get_in, lcat, once, project @@ -17,6 +17,8 @@ from .params import StageParams from .utils import fill_stage_dependencies, resolve_paths if TYPE_CHECKING: + from collections.abc import Iterable + from dvc.dvcfile import ProjectFile, SingleStageFile logger = logger.getChild(__name__) @@ -39,7 +41,7 @@ class StageLoader(Mapping): self._lockfile_data = lockfile_data.get("stages", {}) @cached_property - def lockfile_data(self) -> Dict[str, Any]: + def lockfile_data(self) -> dict[str, Any]: if not self._lockfile_data: logger.debug("Lockfile for '%s' not found", self.dvcfile.relpath) return self._lockfile_data @@ -53,7 +55,7 @@ class StageLoader(Mapping): from dvc.output import Output, merge_file_meta_from_cloud assert isinstance(lock_data, dict) - items: Iterable[Tuple[str, "Output"]] = chain( + items: Iterable[tuple[str, "Output"]] = chain( ((StageParams.PARAM_DEPS, dep) for dep in stage.deps), ((StageParams.PARAM_OUTS, out) for out in stage.outs), ) @@ -174,7 +176,7 @@ class SingleStageLoader(Mapping): def __init__( self, dvcfile: "SingleStageFile", - stage_data: Dict[Any, str], + stage_data: dict[Any, str], stage_text: Optional[str] = None, ): self.dvcfile = dvcfile @@ -196,7 +198,7 @@ class SingleStageLoader(Mapping): def load_stage( cls, dvcfile: "SingleStageFile", - d: Dict[str, Any], + d: dict[str, Any], stage_text: Optional[str], ) -> Stage: path, wdir = resolve_paths( diff --git a/dvc/stage/serialize.py b/dvc/stage/serialize.py index 8038f6513..4b7c4e89b 100644 --- a/dvc/stage/serialize.py +++ b/dvc/stage/serialize.py @@ -1,11 +1,9 @@ from collections import OrderedDict +from collections.abc import Iterable from operator import attrgetter from typing import ( TYPE_CHECKING, Any, - Dict, - Iterable, - List, Optional, Union, no_type_check, @@ -67,7 +65,7 @@ def _serialize_out(out): @no_type_check -def _serialize_outs(outputs: List[Output]): +def _serialize_outs(outputs: list[Output]): outs, metrics, plots = [], [], [] for out in sorted(outputs, key=attrgetter("def_path")): bucket = outs @@ -88,10 +86,10 @@ def _serialize_params_keys(params: Iterable["ParamsDependency"]): at the first, and then followed by entry of other files in lexicographic order. The keys of those custom files are also sorted in the same order. """ - keys: List[Union[str, Dict[str, Optional[List[str]]]]] = [] + keys: list[Union[str, dict[str, Optional[list[str]]]]] = [] for param_dep in sorted(params, key=attrgetter("def_path")): # when on no_exec, params are not filled and are saved as list - k: List[str] = sorted(param_dep.params) + k: list[str] = sorted(param_dep.params) if k and param_dep.def_path == DEFAULT_PARAMS_FILE: keys = k + keys # type: ignore[operator,assignment] else: @@ -100,7 +98,7 @@ def _serialize_params_keys(params: Iterable["ParamsDependency"]): @no_type_check -def _serialize_params_values(params: List[ParamsDependency]): +def _serialize_params_values(params: list[ParamsDependency]): """Returns output of following format, used for lockfile: {'params.yaml': {'lr': '1', 'train': 2}, {'params2.yaml': {'lr': '1'}} @@ -160,7 +158,7 @@ def to_single_stage_lockfile(stage: "Stage", **kwargs) -> dict: assert stage.cmd def _dumpd(item: "Output"): - ret: Dict[str, Any] = {item.PARAM_PATH: item.def_path} + ret: dict[str, Any] = {item.PARAM_PATH: item.def_path} if item.hash_name not in LEGACY_HASH_NAMES: ret[item.PARAM_HASH] = "md5" if item.hash_info.isdir and kwargs.get("with_files"): diff --git a/dvc/stage/utils.py b/dvc/stage/utils.py index 4722d9ba1..70b2d249a 100644 --- a/dvc/stage/utils.py +++ b/dvc/stage/utils.py @@ -1,6 +1,6 @@ import os import pathlib -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Optional, Union from funcy import concat, first, lsplit, rpartial @@ -210,7 +210,7 @@ def get_dump(stage: "Stage", **kwargs): def split_params_deps( stage: "Stage", -) -> Tuple[List["ParamsDependency"], List["Dependency"]]: +) -> tuple[list["ParamsDependency"], list["Dependency"]]: from dvc.dependency import ParamsDependency return lsplit(rpartial(isinstance, ParamsDependency), stage.deps) @@ -265,7 +265,7 @@ def check_stage_exists(repo: "Repo", stage: Union["Stage", "PipelineStage"], pat def validate_kwargs( single_stage: bool = False, fname: Optional[str] = None, **kwargs -) -> Dict[str, Any]: +) -> dict[str, Any]: """Prepare, validate and process kwargs passed from cli""" cmd = kwargs.get("cmd") if not cmd and not single_stage: @@ -288,11 +288,11 @@ def validate_kwargs( return kwargs -def _get_stage_files(stage: "Stage") -> List[str]: +def _get_stage_files(stage: "Stage") -> list[str]: from dvc.dvcfile import ProjectFile from dvc.utils import relpath - ret: List[str] = [] + ret: list[str] = [] file = stage.dvcfile ret.append(file.relpath) if isinstance(file, ProjectFile): diff --git a/dvc/testing/benchmarks/fixtures.py b/dvc/testing/benchmarks/fixtures.py index d59843c5a..bf3adb35e 100644 --- a/dvc/testing/benchmarks/fixtures.py +++ b/dvc/testing/benchmarks/fixtures.py @@ -2,7 +2,7 @@ import os import shutil from pathlib import Path from subprocess import check_output -from typing import Dict, Optional +from typing import Optional import pytest from dulwich.porcelain import clone @@ -27,7 +27,7 @@ class VirtualEnv: virtualenv.cli_run([os.fspath(self.path)]) - def run(self, cmd: str, *args: str, env: Optional[Dict[str, str]] = None) -> None: + def run(self, cmd: str, *args: str, env: Optional[dict[str, str]] = None) -> None: exe = self.which(cmd) check_output([exe, *args], env=env) # noqa: S603 diff --git a/dvc/testing/fixtures.py b/dvc/testing/fixtures.py index 551f9e52c..cc1bc4f40 100644 --- a/dvc/testing/fixtures.py +++ b/dvc/testing/fixtures.py @@ -1,7 +1,6 @@ import os import pathlib import subprocess -from typing import Dict, Tuple import pytest @@ -33,7 +32,7 @@ __all__ = [ "workspace", ] -CACHE: Dict[Tuple[bool, bool, bool], str] = {} +CACHE: dict[tuple[bool, bool, bool], str] = {} @pytest.fixture(scope="session") diff --git a/dvc/testing/path_info.py b/dvc/testing/path_info.py index 820d7f0d2..ef98f69c7 100644 --- a/dvc/testing/path_info.py +++ b/dvc/testing/path_info.py @@ -1,7 +1,7 @@ import os import pathlib import posixpath -from typing import Callable, ClassVar, Dict +from typing import Callable, ClassVar from urllib.parse import urlparse from dvc.utils import relpath @@ -120,7 +120,7 @@ class _URLPathParents: class URLInfo(_BasePath): - DEFAULT_PORTS: ClassVar[Dict[str, int]] = { + DEFAULT_PORTS: ClassVar[dict[str, int]] = { "http": 80, "https": 443, "ssh": 22, diff --git a/dvc/testing/workspace_tests.py b/dvc/testing/workspace_tests.py index 13bdaf7d3..fcf48ed53 100644 --- a/dvc/testing/workspace_tests.py +++ b/dvc/testing/workspace_tests.py @@ -1,5 +1,5 @@ import os -from typing import Dict, Union +from typing import Union import pytest from funcy import first @@ -73,7 +73,7 @@ class TestImport: # of directories. So instead we create an empty file that ends with a # trailing slash in order to actually support this operation if is_object_storage: - contents: Union[str, Dict[str, str]] = "" + contents: Union[str, dict[str, str]] = "" else: contents = {} diff --git a/dvc/types.py b/dvc/types.py index b061af6e4..718e49208 100644 --- a/dvc/types.py +++ b/dvc/types.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, AnyStr, Dict, List, Union +from typing import TYPE_CHECKING, Any, AnyStr, Union if TYPE_CHECKING: from os import PathLike @@ -8,5 +8,5 @@ BytesPath = Union[bytes, "PathLike[bytes]"] GenericPath = Union[AnyStr, "PathLike[AnyStr]"] StrOrBytesPath = Union[str, bytes, "PathLike[str]", "PathLike[bytes]"] -TargetType = Union[List[str], str] -DictStrAny = Dict[str, Any] +TargetType = Union[list[str], str] +DictStrAny = dict[str, Any] diff --git a/dvc/ui/__init__.py b/dvc/ui/__init__.py index 408eb70d0..93da4190d 100644 --- a/dvc/ui/__init__.py +++ b/dvc/ui/__init__.py @@ -1,15 +1,11 @@ +from collections.abc import Iterable, Iterator, Sequence from contextlib import contextmanager, nullcontext from typing import ( TYPE_CHECKING, Any, Callable, - Dict, - Iterable, - Iterator, Optional, - Sequence, TextIO, - Type, Union, ) @@ -46,7 +42,7 @@ def disable_colorama(): class Formatter: def __init__( - self, theme: Optional[Dict] = None, defaults: Optional[Dict] = None + self, theme: Optional[dict] = None, defaults: Optional[dict] = None ) -> None: from collections import defaultdict @@ -230,7 +226,7 @@ class Console: return print(*values, sep=sep, end=end, file=file) @property - def rich_text(self) -> "Type[RichText]": + def rich_text(self) -> "type[RichText]": from rich.text import Text return Text @@ -308,7 +304,7 @@ class Console: rich_table: bool = False, force: bool = True, pager: bool = False, - header_styles: Optional[Union[Dict[str, "Styles"], Sequence["Styles"]]] = None, + header_styles: Optional[Union[dict[str, "Styles"], Sequence["Styles"]]] = None, row_styles: Optional[Sequence["Styles"]] = None, borders: Union[bool, str] = False, ) -> None: diff --git a/dvc/ui/table.py b/dvc/ui/table.py index 55c64f423..2ceece9d2 100644 --- a/dvc/ui/table.py +++ b/dvc/ui/table.py @@ -1,7 +1,8 @@ from collections import abc +from collections.abc import Iterator, Sequence from contextlib import ExitStack, contextmanager from itertools import zip_longest -from typing import TYPE_CHECKING, Dict, Iterator, Optional, Sequence, Union +from typing import TYPE_CHECKING, Optional, Union from dvc.types import DictStrAny @@ -75,7 +76,7 @@ def rich_table( data: TableData, headers: Optional[Headers] = None, pager: bool = False, - header_styles: Optional[Union[Dict[str, Styles], Sequence[Styles]]] = None, + header_styles: Optional[Union[dict[str, Styles], Sequence[Styles]]] = None, row_styles: Optional[Sequence[Styles]] = None, borders: Union[bool, str] = False, ) -> None: @@ -94,7 +95,7 @@ def rich_table( table = Table(box=border_style[borders]) if isinstance(header_styles, abc.Sequence): - hs: Dict[str, Styles] = dict(zip(headers or [], header_styles)) + hs: dict[str, Styles] = dict(zip(headers or [], header_styles)) else: hs = header_styles or {} diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index 3eba21e20..c7f222c3c 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -5,7 +5,8 @@ import json import os import re import sys -from typing import TYPE_CHECKING, Dict, Iterable, Iterator, List, Optional, Tuple +from collections.abc import Iterable, Iterator +from typing import TYPE_CHECKING, Optional import colorama @@ -323,7 +324,7 @@ def error_link(name): def parse_target( target: str, default: Optional[str] = None, isa_glob: bool = False -) -> Tuple[Optional[str], Optional[str]]: +) -> tuple[Optional[str], Optional[str]]: from dvc.dvcfile import LOCK_FILE, PROJECT_FILE, is_valid_filename from dvc.exceptions import DvcException from dvc.parsing import JOIN @@ -399,7 +400,7 @@ def error_handler(func): return wrapper -def errored_revisions(rev_data: Dict) -> List: +def errored_revisions(rev_data: dict) -> list: from dvc.utils.collections import nested_contains result = [] diff --git a/dvc/utils/cli_parse.py b/dvc/utils/cli_parse.py index d894eebe2..83d994a96 100644 --- a/dvc/utils/cli_parse.py +++ b/dvc/utils/cli_parse.py @@ -1,12 +1,12 @@ from collections import defaultdict -from typing import Dict, Iterable, List +from collections.abc import Iterable -def parse_params(path_params: Iterable[str]) -> List[Dict[str, List[str]]]: +def parse_params(path_params: Iterable[str]) -> list[dict[str, list[str]]]: """Normalizes the shape of params from the CLI to dict.""" from dvc.dependency.param import ParamsDependency - ret: Dict[str, List[str]] = defaultdict(list) + ret: dict[str, list[str]] = defaultdict(list) for path_param in path_params: path, _, params_str = path_param.rpartition(":") # remove empty strings from params, on condition such as `-p "file1:"` @@ -19,7 +19,7 @@ def parse_params(path_params: Iterable[str]) -> List[Dict[str, List[str]]]: def to_path_overrides( path_params: Iterable[str], -) -> Dict[str, List[str]]: +) -> dict[str, list[str]]: """Group overrides by path""" from dvc.dependency.param import ParamsDependency diff --git a/dvc/utils/collections.py b/dvc/utils/collections.py index 36a9fb807..d8c38c317 100644 --- a/dvc/utils/collections.py +++ b/dvc/utils/collections.py @@ -1,5 +1,5 @@ -from collections.abc import Mapping -from typing import Dict, Iterable, List, Union, no_type_check +from collections.abc import Iterable, Mapping +from typing import Union, no_type_check @no_type_check @@ -80,7 +80,7 @@ def _merge_item(d, key, value): d[key] = value -def merge_dicts(src: Dict, to_update: Dict) -> Dict: +def merge_dicts(src: dict, to_update: dict) -> dict: """Recursively merges dictionaries. Args: @@ -92,7 +92,7 @@ def merge_dicts(src: Dict, to_update: Dict) -> Dict: return src -def ensure_list(item: Union[Iterable[str], str, None]) -> List[str]: +def ensure_list(item: Union[Iterable[str], str, None]) -> list[str]: if item is None: return [] if isinstance(item, str): @@ -100,7 +100,7 @@ def ensure_list(item: Union[Iterable[str], str, None]) -> List[str]: return list(item) -def nested_contains(dictionary: Dict, phrase: str) -> bool: +def nested_contains(dictionary: dict, phrase: str) -> bool: for key, val in dictionary.items(): if key == phrase and val: return True diff --git a/dvc/utils/diff.py b/dvc/utils/diff.py index de217b5dc..8dccea977 100644 --- a/dvc/utils/diff.py +++ b/dvc/utils/diff.py @@ -1,6 +1,5 @@ import json from collections import defaultdict -from typing import Dict from .flatten import flatten @@ -44,7 +43,7 @@ def _diff_dicts(old_dict, new_dict, with_unchanged): new = _flatten(new_dict) old = _flatten(old_dict) - res: Dict[str, Dict] = defaultdict(dict) + res: dict[str, dict] = defaultdict(dict) xpaths = set(old.keys()) xpaths.update(set(new.keys())) @@ -75,7 +74,7 @@ def diff(old, new, with_unchanged=False): paths = set(old.keys()) paths.update(set(new.keys())) - res: Dict[str, Dict] = defaultdict(dict) + res: dict[str, dict] = defaultdict(dict) for path in paths: path_diff = _diff( old.get(path, {}).get("data", {}), diff --git a/dvc/utils/hydra.py b/dvc/utils/hydra.py index 721b0bb17..09ade0822 100644 --- a/dvc/utils/hydra.py +++ b/dvc/utils/hydra.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import TYPE_CHECKING, List, Optional +from typing import TYPE_CHECKING, Optional from dvc.exceptions import InvalidArgumentError from dvc.log import logger @@ -31,7 +31,7 @@ def compose_and_dump( config_module: Optional[str], config_name: str, plugins_path: str, - overrides: List[str], + overrides: list[str], ) -> None: """Compose Hydra config and dumpt it to `output_file`. @@ -80,7 +80,7 @@ def compose_and_dump( ) -def apply_overrides(path: "StrPath", overrides: List[str]) -> None: +def apply_overrides(path: "StrPath", overrides: list[str]) -> None: """Update `path` params with the provided `Hydra Override`_ patterns. Args: diff --git a/dvc/utils/plots.py b/dvc/utils/plots.py index ee7f988c1..b6a8c16e4 100644 --- a/dvc/utils/plots.py +++ b/dvc/utils/plots.py @@ -1,5 +1,4 @@ from collections import defaultdict -from typing import Dict, Tuple def get_plot_id(config_plot_id: str, config_file_path: str = ""): @@ -9,8 +8,8 @@ def get_plot_id(config_plot_id: str, config_file_path: str = ""): def group_definitions_by_id( - definitions: Dict[str, Dict], -) -> Dict[str, Tuple[str, Dict]]: + definitions: dict[str, dict], +) -> dict[str, tuple[str, dict]]: """ Format ID and extracts plot_definition for each plot. @@ -20,8 +19,8 @@ def group_definitions_by_id( Returns: Dict of {plot_id: (original_plot_id, plot_definition)}. """ - groups_by_config: Dict = defaultdict(dict) - groups_by_id: Dict = {} + groups_by_config: dict = defaultdict(dict) + groups_by_id: dict = {} for config_file, config_file_content in definitions.items(): for plot_id, plot_definition in config_file_content.get("data", {}).items(): groups_by_config[plot_id][config_file] = (plot_id, plot_definition) diff --git a/dvc/utils/serialize/__init__.py b/dvc/utils/serialize/__init__.py index e42cb8cae..58539209c 100644 --- a/dvc/utils/serialize/__init__.py +++ b/dvc/utils/serialize/__init__.py @@ -1,5 +1,4 @@ from collections import defaultdict -from typing import DefaultDict from ._common import * # noqa: F403 from ._json import * # noqa: F403 @@ -7,12 +6,12 @@ from ._py import * # noqa: F403 from ._toml import * # noqa: F403 from ._yaml import * # noqa: F403 -LOADERS: DefaultDict[str, LoaderFn] = defaultdict( # noqa: F405 +LOADERS: defaultdict[str, LoaderFn] = defaultdict( # noqa: F405 lambda: load_yaml # noqa: F405 ) LOADERS.update({".toml": load_toml, ".json": load_json, ".py": load_py}) # noqa: F405 -PARSERS: DefaultDict[str, ParserFn] = defaultdict( # noqa: F405 +PARSERS: defaultdict[str, ParserFn] = defaultdict( # noqa: F405 lambda: parse_yaml # noqa: F405 ) PARSERS.update( @@ -26,12 +25,12 @@ def load_path(fs_path, fs, **kwargs): return loader(fs_path, fs=fs, **kwargs) -DUMPERS: DefaultDict[str, DumperFn] = defaultdict( # noqa: F405 +DUMPERS: defaultdict[str, DumperFn] = defaultdict( # noqa: F405 lambda: dump_yaml # noqa: F405 ) DUMPERS.update({".toml": dump_toml, ".json": dump_json, ".py": dump_py}) # noqa: F405 -MODIFIERS: DefaultDict[str, ModifierFn] = defaultdict( # noqa: F405 +MODIFIERS: defaultdict[str, ModifierFn] = defaultdict( # noqa: F405 lambda: modify_yaml # noqa: F405 ) MODIFIERS.update( diff --git a/dvc/utils/serialize/_common.py b/dvc/utils/serialize/_common.py index bbf635c3f..3c90bf641 100644 --- a/dvc/utils/serialize/_common.py +++ b/dvc/utils/serialize/_common.py @@ -1,12 +1,10 @@ """Common utilities for serialize.""" import os -from contextlib import contextmanager +from contextlib import AbstractContextManager, contextmanager from typing import ( TYPE_CHECKING, Any, Callable, - ContextManager, - Dict, Optional, Protocol, TextIO, @@ -37,7 +35,7 @@ class DumpersFn(Protocol): class ModifierFn(Protocol): def __call__( self, path: "StrPath", fs: Optional["FileSystem"] = None - ) -> ContextManager[Dict]: + ) -> AbstractContextManager[dict]: ... diff --git a/dvc/utils/strictyaml.py b/dvc/utils/strictyaml.py index 313818169..7d6a11ce6 100644 --- a/dvc/utils/strictyaml.py +++ b/dvc/utils/strictyaml.py @@ -9,7 +9,7 @@ Not to be confused with strictyaml, a python library with similar motivations. import re import typing from contextlib import suppress -from typing import TYPE_CHECKING, Any, Callable, List, Optional, Tuple, TypeVar +from typing import TYPE_CHECKING, Any, Callable, Optional, TypeVar from dvc.exceptions import PrettyDvcException from dvc.ui import ui @@ -111,7 +111,7 @@ class YAMLSyntaxError(PrettyDvcException, YAMLFileCorruptedError): code = "" if line > len(source) else source[line - 1] return _prepare_code_snippets(code, line) - lines: List[object] = [] + lines: list[object] = [] if hasattr(exc, "context"): if exc.context_mark is not None: lines.append(prepare_message(str(exc.context), exc.context_mark)) @@ -143,7 +143,7 @@ class YAMLSyntaxError(PrettyDvcException, YAMLFileCorruptedError): def determine_linecol( data, paths, max_steps=5 -) -> Tuple[Optional[int], Optional[int], int]: +) -> tuple[Optional[int], Optional[int], int]: """Determine linecol from the CommentedMap for the `paths` location. CommentedMap from `ruamel.yaml` has `.lc` property from which we can read @@ -206,8 +206,8 @@ class YAMLValidationError(PrettyDvcException): message += f": {len(self.exc.errors)} errors" super().__init__(f"{message}") - def _prepare_context(self, data: typing.Mapping) -> List[object]: - lines: List[object] = [] + def _prepare_context(self, data: typing.Mapping) -> list[object]: + lines: list[object] = [] for index, error in enumerate(self.exc.errors): if index and lines[-1]: lines.append("") @@ -234,7 +234,7 @@ class YAMLValidationError(PrettyDvcException): """Prettify exception message.""" from collections.abc import Mapping - lines: List[object] = [] + lines: list[object] = [] data = parse_yaml_for_update(self.text, self.path) if isinstance(data, Mapping): lines.extend(self._prepare_context(data)) diff --git a/dvc/utils/studio.py b/dvc/utils/studio.py index 2530db78b..45aa7b17e 100644 --- a/dvc/utils/studio.py +++ b/dvc/utils/studio.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Optional from urllib.parse import urljoin import requests @@ -24,7 +24,7 @@ STUDIO_URL = "https://studio.iterative.ai" def post( url: str, token: str, - data: Dict[str, Any], + data: dict[str, Any], base_url: Optional[str] = STUDIO_URL, max_retries: int = 3, timeout: int = 5, @@ -48,8 +48,8 @@ def notify_refs( token: str, *, base_url: Optional[str] = STUDIO_URL, - **refs: List[str], -) -> Dict[str, Any]: + **refs: list[str], +) -> dict[str, Any]: extra_keys = refs.keys() - {"pushed", "removed"} assert not extra_keys, f"got extra args: {extra_keys}" @@ -87,7 +87,7 @@ def notify_refs( return d -def config_to_env(config: Dict[str, Any]) -> Dict[str, Any]: +def config_to_env(config: dict[str, Any]) -> dict[str, Any]: env = {} if "offline" in config: env[DVC_STUDIO_OFFLINE] = config["offline"] @@ -100,7 +100,7 @@ def config_to_env(config: Dict[str, Any]) -> Dict[str, Any]: return env -def env_to_config(env: Dict[str, Any]) -> Dict[str, Any]: +def env_to_config(env: dict[str, Any]) -> dict[str, Any]: config = {} if DVC_STUDIO_OFFLINE in env: config["offline"] = env[DVC_STUDIO_OFFLINE] diff --git a/dvc/utils/table.py b/dvc/utils/table.py index 3ab4a80d9..eb42f81a1 100644 --- a/dvc/utils/table.py +++ b/dvc/utils/table.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, List +from typing import TYPE_CHECKING, Any from rich.table import Table as RichTable @@ -13,7 +13,7 @@ class Table(RichTable): def _calculate_column_widths( self, console: "Console", options: "ConsoleOptions" - ) -> List[int]: + ) -> list[int]: """Calculate the widths of each column, including padding, not including borders. @@ -47,10 +47,10 @@ class Table(RichTable): def _collapse_widths( # type: ignore[override] self, - widths: List[int], - wrapable: List[bool], + widths: list[int], + wrapable: list[bool], max_width: int, - ) -> List[int]: + ) -> list[int]: """Collapse columns right-to-left if possible to fit table into max_width. diff --git a/dvc/utils/threadpool.py b/dvc/utils/threadpool.py index 73e0e0d79..b8b9e6b25 100644 --- a/dvc/utils/threadpool.py +++ b/dvc/utils/threadpool.py @@ -1,8 +1,9 @@ import queue import sys +from collections.abc import Iterable, Iterator from concurrent import futures from itertools import islice -from typing import Any, Callable, Iterable, Iterator, Optional, Set, TypeVar +from typing import Any, Callable, Optional, TypeVar _T = TypeVar("_T") @@ -31,7 +32,7 @@ class ThreadPoolExecutor(futures.ThreadPoolExecutor): It does not create all the futures at once to reduce memory usage. """ - def create_taskset(n: int) -> Set[futures.Future]: + def create_taskset(n: int) -> set[futures.Future]: return {self.submit(fn, *args) for args in islice(it, n)} it = zip(*iterables) diff --git a/pyproject.toml b/pyproject.toml index 365dcdf6e..749855aa8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,14 +19,14 @@ keywords = [ license = { text = "Apache License 2.0" } maintainers = [{ name = "Iterative", email = "support@dvc.org" }] authors = [{ name = "Dmitry Petrov", email = "dmitry@dvc.org" }] -requires-python = ">=3.8" +requires-python = ">=3.9" classifiers = [ "Development Status :: 4 - Beta", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] dynamic = ["version"] @@ -86,6 +86,7 @@ lint = [ "types-tabulate", "types-toml", "types-tqdm", + "typing-extensions", ] oss = ["dvc-oss>=3,<4"] s3 = ["dvc-s3>=3,<4"] diff --git a/tests/func/api/test_show.py b/tests/func/api/test_show.py index 6b0c0e091..4833af59a 100644 --- a/tests/func/api/test_show.py +++ b/tests/func/api/test_show.py @@ -1,13 +1,12 @@ import json import os from textwrap import dedent -from typing import Dict, List import pytest from dvc import api -TRAIN_METRICS: List[Dict[str, Dict[str, float]]] = [ +TRAIN_METRICS: list[dict[str, dict[str, float]]] = [ { "avg_prec": {"train": 0.85, "val": 0.75}, "roc_auc": {"train": 0.80, "val": 0.70}, @@ -17,7 +16,7 @@ TRAIN_METRICS: List[Dict[str, Dict[str, float]]] = [ "roc_auc": {"train": 0.98, "val": 0.94}, }, ] -TEST_METRICS: List[Dict[str, Dict[str, float]]] = [ +TEST_METRICS: list[dict[str, dict[str, float]]] = [ {"avg_prec": {"test": 0.72}, "roc_auc": {"test": 0.77}}, { "avg_prec": {"test": 0.91}, diff --git a/tests/func/test_daemon.py b/tests/func/test_daemon.py index 291aca04e..33f024e83 100644 --- a/tests/func/test_daemon.py +++ b/tests/func/test_daemon.py @@ -4,11 +4,12 @@ import re import subprocess import sys from collections import defaultdict +from collections.abc import Iterator from contextlib import contextmanager, suppress from http.server import HTTPServer, SimpleHTTPRequestHandler from pathlib import Path from threading import Thread -from typing import ClassVar, Dict, Iterator +from typing import ClassVar import psutil import pytest @@ -42,7 +43,7 @@ UPDATER_INFO_STR = json.dumps(UPDATER_INFO).encode("utf8") def make_request_handler(): class RequestHandler(SimpleHTTPRequestHandler): # save requests count for each method - hits: ClassVar[Dict[str, int]] = defaultdict(int) + hits: ClassVar[dict[str, int]] = defaultdict(int) def log_message(self, format, *args) -> None: # noqa: A002 super().log_message(format, *args) diff --git a/tests/func/test_ignore.py b/tests/func/test_ignore.py index d4ca3eb1d..b40630faa 100644 --- a/tests/func/test_ignore.py +++ b/tests/func/test_ignore.py @@ -1,7 +1,6 @@ import os import shutil from pathlib import Path -from typing import List import pytest @@ -14,7 +13,7 @@ from dvc_data.hashfile.build import IgnoreInCollectedDirError from dvc_data.hashfile.utils import get_mtime_and_size -def _to_pattern_info_list(str_list: List): +def _to_pattern_info_list(str_list: list): return [PatternInfo(a, "") for a in str_list] diff --git a/tests/integration/plots/test_plots.py b/tests/integration/plots/test_plots.py index 4232f89ab..144de0f14 100644 --- a/tests/integration/plots/test_plots.py +++ b/tests/integration/plots/test_plots.py @@ -1,7 +1,6 @@ import json import os from copy import deepcopy -from typing import Dict, List from urllib.parse import urlparse from urllib.request import url2pathname @@ -57,7 +56,7 @@ def extract_vega_specs(html_path, plots_ids): return result -def drop_fields(datapoints: List[Dict], fields: List[str]): +def drop_fields(datapoints: list[dict], fields: list[str]): tmp = deepcopy(datapoints) for datapoint in tmp: keys = set(datapoint.keys()) @@ -192,7 +191,7 @@ def verify_vega_props(plot_id, json_result, title, x, y, **kwargs): assert plot_y == y -def _update_datapoints(datapoints: List, update: Dict): +def _update_datapoints(datapoints: list, update: dict): result = [] for dp in datapoints: tmp = dp.copy() @@ -501,7 +500,7 @@ def test_repo_with_dvclive_plots(tmp_dir, capsys, repo_with_dvclive_plots): for s in ("show", "diff"): _, json_result, split_json_result = call(capsys, subcommand=s) - expected_result: Dict[str, Dict[str, list[str]]] = { + expected_result: dict[str, dict[str, list[str]]] = { "data": { "dvclive/plots/metrics/metric.tsv": [], }, diff --git a/tests/unit/command/test_help.py b/tests/unit/command/test_help.py index 3eb453ede..0350c11e2 100644 --- a/tests/unit/command/test_help.py +++ b/tests/unit/command/test_help.py @@ -2,7 +2,6 @@ import logging import re from argparse import SUPPRESS, ArgumentParser from itertools import takewhile -from typing import Tuple import pytest import shtab @@ -12,10 +11,10 @@ from dvc.cli.parser import get_main_parser def command_tuples(): - root: Tuple[str, ...] = () + root: tuple[str, ...] = () commands = [root] - def recurse_parser(parser: ArgumentParser, parents: Tuple[str, ...] = root) -> None: + def recurse_parser(parser: ArgumentParser, parents: tuple[str, ...] = root) -> None: for positional in parser._get_positional_actions(): if positional.help != SUPPRESS and isinstance(positional.choices, dict): public_cmds = shtab.get_public_subcommands(positional) diff --git a/tests/unit/repo/experiments/test_collect.py b/tests/unit/repo/experiments/test_collect.py index e1342ed2c..aef1b62eb 100644 --- a/tests/unit/repo/experiments/test_collect.py +++ b/tests/unit/repo/experiments/test_collect.py @@ -1,6 +1,5 @@ import datetime import random -from typing import Dict, List import pytest @@ -22,7 +21,7 @@ def test_collect_stable_sorting(dvc, scm, mocker): "7" * 40, ] - def collect_queued_patched(_, baseline_revs) -> Dict[str, List["ExpRange"]]: + def collect_queued_patched(_, baseline_revs) -> dict[str, list["ExpRange"]]: single_timestamp = datetime.datetime(2023, 6, 20, 0, 0, 0) exp_ranges = [ @@ -59,8 +58,8 @@ def test_collect_stable_sorting(dvc, scm, mocker): def _assert_experiment_rev_order( - actual: List["ExpRange"], - expected_revs: List[str], + actual: list["ExpRange"], + expected_revs: list[str], ): expected_revs = expected_revs.copy() diff --git a/tests/utils/asserts.py b/tests/utils/asserts.py index 301c7422a..f7ae07452 100644 --- a/tests/utils/asserts.py +++ b/tests/utils/asserts.py @@ -1,11 +1,11 @@ -from typing import TYPE_CHECKING, Any, Dict +from typing import TYPE_CHECKING, Any from unittest.mock import ANY if TYPE_CHECKING: from unittest.mock import Mock -def issubset(subset: Dict, superset: Dict) -> bool: +def issubset(subset: dict, superset: dict) -> bool: assert superset == {**superset, **subset} return True