drop Python 3.8 support (#10265)

* drop Python 3.8

Note that this PR only makes typing related changes.
Other changes can be done once this gets merged, they
are relatively few in numbers.

* bump build python version

* add typing_extensions to lint group
This commit is contained in:
skshetry
2024-01-30 11:33:25 +02:00
committed by GitHub
parent 966adec21c
commit b647475a64
126 changed files with 843 additions and 905 deletions
+1 -1
View File
@@ -25,7 +25,7 @@ jobs:
- uses: actions/setup-python@v5
with:
python-version: 3.8
python-version: 3.12
- run: python -m pip install --upgrade pip setuptools_scm build twine
+1 -1
View File
@@ -23,7 +23,7 @@ jobs:
fail-fast: false
matrix:
os: [ubuntu-20.04, windows-latest, macos-latest]
pyv: ["3.8", "3.9", "3.10", "3.11", "3.12"]
pyv: ["3.9", "3.10", "3.11", "3.12"]
steps:
- uses: actions/checkout@v4
+8 -8
View File
@@ -1,5 +1,5 @@
from dataclasses import asdict, dataclass, field, fields
from typing import Any, ClassVar, Dict, List, Optional
from typing import Any, ClassVar, Optional
from funcy import compact
from voluptuous import Required
@@ -14,10 +14,10 @@ class Annotation:
desc: Optional[str] = None
type: Optional[str] = None
labels: List[str] = field(default_factory=list)
meta: Dict[str, Any] = field(default_factory=dict)
labels: list[str] = field(default_factory=list)
meta: dict[str, Any] = field(default_factory=dict)
def to_dict(self) -> Dict[str, str]:
def to_dict(self) -> dict[str, str]:
return compact(asdict(self))
@@ -32,10 +32,10 @@ class Artifact:
path: str
desc: Optional[str] = None
type: Optional[str] = None
labels: List[str] = field(default_factory=list)
meta: Dict[str, Any] = field(default_factory=dict)
labels: list[str] = field(default_factory=list)
meta: dict[str, Any] = field(default_factory=dict)
def to_dict(self) -> Dict[str, str]:
def to_dict(self) -> dict[str, str]:
return compact(asdict(self))
@@ -46,7 +46,7 @@ ANNOTATION_SCHEMA = {
Annotation.PARAM_LABELS: [str],
Annotation.PARAM_META: object,
}
ARTIFACT_SCHEMA: Dict[Any, Any] = {
ARTIFACT_SCHEMA: dict[Any, Any] = {
Required(Artifact.PARAM_PATH): str,
**ANNOTATION_SCHEMA, # type: ignore[arg-type]
}
+3 -3
View File
@@ -1,4 +1,4 @@
from typing import Any, Dict, Optional
from typing import Any, Optional
from dvc.repo import Repo
@@ -8,7 +8,7 @@ def artifacts_show(
version: Optional[str] = None,
stage: Optional[str] = None,
repo: Optional[str] = None,
) -> Dict[str, str]:
) -> dict[str, str]:
"""
Return path and Git revision for an artifact in a DVC project.
@@ -36,7 +36,7 @@ def artifacts_show(
if version and stage:
raise ValueError("Artifact version and stage are mutually exclusive.")
repo_kwargs: Dict[str, Any] = {
repo_kwargs: dict[str, Any] = {
"subrepos": True,
"uninitialized": True,
}
+5 -5
View File
@@ -1,6 +1,6 @@
from contextlib import _GeneratorContextManager as GCM
from contextlib import contextmanager
from typing import Any, Dict, Optional
from typing import Any, Optional
from funcy import reraise
@@ -39,7 +39,7 @@ def get_url(path, repo=None, rev=None, remote=None):
from dvc.config import NoRemoteError
from dvc_data.index import StorageKeyError
repo_kwargs: Dict[str, Any] = {}
repo_kwargs: dict[str, Any] = {}
if remote:
repo_kwargs["config"] = {"core": {"remote": remote}}
with Repo.open(
@@ -74,8 +74,8 @@ def open( # noqa: A001
remote: Optional[str] = None,
mode: str = "r",
encoding: Optional[str] = None,
config: Optional[Dict[str, Any]] = None,
remote_config: Optional[Dict[str, Any]] = None,
config: Optional[dict[str, Any]] = None,
remote_config: Optional[dict[str, Any]] = None,
):
"""
Opens a file tracked in a DVC project.
@@ -233,7 +233,7 @@ def _open(
config=None,
remote_config=None,
):
repo_kwargs: Dict[str, Any] = {
repo_kwargs: dict[str, Any] = {
"subrepos": True,
"uninitialized": True,
"remote": remote,
+5 -5
View File
@@ -1,4 +1,4 @@
from typing import Dict, List, Optional, Union
from typing import Optional, Union
from rich.text import Text
@@ -9,7 +9,7 @@ from dvc.repo.experiments.show import tabulate
def exp_save(
name: Optional[str] = None,
force: bool = False,
include_untracked: Optional[List[str]] = None,
include_untracked: Optional[list[str]] = None,
):
"""
Create a new DVC experiment using `exp save`.
@@ -61,12 +61,12 @@ def _postprocess(exp_rows):
def exp_show(
repo: Optional[str] = None,
revs: Optional[Union[str, List[str]]] = None,
revs: Optional[Union[str, list[str]]] = None,
num: int = 1,
param_deps: bool = False,
force: bool = False,
config: Optional[Dict] = None,
) -> List[Dict]:
config: Optional[dict] = None,
) -> list[dict]:
"""Get DVC experiments tracked in `repo`.
Without arguments, this function will retrieve all experiments derived from
+4 -4
View File
@@ -1,9 +1,9 @@
from typing import List, Optional
from typing import Optional
from dvc.repo import Repo
def all_branches(repo: Optional[str] = None) -> List[str]:
def all_branches(repo: Optional[str] = None) -> list[str]:
"""Get all Git branches in a DVC repository.
Args:
@@ -20,7 +20,7 @@ def all_branches(repo: Optional[str] = None) -> List[str]:
return _repo.scm.list_branches()
def all_commits(repo: Optional[str] = None) -> List[str]:
def all_commits(repo: Optional[str] = None) -> list[str]:
"""Get all Git commits in a DVC repository.
Args:
@@ -37,7 +37,7 @@ def all_commits(repo: Optional[str] = None) -> List[str]:
return _repo.scm.list_all_commits()
def all_tags(repo: Optional[str] = None) -> List[str]:
def all_tags(repo: Optional[str] = None) -> list[str]:
"""Get all Git tags in a DVC repository.
Args:
+7 -6
View File
@@ -1,6 +1,7 @@
import typing
from collections import Counter
from typing import Dict, Iterable, Optional, Union
from collections.abc import Iterable
from typing import Optional, Union
from funcy import first
@@ -8,7 +9,7 @@ from dvc.repo import Repo
def _postprocess(results):
processed: Dict[str, Dict] = {}
processed: dict[str, dict] = {}
for rev, rev_data in results.items():
if not rev_data:
continue
@@ -36,8 +37,8 @@ def metrics_show(
*targets: str,
repo: Optional[str] = None,
rev: Optional[str] = None,
config: Optional[Dict] = None,
) -> Dict:
config: Optional[dict] = None,
) -> dict:
"""Get metrics tracked in `repo`.
Without arguments, this function will retrieve all metrics from all tracked
@@ -159,8 +160,8 @@ def params_show(
stages: Optional[Union[str, Iterable[str]]] = None,
rev: Optional[str] = None,
deps: bool = False,
config: Optional[Dict] = None,
) -> Dict:
config: Optional[dict] = None,
) -> dict:
"""Get parameters tracked in `repo`.
Without arguments, this function will retrieve all params from all tracked
+2 -2
View File
@@ -1,5 +1,5 @@
import os
from typing import TYPE_CHECKING, Optional, Tuple
from typing import TYPE_CHECKING, Optional
from dvc.fs import GitFileSystem, Schemes
from dvc_data.hashfile.db import get_odb
@@ -15,7 +15,7 @@ def _get_odb(
repo,
settings,
fs=None,
prefix: Optional[Tuple[str, ...]] = None,
prefix: Optional[tuple[str, ...]] = None,
hash_name: Optional[str] = None,
**kwargs,
):
+4 -4
View File
@@ -1,4 +1,4 @@
from typing import TYPE_CHECKING, ClassVar, Dict, Tuple
from typing import TYPE_CHECKING, ClassVar
from funcy import chunks, compact, log_durations
@@ -17,14 +17,14 @@ logger = logger.getChild(__name__)
class CmdDataStatus(CmdBase):
COLORS: ClassVar[Dict[str, str]] = {
COLORS: ClassVar[dict[str, str]] = {
"not_in_remote": "red",
"not_in_cache": "red",
"committed": "green",
"uncommitted": "yellow",
"untracked": "cyan",
}
LABELS: ClassVar[Dict[str, str]] = {
LABELS: ClassVar[dict[str, str]] = {
"not_in_remote": "Not in remote",
"not_in_cache": "Not in cache",
"committed": "DVC committed changes",
@@ -32,7 +32,7 @@ class CmdDataStatus(CmdBase):
"untracked": "Untracked files",
"unchanged": "DVC unchanged files",
}
HINTS: ClassVar[Dict[str, Tuple[str, ...]]] = {
HINTS: ClassVar[dict[str, tuple[str, ...]]] = {
"not_in_remote": ('use "dvc push <file>..." to upload files',),
"not_in_cache": ('use "dvc fetch <file>..." to download files',),
"committed": ("git commit the corresponding dvc files to update the repo",),
+2 -2
View File
@@ -1,4 +1,4 @@
from typing import Any, Dict
from typing import Any
from dvc.cli import completion, formatter
from dvc.cli.command import CmdBase
@@ -11,7 +11,7 @@ logger = logger.getChild(__name__)
class CmdExperimentsPush(CmdBase):
@staticmethod
def log_result(result: Dict[str, Any], remote: str):
def log_result(result: dict[str, Any], remote: str):
from dvc.utils import humanize
def join_exps(exps):
+3 -2
View File
@@ -1,7 +1,8 @@
import argparse
import re
from collections.abc import Iterable
from datetime import date, datetime
from typing import TYPE_CHECKING, Dict, Iterable
from typing import TYPE_CHECKING
from funcy import lmap
@@ -57,7 +58,7 @@ def baseline_styler(typ):
def show_experiments(
td: "TabularData",
headers: Dict[str, Iterable[str]],
headers: dict[str, Iterable[str]],
keep=None,
drop=None,
pager=True,
+4 -4
View File
@@ -1,6 +1,6 @@
import argparse
import os
from typing import TYPE_CHECKING, Dict, List, Optional
from typing import TYPE_CHECKING, Optional
from funcy import compact, first, get_in
@@ -20,14 +20,14 @@ logger = logger.getChild(__name__)
def _show_json(
renderers_with_errors: List["RendererWithErrors"],
renderers_with_errors: list["RendererWithErrors"],
split=False,
errors: Optional[Dict[str, Exception]] = None,
errors: Optional[dict[str, Exception]] = None,
):
from dvc.render.convert import to_json
from dvc.utils.serialize import encode_exception
all_errors: List[Dict] = []
all_errors: list[dict] = []
data = {}
for renderer, src_errors, def_errors in renderers_with_errors:
+6 -5
View File
@@ -1,8 +1,9 @@
import argparse
import logging
from collections.abc import Iterable
from contextlib import contextmanager
from itertools import chain, filterfalse
from typing import TYPE_CHECKING, Dict, Iterable, List
from typing import TYPE_CHECKING
from dvc.cli import completion, formatter
from dvc.cli.command import CmdBase
@@ -33,7 +34,7 @@ def generate_description(stage: "Stage") -> str:
def is_plot_or_metric(out: "Output"):
return bool(out.plot) or bool(out.metric)
desc: List[str] = []
desc: list[str] = []
outs = list(filterfalse(is_plot_or_metric, stage.outs))
if outs:
@@ -55,7 +56,7 @@ def prepare_stages_data(
stages: Iterable["Stage"],
description: bool = True,
max_length: int = MAX_TEXT_LENGTH,
) -> Dict[str, str]:
) -> dict[str, str]:
return {
stage.addressing: (
prepare_description(stage, max_length=max_length) if description else ""
@@ -67,7 +68,7 @@ def prepare_stages_data(
class CmdStageList(CmdBase):
def _get_stages(self) -> Iterable["Stage"]:
if self.args.all:
stages: List["Stage"] = self.repo.index.stages
stages: list["Stage"] = self.repo.index.stages
logger.trace("%d no. of stages found", len(stages))
return stages
@@ -96,7 +97,7 @@ class CmdStageList(CmdBase):
return 0
def parse_cmd(commands: List[str]) -> str:
def parse_cmd(commands: list[str]) -> str:
"""
We need to take into account two cases:
+24 -26
View File
@@ -1,20 +1,18 @@
from collections import abc
from collections.abc import (
ItemsView,
Iterable,
Iterator,
Mapping,
MutableSequence,
Sequence,
)
from itertools import chain, repeat, zip_longest
from operator import itemgetter
from typing import (
TYPE_CHECKING,
Any,
Dict,
ItemsView,
Iterable,
Iterator,
List,
Mapping,
MutableSequence,
Optional,
Sequence,
Set,
Tuple,
Union,
overload,
)
@@ -25,7 +23,7 @@ if TYPE_CHECKING:
from dvc.ui.table import CellT
class Column(List["CellT"]):
class Column(list["CellT"]):
pass
@@ -35,13 +33,13 @@ def with_value(value, default):
class TabularData(MutableSequence[Sequence["CellT"]]):
def __init__(self, columns: Sequence[str], fill_value: Optional[str] = ""):
self._columns: Dict[str, Column] = {name: Column() for name in columns}
self._keys: List[str] = list(columns)
self._columns: dict[str, Column] = {name: Column() for name in columns}
self._keys: list[str] = list(columns)
self._fill_value = fill_value
self._protected: Set[str] = set()
self._protected: set[str] = set()
@property
def columns(self) -> List[Column]:
def columns(self) -> list[Column]:
return list(map(self.column, self.keys()))
def is_protected(self, col_name) -> bool:
@@ -60,10 +58,10 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
projection = {k: self.column(k) for k in self.keys()}
return projection.items()
def keys(self) -> List[str]:
def keys(self) -> list[str]:
return self._keys
def _iter_col_row(self, row: Sequence["CellT"]) -> Iterator[Tuple["CellT", Column]]:
def _iter_col_row(self, row: Sequence["CellT"]) -> Iterator[tuple["CellT", Column]]:
for val, col in zip_longest(row, self.columns):
if col is None:
break
@@ -81,7 +79,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
for val, col in self._iter_col_row(value):
col.insert(index, val)
def __iter__(self) -> Iterator[List["CellT"]]:
def __iter__(self) -> Iterator[list["CellT"]]:
return map(list, zip(*self.columns))
def __getattr__(self, item: str) -> Column:
@@ -127,7 +125,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
return len(self.columns[0])
@property
def shape(self) -> Tuple[int, int]:
def shape(self) -> tuple[int, int]:
return len(self.columns), len(self)
def drop(self, *col_names: str) -> None:
@@ -170,7 +168,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
if key not in keys:
self.add_column(key)
row: List["CellT"] = [
row: list["CellT"] = [
with_value(d.get(key), self._fill_value) for key in self.keys()
]
self.append(row)
@@ -185,7 +183,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
def as_dict(
self, cols: Optional[Iterable[str]] = None
) -> Iterable[Dict[str, "CellT"]]:
) -> Iterable[dict[str, "CellT"]]:
keys = self.keys() if cols is None else set(cols)
return [{k: self._columns[k][i] for k in keys} for i in range(len(self))]
@@ -202,7 +200,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
if how not in ["any", "all"]:
raise ValueError(f"Invalid 'how' value {how}. Choose one of ['any', 'all']")
match_line: Set = set()
match_line: set = set()
match_any = True
if how == "all":
match_any = False
@@ -245,7 +243,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
)
if axis == "cols":
cols_to_drop: List[str] = []
cols_to_drop: list[str] = []
for n_col, col in enumerate(self.columns):
if subset and self.keys()[n_col] not in subset:
continue
@@ -259,7 +257,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
elif axis == "rows":
unique_rows = []
rows_to_drop: List[int] = []
rows_to_drop: list[int] = []
for n_row, row in enumerate(self):
if subset:
row = [
@@ -319,7 +317,7 @@ def diff_table(
) -> TabularData:
a_rev = a_rev or "HEAD"
b_rev = b_rev or "workspace"
headers: List[str] = ["Path", title, a_rev, b_rev, "Change"]
headers: list[str] = ["Path", title, a_rev, b_rev, "Change"]
fill_value = "-"
td = TabularData(headers, fill_value=fill_value)
@@ -394,7 +392,7 @@ def metrics_table(
for branch, val in metrics.items():
for fname, metric in val.get("data", {}).items():
row_data: Dict[str, str] = {"Revision": branch, "Path": fname}
row_data: dict[str, str] = {"Revision": branch, "Path": fname}
metric = metric.get("data", {})
flattened = (
flatten(format_dict(metric))
+4 -4
View File
@@ -3,7 +3,7 @@ import os
import re
from contextlib import contextmanager
from functools import partial
from typing import TYPE_CHECKING, Dict, Optional
from typing import TYPE_CHECKING, Optional
from funcy import compact, memoize, re_find
@@ -138,7 +138,7 @@ class Config(dict):
return system_config_dir()
@cached_property
def files(self) -> Dict[str, str]:
def files(self) -> dict[str, str]:
files = {
level: os.path.join(self.get_dir(level), self.CONFIG)
for level in ("system", "global")
@@ -344,7 +344,7 @@ class Config(dict):
return Schema(dirs_schema, extra=ALLOW_EXTRA)(conf)
def load_config_to_level(self, level=None):
merged_conf: Dict = {}
merged_conf: dict = {}
for merge_level in self.LEVELS:
if merge_level == level:
break
@@ -390,7 +390,7 @@ class Config(dict):
def _parse_named(conf):
result: Dict[str, Dict] = {"remote": {}, "machine": {}, "db": {}}
result: dict[str, dict] = {"remote": {}, "machine": {}, "db": {}}
for section, val in conf.items():
match = re_find(r'^\s*(remote|machine|db)\s*"(.*)"\s*$', section)
+9 -8
View File
@@ -5,13 +5,14 @@ import logging
import os
import subprocess
import sys
from collections.abc import Mapping, Sequence
from contextlib import nullcontext
from typing import TYPE_CHECKING, Any, List, Mapping, Optional, Sequence, Union
from typing import TYPE_CHECKING, Any, Optional, Union
from dvc.log import logger
if TYPE_CHECKING:
from typing import ContextManager
from contextlib import AbstractContextManager
from dvc.env import DVC_DAEMON, DVC_DAEMON_LOGFILE
from dvc.utils import fix_env, is_binary
@@ -55,7 +56,7 @@ def _win_detached_subprocess(args: Sequence[str], **kwargs) -> int:
return popen.pid
def _get_dvc_args() -> List[str]:
def _get_dvc_args() -> list[str]:
args = [sys.executable]
if not is_binary():
root_dir = os.path.abspath(os.path.dirname(__file__))
@@ -136,7 +137,7 @@ def _map_log_level_to_flag() -> Optional[str]:
return flags.get(logger.getEffectiveLevel())
def daemon(args: List[str]) -> None:
def daemon(args: list[str]) -> None:
"""Launch a `dvc daemon` command in a detached process.
Args:
@@ -148,12 +149,12 @@ def daemon(args: List[str]) -> None:
def _spawn(
args: List[str],
executable: Optional[Union[str, List[str]]] = None,
args: list[str],
executable: Optional[Union[str, list[str]]] = None,
env: Optional[Mapping[str, str]] = None,
output_file: Optional[str] = None,
) -> int:
file: "ContextManager[Any]" = nullcontext()
file: "AbstractContextManager[Any]" = nullcontext()
kwargs = {}
if output_file:
file = open(output_file, "ab") # noqa: SIM115
@@ -168,7 +169,7 @@ def _spawn(
return _detached_subprocess(executable + args, env=env, **kwargs)
def daemonize(args: List[str], executable: Union[None, str, List[str]] = None) -> None:
def daemonize(args: list[str], executable: Union[None, str, list[str]] = None) -> None:
if os.name not in ("posix", "nt"):
return
+3 -2
View File
@@ -1,6 +1,7 @@
"""Manages dvc remotes that user can use with push/pull/status commands."""
from typing import TYPE_CHECKING, Iterable, Optional, Set, Tuple
from collections.abc import Iterable
from typing import TYPE_CHECKING, Optional
from dvc.config import NoRemoteError, RemoteConfigError
from dvc.log import logger
@@ -50,7 +51,7 @@ class Remote:
def _split_legacy_hash_infos(
hash_infos: Iterable["HashInfo"],
) -> Tuple[Set["HashInfo"], Set["HashInfo"]]:
) -> tuple[set["HashInfo"], set["HashInfo"]]:
from dvc.cachemgr import LEGACY_HASH_NAMES
legacy = set()
+4 -3
View File
@@ -1,8 +1,9 @@
import os
from collections.abc import Iterator
from contextlib import contextmanager
from dataclasses import dataclass
from tempfile import NamedTemporaryFile
from typing import TYPE_CHECKING, Any, Callable, Dict, Iterator, Optional, Union
from typing import TYPE_CHECKING, Any, Callable, Optional, Union
from sqlalchemy import create_engine
from sqlalchemy.engine import make_url as _make_url
@@ -30,7 +31,7 @@ def make_url(url: Union["URL", str], **kwargs: Any) -> "URL":
return _make_url(url).set(**kwargs)
def url_from_config(config: Union[str, "URL", Dict[str, str]]) -> "URL":
def url_from_config(config: Union[str, "URL", dict[str, str]]) -> "URL":
if isinstance(config, dict):
return make_url(**config)
return make_url(config)
@@ -118,7 +119,7 @@ def handle_error(url: "URL"):
@contextmanager
def client(
url_or_config: Union[str, "URL", Dict[str, str]], **engine_kwargs: Any
url_or_config: Union[str, "URL", dict[str, str]], **engine_kwargs: Any
) -> Iterator[Client]:
url = url_from_config(url_or_config)
echo = env2bool(env.DVC_SQLALCHEMY_ECHO, False)
+4 -3
View File
@@ -1,5 +1,6 @@
from collections import defaultdict
from typing import Any, Dict, List, Mapping, Set
from collections.abc import Mapping
from typing import Any
from dvc.output import ARTIFACT_SCHEMA, DIR_FILES_SCHEMA, Output
@@ -66,12 +67,12 @@ def loads_from(stage, s_list, erepo=None, fs_config=None, db=None):
]
def _merge_params(s_list) -> Dict[str, List[str]]:
def _merge_params(s_list) -> dict[str, list[str]]:
d = defaultdict(list)
default_file = ParamsDependency.DEFAULT_PARAMS_FILE
# figure out completely tracked params file, and ignore specific keys
wholly_tracked: Set[str] = set()
wholly_tracked: set[str] = set()
for key in s_list:
if not isinstance(key, dict):
continue
+4 -6
View File
@@ -1,5 +1,3 @@
from typing import Dict, Type
from dvc.exceptions import DvcException
from dvc.fs import download as fs_download
from dvc.output import Output
@@ -25,11 +23,11 @@ class DependencyIsStageFileError(DvcException):
class Dependency(Output):
IS_DEPENDENCY = True
DoesNotExistError: Type[DvcException] = DependencyDoesNotExistError
IsNotFileOrDirError: Type[DvcException] = DependencyIsNotFileOrDirError
IsStageFileError: Type[DvcException] = DependencyIsStageFileError
DoesNotExistError: type[DvcException] = DependencyDoesNotExistError
IsNotFileOrDirError: type[DvcException] = DependencyIsNotFileOrDirError
IsStageFileError: type[DvcException] = DependencyIsStageFileError
def workspace_status(self) -> Dict[str, str]:
def workspace_status(self) -> dict[str, str]:
if self.fs.version_aware:
old_fs_path = self.fs_path
try:
+5 -4
View File
@@ -1,5 +1,6 @@
from collections.abc import Iterator
from contextlib import contextmanager
from typing import TYPE_CHECKING, Any, Callable, ClassVar, Dict, Iterator, Optional
from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional
from funcy import compact, log_durations
@@ -31,7 +32,7 @@ def download_progress(to: "Output") -> Iterator[Callable[[int], Any]]:
class AbstractDependency(Dependency):
"""Dependency without workspace/fs/fs_path"""
def __init__(self, stage: "Stage", info: Dict[str, Any], *args, **kwargs):
def __init__(self, stage: "Stage", info: dict[str, Any], *args, **kwargs):
self.repo = stage.repo
self.stage = stage
self.fs = None
@@ -46,7 +47,7 @@ class DbDependency(AbstractDependency):
PARAM_QUERY = "query"
PARAM_TABLE = "table"
PARAM_FILE_FORMAT = "file_format"
DB_SCHEMA: ClassVar[Dict] = {
DB_SCHEMA: ClassVar[dict] = {
PARAM_DB: {
PARAM_QUERY: str,
PARAM_CONNECTION: str,
@@ -57,7 +58,7 @@ class DbDependency(AbstractDependency):
def __init__(self, stage: "Stage", info, *args, **kwargs):
super().__init__(stage, info, *args, **kwargs)
self.db_info: Dict[str, str] = self.info.get(self.PARAM_DB, {})
self.db_info: dict[str, str] = self.info.get(self.PARAM_DB, {})
self.connection = self.db_info.get(self.PARAM_CONNECTION)
@property
+4 -4
View File
@@ -1,7 +1,7 @@
import os
import typing
from collections import defaultdict
from typing import TYPE_CHECKING, Any, Dict, List, Optional
from typing import TYPE_CHECKING, Any, Optional
import dpath
@@ -37,7 +37,7 @@ class BadParamFileError(DvcException):
def read_param_file(
fs: "FileSystem",
path: str,
key_paths: Optional[List[str]] = None,
key_paths: Optional[list[str]] = None,
flatten: bool = False,
**load_kwargs,
) -> Any:
@@ -110,7 +110,7 @@ class ParamsDependency(Dependency):
def read_params(
self, flatten: bool = True, **kwargs: typing.Any
) -> Dict[str, typing.Any]:
) -> dict[str, typing.Any]:
try:
self.validate_filepath()
except MissingParamsFile:
@@ -134,7 +134,7 @@ class ParamsDependency(Dependency):
from funcy import ldistinct
status: Dict[str, Any] = defaultdict(dict)
status: dict[str, Any] = defaultdict(dict)
info = self.hash_info.value if self.hash_info else {}
assert isinstance(info, dict)
actual = self.read_params()
+5 -5
View File
@@ -1,5 +1,5 @@
from copy import deepcopy
from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union
from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union
import voluptuous as vol
@@ -20,7 +20,7 @@ class RepoDependency(Dependency):
PARAM_CONFIG = "config"
PARAM_REMOTE = "remote"
REPO_SCHEMA: ClassVar[Dict] = {
REPO_SCHEMA: ClassVar[dict] = {
PARAM_REPO: {
vol.Required(PARAM_URL): str,
PARAM_REV: str,
@@ -30,7 +30,7 @@ class RepoDependency(Dependency):
}
}
def __init__(self, def_repo: Dict[str, Any], stage: "Stage", *args, **kwargs):
def __init__(self, def_repo: dict[str, Any], stage: "Stage", *args, **kwargs):
self.def_repo = def_repo
super().__init__(stage, *args, **kwargs)
@@ -65,7 +65,7 @@ class RepoDependency(Dependency):
self.def_repo[self.PARAM_REV_LOCK] = rev
@classmethod
def _dump_def_repo(cls, def_repo) -> Dict[str, str]:
def _dump_def_repo(cls, def_repo) -> dict[str, str]:
repo = {cls.PARAM_URL: def_repo[cls.PARAM_URL]}
rev = def_repo.get(cls.PARAM_REV)
@@ -85,7 +85,7 @@ class RepoDependency(Dependency):
repo[cls.PARAM_REMOTE] = remote
return repo
def dumpd(self, **kwargs) -> Dict[str, Union[str, Dict[str, str]]]:
def dumpd(self, **kwargs) -> dict[str, Union[str, dict[str, str]]]:
return {
self.PARAM_PATH: self.def_path,
self.PARAM_REPO: self._dump_def_repo(self.def_repo),
+10 -13
View File
@@ -5,10 +5,7 @@ from typing import (
Any,
Callable,
ClassVar,
Dict,
List,
Optional,
Tuple,
TypeVar,
Union,
)
@@ -134,7 +131,7 @@ class FileMixin:
d, _ = self._load(**kwargs)
return d
def _load(self, **kwargs: Any) -> Tuple[Any, str]:
def _load(self, **kwargs: Any) -> tuple[Any, str]:
# it raises the proper exceptions by priority:
# 1. when the file doesn't exists
# 2. filename is not a DVC file
@@ -157,7 +154,7 @@ class FileMixin:
return validate(d, cls.SCHEMA, path=fname) # type: ignore[arg-type]
def _load_yaml(self, **kwargs: Any) -> Tuple[Any, str]:
def _load_yaml(self, **kwargs: Any) -> tuple[Any, str]:
from dvc.utils import strictyaml
return strictyaml.load(
@@ -182,10 +179,10 @@ class SingleStageFile(FileMixin):
from dvc.schema import COMPILED_SINGLE_STAGE_SCHEMA as SCHEMA
from dvc.stage.loader import SingleStageLoader as LOADER # noqa: N814
metrics: ClassVar[List[str]] = []
metrics: ClassVar[list[str]] = []
plots: ClassVar[Any] = {}
params: ClassVar[List[str]] = []
artifacts: ClassVar[Dict[str, Optional[Dict[str, Any]]]] = {}
params: ClassVar[list[str]] = []
artifacts: ClassVar[dict[str, Optional[dict[str, Any]]]] = {}
@property
def stage(self) -> "Stage":
@@ -284,11 +281,11 @@ class ProjectFile(FileMixin):
raise DvcException("ProjectFile has multiple stages. Please specify it's name.")
@cached_property
def contents(self) -> Dict[str, Any]:
def contents(self) -> dict[str, Any]:
return self._load()[0]
@cached_property
def lockfile_contents(self) -> Dict[str, Any]:
def lockfile_contents(self) -> dict[str, Any]:
return self._lockfile.load()
@cached_property
@@ -303,7 +300,7 @@ class ProjectFile(FileMixin):
return self.LOADER(self, self.contents, self.lockfile_contents)
@property
def metrics(self) -> List[str]:
def metrics(self) -> list[str]:
return self.contents.get("metrics", [])
@property
@@ -311,11 +308,11 @@ class ProjectFile(FileMixin):
return self.contents.get("plots", {})
@property
def params(self) -> List[str]:
def params(self) -> list[str]:
return self.contents.get("params", [])
@property
def artifacts(self) -> Dict[str, Optional[Dict[str, Any]]]:
def artifacts(self) -> dict[str, Optional[dict[str, Any]]]:
return self.contents.get("artifacts", {})
def remove(self, force=False):
+3 -3
View File
@@ -1,6 +1,6 @@
"""Exceptions raised by the dvc."""
import errno
from typing import TYPE_CHECKING, Dict, List, Optional, Set
from typing import TYPE_CHECKING, Optional
from dvc.utils import format_link
@@ -34,7 +34,7 @@ class OutputDuplicationError(DvcException):
stages (list): list of paths to stages.
"""
def __init__(self, output: str, stages: Set["Stage"]):
def __init__(self, output: str, stages: set["Stage"]):
from funcy import first
assert isinstance(output, str)
@@ -241,7 +241,7 @@ class UploadError(FileTransferError):
class CheckoutError(DvcException):
def __init__(self, target_infos: List[str], stats: Dict[str, List[str]]):
def __init__(self, target_infos: list[str], stats: dict[str, list[str]]):
from dvc.utils import error_link
self.target_infos = target_infos
+3 -3
View File
@@ -1,5 +1,5 @@
from contextlib import ExitStack
from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optional, Union
from typing import TYPE_CHECKING, Any, BinaryIO, Optional, Union
from dvc.progress import Tqdm
from dvc.utils.objects import cached_property
@@ -55,7 +55,7 @@ class TqdmCallback(Callback):
self,
path_1: "Union[str, BinaryIO]",
path_2: str,
kwargs: Dict[str, Any],
kwargs: dict[str, Any],
child: Optional[Callback] = None,
):
child = child or TqdmCallback(
@@ -130,7 +130,7 @@ class RichCallback(Callback):
self,
path_1: Union[str, BinaryIO],
path_2: str,
kwargs: Dict[str, Any],
kwargs: dict[str, Any],
child: Optional["Callback"] = None,
):
child = child or RichCallback(
+9 -9
View File
@@ -6,7 +6,7 @@ import posixpath
import threading
from collections import deque
from contextlib import ExitStack, suppress
from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple, Type, Union
from typing import TYPE_CHECKING, Any, Callable, Optional, Union
from fsspec.spec import AbstractFileSystem
from funcy import wrap_with
@@ -22,8 +22,8 @@ if TYPE_CHECKING:
logger = logger.getChild(__name__)
RepoFactory = Union[Callable[..., "Repo"], Type["Repo"]]
Key = Tuple[str, ...]
RepoFactory = Union[Callable[..., "Repo"], type["Repo"]]
Key = tuple[str, ...]
def as_posix(path: str) -> str:
@@ -85,7 +85,7 @@ class _DVCFileSystem(AbstractFileSystem):
subrepos: bool = False,
repo_factory: Optional[RepoFactory] = None,
fo: Optional[str] = None,
target_options: Optional[Dict[str, Any]] = None, # noqa: ARG002
target_options: Optional[dict[str, Any]] = None, # noqa: ARG002
target_protocol: Optional[str] = None, # noqa: ARG002
config: Optional["DictStrAny"] = None,
remote: Optional[str] = None,
@@ -145,7 +145,7 @@ class _DVCFileSystem(AbstractFileSystem):
}
def getcwd(self):
relparts: Tuple[str, ...] = ()
relparts: tuple[str, ...] = ()
assert self.repo is not None
if self.repo.fs.isin(self.repo.fs.getcwd(), self.repo.root_dir):
relparts = self.repo.fs.relparts(self.repo.fs.getcwd(), self.repo.root_dir)
@@ -156,7 +156,7 @@ class _DVCFileSystem(AbstractFileSystem):
return posixpath.join(*parts)
@classmethod
def parts(cls, path: str) -> Tuple[str, ...]:
def parts(cls, path: str) -> tuple[str, ...]:
ret = []
while True:
path, part = posixpath.split(path)
@@ -187,7 +187,7 @@ class _DVCFileSystem(AbstractFileSystem):
start = "."
return posixpath.relpath(self.abspath(path), start=self.abspath(start))
def relparts(self, path: str, start: Optional[str] = None) -> Tuple[str, ...]:
def relparts(self, path: str, start: Optional[str] = None) -> tuple[str, ...]:
return self.parts(self.relpath(path, start=start))
@functools.cached_property
@@ -321,7 +321,7 @@ class _DVCFileSystem(AbstractFileSystem):
def _get_subrepo_info(
self, key: Key
) -> Tuple["Repo", Optional[DataFileSystem], Key]:
) -> tuple["Repo", Optional[DataFileSystem], Key]:
"""
Returns information about the subrepo the key is part of.
"""
@@ -525,7 +525,7 @@ class DVCFileSystem(FileSystem):
protocol = "local"
PARAM_CHECKSUM = "md5"
def _prepare_credentials(self, **config) -> Dict[str, Any]:
def _prepare_credentials(self, **config) -> dict[str, Any]:
return config
@functools.cached_property
+4 -4
View File
@@ -2,7 +2,7 @@ import os
import re
from collections import namedtuple
from itertools import chain, groupby, takewhile
from typing import TYPE_CHECKING, List, Optional
from typing import TYPE_CHECKING, Optional
from pathspec.patterns import GitWildMatchPattern
from pathspec.util import normalize_file
@@ -62,7 +62,7 @@ class DvcIgnorePatterns(DvcIgnore):
return cls(path_spec_lines, dirname, fs.sep)
def __call__(self, root: List[str], dirs: List[str], files: List[str]):
def __call__(self, root: list[str], dirs: list[str], files: list[str]):
files = [f for f in files if not self.matches(root, f)]
dirs = [d for d in dirs if not self.matches(root, d, True)]
@@ -226,7 +226,7 @@ class DvcIgnoreFilter:
self,
dirname: str,
ignore_trie: Trie,
dnames: Optional["List"],
dnames: Optional["list"],
ignore_subrepos: bool,
) -> None:
self._update_trie(dirname, ignore_trie)
@@ -331,7 +331,7 @@ class DvcIgnoreFilter:
yield from fs.find(path)
def _get_trie_pattern(
self, dirname, dnames: Optional["List"] = None, ignore_subrepos=True
self, dirname, dnames: Optional["list"] = None, ignore_subrepos=True
) -> Optional["DvcIgnorePatterns"]:
if ignore_subrepos:
ignores_trie = self.ignores_trie_fs
+2 -2
View File
@@ -5,7 +5,7 @@ import logging.config
import logging.handlers
import os
import sys
from typing import ClassVar, Dict
from typing import ClassVar
import colorama
@@ -79,7 +79,7 @@ class ColorFormatter(logging.Formatter):
"""
reset = colorama.Fore.RESET
color_codes: ClassVar[Dict[str, str]] = {
color_codes: ClassVar[dict[str, str]] = {
"TRACE": colorama.Fore.GREEN,
"DEBUG": colorama.Fore.BLUE,
"WARNING": colorama.Fore.YELLOW,
+23 -23
View File
@@ -4,7 +4,7 @@ import posixpath
from collections import defaultdict
from contextlib import suppress
from operator import itemgetter
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Type, Union
from typing import TYPE_CHECKING, Any, Optional, Union
from urllib.parse import urlparse
import voluptuous as vol
@@ -148,7 +148,7 @@ def _split_dict(d, keys):
def _merge_data(s_list):
d: Dict[str, Dict] = defaultdict(dict)
d: dict[str, dict] = defaultdict(dict)
for key in s_list:
if isinstance(key, str):
d[key].update({})
@@ -207,7 +207,7 @@ def load_from_pipeline(stage, data, typ="outs"):
)
def split_file_meta_from_cloud(entry: Dict) -> Dict:
def split_file_meta_from_cloud(entry: dict) -> dict:
if remote_name := entry.pop(Meta.PARAM_REMOTE, None):
remote_meta = {}
for key in (
@@ -223,7 +223,7 @@ def split_file_meta_from_cloud(entry: Dict) -> Dict:
return entry
def merge_file_meta_from_cloud(entry: Dict) -> Dict:
def merge_file_meta_from_cloud(entry: dict) -> dict:
cloud_meta = entry.pop(Output.PARAM_CLOUD, {})
if remote_name := first(cloud_meta):
entry.update(cloud_meta[remote_name])
@@ -231,7 +231,7 @@ def merge_file_meta_from_cloud(entry: Dict) -> Dict:
return entry
def _serialize_tree_obj_to_files(obj: Tree) -> List[Dict[str, Any]]:
def _serialize_tree_obj_to_files(obj: Tree) -> list[dict[str, Any]]:
key = obj.PARAM_RELPATH
return sorted(
(
@@ -246,7 +246,7 @@ def _serialize_tree_obj_to_files(obj: Tree) -> List[Dict[str, Any]]:
)
def _serialize_hi_to_dict(hash_info: Optional[HashInfo]) -> Dict[str, Any]:
def _serialize_hi_to_dict(hash_info: Optional[HashInfo]) -> dict[str, Any]:
if hash_info:
if hash_info.name == "md5-dos2unix":
return {"md5": hash_info.value}
@@ -316,10 +316,10 @@ class Output:
PARAM_HASH = "hash"
PARAM_FS_CONFIG = "fs_config"
DoesNotExistError: Type[DvcException] = OutputDoesNotExistError
IsNotFileOrDirError: Type[DvcException] = OutputIsNotFileOrDirError
IsStageFileError: Type[DvcException] = OutputIsStageFileError
IsIgnoredError: Type[DvcException] = OutputIsIgnoredError
DoesNotExistError: type[DvcException] = OutputDoesNotExistError
IsNotFileOrDirError: type[DvcException] = OutputIsNotFileOrDirError
IsStageFileError: type[DvcException] = OutputIsStageFileError
IsIgnoredError: type[DvcException] = OutputIsIgnoredError
def __init__( # noqa: PLR0913
self,
@@ -337,7 +337,7 @@ class Output:
remote=None,
repo=None,
fs_config=None,
files: Optional[List[Dict[str, Any]]] = None,
files: Optional[list[dict[str, Any]]] = None,
push: bool = True,
hash_name: Optional[str] = DEFAULT_ALGORITHM,
):
@@ -420,7 +420,7 @@ class Output:
def _compute_hash_info_from_meta(
self, hash_name: Optional[str]
) -> Tuple[str, HashInfo]:
) -> tuple[str, HashInfo]:
if self.is_in_repo:
if hash_name is None:
# Legacy 2.x output, use "md5-dos2unix" but read "md5" from
@@ -555,7 +555,7 @@ class Output:
def _build(
self, *args, no_progress_bar=False, **kwargs
) -> Tuple["HashFileDB", "Meta", "HashFile"]:
) -> tuple["HashFileDB", "Meta", "HashFile"]:
from dvc.ui import ui
with ui.progress(
@@ -601,7 +601,7 @@ class Output:
return self.fs.exists(self.fs_path)
@cached_property
def index_key(self) -> Tuple[str, "DataIndexKey"]:
def index_key(self) -> tuple[str, "DataIndexKey"]:
if self.is_in_repo:
workspace = "repo"
key = self.repo.fs.relparts(self.fs_path, self.repo.root_dir)
@@ -633,7 +633,7 @@ class Output:
return self.meta.version_id != self.get_meta().version_id
return False
def workspace_status(self) -> Dict[str, str]:
def workspace_status(self) -> dict[str, str]:
if not self.exists:
return {str(self): "deleted"}
@@ -645,7 +645,7 @@ class Output:
return {}
def status(self) -> Dict[str, str]:
def status(self) -> dict[str, str]:
if self.hash_info and self.use_cache and self.changed_cache():
return {str(self): "not in cache"}
@@ -840,7 +840,7 @@ class Output:
def dumpd(self, **kwargs): # noqa: C901, PLR0912
from dvc.cachemgr import LEGACY_HASH_NAMES
ret: Dict[str, Any] = {}
ret: dict[str, Any] = {}
with_files = (
(not self.IS_DEPENDENCY or self.stage.is_import)
and self.hash_info.isdir
@@ -962,7 +962,7 @@ class Output:
filter_info: Optional[str] = None,
allow_missing: bool = False,
**kwargs,
) -> Optional[Tuple[bool, Optional[bool]]]:
) -> Optional[tuple[bool, Optional[bool]]]:
# callback passed act as a aggregate callback.
# do not let checkout to call set_size and change progressbar.
class CallbackProxy(Callback):
@@ -1157,7 +1157,7 @@ class Output:
def get_used_objs( # noqa: PLR0911
self, **kwargs
) -> Dict[Optional["HashFileDB"], Set["HashInfo"]]:
) -> dict[Optional["HashFileDB"], set["HashInfo"]]:
"""Return filtered set of used object IDs for this out."""
from dvc.cachemgr import LEGACY_HASH_NAMES
@@ -1291,7 +1291,7 @@ class Output:
nfiles=len(merged),
)
def unstage(self, path: str) -> Tuple["Meta", "Tree"]:
def unstage(self, path: str) -> tuple["Meta", "Tree"]:
from pygtrie import Trie
rel_key = tuple(self.fs.parts(self.fs.relpath(path, self.fs_path)))
@@ -1324,7 +1324,7 @@ class Output:
path: str,
obj: Union["Tree", "HashFile"],
meta: "Meta",
) -> Tuple["Meta", "Tree"]:
) -> tuple["Meta", "Tree"]:
from pygtrie import Trie
append_only = True
@@ -1514,7 +1514,7 @@ META_SCHEMA = {
CLOUD_SCHEMA = vol.All({str: {**META_SCHEMA, **CHECKSUMS_SCHEMA}}, vol.Length(max=1))
ARTIFACT_SCHEMA: Dict[Any, Any] = {
ARTIFACT_SCHEMA: dict[Any, Any] = {
**CHECKSUMS_SCHEMA,
**META_SCHEMA,
Output.PARAM_PATH: str,
@@ -1523,7 +1523,7 @@ ARTIFACT_SCHEMA: Dict[Any, Any] = {
Output.PARAM_HASH: str,
}
DIR_FILES_SCHEMA: Dict[Any, Any] = {
DIR_FILES_SCHEMA: dict[Any, Any] = {
**CHECKSUMS_SCHEMA,
**META_SCHEMA,
vol.Required(Tree.PARAM_RELPATH): str,
+13 -17
View File
@@ -6,12 +6,8 @@ from itertools import product
from typing import (
TYPE_CHECKING,
Any,
Dict,
List,
NamedTuple,
Optional,
Tuple,
Type,
Union,
)
@@ -85,7 +81,7 @@ def format_and_raise(exc: Exception, msg: str, path: str) -> "NoReturn":
def _reraise_err(
exc_cls: Type[Exception], *args, from_exc: Optional[Exception] = None
exc_cls: type[Exception], *args, from_exc: Optional[Exception] = None
) -> "NoReturn":
err = exc_cls(*args)
if from_exc and logger.isEnabledFor(logging.DEBUG):
@@ -108,7 +104,7 @@ def is_map_or_seq(data: Any) -> bool:
return not isinstance(data, str) and _is_map_or_seq(data)
def split_group_name(name: str) -> Tuple[str, Optional[str]]:
def split_group_name(name: str) -> tuple[str, Optional[str]]:
group, *keys = name.rsplit(JOIN, maxsplit=1)
return group, first(keys)
@@ -161,14 +157,14 @@ class DataResolver:
# we use `tracked_vars` to keep a dictionary of used variables
# by the interpolated entries.
self.tracked_vars: Dict[str, Mapping] = {}
self.tracked_vars: dict[str, Mapping] = {}
stages_data = d.get(STAGES_KWD, {})
# we wrap the definitions into:
# ForeachDefinition, MatrixDefinition, and EntryDefinition
# that helps us to optimize, cache and selectively load each one of
# them as we need, and simplify all of this DSL/parsing logic.
self.definitions: Dict[str, Definition] = {
self.definitions: dict[str, Definition] = {
name: make_definition(self, name, definition)
for name, definition in stages_data.items()
}
@@ -376,7 +372,7 @@ class ForeachDefinition:
f" in '{self.relpath}': expected list/dictionary, got " + typ
)
def _warn_if_overwriting(self, keys: List[str]):
def _warn_if_overwriting(self, keys: list[str]):
warn_for = [k for k in keys if k in self.context]
if warn_for:
linking_verb = "is" if len(warn_for) == 1 else "are"
@@ -390,7 +386,7 @@ class ForeachDefinition:
self.name,
)
def _inserted_keys(self, iterable) -> List[str]:
def _inserted_keys(self, iterable) -> list[str]:
keys = [self.pair.value]
if isinstance(iterable, Mapping):
keys.append(self.pair.key)
@@ -490,10 +486,10 @@ class MatrixDefinition:
return self._template
@cached_property
def resolved_iterable(self) -> Dict[str, List]:
def resolved_iterable(self) -> dict[str, list]:
return self._resolve_matrix_data()
def _resolve_matrix_data(self) -> Dict[str, List]:
def _resolve_matrix_data(self) -> dict[str, list]:
try:
iterable = self.context.resolve(self.matrix_data, unwrap=False)
except (ContextError, ParseError) as exc:
@@ -504,7 +500,7 @@ class MatrixDefinition:
self._warn_if_overwriting([self.pair.key, self.pair.value])
return iterable
def _warn_if_overwriting(self, keys: List[str]):
def _warn_if_overwriting(self, keys: list[str]):
warn_for = [k for k in keys if k in self.context]
if warn_for:
linking_verb = "is" if len(warn_for) == 1 else "are"
@@ -519,16 +515,16 @@ class MatrixDefinition:
)
@cached_property
def normalized_iterable(self) -> Dict[str, "DictStrAny"]:
def normalized_iterable(self) -> dict[str, "DictStrAny"]:
"""Convert sequence to Mapping with keys normalized."""
iterable = self.resolved_iterable
assert isinstance(iterable, Mapping)
ret: Dict[str, "DictStrAny"] = {}
ret: dict[str, "DictStrAny"] = {}
matrix = {key: enumerate(v) for key, v in iterable.items()}
for combination in product(*matrix.values()):
d: "DictStrAny" = {}
fragments: List[str] = []
fragments: list[str] = []
for k, (i, v) in zip(matrix.keys(), combination):
d[k] = v
fragments.append(f"{k}{i}" if is_map_or_seq(v) else to_str(v))
@@ -540,7 +536,7 @@ class MatrixDefinition:
def has_member(self, key: str) -> bool:
return key in self.normalized_iterable
def get_generated_names(self) -> List[str]:
def get_generated_names(self) -> list[str]:
return list(map(self._generate_name, self.normalized_iterable))
def _generate_name(self, key: str) -> str:
+6 -6
View File
@@ -4,7 +4,7 @@ from collections.abc import Mapping, MutableMapping, MutableSequence, Sequence
from contextlib import contextmanager
from copy import deepcopy
from dataclasses import dataclass, field, replace
from typing import Any, Dict, List, Optional, Union
from typing import Any, Optional, Union
from funcy import identity, lfilter, nullcontext, select
@@ -22,7 +22,7 @@ from dvc.parsing.interpolate import (
logger = logger.getChild(__name__)
SeqOrMap = Union[Sequence, Mapping]
DictStr = Dict[str, Any]
DictStr = dict[str, Any]
class ContextError(DvcException):
@@ -100,7 +100,7 @@ def recurse_not_a_node(data: dict):
@dataclass
class Meta:
source: Optional[str] = None
dpaths: List[str] = field(default_factory=list)
dpaths: list[str] = field(default_factory=list)
local: bool = True
@staticmethod
@@ -297,7 +297,7 @@ class Context(CtxDict):
"""
super().__init__(*args, **kwargs)
self._track = False
self._tracked_data: Dict[str, Dict] = defaultdict(dict)
self._tracked_data: dict[str, dict] = defaultdict(dict)
self.imports = {}
self._reserved_keys = {}
@@ -348,7 +348,7 @@ class Context(CtxDict):
@classmethod
def load_from(
cls, fs, path: str, select_keys: Optional[List[str]] = None
cls, fs, path: str, select_keys: Optional[list[str]] = None
) -> "Context":
from dvc.utils.serialize import load_path
@@ -423,7 +423,7 @@ class Context(CtxDict):
def load_from_vars(
self,
fs,
vars_: List,
vars_: list,
wdir: str,
stage_name: Optional[str] = None,
default: Optional[str] = None,
+4 -3
View File
@@ -10,7 +10,8 @@ from dvc.exceptions import DvcException
from dvc.utils.flatten import flatten
if typing.TYPE_CHECKING:
from typing import List, Match, NoReturn
from re import Match
from typing import NoReturn
from pyparsing import ParseException
@@ -205,7 +206,7 @@ def validate_value(value, key):
def str_interpolate(
template: str,
matches: "List[Match]",
matches: "list[Match]",
context: "Context",
skip_checks: bool = False,
key=None,
@@ -225,5 +226,5 @@ def str_interpolate(
return buf.replace(r"\${", BRACE_OPEN)
def is_exact_string(src: str, matches: "List[Match]"):
def is_exact_string(src: str, matches: "list[Match]"):
return len(matches) == 1 and src == matches[0].group(0)
+2 -2
View File
@@ -2,7 +2,7 @@
import logging
import sys
from threading import RLock
from typing import TYPE_CHECKING, Any, ClassVar, Dict
from typing import TYPE_CHECKING, Any, ClassVar
from tqdm import tqdm
@@ -33,7 +33,7 @@ class Tqdm(tqdm):
" [{elapsed}<{remaining}, {rate_fmt:>11}]"
)
BAR_FMT_NOTOTAL = "{desc}{bar:b}|{postfix[info]}{n_fmt} [{elapsed}, {rate_fmt:>11}]"
BYTES_DEFAULTS: ClassVar[Dict[str, Any]] = {
BYTES_DEFAULTS: ClassVar[dict[str, Any]] = {
"unit": "B",
"unit_scale": True,
"unit_divisor": 1024,
+2 -1
View File
@@ -1,7 +1,8 @@
"""Manages user prompts."""
from collections.abc import Collection
from getpass import getpass
from typing import Collection, Optional
from typing import Optional
from dvc.log import logger
+2 -2
View File
@@ -1,4 +1,4 @@
from typing import Dict, List, Union
from typing import Union
from dvc.render import REVISION, REVISIONS, SRC, TYPE_KEY
from dvc.render.converter.image import ImageConverter
@@ -18,7 +18,7 @@ def _get_converter(
raise ValueError(f"Invalid renderer class {renderer_class}")
def to_json(renderer, split: bool = False) -> List[Dict]:
def to_json(renderer, split: bool = False) -> list[dict]:
if renderer.TYPE == "vega":
if not renderer.datapoints:
return []
+5 -5
View File
@@ -1,19 +1,19 @@
from typing import Any, Dict, List, Optional, Tuple
from typing import Any, Optional
class Converter:
def __init__(
self,
plot_id: str,
data: Optional[Dict[str, Any]] = None,
properties: Optional[Dict] = None,
data: Optional[dict[str, Any]] = None,
properties: Optional[dict] = None,
):
self.plot_id = plot_id
self.properties = properties or {}
self.data = data or {}
def convert(self) -> Tuple[List[Tuple[str, str, Any]], Dict]:
def convert(self) -> tuple[list[tuple[str, str, Any]], dict]:
raise NotImplementedError
def flat_datapoints(self, revision: str) -> Tuple[List[Dict], Dict]:
def flat_datapoints(self, revision: str) -> tuple[list[dict], dict]:
raise NotImplementedError
+3 -3
View File
@@ -1,6 +1,6 @@
import base64
import os
from typing import TYPE_CHECKING, Any, Dict, List, Tuple
from typing import TYPE_CHECKING, Any
from dvc.render import FILENAME, REVISION, SRC
@@ -34,13 +34,13 @@ class ImageConverter(Converter):
base64_str = base64.b64encode(image_data).decode()
return f"data:image;base64,{base64_str}"
def convert(self) -> Tuple[List[Tuple[str, str, Any]], Dict]:
def convert(self) -> tuple[list[tuple[str, str, Any]], dict]:
datas = []
for filename, image_data in self.data.items():
datas.append((filename, "", image_data))
return datas, self.properties
def flat_datapoints(self, revision: str) -> Tuple[List[Dict], Dict]:
def flat_datapoints(self, revision: str) -> tuple[list[dict], dict]:
"""
Convert the DVC Plots content to DVC Render datapoints.
Return both generated datapoints and updated properties.
+18 -17
View File
@@ -1,5 +1,6 @@
import os
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
from collections.abc import Iterable
from typing import Any, Optional, Union
from funcy import first, last
@@ -18,7 +19,7 @@ class FieldNotFoundError(DvcException):
)
def _lists(blob: Union[Dict, List]) -> Iterable[List]:
def _lists(blob: Union[dict, list]) -> Iterable[list]:
if isinstance(blob, list):
yield blob
else:
@@ -43,7 +44,7 @@ def _file_field(*args):
def _find(
filename: str,
field: str,
data_series: List[Tuple[str, str, Any]],
data_series: list[tuple[str, str, Any]],
):
for data_file, data_field, data in data_series:
if data_file == filename and data_field == field:
@@ -51,14 +52,14 @@ def _find(
return None
def _verify_field(file2datapoints: Dict[str, List], filename: str, field: str):
def _verify_field(file2datapoints: dict[str, list], filename: str, field: str):
if filename in file2datapoints:
datapoint = first(file2datapoints[filename])
if field not in datapoint:
raise FieldNotFoundError(field, datapoint.keys())
def _get_xs(properties: Dict, file2datapoints: Dict[str, List[Dict]]):
def _get_xs(properties: dict, file2datapoints: dict[str, list[dict]]):
x = properties.get("x")
if x is not None and isinstance(x, dict):
for filename, field in _file_field(x):
@@ -66,7 +67,7 @@ def _get_xs(properties: Dict, file2datapoints: Dict[str, List[Dict]]):
yield filename, field
def _get_ys(properties, file2datapoints: Dict[str, List[Dict]]):
def _get_ys(properties, file2datapoints: dict[str, list[dict]]):
y = properties.get("y", None)
if y is not None:
for filename, field in _file_field(y):
@@ -74,7 +75,7 @@ def _get_ys(properties, file2datapoints: Dict[str, List[Dict]]):
yield filename, field
def _is_datapoints(lst: List[Dict]):
def _is_datapoints(lst: list[dict]):
"""
check if dict keys match, datapoints with different keys mgiht lead
to unexpected behavior
@@ -85,8 +86,8 @@ def _is_datapoints(lst: List[Dict]):
}
def get_datapoints(file_content: Dict):
result: List[Dict[str, Any]] = []
def get_datapoints(file_content: dict):
result: list[dict[str, Any]] = []
for lst in _lists(file_content):
if _is_datapoints(lst):
for index, datapoint in enumerate(lst):
@@ -107,8 +108,8 @@ class VegaConverter(Converter):
def __init__(
self,
plot_id: str,
data: Optional[Dict] = None,
properties: Optional[Dict] = None,
data: Optional[dict] = None,
properties: Optional[dict] = None,
):
super().__init__(plot_id, data, properties)
self.plot_id = plot_id
@@ -126,7 +127,7 @@ class VegaConverter(Converter):
x = self.properties.get("x", None)
y = self.properties.get("y", None)
inferred_properties: Dict = {}
inferred_properties: dict = {}
# Infer x.
if isinstance(x, str):
@@ -195,7 +196,7 @@ class VegaConverter(Converter):
def flat_datapoints(self, revision): # noqa: C901, PLR0912
file2datapoints, properties = self.convert()
props_update: Dict[str, Union[str, List[Dict[str, str]]]] = {}
props_update: dict[str, Union[str, list[dict[str, str]]]] = {}
xs = list(_get_xs(properties, file2datapoints))
@@ -322,9 +323,9 @@ def _get_short_y_file(y_file, common_prefix_len):
def _update_from_field(
target_datapoints: List[Dict],
target_datapoints: list[dict],
field: str,
source_datapoints: Optional[List[Dict]] = None,
source_datapoints: Optional[list[dict]] = None,
source_field: Optional[str] = None,
):
if source_datapoints is None:
@@ -341,11 +342,11 @@ def _update_from_field(
datapoint[field] = source_datapoint[source_field]
def _update_from_index(datapoints: List[Dict], new_field: str):
def _update_from_index(datapoints: list[dict], new_field: str):
for index, datapoint in enumerate(datapoints):
datapoint[new_field] = index
def _update_all(datapoints: List[Dict], update_dict: Dict):
def _update_all(datapoints: list[dict], update_dict: dict):
for datapoint in datapoints:
datapoint.update(update_dict)
+11 -11
View File
@@ -1,6 +1,6 @@
import os
from collections import defaultdict
from typing import TYPE_CHECKING, DefaultDict, Dict, List, NamedTuple, Optional
from typing import TYPE_CHECKING, NamedTuple, Optional
import dpath
import dpath.options
@@ -21,16 +21,16 @@ dpath.options.ALLOW_EMPTY_STRING_KEYS = True
logger = logger.getChild(__name__)
def _squash_plots_properties(data: List) -> Dict:
def _squash_plots_properties(data: list) -> dict:
configs = [last(group) for group in data]
resolved: Dict = {}
resolved: dict = {}
for config in reversed(configs):
resolved = {**resolved, **config}
return resolved
class PlotsData:
def __init__(self, data: Dict):
def __init__(self, data: dict):
self.data = data
def group_definitions(self):
@@ -62,15 +62,15 @@ class PlotsData:
class RendererWithErrors(NamedTuple):
renderer: "Renderer"
source_errors: Dict[str, Dict[str, Exception]]
definition_errors: Dict[str, Exception]
source_errors: dict[str, dict[str, Exception]]
definition_errors: dict[str, Exception]
def match_defs_renderers( # noqa: C901, PLR0912
data,
out=None,
templates_dir: Optional["StrPath"] = None,
) -> List[RendererWithErrors]:
) -> list[RendererWithErrors]:
from dvc_render import ImageRenderer, VegaRenderer
plots_data = PlotsData(data)
@@ -78,12 +78,12 @@ def match_defs_renderers( # noqa: C901, PLR0912
renderer_cls = None
for plot_id, group in plots_data.group_definitions().items():
plot_datapoints: List[Dict] = []
plot_datapoints: list[dict] = []
props = _squash_plots_properties(group)
first_props: Dict = {}
first_props: dict = {}
def_errors: Dict[str, Exception] = {}
src_errors: DefaultDict[str, Dict[str, Exception]] = defaultdict(dict)
def_errors: dict[str, Exception] = {}
src_errors: defaultdict[str, dict[str, Exception]] = defaultdict(dict)
if out is not None:
props["out"] = out
+7 -9
View File
@@ -1,14 +1,12 @@
import os
from collections import defaultdict
from contextlib import contextmanager
from collections.abc import Iterable
from contextlib import AbstractContextManager, contextmanager
from functools import wraps
from typing import (
TYPE_CHECKING,
Callable,
ContextManager,
Iterable,
Optional,
Tuple,
Union,
)
@@ -111,7 +109,7 @@ class Repo:
fs: Optional["FileSystem"] = None,
uninitialized: bool = False,
scm: Optional[Union["Git", "NoSCM"]] = None,
) -> Tuple[str, Optional[str]]:
) -> tuple[str, Optional[str]]:
from dvc.fs import localfs
from dvc.scm import SCM, SCMError
@@ -268,7 +266,7 @@ class Repo:
if not isinstance(self.fs, GitFileSystem):
return None
relparts: Tuple[str, ...] = ()
relparts: tuple[str, ...] = ()
if self.root_dir != "/":
# subrepo
relparts = self.fs.relparts(self.root_dir, "/")
@@ -392,7 +390,7 @@ class Repo:
self,
path: str,
workspace: str = "repo",
) -> Tuple["DataIndex", "DataIndexEntry"]:
) -> tuple["DataIndex", "DataIndexEntry"]:
if self.subrepos:
fs_path = self.dvcfs.from_os_path(path)
fs = self.dvcfs.fs
@@ -476,7 +474,7 @@ class Repo:
return brancher(self, *args, **kwargs)
def switch(self, rev: str) -> ContextManager[str]:
def switch(self, rev: str) -> AbstractContextManager[str]:
from dvc.repo.brancher import switch
return switch(self, rev)
@@ -626,7 +624,7 @@ class Repo:
cache_dir = self.config["core"].get("site_cache_dir") or site_cache_dir()
if isinstance(self.fs, GitFileSystem):
relparts: Tuple[str, ...] = ()
relparts: tuple[str, ...] = ()
if self.root_dir != "/":
# subrepo
relparts = self.fs.relparts(self.root_dir, "/")
+7 -10
View File
@@ -1,13 +1,10 @@
import os
from collections.abc import Iterator
from contextlib import contextmanager
from typing import (
TYPE_CHECKING,
Dict,
Iterator,
List,
NamedTuple,
Optional,
Tuple,
Union,
)
@@ -37,7 +34,7 @@ class StageInfo(NamedTuple):
def find_targets(
targets: Union["StrOrBytesPath", Iterator["StrOrBytesPath"]], glob: bool = False
) -> List[str]:
) -> list[str]:
if isinstance(targets, (str, bytes, os.PathLike)):
targets_list = [os.fsdecode(targets)]
else:
@@ -99,7 +96,7 @@ OVERLAPPING_PARENT_FMT = (
@contextmanager
def translate_graph_error(stages: List["Stage"]) -> Iterator[None]:
def translate_graph_error(stages: list["Stage"]) -> Iterator[None]:
try:
yield
except OverlappingOutputPathsError as exc:
@@ -124,7 +121,7 @@ def translate_graph_error(stages: List["Stage"]) -> Iterator[None]:
)
def progress_iter(stages: Dict[str, StageInfo]) -> Iterator[Tuple[str, StageInfo]]:
def progress_iter(stages: dict[str, StageInfo]) -> Iterator[tuple[str, StageInfo]]:
total = len(stages)
desc = "Adding..."
with ui.progress(
@@ -151,8 +148,8 @@ LINK_FAILURE_MESSAGE = (
@contextmanager
def warn_link_failures() -> Iterator[List[str]]:
link_failures: List[str] = []
def warn_link_failures() -> Iterator[list[str]]:
link_failures: list[str] = []
try:
yield link_failures
finally:
@@ -202,7 +199,7 @@ def add(
to_remote: bool = False,
remote_jobs: Optional[int] = None,
force: bool = False,
) -> List["Stage"]:
) -> list["Stage"]:
add_targets = find_targets(targets, glob=glob)
if not add_targets:
return []
+12 -12
View File
@@ -1,7 +1,7 @@
import os
import posixpath
from pathlib import Path
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
from typing import TYPE_CHECKING, Any, Optional, Union
from dvc.annotations import Artifact
from dvc.dvcfile import PROJECT_FILE
@@ -95,9 +95,9 @@ class Artifacts:
return self.repo.scm
return None
def read(self) -> Dict[str, Dict[str, Artifact]]:
def read(self) -> dict[str, dict[str, Artifact]]:
"""Read artifacts from dvc.yaml."""
artifacts: Dict[str, Dict[str, Artifact]] = {}
artifacts: dict[str, dict[str, Artifact]] = {}
for (
dvcfile,
dvcfile_artifacts,
@@ -140,14 +140,14 @@ class Artifacts:
assert not (version and stage)
name = _reformat_name(name)
tags: List["GitTag"] = find(
tags: list["GitTag"] = find(
name=name, version=version, stage=stage, scm=self.scm
)
if not tags:
raise ArtifactNotFoundError(name, version=version, stage=stage)
if version or stage:
return tags[-1].target
gto_tags: List["GTOTag"] = sort_versions(parse_tag(tag) for tag in tags)
gto_tags: list["GTOTag"] = sort_versions(parse_tag(tag) for tag in tags)
return gto_tags[0].tag.target
def get_path(self, name: str):
@@ -177,7 +177,7 @@ class Artifacts:
out: Optional[str] = None,
force: bool = False,
jobs: Optional[int] = None,
) -> Tuple[int, str]:
) -> tuple[int, str]:
"""Download the specified artifact."""
from dvc.fs import download as fs_download
@@ -205,9 +205,9 @@ class Artifacts:
out: Optional[str] = None,
force: bool = False,
jobs: Optional[int] = None,
dvc_studio_config: Optional[Dict[str, Any]] = None,
dvc_studio_config: Optional[dict[str, Any]] = None,
**kwargs,
) -> Tuple[int, str]:
) -> tuple[int, str]:
from dvc_studio_client.model_registry import get_download_uris
from dvc.fs import HTTPFileSystem, generic, localfs
@@ -215,8 +215,8 @@ class Artifacts:
logger.debug("Trying to download artifact '%s' via studio", name)
out = out or os.getcwd()
to_infos: List[str] = []
from_infos: List[str] = []
to_infos: list[str] = []
from_infos: list[str] = []
if dvc_studio_config is None:
dvc_studio_config = {}
dvc_studio_config["repo_url"] = repo_url
@@ -263,9 +263,9 @@ class Artifacts:
name: str,
version: Optional[str] = None,
stage: Optional[str] = None,
config: Optional[Union[str, Dict[str, Any]]] = None,
config: Optional[Union[str, dict[str, Any]]] = None,
remote: Optional[str] = None,
remote_config: Optional[Union[str, Dict[str, Any]]] = None,
remote_config: Optional[Union[str, dict[str, Any]]] = None,
out: Optional[str] = None,
force: bool = False,
jobs: Optional[int] = None,
+8 -7
View File
@@ -1,5 +1,6 @@
from collections.abc import Iterator
from contextlib import contextmanager
from typing import TYPE_CHECKING, Iterator, Optional, Tuple
from typing import TYPE_CHECKING, Optional
from scmrepo.git import Git
@@ -59,11 +60,11 @@ def brancher(
from dvc.fs import LocalFileSystem
repo_root_parts: Tuple[str, ...] = ()
repo_root_parts: tuple[str, ...] = ()
if self.fs.isin(self.root_dir, self.scm.root_dir):
repo_root_parts = self.fs.relparts(self.root_dir, self.scm.root_dir)
cwd_parts: Tuple[str, ...] = ()
cwd_parts: tuple[str, ...] = ()
if self.fs.isin(self.fs.getcwd(), self.scm.root_dir):
cwd_parts = self.fs.relparts(self.fs.getcwd(), self.scm.root_dir)
@@ -112,8 +113,8 @@ def brancher(
def _switch_fs(
repo: "Repo",
rev: str,
repo_root_parts: Tuple[str, ...],
cwd_parts: Tuple[str, ...],
repo_root_parts: tuple[str, ...],
cwd_parts: tuple[str, ...],
):
from dvc.fs import GitFileSystem, LocalFileSystem
@@ -147,11 +148,11 @@ def switch(repo: "Repo", rev: str) -> Iterator[str]:
if rev != "workspace":
rev = resolve_rev(repo.scm, rev)
repo_root_parts: Tuple[str, ...] = ()
repo_root_parts: tuple[str, ...] = ()
if repo.fs.isin(repo.root_dir, repo.scm.root_dir):
repo_root_parts = repo.fs.relparts(repo.root_dir, repo.scm.root_dir)
cwd_parts: Tuple[str, ...] = ()
cwd_parts: tuple[str, ...] = ()
if repo.fs.isin(repo.fs.getcwd(), repo.scm.root_dir):
cwd_parts = repo.fs.relparts(repo.fs.getcwd(), repo.scm.root_dir)
+3 -3
View File
@@ -1,5 +1,5 @@
import os
from typing import TYPE_CHECKING, Dict, List
from typing import TYPE_CHECKING
from dvc.exceptions import (
CheckoutError,
@@ -66,7 +66,7 @@ def _build_out_changes(index, changes):
def _check_can_delete(
entries: List["DataIndexEntry"],
entries: list["DataIndexEntry"],
index: "BaseDataIndex",
path: str,
fs: "FileSystem",
@@ -107,7 +107,7 @@ def checkout( # noqa: C901
from dvc.stage.exceptions import StageFileBadNameError, StageFileDoesNotExistError
from dvc_data.index.checkout import ADD, DELETE, MODIFY, apply, compare
stats: Dict[str, List[str]] = {
stats: dict[str, list[str]] = {
"added": [],
"deleted": [],
"modified": [],
+6 -5
View File
@@ -1,4 +1,5 @@
from typing import TYPE_CHECKING, Callable, Iterable, List, Optional, Tuple
from collections.abc import Iterable
from typing import TYPE_CHECKING, Callable, Optional
from dvc.log import logger
@@ -10,8 +11,8 @@ logger = logger.getChild(__name__)
FilterFn = Callable[["Output"], bool]
Outputs = List["Output"]
StrPaths = List[str]
Outputs = list["Output"]
StrPaths = list[str]
def _collect_outs(
@@ -43,7 +44,7 @@ def _collect_paths(
def _filter_outs(
outs: Outputs, fs_paths: StrPaths, duplicates=False
) -> Tuple[Outputs, StrPaths]:
) -> tuple[Outputs, StrPaths]:
res_outs: Outputs = []
fs_res_paths = fs_paths
@@ -65,7 +66,7 @@ def collect(
output_filter: Optional[FilterFn] = None,
recursive: bool = False,
duplicates: bool = False,
) -> Tuple[Outputs, StrPaths]:
) -> tuple[Outputs, StrPaths]:
assert targets or output_filter
outs: Outputs = _collect_outs(repo, output_filter=output_filter, deps=deps)
+2 -2
View File
@@ -1,4 +1,4 @@
from typing import TYPE_CHECKING, Set
from typing import TYPE_CHECKING
from dvc import prompt
@@ -122,7 +122,7 @@ def commit_2_to_3(repo: "Repo", dry: bool = False):
stage.dump(update_pipeline=False)
def _migrateable_dvcfiles(view: "IndexView") -> Set[str]:
def _migrateable_dvcfiles(view: "IndexView") -> set[str]:
from dvc.dvcfile import ProjectFile
migrated = set()
+16 -15
View File
@@ -1,6 +1,7 @@
import os
import posixpath
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, TypedDict, Union
from collections.abc import Iterable
from typing import TYPE_CHECKING, Any, TypedDict, Union
from dvc.fs.callbacks import DEFAULT_CALLBACK
from dvc.ui import ui
@@ -53,11 +54,11 @@ def _diff(
not_in_remote: bool = False,
remote_refresh: bool = False,
callback: "Callback" = DEFAULT_CALLBACK,
) -> Dict[str, List[str]]:
) -> dict[str, list[str]]:
from dvc_data.index import StorageError
from dvc_data.index.diff import UNCHANGED, UNKNOWN, diff
ret: Dict[str, List[str]] = {}
ret: dict[str, list[str]] = {}
def _add_change(typ, change):
typ = _adapt_typ(typ)
@@ -115,9 +116,9 @@ def _diff(
class GitInfo(TypedDict, total=False):
staged: Dict[str, List[str]]
unstaged: Dict[str, List[str]]
untracked: List[str]
staged: dict[str, list[str]]
unstaged: dict[str, list[str]]
untracked: list[str]
is_empty: bool
is_dirty: bool
@@ -150,7 +151,7 @@ def _git_info(scm: Union["Git", "NoSCM"], untracked_files: str = "all") -> GitIn
)
def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> Dict[str, List[str]]:
def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> dict[str, list[str]]:
from .index import build_data_index
with ui.progress(
@@ -180,7 +181,7 @@ def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> Dict[str, List[str]]:
def _diff_head_to_index(
repo: "Repo", head: str = "HEAD", **kwargs: Any
) -> Dict[str, List[str]]:
) -> dict[str, list[str]]:
index = repo.index.data["repo"]
with repo.switch(head):
@@ -194,16 +195,16 @@ def _diff_head_to_index(
class Status(TypedDict):
not_in_cache: List[str]
not_in_remote: List[str]
committed: Dict[str, List[str]]
uncommitted: Dict[str, List[str]]
untracked: List[str]
unchanged: List[str]
not_in_cache: list[str]
not_in_remote: list[str]
committed: dict[str, list[str]]
uncommitted: dict[str, list[str]]
untracked: list[str]
unchanged: list[str]
git: GitInfo
def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> List[str]:
def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> list[str]:
"""Transform files rel. to Git root to DVC root, and drop outside files."""
rel = repo.fs.relpath(repo.root_dir, repo.scm.root_dir).rstrip("/")
+3 -3
View File
@@ -1,7 +1,7 @@
import errno
import os
from collections import defaultdict
from typing import Dict, List, Optional
from typing import Optional
from dvc.log import logger
from dvc.repo import locked
@@ -26,7 +26,7 @@ def _diff(old, new, data_keys, with_missing=False):
from dvc_data.index.diff import ADD, DELETE, MODIFY, RENAME
from dvc_data.index.diff import diff as idiff
ret: "Dict[str, List[Dict]]" = {
ret: "dict[str, list[dict]]" = {
"added": [],
"deleted": [],
"modified": [],
@@ -107,7 +107,7 @@ def diff(
self,
a_rev: str = "HEAD",
b_rev: Optional[str] = None,
targets: Optional[List[str]] = None,
targets: Optional[list[str]] = None,
recursive: bool = False,
):
"""
+2 -2
View File
@@ -1,4 +1,4 @@
from typing import Any, Dict, Optional, Union
from typing import Any, Optional, Union
def du(
@@ -6,7 +6,7 @@ def du(
path: Optional[str] = None,
rev: Optional[str] = None,
summarize: bool = False,
config: Union[None, Dict[str, Any], str] = None,
config: Union[None, dict[str, Any], str] = None,
remote: Optional[str] = None,
remote_config: Optional[dict] = None,
):
+11 -10
View File
@@ -1,6 +1,7 @@
import os
import re
from typing import TYPE_CHECKING, Dict, Iterable, List, Optional
from collections.abc import Iterable
from typing import TYPE_CHECKING, Optional
from funcy import chain, first
@@ -104,7 +105,7 @@ class Experiments:
return ExpCache(self.repo)
@property
def stash_revs(self) -> Dict[str, "ExpStashEntry"]:
def stash_revs(self) -> dict[str, "ExpStashEntry"]:
revs = {}
for queue in (self.workspace_queue, self.celery_queue):
revs.update(queue.stash.stash_revs)
@@ -113,7 +114,7 @@ class Experiments:
def reproduce_one(
self,
tmp_dir: bool = False,
copy_paths: Optional[List[str]] = None,
copy_paths: Optional[list[str]] = None,
message: Optional[str] = None,
**kwargs,
):
@@ -143,8 +144,8 @@ class Experiments:
def reproduce_celery(
self, entries: Optional[Iterable["QueueEntry"]] = None, **kwargs
) -> Dict[str, str]:
results: Dict[str, str] = {}
) -> dict[str, str]:
results: dict[str, str] = {}
if entries is None:
entries = list(
chain(
@@ -250,10 +251,10 @@ class Experiments:
def _reproduce_queue(
self,
queue: "BaseStashQueue",
copy_paths: Optional[List[str]] = None,
copy_paths: Optional[list[str]] = None,
message: Optional[str] = None,
**kwargs,
) -> Dict[str, str]:
) -> dict[str, str]:
"""Reproduce queued experiments.
Arguments:
@@ -265,7 +266,7 @@ class Experiments:
"""
exec_results = queue.reproduce(copy_paths=copy_paths, message=message)
results: Dict[str, str] = {}
results: dict[str, str] = {}
for _, exp_result in exec_results.items():
results.update(exp_result)
return results
@@ -319,12 +320,12 @@ class Experiments:
raise MultipleBranchError(rev, ref_infos)
return str(ref_infos[0])
def get_exact_name(self, revs: Iterable[str]) -> Dict[str, Optional[str]]:
def get_exact_name(self, revs: Iterable[str]) -> dict[str, Optional[str]]:
"""Returns preferred name for the specified revision.
Prefers tags, branches (heads), experiments in that order.
"""
result: Dict[str, Optional[str]] = {}
result: dict[str, Optional[str]] = {}
exclude = f"{EXEC_NAMESPACE}/*"
ref_dict = self.scm.describe(revs, base=EXPS_NAMESPACE, exclude=exclude)
for rev in revs:
+3 -2
View File
@@ -1,5 +1,6 @@
from collections.abc import Iterator
from contextlib import ExitStack, contextmanager
from typing import TYPE_CHECKING, Iterator, Tuple
from typing import TYPE_CHECKING
from dvc.repo.experiments.exceptions import InvalidExpRevError
from dvc.scm import RevError
@@ -12,7 +13,7 @@ if TYPE_CHECKING:
def switch_repo(
repo: "Repo",
rev: str,
) -> Iterator[Tuple["Repo", str]]:
) -> Iterator[tuple["Repo", str]]:
"""Return a repo instance (brancher) switched to rev.
If rev is the name of a running experiment, the returned instance will be
+13 -18
View File
@@ -1,16 +1,11 @@
import itertools
import os
from collections.abc import Collection, Iterable, Iterator
from dataclasses import fields
from datetime import datetime
from typing import (
TYPE_CHECKING,
Collection,
Dict,
Iterable,
Iterator,
List,
Optional,
Tuple,
Union,
)
@@ -142,7 +137,7 @@ def collect_queued(
repo: "Repo",
baseline_revs: Collection[str],
**kwargs,
) -> Dict[str, List["ExpRange"]]:
) -> dict[str, list["ExpRange"]]:
"""Collect queued experiments derived from the specified revisions.
Args:
@@ -177,7 +172,7 @@ def collect_active(
repo: "Repo",
baseline_revs: Collection[str],
**kwargs,
) -> Dict[str, List["ExpRange"]]:
) -> dict[str, list["ExpRange"]]:
"""Collect active (running) experiments derived from the specified revisions.
Args:
@@ -189,7 +184,7 @@ def collect_active(
"""
if not baseline_revs:
return {}
result: Dict[str, List["ExpRange"]] = {}
result: dict[str, list["ExpRange"]] = {}
exps = repo.experiments
for queue in (exps.workspace_queue, exps.tempdir_queue, exps.celery_queue):
for baseline, active_exps in queue.collect_active_data(
@@ -206,7 +201,7 @@ def collect_failed(
repo: "Repo",
baseline_revs: Collection[str],
**kwargs,
) -> Dict[str, List["ExpRange"]]:
) -> dict[str, list["ExpRange"]]:
"""Collect failed experiments derived from the specified revisions.
Args:
@@ -225,7 +220,7 @@ def collect_successful(
repo: "Repo",
baseline_revs: Collection[str],
**kwargs,
) -> Dict[str, List["ExpRange"]]:
) -> dict[str, list["ExpRange"]]:
"""Collect successful experiments derived from the specified revisions.
Args:
@@ -235,7 +230,7 @@ def collect_successful(
Returns:
Dict mapping baseline revision to successful experiments.
"""
result: Dict[str, List["ExpRange"]] = {}
result: dict[str, list["ExpRange"]] = {}
for baseline_rev in baseline_revs:
result[baseline_rev] = list(_collect_baseline(repo, baseline_rev, **kwargs))
return result
@@ -282,7 +277,7 @@ def _collect_baseline(
def collect(
repo: "Repo",
revs: Union[List[str], str, None] = None,
revs: Union[list[str], str, None] = None,
all_branches: bool = False,
all_tags: bool = False,
all_commits: bool = False,
@@ -291,7 +286,7 @@ def collect(
hide_failed: bool = False,
sha_only: bool = False,
**kwargs,
) -> List["ExpState"]:
) -> list["ExpState"]:
"""Collect baseline revisions and derived experiments."""
assert isinstance(repo.scm, Git)
if repo.scm.no_commits:
@@ -312,14 +307,14 @@ def collect(
)
)
if sha_only:
baseline_names: Dict[str, Optional[str]] = {}
baseline_names: dict[str, Optional[str]] = {}
else:
baseline_names = describe(
repo.scm, baseline_revs, refs=cached_refs, logger=logger
)
workspace_data = collect_rev(repo, "workspace", **kwargs)
result: List["ExpState"] = [workspace_data]
result: list["ExpState"] = [workspace_data]
queued = collect_queued(repo, baseline_revs, **kwargs) if not hide_queued else {}
active = collect_active(repo, baseline_revs, **kwargs)
failed = collect_failed(repo, baseline_revs, **kwargs) if not hide_failed else {}
@@ -345,10 +340,10 @@ def collect(
return result
def _sorted_ranges(exp_ranges: Iterable["ExpRange"]) -> List["ExpRange"]:
def _sorted_ranges(exp_ranges: Iterable["ExpRange"]) -> list["ExpRange"]:
"""Return list of ExpRange sorted by (timestamp, rev)."""
def _head_timestamp(exp_range: "ExpRange") -> Tuple[datetime, str]:
def _head_timestamp(exp_range: "ExpRange") -> tuple[datetime, str]:
head_exp = first(exp_range.revs)
if head_exp and head_exp.data and head_exp.data.timestamp:
return head_exp.data.timestamp, head_exp.rev
+2 -1
View File
@@ -1,4 +1,5 @@
from typing import TYPE_CHECKING, Collection, Iterable, Optional
from collections.abc import Collection, Iterable
from typing import TYPE_CHECKING, Optional
from dvc.exceptions import DvcException, InvalidArgumentError
+16 -23
View File
@@ -3,6 +3,7 @@ import os
import pickle
import shutil
from abc import ABC, abstractmethod
from collections.abc import Iterable, Iterator
from contextlib import contextmanager
from dataclasses import asdict, dataclass
from enum import IntEnum
@@ -11,15 +12,8 @@ from typing import (
TYPE_CHECKING,
Any,
Callable,
Dict,
Iterable,
Iterator,
List,
NamedTuple,
Optional,
Tuple,
Type,
TypeVar,
Union,
)
@@ -41,6 +35,8 @@ from dvc.utils.studio import env_to_config
if TYPE_CHECKING:
from queue import Queue
from typing_extensions import Self
from dvc.repo import Repo
from dvc.repo.experiments.stash import ExpStashEntry
from dvc.scm import Git
@@ -112,9 +108,6 @@ class ExecutorInfo:
return cls.from_dict(load_json(filename))
_T = TypeVar("_T", bound="BaseExecutor")
class BaseExecutor(ABC):
"""Base class for executing experiments in parallel.
@@ -185,7 +178,7 @@ class BaseExecutor(ABC):
@property
def info(self) -> "ExecutorInfo":
if self.result is not None:
result_dict: Dict[str, Any] = {
result_dict: dict[str, Any] = {
"result_hash": self.result.exp_hash,
"result_ref": (
str(self.result.ref_info) if self.result.ref_info else None
@@ -207,7 +200,7 @@ class BaseExecutor(ABC):
)
@classmethod
def from_info(cls: Type[_T], info: "ExecutorInfo") -> _T:
def from_info(cls, info: "ExecutorInfo") -> "Self":
if info.result_hash:
result: Optional["ExecutorResult"] = ExecutorResult(
info.result_hash,
@@ -229,21 +222,21 @@ class BaseExecutor(ABC):
@classmethod
@abstractmethod
def from_stash_entry(
cls: Type[_T],
cls,
repo: "Repo",
entry: "ExpStashEntry",
**kwargs,
) -> _T:
) -> "Self":
pass
@classmethod
def _from_stash_entry(
cls: Type[_T],
cls,
repo: "Repo",
entry: "ExpStashEntry",
root_dir: str,
**kwargs,
) -> _T:
) -> "Self":
return cls(
root_dir=root_dir,
dvc_dir=relpath(repo.dvc_dir, repo.scm.root_dir),
@@ -255,7 +248,7 @@ class BaseExecutor(ABC):
)
@classmethod
def _get_top_level_paths(cls, repo: "Repo") -> List["str"]:
def _get_top_level_paths(cls, repo: "Repo") -> list["str"]:
return list(
chain(
_collect_top_level_metrics(repo),
@@ -271,7 +264,7 @@ class BaseExecutor(ABC):
targets: Optional[Iterable[str]] = None,
recursive: bool = False,
force: bool = False,
include_untracked: Optional[List[str]] = None,
include_untracked: Optional[list[str]] = None,
message: Optional[str] = None,
) -> ExecutorResult:
from dvc.dvcfile import LOCK_FILE
@@ -382,7 +375,7 @@ class BaseExecutor(ABC):
def fetch_exps(
self,
dest_scm: "Git",
refs: List[str],
refs: list[str],
force: bool = False,
on_diverged: Optional[Callable[[str], None]] = None,
**kwargs,
@@ -457,7 +450,7 @@ class BaseExecutor(ABC):
infofile: Optional[str] = None,
log_errors: bool = True,
log_level: Optional[int] = None,
copy_paths: Optional[List[str]] = None,
copy_paths: Optional[list[str]] = None,
message: Optional[str] = None,
**kwargs,
) -> "ExecutorResult":
@@ -565,7 +558,7 @@ class BaseExecutor(ABC):
git_remote,
repro_force,
message: Optional[str] = None,
) -> Tuple[Optional[str], Optional["ExpRefInfo"], bool]:
) -> tuple[Optional[str], Optional["ExpRefInfo"], bool]:
cls.commit(
dvc.scm,
exp_hash,
@@ -598,7 +591,7 @@ class BaseExecutor(ABC):
info: "ExecutorInfo",
infofile: Optional[str] = None,
log_errors: bool = True,
copy_paths: Optional[List[str]] = None,
copy_paths: Optional[list[str]] = None,
message: Optional[str] = None,
**kwargs,
) -> Iterator["Repo"]:
@@ -790,7 +783,7 @@ class BaseExecutor(ABC):
raise DvcException(f"Unable to copy '{src}' to '{dst}'.") from exc
@contextmanager
def set_temp_refs(self, scm: "Git", temp_dict: Dict[str, str]):
def set_temp_refs(self, scm: "Git", temp_dict: dict[str, str]):
try:
for ref, rev in temp_dict.items():
scm.set_ref(ref, rev)
+3 -3
View File
@@ -1,5 +1,5 @@
from collections import defaultdict
from typing import Dict, List, Optional, Tuple, Union
from typing import Optional, Union
from dvc.log import logger
from dvc.repo import locked
@@ -15,11 +15,11 @@ logger = logger.getChild(__name__)
@scm_context
def ls(
repo,
rev: Optional[Union[List[str], str]] = None,
rev: Optional[Union[list[str], str]] = None,
all_commits: bool = False,
num: int = 1,
git_remote: Optional[str] = None,
) -> Dict[str, List[Tuple[str, Optional[str]]]]:
) -> dict[str, list[tuple[str, Optional[str]]]]:
"""List experiments.
Returns a dict mapping baseline revs to a list of (exp_name, exp_sha) tuples.
+6 -5
View File
@@ -1,4 +1,5 @@
from typing import Iterable, List, Mapping, Optional, Set, Union
from collections.abc import Iterable, Mapping
from typing import Optional, Union
from funcy import group_by
from scmrepo.git.backend.base import SyncStatus
@@ -23,13 +24,13 @@ def pull( # noqa: C901
git_remote: str,
exp_names: Optional[Union[Iterable[str], str]] = None,
all_commits=False,
rev: Optional[Union[List[str], str]] = None,
rev: Optional[Union[list[str], str]] = None,
num=1,
force: bool = False,
pull_cache: bool = False,
**kwargs,
) -> Iterable[str]:
exp_ref_set: Set["ExpRefInfo"] = set()
exp_ref_set: set["ExpRefInfo"] = set()
if all_commits:
exp_ref_set.update(exp_refs(repo.scm, git_remote))
elif exp_names:
@@ -81,7 +82,7 @@ def _pull(
git_remote: str,
refs: Iterable["ExpRefInfo"],
force: bool,
) -> Mapping[SyncStatus, List["ExpRefInfo"]]:
) -> Mapping[SyncStatus, list["ExpRefInfo"]]:
refspec_list = [f"{exp_ref}:{exp_ref}" for exp_ref in refs]
logger.debug("git pull experiment '%s' -> '%s'", git_remote, refspec_list)
@@ -96,7 +97,7 @@ def _pull(
def group_result(refspec):
return results[str(refspec)]
pull_result: Mapping[SyncStatus, List["ExpRefInfo"]] = group_by(group_result, refs)
pull_result: Mapping[SyncStatus, list["ExpRefInfo"]] = group_by(group_result, refs)
return pull_result
+11 -15
View File
@@ -1,12 +1,8 @@
from collections.abc import Iterable, Mapping
from typing import (
TYPE_CHECKING,
Any,
Dict,
Iterable,
List,
Mapping,
Optional,
Set,
Union,
)
@@ -39,7 +35,7 @@ class UploadError(DvcException):
def notify_refs_to_studio(
repo: "Repo", git_remote: str, **refs: List[str]
repo: "Repo", git_remote: str, **refs: list[str]
) -> Optional[str]:
import os
@@ -67,7 +63,7 @@ def notify_refs_to_studio(
return d.get("url")
def exp_refs_from_names(scm: "Git", exp_names: List[str]) -> Set["ExpRefInfo"]:
def exp_refs_from_names(scm: "Git", exp_names: list[str]) -> set["ExpRefInfo"]:
exp_ref_set = set()
exp_ref_dict = resolve_name(scm, exp_names)
unresolved_exp_names = []
@@ -82,7 +78,7 @@ def exp_refs_from_names(scm: "Git", exp_names: List[str]) -> Set["ExpRefInfo"]:
return exp_ref_set
def exp_refs_from_rev(scm: "Git", rev: List[str], num: int = 1) -> Set["ExpRefInfo"]:
def exp_refs_from_rev(scm: "Git", rev: list[str], num: int = 1) -> set["ExpRefInfo"]:
exp_ref_set = set()
rev_dict = iter_revs(scm, rev, num)
rev_set = set(rev_dict.keys())
@@ -97,15 +93,15 @@ def exp_refs_from_rev(scm: "Git", rev: List[str], num: int = 1) -> Set["ExpRefIn
def push(
repo: "Repo",
git_remote: str,
exp_names: Optional[Union[List[str], str]] = None,
exp_names: Optional[Union[list[str], str]] = None,
all_commits: bool = False,
rev: Optional[Union[List[str], str]] = None,
rev: Optional[Union[list[str], str]] = None,
num: int = 1,
force: bool = False,
push_cache: bool = False,
**kwargs: Any,
) -> Dict[str, Any]:
exp_ref_set: Set["ExpRefInfo"] = set()
) -> dict[str, Any]:
exp_ref_set: set["ExpRefInfo"] = set()
assert isinstance(repo.scm, Git)
if all_commits:
exp_ref_set.update(exp_refs(repo.scm))
@@ -123,7 +119,7 @@ def push(
status.name.lower(): [ref.name for ref in ref_list]
for status, ref_list in push_result.items()
}
result: Dict[str, Any] = {**refs, "uploaded": 0}
result: dict[str, Any] = {**refs, "uploaded": 0}
pushed_refs_info = (
push_result[SyncStatus.UP_TO_DATE] + push_result[SyncStatus.SUCCESS]
@@ -149,7 +145,7 @@ def _push(
git_remote: str,
refs: Iterable["ExpRefInfo"],
force: bool,
) -> Mapping[SyncStatus, List["ExpRefInfo"]]:
) -> Mapping[SyncStatus, list["ExpRefInfo"]]:
from scmrepo.exceptions import AuthError
from dvc.scm import GitAuthError
@@ -171,7 +167,7 @@ def _push(
def group_result(refspec):
return results[str(refspec)]
pull_result: Mapping[SyncStatus, List["ExpRefInfo"]] = group_by(group_result, refs)
pull_result: Mapping[SyncStatus, list["ExpRefInfo"]] = group_by(group_result, refs)
return pull_result
+23 -29
View File
@@ -1,18 +1,12 @@
import os
from abc import ABC, abstractmethod
from collections.abc import Collection, Generator, Iterable, Mapping
from dataclasses import asdict, dataclass
from typing import (
TYPE_CHECKING,
Any,
Collection,
Dict,
Generator,
Iterable,
List,
Mapping,
NamedTuple,
Optional,
Type,
Union,
)
@@ -69,11 +63,11 @@ class QueueEntry:
and self.stash_rev == other.stash_rev
)
def asdict(self) -> Dict[str, Any]:
def asdict(self) -> dict[str, Any]:
return asdict(self)
@classmethod
def from_dict(cls, d: Dict[str, Any]) -> "QueueEntry":
def from_dict(cls, d: dict[str, Any]) -> "QueueEntry":
return cls(**d)
@@ -150,7 +144,7 @@ class BaseStashQueue(ABC):
all_: bool = False,
queued: bool = False,
**kwargs,
) -> List[str]:
) -> list[str]:
"""Remove the specified entries from the queue.
Arguments:
@@ -165,8 +159,8 @@ class BaseStashQueue(ABC):
if all_ or queued:
return self.clear()
name_to_remove: List[str] = []
entry_to_remove: List[ExpStashEntry] = []
name_to_remove: list[str] = []
entry_to_remove: list[ExpStashEntry] = []
queue_entries = self.match_queue_entry_by_name(revs, self.iter_queued())
for name, entry in queue_entries.items():
if entry:
@@ -176,7 +170,7 @@ class BaseStashQueue(ABC):
self.stash.remove_revs(entry_to_remove)
return name_to_remove
def clear(self, **kwargs) -> List[str]:
def clear(self, **kwargs) -> list[str]:
"""Remove all entries from the queue."""
stash_revs = self.stash.stash_revs
name_to_remove = list(stash_revs)
@@ -184,11 +178,11 @@ class BaseStashQueue(ABC):
return name_to_remove
def status(self) -> List[Dict[str, Any]]:
def status(self) -> list[dict[str, Any]]:
"""Show the status of exp tasks in queue"""
from datetime import datetime
result: List[Dict[str, Optional[str]]] = []
result: list[dict[str, Optional[str]]] = []
def _get_timestamp(rev: str) -> datetime:
commit = self.scm.resolve_commit(rev)
@@ -198,7 +192,7 @@ class BaseStashQueue(ABC):
entry: QueueEntry,
exp_result: Optional["ExecutorResult"] = None,
status: str = "Unknown",
) -> Dict[str, Any]:
) -> dict[str, Any]:
name = entry.name
if not name and exp_result and exp_result.ref_info:
name = exp_result.ref_info.name
@@ -251,7 +245,7 @@ class BaseStashQueue(ABC):
@abstractmethod
def reproduce(
self, copy_paths: Optional[List[str]] = None, message: Optional[str] = None
self, copy_paths: Optional[list[str]] = None, message: Optional[str] = None
) -> Mapping[str, Mapping[str, str]]:
"""Reproduce queued experiments sequentially."""
@@ -300,7 +294,7 @@ class BaseStashQueue(ABC):
def _stash_exp(
self,
*args,
params: Optional[Dict[str, List[str]]] = None,
params: Optional[dict[str, list[str]]] = None,
baseline_rev: Optional[str] = None,
branch: Optional[str] = None,
name: Optional[str] = None,
@@ -460,7 +454,7 @@ class BaseStashQueue(ABC):
f"from '{config_path}': {param_list}"
)
def _update_params(self, params: Dict[str, List[str]]):
def _update_params(self, params: dict[str, list[str]]):
"""Update param files with the provided `Hydra Override`_ patterns.
Args:
@@ -528,7 +522,7 @@ class BaseStashQueue(ABC):
cls,
exp: "Experiments",
queue_entry: QueueEntry,
executor_cls: Type[BaseExecutor] = WorkspaceExecutor,
executor_cls: type[BaseExecutor] = WorkspaceExecutor,
**kwargs,
) -> BaseExecutor:
stash_entry = cls.get_stash_entry(exp, queue_entry)
@@ -563,7 +557,7 @@ class BaseStashQueue(ABC):
exp: "Experiments",
executor: BaseExecutor,
exec_result: "ExecutorResult",
) -> Dict[str, str]:
) -> dict[str, str]:
results = {}
def on_diverged(ref: str):
@@ -593,7 +587,7 @@ class BaseStashQueue(ABC):
exp: "Experiments",
executor: BaseExecutor,
exec_result: "ExecutorResult",
) -> Dict[str, str]:
) -> dict[str, str]:
results = cls.collect_git(exp, executor, exec_result)
if exec_result.ref_info is not None:
@@ -605,11 +599,11 @@ class BaseStashQueue(ABC):
self,
exp_names: Collection[str],
*entries: Iterable[Union[QueueEntry, QueueDoneResult]],
) -> Dict[str, Optional[QueueEntry]]:
) -> dict[str, Optional[QueueEntry]]:
from funcy import concat
entry_name_dict: Dict[str, QueueEntry] = {}
entry_rev_dict: Dict[str, QueueEntry] = {}
entry_name_dict: dict[str, QueueEntry] = {}
entry_rev_dict: dict[str, QueueEntry] = {}
for entry in concat(*entries):
if isinstance(entry, QueueDoneResult):
queue_entry: QueueEntry = entry.entry
@@ -624,7 +618,7 @@ class BaseStashQueue(ABC):
entry_name_dict[name] = queue_entry
entry_rev_dict[queue_entry.stash_rev] = queue_entry
result: Dict[str, Optional[QueueEntry]] = {}
result: dict[str, Optional[QueueEntry]] = {}
for exp_name in exp_names:
result[exp_name] = None
if exp_name in entry_name_dict:
@@ -666,7 +660,7 @@ class BaseStashQueue(ABC):
baseline_revs: Optional[Collection[str]],
fetch_refs: bool = False,
**kwargs,
) -> Dict[str, List["ExpRange"]]:
) -> dict[str, list["ExpRange"]]:
"""Collect data for active (running) experiments.
Args:
@@ -685,7 +679,7 @@ class BaseStashQueue(ABC):
self,
baseline_revs: Optional[Collection[str]],
**kwargs,
) -> Dict[str, List["ExpRange"]]:
) -> dict[str, list["ExpRange"]]:
"""Collect data for queued experiments.
Args:
@@ -702,7 +696,7 @@ class BaseStashQueue(ABC):
self,
baseline_revs: Optional[Collection[str]],
**kwargs,
) -> Dict[str, List["ExpRange"]]:
) -> dict[str, list["ExpRange"]]:
"""Collect data for failed experiments.
Args:
+31 -36
View File
@@ -3,16 +3,11 @@ import locale
import logging
import os
from collections import defaultdict
from collections.abc import Collection, Generator, Mapping
from typing import (
TYPE_CHECKING,
Collection,
Dict,
Generator,
List,
Mapping,
NamedTuple,
Optional,
Set,
Union,
)
@@ -166,7 +161,7 @@ class LocalCeleryQueue(BaseStashQueue):
"""
logger.debug("Spawning %s exp queue workers", count)
active_worker: Dict = self.worker_status()
active_worker: dict = self.worker_status()
started = 0
for num in range(1, 1 + count):
@@ -183,7 +178,7 @@ class LocalCeleryQueue(BaseStashQueue):
def put(
self,
*args,
copy_paths: Optional[List[str]] = None,
copy_paths: Optional[list[str]] = None,
message: Optional[str] = None,
**kwargs,
) -> QueueEntry:
@@ -265,7 +260,7 @@ class LocalCeleryQueue(BaseStashQueue):
yield QueueDoneResult(queue_entry, exp_result)
def reproduce(
self, copy_paths: Optional[List[str]] = None, message: Optional[str] = None
self, copy_paths: Optional[list[str]] = None, message: Optional[str] = None
) -> Mapping[str, Mapping[str, str]]:
raise NotImplementedError
@@ -333,8 +328,8 @@ class LocalCeleryQueue(BaseStashQueue):
while not self.proc.get(entry.stash_rev):
time.sleep(sleep_interval)
def _get_running_task_ids(self) -> Set[str]:
running_task_ids: Set[str] = set()
def _get_running_task_ids(self) -> set[str]:
running_task_ids: set[str] = set()
active_workers = self.worker_status()
for _, tasks in active_workers.items():
task = first(tasks)
@@ -343,9 +338,9 @@ class LocalCeleryQueue(BaseStashQueue):
return running_task_ids
def _try_to_kill_tasks(
self, to_kill: Dict[QueueEntry, str], force: bool
) -> Dict[QueueEntry, str]:
fail_to_kill_entries: Dict[QueueEntry, str] = {}
self, to_kill: dict[QueueEntry, str], force: bool
) -> dict[QueueEntry, str]:
fail_to_kill_entries: dict[QueueEntry, str] = {}
for queue_entry, rev in to_kill.items():
try:
if force:
@@ -358,9 +353,9 @@ class LocalCeleryQueue(BaseStashQueue):
return fail_to_kill_entries
def _mark_inactive_tasks_failure(
self, remained_entries: Dict[QueueEntry, str]
self, remained_entries: dict[QueueEntry, str]
) -> None:
remained_revs: List[str] = []
remained_revs: list[str] = []
running_ids = self._get_running_task_ids()
logger.debug("Current running tasks ids: %s.", running_ids)
for msg, entry in self._iter_processed():
@@ -383,12 +378,12 @@ class LocalCeleryQueue(BaseStashQueue):
if remained_revs:
raise CannotKillTasksError(remained_revs)
def _kill_entries(self, entries: Dict[QueueEntry, str], force: bool) -> None:
def _kill_entries(self, entries: dict[QueueEntry, str], force: bool) -> None:
logger.debug(
"Found active tasks: '%s' to kill",
list(entries.values()),
)
inactive_entries: Dict[QueueEntry, str] = self._try_to_kill_tasks(
inactive_entries: dict[QueueEntry, str] = self._try_to_kill_tasks(
entries, force
)
@@ -396,12 +391,12 @@ class LocalCeleryQueue(BaseStashQueue):
self._mark_inactive_tasks_failure(inactive_entries)
def kill(self, revs: Collection[str], force: bool = False) -> None:
name_dict: Dict[str, Optional[QueueEntry]] = self.match_queue_entry_by_name(
name_dict: dict[str, Optional[QueueEntry]] = self.match_queue_entry_by_name(
set(revs), self.iter_active()
)
missing_revs: List[str] = []
to_kill: Dict[QueueEntry, str] = {}
missing_revs: list[str] = []
to_kill: dict[QueueEntry, str] = {}
for rev, queue_entry in name_dict.items():
if queue_entry is None:
missing_revs.append(rev)
@@ -417,7 +412,7 @@ class LocalCeleryQueue(BaseStashQueue):
def shutdown(self, kill: bool = False):
self.celery.control.shutdown()
if kill:
to_kill: Dict[QueueEntry, str] = {}
to_kill: dict[QueueEntry, str] = {}
for entry in self.iter_active():
to_kill[entry] = entry.name or entry.stash_rev
if to_kill:
@@ -468,7 +463,7 @@ class LocalCeleryQueue(BaseStashQueue):
) as fobj:
ui.write(fobj.read())
def worker_status(self) -> Dict[str, List[Dict]]:
def worker_status(self) -> dict[str, list[dict]]:
"""Return the current active celery worker"""
status = self.celery.control.inspect().active() or {}
logger.debug("Worker status: %s", status)
@@ -486,21 +481,21 @@ class LocalCeleryQueue(BaseStashQueue):
def get_ref_and_entry_by_names(
self,
exp_names: Union[str, List[str]],
exp_names: Union[str, list[str]],
git_remote: Optional[str] = None,
) -> Dict[str, ExpRefAndQueueEntry]:
) -> dict[str, ExpRefAndQueueEntry]:
"""Find finished ExpRefInfo or queued or failed QueueEntry by name"""
from dvc.repo.experiments.utils import resolve_name
if isinstance(exp_names, str):
exp_names = [exp_names]
results: Dict[str, ExpRefAndQueueEntry] = {}
results: dict[str, ExpRefAndQueueEntry] = {}
exp_ref_match: Dict[str, Optional["ExpRefInfo"]] = resolve_name(
exp_ref_match: dict[str, Optional["ExpRefInfo"]] = resolve_name(
self.scm, exp_names, git_remote
)
if not git_remote:
queue_entry_match: Dict[
queue_entry_match: dict[
str, Optional["QueueEntry"]
] = self.match_queue_entry_by_name(
exp_names, self.iter_queued(), self.iter_done()
@@ -517,7 +512,7 @@ class LocalCeleryQueue(BaseStashQueue):
baseline_revs: Optional[Collection[str]],
fetch_refs: bool = False,
**kwargs,
) -> Dict[str, List["ExpRange"]]:
) -> dict[str, list["ExpRange"]]:
from dvc.repo import Repo
from dvc.repo.experiments.collect import collect_exec_branch
from dvc.repo.experiments.serialize import (
@@ -526,7 +521,7 @@ class LocalCeleryQueue(BaseStashQueue):
LocalExpExecutor,
)
result: Dict[str, List[ExpRange]] = defaultdict(list)
result: dict[str, list[ExpRange]] = defaultdict(list)
for entry in self.iter_active():
if baseline_revs and entry.baseline_rev not in baseline_revs:
continue
@@ -568,7 +563,7 @@ class LocalCeleryQueue(BaseStashQueue):
self,
baseline_revs: Optional[Collection[str]],
**kwargs,
) -> Dict[str, List["ExpRange"]]:
) -> dict[str, list["ExpRange"]]:
from dvc.repo.experiments.collect import collect_rev
from dvc.repo.experiments.serialize import (
ExpExecutor,
@@ -576,7 +571,7 @@ class LocalCeleryQueue(BaseStashQueue):
LocalExpExecutor,
)
result: Dict[str, List[ExpRange]] = defaultdict(list)
result: dict[str, list[ExpRange]] = defaultdict(list)
for entry in self.iter_queued():
if baseline_revs and entry.baseline_rev not in baseline_revs:
continue
@@ -598,7 +593,7 @@ class LocalCeleryQueue(BaseStashQueue):
self,
baseline_revs: Optional[Collection[str]],
**kwargs,
) -> Dict[str, List["ExpRange"]]:
) -> dict[str, list["ExpRange"]]:
from dvc.repo.experiments.collect import collect_rev
from dvc.repo.experiments.serialize import (
ExpExecutor,
@@ -607,7 +602,7 @@ class LocalCeleryQueue(BaseStashQueue):
SerializableError,
)
result: Dict[str, List[ExpRange]] = defaultdict(list)
result: dict[str, list[ExpRange]] = defaultdict(list)
for entry, _ in self.iter_failed():
if baseline_revs and entry.baseline_rev not in baseline_revs:
continue
@@ -637,11 +632,11 @@ class LocalCeleryQueue(BaseStashQueue):
self,
baseline_revs: Optional[Collection[str]],
**kwargs,
) -> Dict[str, "ExpExecutor"]:
) -> dict[str, "ExpExecutor"]:
"""Map exp refs to any available successful executors."""
from dvc.repo.experiments.serialize import ExpExecutor, LocalExpExecutor
result: Dict[str, "ExpExecutor"] = {}
result: dict[str, "ExpExecutor"] = {}
for entry, exec_result in self.iter_success():
if baseline_revs and entry.baseline_rev not in baseline_revs:
continue
+1 -1
View File
@@ -1,4 +1,4 @@
from typing import Collection
from collections.abc import Collection
from dvc.exceptions import DvcException
+17 -16
View File
@@ -1,4 +1,5 @@
from typing import TYPE_CHECKING, Collection, Dict, Iterable, List, Set, Union
from collections.abc import Collection, Iterable
from typing import TYPE_CHECKING, Union
from dvc.repo.experiments.exceptions import UnresolvedExpNamesError
from dvc.repo.experiments.queue.base import QueueDoneResult
@@ -20,11 +21,11 @@ def remove_tasks( # noqa: C901, PLR0912
"""
from celery.result import AsyncResult
stash_revs: Dict[str, "ExpStashEntry"] = {}
failed_stash_revs: List["ExpStashEntry"] = []
done_entry_set: Set["QueueEntry"] = set()
stash_revs: dict[str, "ExpStashEntry"] = {}
failed_stash_revs: list["ExpStashEntry"] = []
done_entry_set: set["QueueEntry"] = set()
stash_rev_all = celery_queue.stash.stash_revs
failed_rev_all: Dict[str, "ExpStashEntry"] = {}
failed_rev_all: dict[str, "ExpStashEntry"] = {}
if celery_queue.failed_stash:
failed_rev_all = celery_queue.failed_stash.stash_revs
for entry in queue_entries:
@@ -64,7 +65,7 @@ def remove_tasks( # noqa: C901, PLR0912
def _get_names(entries: Iterable[Union["QueueEntry", "QueueDoneResult"]]):
names: List[str] = []
names: list[str] = []
for entry in entries:
if isinstance(entry, QueueDoneResult):
if entry.result and entry.result.ref_info:
@@ -82,7 +83,7 @@ def celery_clear(
queued: bool = False,
failed: bool = False,
success: bool = False,
) -> List[str]:
) -> list[str]:
"""Remove entries from the queue.
Arguments:
@@ -94,18 +95,18 @@ def celery_clear(
Revisions which were removed.
"""
removed: List[str] = []
entry_list: List["QueueEntry"] = []
removed: list[str] = []
entry_list: list["QueueEntry"] = []
if queued:
queue_entries: List["QueueEntry"] = list(self.iter_queued())
queue_entries: list["QueueEntry"] = list(self.iter_queued())
entry_list.extend(queue_entries)
removed.extend(_get_names(queue_entries))
if failed:
failed_tasks: List["QueueDoneResult"] = list(self.iter_failed())
failed_tasks: list["QueueDoneResult"] = list(self.iter_failed())
entry_list.extend([result.entry for result in failed_tasks])
removed.extend(_get_names(failed_tasks))
if success:
success_tasks: List["QueueDoneResult"] = list(self.iter_success())
success_tasks: list["QueueDoneResult"] = list(self.iter_success())
entry_list.extend([result.entry for result in success_tasks])
removed.extend(_get_names(success_tasks))
@@ -117,7 +118,7 @@ def celery_clear(
def celery_remove(
self: "LocalCeleryQueue",
revs: Collection[str],
) -> List[str]:
) -> list[str]:
"""Remove the specified entries from the queue.
Arguments:
@@ -131,9 +132,9 @@ def celery_remove(
revs, self.iter_queued(), self.iter_done()
)
remained: List[str] = []
removed: List[str] = []
entry_to_remove: List["QueueEntry"] = []
remained: list[str] = []
removed: list[str] = []
entry_to_remove: list["QueueEntry"] = []
for name, entry in match_results.items():
if entry:
entry_to_remove.append(entry)
+6 -6
View File
@@ -1,4 +1,4 @@
from typing import TYPE_CHECKING, Any, Dict, List, Optional
from typing import TYPE_CHECKING, Any, Optional
from celery import shared_task
from celery.utils.log import get_task_logger
@@ -16,7 +16,7 @@ logger = get_task_logger(__name__)
@shared_task
def setup_exp(entry_dict: Dict[str, Any]) -> "BaseExecutor":
def setup_exp(entry_dict: dict[str, Any]) -> "BaseExecutor":
"""Setup an experiment.
Arguments:
@@ -44,8 +44,8 @@ def setup_exp(entry_dict: Dict[str, Any]) -> "BaseExecutor":
@shared_task
def collect_exp(
proc_dict: Dict[str, Any], # noqa: ARG001
entry_dict: Dict[str, Any],
proc_dict: dict[str, Any], # noqa: ARG001
entry_dict: dict[str, Any],
) -> str:
"""Collect results for an experiment.
@@ -92,8 +92,8 @@ def cleanup_exp(executor: TempDirExecutor, infofile: str) -> None:
@shared_task
def run_exp(
entry_dict: Dict[str, Any],
copy_paths: Optional[List[str]] = None,
entry_dict: dict[str, Any],
copy_paths: Optional[list[str]] = None,
message: Optional[str] = None,
) -> None:
"""Run a full experiment.
+8 -7
View File
@@ -1,6 +1,7 @@
import os
from collections import defaultdict
from typing import TYPE_CHECKING, Collection, Dict, Generator, List, Optional
from collections.abc import Collection, Generator
from typing import TYPE_CHECKING, Optional
from funcy import first
@@ -96,11 +97,11 @@ class TempDirQueue(WorkspaceQueue):
self,
entry: QueueEntry,
executor: "BaseExecutor",
copy_paths: Optional[List[str]] = None,
copy_paths: Optional[list[str]] = None,
message: Optional[str] = None,
**kwargs,
) -> Dict[str, Dict[str, str]]:
results: Dict[str, Dict[str, str]] = defaultdict(dict)
) -> dict[str, dict[str, str]]:
results: dict[str, dict[str, str]] = defaultdict(dict)
exec_name = self._EXEC_NAME or entry.stash_rev
infofile = self.get_infofile_path(exec_name)
try:
@@ -135,7 +136,7 @@ class TempDirQueue(WorkspaceQueue):
exp: "Experiments",
executor: "BaseExecutor",
exec_result: "ExecutorResult",
) -> Dict[str, str]:
) -> dict[str, str]:
return BaseStashQueue.collect_executor(exp, executor, exec_result)
def collect_active_data(
@@ -143,7 +144,7 @@ class TempDirQueue(WorkspaceQueue):
baseline_revs: Optional[Collection[str]],
fetch_refs: bool = False,
**kwargs,
) -> Dict[str, List["ExpRange"]]:
) -> dict[str, list["ExpRange"]]:
from dvc.repo import Repo
from dvc.repo.experiments.collect import collect_exec_branch
from dvc.repo.experiments.serialize import (
@@ -152,7 +153,7 @@ class TempDirQueue(WorkspaceQueue):
LocalExpExecutor,
)
result: Dict[str, List[ExpRange]] = defaultdict(list)
result: dict[str, list[ExpRange]] = defaultdict(list)
for entry in self.iter_active():
if baseline_revs and entry.baseline_rev not in baseline_revs:
continue
+4 -4
View File
@@ -1,4 +1,4 @@
from typing import TYPE_CHECKING, Dict, List
from typing import TYPE_CHECKING
from scmrepo.exceptions import SCMError
@@ -16,7 +16,7 @@ if TYPE_CHECKING:
from .base import BaseStashQueue
def get_remote_executor_refs(scm: "Git", remote_url: str) -> List[str]:
def get_remote_executor_refs(scm: "Git", remote_url: str) -> list[str]:
"""Get result list refs from a remote repository
Args:
@@ -35,7 +35,7 @@ def get_remote_executor_refs(scm: "Git", remote_url: str) -> List[str]:
def fetch_running_exp_from_temp_dir(
queue: "BaseStashQueue", rev: str, fetch_refs: bool
) -> Dict[str, Dict]:
) -> dict[str, dict]:
"""Fetch status of running exps out of current working directory
Args:
@@ -50,7 +50,7 @@ def fetch_running_exp_from_temp_dir(
from dvc.scm import InvalidRemoteSCMRepo
from dvc.utils.serialize import load_json
result: Dict[str, Dict] = {}
result: dict[str, dict] = {}
infofile = queue.get_infofile_path(rev)
try:
info = ExecutorInfo.from_dict(load_json(infofile))
+13 -12
View File
@@ -1,7 +1,8 @@
import json
import os
from collections import defaultdict
from typing import TYPE_CHECKING, Collection, Dict, Generator, List, Optional
from collections.abc import Collection, Generator
from typing import TYPE_CHECKING, Optional
import psutil
from funcy import first
@@ -82,9 +83,9 @@ class WorkspaceQueue(BaseStashQueue):
raise NotImplementedError
def reproduce(
self, copy_paths: Optional[List[str]] = None, message: Optional[str] = None
) -> Dict[str, Dict[str, str]]:
results: Dict[str, Dict[str, str]] = defaultdict(dict)
self, copy_paths: Optional[list[str]] = None, message: Optional[str] = None
) -> dict[str, dict[str, str]]:
results: dict[str, dict[str, str]] = defaultdict(dict)
try:
while True:
entry, executor = self.get()
@@ -99,11 +100,11 @@ class WorkspaceQueue(BaseStashQueue):
def _reproduce_entry(
self, entry: QueueEntry, executor: "BaseExecutor", **kwargs
) -> Dict[str, Dict[str, str]]:
) -> dict[str, dict[str, str]]:
kwargs.pop("copy_paths", None)
from dvc_task.proc.process import ProcessInfo
results: Dict[str, Dict[str, str]] = defaultdict(dict)
results: dict[str, dict[str, str]] = defaultdict(dict)
exec_name = self._EXEC_NAME or entry.stash_rev
proc_info = ProcessInfo(os.getpid(), None, None, None, None)
proc_info_path = self._proc_info_path(exec_name)
@@ -162,8 +163,8 @@ class WorkspaceQueue(BaseStashQueue):
exp: "Experiments",
executor: "BaseExecutor", # noqa: ARG004
exec_result: "ExecutorResult",
) -> Dict[str, str]:
results: Dict[str, str] = {}
) -> dict[str, str]:
results: dict[str, str] = {}
exp_rev = exp.scm.get_ref(EXEC_BRANCH)
if exp_rev:
assert exec_result.exp_hash
@@ -207,7 +208,7 @@ class WorkspaceQueue(BaseStashQueue):
baseline_revs: Optional[Collection[str]],
fetch_refs: bool = False, # noqa: ARG002
**kwargs,
) -> Dict[str, List["ExpRange"]]:
) -> dict[str, list["ExpRange"]]:
from dvc.repo.experiments.collect import collect_exec_branch
from dvc.repo.experiments.serialize import (
ExpExecutor,
@@ -215,7 +216,7 @@ class WorkspaceQueue(BaseStashQueue):
LocalExpExecutor,
)
result: Dict[str, List[ExpRange]] = defaultdict(list)
result: dict[str, list[ExpRange]] = defaultdict(list)
pid = self._active_pid
if pid is None:
return result
@@ -248,12 +249,12 @@ class WorkspaceQueue(BaseStashQueue):
self,
baseline_revs: Optional[Collection[str]],
**kwargs,
) -> Dict[str, List["ExpRange"]]:
) -> dict[str, list["ExpRange"]]:
raise NotImplementedError
def collect_failed_data(
self,
baseline_revs: Optional[Collection[str]],
**kwargs,
) -> Dict[str, List["ExpRange"]]:
) -> dict[str, list["ExpRange"]]:
raise NotImplementedError
+14 -13
View File
@@ -1,4 +1,5 @@
from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Union
from collections.abc import Iterable
from typing import TYPE_CHECKING, Optional, Union
from dvc.log import logger
from dvc.repo import locked
@@ -23,14 +24,14 @@ logger = logger.getChild(__name__)
@scm_context
def remove( # noqa: C901, PLR0912
repo: "Repo",
exp_names: Union[None, str, List[str]] = None,
rev: Optional[Union[List[str], str]] = None,
exp_names: Union[None, str, list[str]] = None,
rev: Optional[Union[list[str], str]] = None,
all_commits: bool = False,
num: int = 1,
queue: bool = False,
git_remote: Optional[str] = None,
) -> List[str]:
removed: List[str] = []
) -> list[str]:
removed: list[str] = []
if not any([exp_names, queue, all_commits, rev]):
return removed
celery_queue: "LocalCeleryQueue" = repo.experiments.celery_queue
@@ -40,13 +41,13 @@ def remove( # noqa: C901, PLR0912
assert isinstance(repo.scm, Git)
exp_ref_list: List["ExpRefInfo"] = []
queue_entry_list: List["QueueEntry"] = []
exp_ref_list: list["ExpRefInfo"] = []
queue_entry_list: list["QueueEntry"] = []
if exp_names:
results: Dict[
results: dict[
str, "ExpRefAndQueueEntry"
] = celery_queue.get_ref_and_entry_by_names(exp_names, git_remote)
remained: List[str] = []
remained: list[str] = []
for name, result in results.items():
if not result.exp_ref_info and not result.queue_entry:
remained.append(name)
@@ -87,13 +88,13 @@ def remove( # noqa: C901, PLR0912
def _resolve_exp_by_baseline(
repo: "Repo",
rev: List[str],
rev: list[str],
num: int,
git_remote: Optional[str] = None,
) -> Dict[str, "ExpRefInfo"]:
) -> dict[str, "ExpRefInfo"]:
assert isinstance(repo.scm, Git)
commit_ref_dict: Dict[str, "ExpRefInfo"] = {}
commit_ref_dict: dict[str, "ExpRefInfo"] = {}
rev_dict = iter_revs(repo.scm, rev, num)
rev_set = set(rev_dict.keys())
ref_info_dict = exp_refs_by_baseline(repo.scm, rev_set, git_remote)
@@ -105,7 +106,7 @@ def _resolve_exp_by_baseline(
def _remove_commited_exps(
scm: "Git", exp_refs_list: Iterable["ExpRefInfo"], remote: Optional[str]
) -> List[str]:
) -> list[str]:
if remote:
from dvc.scm import TqdmGit
+5 -5
View File
@@ -1,4 +1,4 @@
from typing import TYPE_CHECKING, Dict, List, Optional, Union
from typing import TYPE_CHECKING, Optional, Union
from dvc.log import logger
from dvc.repo.experiments.exceptions import (
@@ -22,16 +22,16 @@ def rename(
exp_name: Union[str, None] = None,
git_remote: Optional[str] = None,
force: bool = False,
) -> Union[List[str], None]:
renamed: List[str] = []
remained: List[str] = []
) -> Union[list[str], None]:
renamed: list[str] = []
remained: list[str] = []
assert isinstance(repo.scm, Git)
if exp_name == new_name:
return None
if exp_name:
results: Dict[str, Union[ExpRefInfo, None]] = resolve_name(
results: dict[str, Union[ExpRefInfo, None]] = resolve_name(
scm=repo.scm, exp_names=exp_name, git_remote=git_remote
)
for name, result in results.items():
+3 -2
View File
@@ -1,4 +1,5 @@
from typing import Dict, Iterable, Optional
from collections.abc import Iterable
from typing import Optional
from dvc.dependency.param import ParamsDependency
from dvc.exceptions import InvalidArgumentError
@@ -22,7 +23,7 @@ def run( # noqa: C901, PLR0912
copy_paths: Optional[Iterable[str]] = None,
message: Optional[str] = None,
**kwargs,
) -> Dict[str, str]:
) -> dict[str, str]:
"""Reproduce the specified targets as an experiment.
Accepts the same additional kwargs as Repo.reproduce.
+3 -2
View File
@@ -1,5 +1,6 @@
import os
from typing import TYPE_CHECKING, Iterable, List, Optional
from collections.abc import Iterable
from typing import TYPE_CHECKING, Optional
from funcy import first
@@ -18,7 +19,7 @@ def save(
name: Optional[str] = None,
recursive: bool = False,
force: bool = False,
include_untracked: Optional[List[str]] = None,
include_untracked: Optional[list[str]] = None,
message: Optional[str] = None,
) -> Optional[str]:
"""Save the current workspace status as an experiment.
+13 -12
View File
@@ -1,7 +1,8 @@
import json
from collections.abc import Iterator
from dataclasses import asdict, dataclass, field
from datetime import datetime
from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Literal, Optional
from typing import TYPE_CHECKING, Any, Literal, Optional
from dvc.exceptions import DvcException
from dvc.repo.metrics.show import _gather_metrics
@@ -30,11 +31,11 @@ class SerializableExp:
rev: str
timestamp: Optional[datetime] = None
params: Dict[str, "FileResult"] = field(default_factory=dict)
metrics: Dict[str, "FileResult"] = field(default_factory=dict)
deps: Dict[str, "ExpDep"] = field(default_factory=dict)
outs: Dict[str, "ExpOut"] = field(default_factory=dict)
meta: Dict[str, Any] = field(default_factory=dict)
params: dict[str, "FileResult"] = field(default_factory=dict)
metrics: dict[str, "FileResult"] = field(default_factory=dict)
deps: dict[str, "ExpDep"] = field(default_factory=dict)
outs: dict[str, "ExpOut"] = field(default_factory=dict)
meta: dict[str, Any] = field(default_factory=dict)
@classmethod
def from_repo(
@@ -83,7 +84,7 @@ class SerializableExp:
**kwargs,
)
def dumpd(self) -> Dict[str, Any]:
def dumpd(self) -> dict[str, Any]:
return asdict(self)
def as_bytes(self) -> bytes:
@@ -131,7 +132,7 @@ class SerializableError:
msg: str
type: str = ""
def dumpd(self) -> Dict[str, Any]:
def dumpd(self) -> dict[str, Any]:
return asdict(self)
def as_bytes(self) -> bytes:
@@ -154,15 +155,15 @@ class ExpState:
name: Optional[str] = None
data: Optional[SerializableExp] = None
error: Optional[SerializableError] = None
experiments: Optional[List["ExpRange"]] = None
experiments: Optional[list["ExpRange"]] = None
def dumpd(self) -> Dict[str, Any]:
def dumpd(self) -> dict[str, Any]:
return asdict(self)
@dataclass
class ExpRange:
revs: List["ExpState"]
revs: list["ExpState"]
executor: Optional["ExpExecutor"] = None
name: Optional[str] = None
@@ -175,7 +176,7 @@ class ExpRange:
def __getitem__(self, index: int) -> "ExpState":
return self.revs[index]
def dumpd(self) -> Dict[str, Any]:
def dumpd(self) -> dict[str, Any]:
return asdict(self)
+20 -26
View File
@@ -1,18 +1,12 @@
from collections import Counter, defaultdict
from collections.abc import Iterable, Iterator, Mapping
from datetime import date, datetime
from typing import (
TYPE_CHECKING,
Any,
Dict,
Iterable,
Iterator,
List,
Literal,
Mapping,
NamedTuple,
Optional,
Set,
Tuple,
Union,
)
@@ -36,7 +30,7 @@ logger = logger.getChild(__name__)
def show(
repo: "Repo",
revs: Union[List[str], str, None] = None,
revs: Union[list[str], str, None] = None,
all_branches: bool = False,
all_tags: bool = False,
all_commits: bool = False,
@@ -45,7 +39,7 @@ def show(
hide_failed: bool = False,
sha_only: bool = False,
**kwargs,
) -> List["ExpState"]:
) -> list["ExpState"]:
return collect(
repo,
revs=revs,
@@ -65,7 +59,7 @@ def tabulate(
fill_value: Optional[str] = "-",
error_value: str = "!",
**kwargs,
) -> Tuple["TabularData", Dict[str, Iterable[str]]]:
) -> tuple["TabularData", dict[str, Iterable[str]]]:
"""Return table data for experiments.
Returns:
@@ -112,7 +106,7 @@ def tabulate(
**kwargs,
)
)
data_headers: Dict[str, Iterable[str]] = {
data_headers: dict[str, Iterable[str]] = {
"metrics": metrics_headers,
"params": params_headers,
"deps": deps_names,
@@ -128,9 +122,9 @@ def _build_rows(
sort_by: Optional[str] = None,
sort_order: Optional[Literal["asc", "desc"]] = None,
**kwargs,
) -> Iterator[Tuple["CellT", ...]]:
) -> Iterator[tuple["CellT", ...]]:
for baseline in baseline_states:
row: Dict[str, "CellT"] = dict.fromkeys(all_headers, fill_value)
row: dict[str, "CellT"] = dict.fromkeys(all_headers, fill_value)
row["Experiment"] = ""
if baseline.name:
row["rev"] = baseline.name
@@ -177,10 +171,10 @@ def _sort_column( # noqa: C901
sort_by: str,
metric_names: Mapping[str, Iterable[str]],
param_names: Mapping[str, Iterable[str]],
) -> Tuple[str, str, str]:
) -> tuple[str, str, str]:
sep = ":"
parts = sort_by.split(sep)
matches: Set[Tuple[str, str, str]] = set()
matches: set[tuple[str, str, str]] = set()
for split_num in range(len(parts)):
path = sep.join(parts[:split_num])
@@ -217,7 +211,7 @@ def _sort_exp(
sort_name: str,
typ: str,
reverse: bool,
) -> List["ExpRange"]:
) -> list["ExpRange"]:
from funcy import first
def _sort(exp_range: "ExpRange"):
@@ -238,14 +232,14 @@ def _exp_range_rows(
fill_value: Optional[str],
is_base: bool = False,
**kwargs,
) -> Iterator[Tuple["CellT", ...]]:
) -> Iterator[tuple["CellT", ...]]:
from funcy import first
if len(exp_range.revs) > 1:
logger.debug("Returning tip commit for legacy checkpoint exp")
exp = first(exp_range.revs)
if exp:
row: Dict[str, "CellT"] = dict.fromkeys(all_headers, fill_value)
row: dict[str, "CellT"] = dict.fromkeys(all_headers, fill_value)
row["Experiment"] = exp.name or ""
row["rev"] = exp.rev[:7] if Git.is_sha(exp.rev) else exp.rev
row["typ"] = "branch_base" if is_base else "branch_commit"
@@ -274,12 +268,12 @@ def _data_cells(
error_value: str = "!",
precision: Optional[int] = None,
**kwargs,
) -> Iterator[Tuple[str, "CellT"]]:
) -> Iterator[tuple[str, "CellT"]]:
def _d_cells(
d: Mapping[str, Any],
names: Mapping[str, Iterable[str]],
headers: Iterable[str],
) -> Iterator[Tuple[str, "CellT"]]:
) -> Iterator[tuple[str, "CellT"]]:
from dvc.compare import _format_field, with_value
for fname, data in d.items():
@@ -329,9 +323,9 @@ def format_time(
class _DataNames(NamedTuple):
# NOTE: we use nested dict instead of set for metrics/params names to
# preserve key ordering
metrics: Dict[str, Dict[str, Any]]
params: Dict[str, Dict[str, Any]]
deps: Set[str]
metrics: dict[str, dict[str, Any]]
params: dict[str, dict[str, Any]]
deps: set[str]
@property
def sorted_deps(self):
@@ -339,7 +333,7 @@ class _DataNames(NamedTuple):
def update(self, other: "_DataNames"):
def _update_d(
d: Dict[str, Dict[str, Any]], other_d: Mapping[str, Mapping[str, Any]]
d: dict[str, dict[str, Any]], other_d: Mapping[str, Mapping[str, Any]]
):
for k, v in other_d.items():
if k in d:
@@ -355,7 +349,7 @@ class _DataNames(NamedTuple):
def _collect_names(exp_states: Iterable["ExpState"]) -> _DataNames:
result = _DataNames(defaultdict(dict), defaultdict(dict), set())
def _collect_d(result_d: Dict[str, Dict[str, Any]], data_d: Dict[str, Any]):
def _collect_d(result_d: dict[str, dict[str, Any]], data_d: dict[str, Any]):
for path, item in data_d.items():
item = item.get("data", {})
if isinstance(item, dict):
@@ -376,7 +370,7 @@ def _collect_names(exp_states: Iterable["ExpState"]) -> _DataNames:
def _normalize_headers(
names: Mapping[str, Mapping[str, Any]], count: Mapping[str, int]
) -> List[str]:
) -> list[str]:
return [
name if count[name] == 1 else f"{path}:{name}"
for path in names
+4 -3
View File
@@ -1,6 +1,7 @@
import re
from collections.abc import Iterable, Iterator
from contextlib import contextmanager
from typing import Dict, Iterable, Iterator, NamedTuple, Optional
from typing import NamedTuple, Optional
from scmrepo.git import Stash
@@ -42,7 +43,7 @@ class ExpStash(Stash):
)
@property
def stash_revs(self) -> Dict[str, ExpStashEntry]:
def stash_revs(self) -> dict[str, ExpStashEntry]:
revs = {}
for i, entry in enumerate(self):
msg = entry.message.decode("utf-8").strip()
@@ -110,7 +111,7 @@ class ApplyStash(Stash):
)
@property
def stash_revs(self) -> Dict[str, ApplyStashEntry]:
def stash_revs(self) -> dict[str, ApplyStashEntry]:
revs = {}
for i, entry in enumerate(self):
msg = entry.message.decode("utf-8").strip()
+13 -19
View File
@@ -2,18 +2,12 @@ import os
import random
import sys
from collections import defaultdict
from collections.abc import Generator, Iterable, Mapping
from functools import wraps
from typing import (
TYPE_CHECKING,
Callable,
Dict,
Generator,
Iterable,
List,
Mapping,
Optional,
Set,
Tuple,
Union,
)
@@ -43,8 +37,8 @@ EXEC_PID_DIR = "run"
def get_exp_rwlock(
repo: "Repo",
reads: Optional[List[str]] = None,
writes: Optional[List[str]] = None,
reads: Optional[list[str]] = None,
writes: Optional[list[str]] = None,
):
reads = reads or []
writes = writes or []
@@ -109,9 +103,9 @@ def exp_refs_by_rev(scm: "Git", rev: str) -> Generator[ExpRefInfo, None, None]:
def exp_refs_by_baseline(
scm: "Git",
revs: Optional[Set[str]] = None,
revs: Optional[set[str]] = None,
url: Optional[str] = None,
) -> Mapping[str, List[ExpRefInfo]]:
) -> Mapping[str, list[ExpRefInfo]]:
"""Iterate over all experiment refs with the specified baseline."""
all_exp_refs = exp_refs(scm, url)
result = defaultdict(list)
@@ -137,7 +131,7 @@ def iter_remote_refs(scm: "Git", url: str, base: Optional[str] = None, **kwargs)
def push_refspec(
scm: "Git",
url: str,
push_list=List[Tuple[Optional[str], str]],
push_list=list[tuple[Optional[str], str]],
force: bool = False,
on_diverged: Optional[Callable[[str, str], bool]] = None,
**kwargs,
@@ -183,8 +177,8 @@ def remote_exp_refs(scm: "Git", url: str) -> Generator[ExpRefInfo, None, None]:
def exp_refs_by_names(
scm: "Git", names: Set[str], url: Optional[str] = None
) -> Dict[str, List[ExpRefInfo]]:
scm: "Git", names: set[str], url: Optional[str] = None
) -> dict[str, list[ExpRefInfo]]:
"""Iterate over all experiment refs matching the specified names."""
resolve_results = defaultdict(list)
ref_info_gen = exp_refs(scm, url)
@@ -210,7 +204,7 @@ def exp_commits(
scm: "Git", ref_infos: Optional[Iterable[ExpRefInfo]] = None
) -> Iterable[str]:
"""Iterate over all experiment commits."""
shas: Set["str"] = set()
shas: set["str"] = set()
refs = ref_infos if ref_infos else exp_refs(scm)
for ref_info in refs:
shas.update(scm.branch_revs(str(ref_info), ref_info.baseline_sha))
@@ -242,7 +236,7 @@ def resolve_name(
scm: "Git",
exp_names: Union[Iterable[str], str],
git_remote: Optional[str] = None,
) -> Dict[str, Optional[ExpRefInfo]]:
) -> dict[str, Optional[ExpRefInfo]]:
"""find the ref_info of specified names."""
if isinstance(exp_names, str):
exp_names = [exp_names]
@@ -338,7 +332,7 @@ def to_studio_params(dvc_params):
"params.yaml": {"foo": 1}
}
"""
result: Dict = {}
result: dict = {}
if not dvc_params:
return result
for rev_data in dvc_params.values():
@@ -353,7 +347,7 @@ def describe(
revs: Iterable[str],
logger,
refs: Optional[Iterable[str]] = None,
) -> Dict[str, Optional[str]]:
) -> dict[str, Optional[str]]:
"""Describe revisions using a tag, branch.
The first matching name will be returned for each rev. Names are preferred in this
@@ -390,7 +384,7 @@ def describe(
if is_branch and rev not in branches:
branches[rev] = ref[len("refs/heads/") :]
names: Dict[str, Optional[str]] = {}
names: dict[str, Optional[str]] = {}
for rev in revs:
if rev == head_rev and head_branch:
names[rev] = head_branch
+4 -4
View File
@@ -1,4 +1,4 @@
from typing import TYPE_CHECKING, List, Tuple
from typing import TYPE_CHECKING
from dvc.exceptions import DownloadError
from dvc.log import logger
@@ -205,9 +205,9 @@ def fetch( # noqa: PLR0913
return transferred_count
def _log_unversioned(data: List["DataIndex"]) -> Tuple[List["DataIndex"], int]:
ret: List["DataIndex"] = []
unversioned: List[str] = []
def _log_unversioned(data: list["DataIndex"]) -> tuple[list["DataIndex"], int]:
ret: list["DataIndex"] = []
unversioned: list[str] = []
for fs_index in data:
remote = fs_index.storage_map[()].remote
if not isinstance(remote, FileStorage) or not remote.fs.version_aware:
+2 -2
View File
@@ -1,4 +1,4 @@
from typing import TYPE_CHECKING, List, Optional
from typing import TYPE_CHECKING, Optional
from dvc.exceptions import InvalidArgumentError
from dvc.log import logger
@@ -61,7 +61,7 @@ def gc( # noqa: PLR0913, C901
all_experiments: bool = False,
force: bool = False,
jobs: Optional[int] = None,
repos: Optional[List[str]] = None,
repos: Optional[list[str]] = None,
workspace: bool = False,
commit_date: Optional[str] = None,
rev: Optional[str] = None,
+5 -4
View File
@@ -1,4 +1,5 @@
from typing import TYPE_CHECKING, Any, Iterator, List, Optional, Set, TypeVar
from collections.abc import Iterator
from typing import TYPE_CHECKING, Any, Optional, TypeVar
from dvc.fs import localfs
from dvc.utils.fs import path_isin
@@ -21,7 +22,7 @@ def check_acyclic(graph: "DiGraph") -> None:
except nx.NetworkXNoCycle:
return
stages: Set["Stage"] = set()
stages: set["Stage"] = set()
for from_node, to_node, _ in edges:
stages.add(from_node)
stages.add(to_node)
@@ -45,7 +46,7 @@ def get_pipelines(graph: "DiGraph"):
def get_subgraph_of_nodes(
graph: "DiGraph", sources: Optional[List[Any]] = None, downstream: bool = False
graph: "DiGraph", sources: Optional[list[Any]] = None, downstream: bool = False
) -> "DiGraph":
from networkx import dfs_postorder_nodes, reverse_view
@@ -69,7 +70,7 @@ def collect_pipeline(stage: "Stage", graph: "DiGraph") -> Iterator["Stage"]:
return nx.dfs_postorder_nodes(pipeline, stage)
def collect_inside_path(path: str, graph: "DiGraph") -> List["Stage"]:
def collect_inside_path(path: str, graph: "DiGraph") -> list["Stage"]:
import networkx as nx
stages = nx.dfs_postorder_nodes(graph)
+2 -2
View File
@@ -1,4 +1,4 @@
from typing import TYPE_CHECKING, Dict, Optional
from typing import TYPE_CHECKING, Optional
from funcy import compact
@@ -33,7 +33,7 @@ def imp_db(
assert sql or table
assert output_format in ("csv", "json")
db: Dict[str, str] = compact(
db: dict[str, str] = compact(
{
"connection": connection,
"file_format": output_format,
+5 -5
View File
@@ -1,7 +1,7 @@
import os
from functools import partial
from tempfile import TemporaryDirectory
from typing import TYPE_CHECKING, List, Set, Tuple, Union
from typing import TYPE_CHECKING, Union
from dvc.log import logger
@@ -19,7 +19,7 @@ logger = logger.getChild(__name__)
def unfetched_view(
index: "Index", targets: "TargetType", unpartial: bool = False, **kwargs
) -> Tuple["IndexView", "IndexView", List["Dependency"]]:
) -> tuple["IndexView", "IndexView", list["Dependency"]]:
"""Return index view of imports which have not been fetched.
Returns:
@@ -28,7 +28,7 @@ def unfetched_view(
"""
from dvc.cachemgr import LEGACY_HASH_NAMES
changed_deps: List["Dependency"] = []
changed_deps: list["Dependency"] = []
def need_fetch(stage: "Stage", legacy: bool = False) -> bool:
if not stage.is_import or (stage.is_partial_import and not unpartial):
@@ -92,7 +92,7 @@ def unpartial_imports(index: Union["Index", "IndexView"]) -> int:
def save_imports(
repo: "Repo", targets: "TargetType", unpartial: bool = False, **kwargs
) -> Set["HashInfo"]:
) -> set["HashInfo"]:
"""Save (download) imports from their original source location.
Imports which are already cached will not be downloaded.
@@ -105,7 +105,7 @@ def save_imports(
from dvc_data.index import md5, save
from dvc_data.index.checkout import apply, compare
downloaded: Set["HashInfo"] = set()
downloaded: set["HashInfo"] = set()
legacy_unfetched, unfetched, changed = unfetched_view(
repo.index, targets, unpartial=unpartial, **kwargs
+35 -40
View File
@@ -1,20 +1,15 @@
import logging
import time
from collections import defaultdict
from collections.abc import Iterable, Iterator
from functools import partial
from itertools import chain
from typing import (
TYPE_CHECKING,
Any,
Callable,
Dict,
Iterable,
Iterator,
List,
NamedTuple,
Optional,
Set,
Tuple,
Union,
)
@@ -44,7 +39,7 @@ if TYPE_CHECKING:
logger = logger.getChild(__name__)
ObjectContainer = Dict[Optional["HashFileDB"], Set["HashInfo"]]
ObjectContainer = dict[Optional["HashFileDB"], set["HashInfo"]]
def log_walk(seq):
@@ -74,7 +69,7 @@ def collect_files(
scm = repo.scm
fs = repo.fs
sep = fs.sep
outs: Set[str] = set()
outs: set[str] = set()
is_local_fs = isinstance(fs, LocalFileSystem)
@@ -288,11 +283,11 @@ class Index:
def __init__(
self,
repo: "Repo",
stages: Optional[List["Stage"]] = None,
metrics: Optional[Dict[str, List[str]]] = None,
plots: Optional[Dict[str, List[str]]] = None,
params: Optional[Dict[str, Any]] = None,
artifacts: Optional[Dict[str, Any]] = None,
stages: Optional[list["Stage"]] = None,
metrics: Optional[dict[str, list[str]]] = None,
plots: Optional[dict[str, list[str]]] = None,
params: Optional[dict[str, Any]] = None,
artifacts: Optional[dict[str, Any]] = None,
) -> None:
self.repo = repo
self.stages = stages or []
@@ -300,7 +295,7 @@ class Index:
self._plots = plots or {}
self._params = params or {}
self._artifacts = artifacts or {}
self._collected_targets: Dict[int, List["StageInfo"]] = {}
self._collected_targets: dict[int, list["StageInfo"]] = {}
@cached_property
def rev(self) -> Optional[str]:
@@ -404,8 +399,8 @@ class Index:
yield from stage.outs
@cached_property
def out_data_keys(self) -> Dict[str, Set["DataIndexKey"]]:
by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
def out_data_keys(self) -> dict[str, set["DataIndexKey"]]:
by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set)
by_workspace["repo"] = set()
by_workspace["local"] = set()
@@ -443,10 +438,10 @@ class Index:
yield from stage.deps
@cached_property
def _plot_sources(self) -> List[str]:
def _plot_sources(self) -> list[str]:
from dvc.repo.plots import _collect_pipeline_files
sources: List[str] = []
sources: list[str] = []
for data in _collect_pipeline_files(self.repo, [], {}).values():
for plot_id, props in data.get("data", {}).items():
if isinstance(props.get("y"), dict):
@@ -458,8 +453,8 @@ class Index:
return sources
@cached_property
def data_keys(self) -> Dict[str, Set["DataIndexKey"]]:
by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
def data_keys(self) -> dict[str, set["DataIndexKey"]]:
by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set)
by_workspace["repo"] = set()
by_workspace["local"] = set()
@@ -474,10 +469,10 @@ class Index:
return dict(by_workspace)
@cached_property
def metric_keys(self) -> Dict[str, Set["DataIndexKey"]]:
def metric_keys(self) -> dict[str, set["DataIndexKey"]]:
from .metrics.show import _collect_top_level_metrics
by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set)
by_workspace["repo"] = set()
@@ -495,10 +490,10 @@ class Index:
return dict(by_workspace)
@cached_property
def param_keys(self) -> Dict[str, Set["DataIndexKey"]]:
def param_keys(self) -> dict[str, set["DataIndexKey"]]:
from .params.show import _collect_top_level_params
by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set)
by_workspace["repo"] = set()
param_paths = _collect_top_level_params(self.repo)
@@ -513,8 +508,8 @@ class Index:
return dict(by_workspace)
@cached_property
def plot_keys(self) -> Dict[str, Set["DataIndexKey"]]:
by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
def plot_keys(self) -> dict[str, set["DataIndexKey"]]:
by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set)
by_workspace["repo"] = set()
@@ -536,7 +531,7 @@ class Index:
return _build_tree_from_outs(self.outs)
@cached_property
def data(self) -> "Dict[str, DataIndex]":
def data(self) -> "dict[str, DataIndex]":
prefix: "DataIndexKey"
loaded = False
@@ -584,7 +579,7 @@ class Index:
def collect_targets(
self, targets: Optional["TargetType"], *, onerror=None, **kwargs: Any
) -> List["StageInfo"]:
) -> list["StageInfo"]:
from dvc.exceptions import DvcException
from dvc.repo.stage import StageInfo
from dvc.utils.collections import ensure_list
@@ -658,7 +653,7 @@ class Index:
stage_filter: Optional[Callable[["Stage"], bool]] = None,
outs_filter: Optional[Callable[["Output"], bool]] = None,
max_size: Optional[int] = None,
types: Optional[List[str]] = None,
types: Optional[list[str]] = None,
**kwargs: Any,
) -> "IndexView":
"""Return read-only view of index for the specified targets.
@@ -698,8 +693,8 @@ class Index:
class _DataPrefixes(NamedTuple):
explicit: Set["DataIndexKey"]
recursive: Set["DataIndexKey"]
explicit: set["DataIndexKey"]
recursive: set["DataIndexKey"]
class IndexView:
@@ -728,7 +723,7 @@ class IndexView:
yield from stage.deps
@property
def _filtered_outs(self) -> Iterator[Tuple["Output", Optional[str]]]:
def _filtered_outs(self) -> Iterator[tuple["Output", Optional[str]]]:
for stage, filter_info in self._stage_infos:
for out in stage.filter_outs(filter_info):
if not self._outs_filter or self._outs_filter(out):
@@ -739,8 +734,8 @@ class IndexView:
yield from {out for (out, _) in self._filtered_outs}
@cached_property
def out_data_keys(self) -> Dict[str, Set["DataIndexKey"]]:
by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
def out_data_keys(self) -> dict[str, set["DataIndexKey"]]:
by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set)
by_workspace["repo"] = set()
by_workspace["local"] = set()
@@ -755,8 +750,8 @@ class IndexView:
return dict(by_workspace)
@cached_property
def _data_prefixes(self) -> Dict[str, "_DataPrefixes"]:
prefixes: Dict[str, "_DataPrefixes"] = defaultdict(
def _data_prefixes(self) -> dict[str, "_DataPrefixes"]:
prefixes: dict[str, "_DataPrefixes"] = defaultdict(
lambda: _DataPrefixes(set(), set())
)
for out, filter_info in self._filtered_outs:
@@ -772,8 +767,8 @@ class IndexView:
return prefixes
@cached_property
def data_keys(self) -> Dict[str, Set["DataIndexKey"]]:
ret: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
def data_keys(self) -> dict[str, set["DataIndexKey"]]:
ret: dict[str, set["DataIndexKey"]] = defaultdict(set)
for out, filter_info in self._filtered_outs:
if not out.use_cache:
@@ -791,7 +786,7 @@ class IndexView:
return _build_tree_from_outs(self.outs)
@cached_property
def data(self) -> Dict[str, Union["DataIndex", "DataIndexView"]]:
def data(self) -> dict[str, Union["DataIndex", "DataIndexView"]]:
from dvc_data.index import DataIndex, view
def key_filter(workspace: str, key: "DataIndexKey"):
@@ -803,7 +798,7 @@ class IndexView:
except KeyError:
return False
data: Dict[str, Union["DataIndex", "DataIndexView"]] = {}
data: dict[str, Union["DataIndex", "DataIndexView"]] = {}
for workspace, data_index in self._index.data.items():
if self.stages:
data[workspace] = view(data_index, partial(key_filter, workspace))
+2 -2
View File
@@ -1,5 +1,5 @@
import os
from typing import TYPE_CHECKING, Any, Dict, Optional, Union
from typing import TYPE_CHECKING, Any, Optional, Union
if TYPE_CHECKING:
from dvc.fs.dvc import DVCFileSystem
@@ -13,7 +13,7 @@ def ls(
rev: Optional[str] = None,
recursive: Optional[bool] = None,
dvc_only: bool = False,
config: Union[None, Dict[str, Any], str] = None,
config: Union[None, dict[str, Any], str] = None,
remote: Optional[str] = None,
remote_config: Optional[dict] = None,
):
+4 -4
View File
@@ -1,4 +1,4 @@
from typing import TYPE_CHECKING, Dict, TypedDict, Union
from typing import TYPE_CHECKING, TypedDict, Union
from funcy import compact
@@ -12,12 +12,12 @@ if TYPE_CHECKING:
class DiffResult(TypedDict, total=False):
errors: Dict[str, Union[Exception, Dict[str, Exception]]]
diff: Dict[str, Dict[str, Dict]]
errors: dict[str, Union[Exception, dict[str, Exception]]]
diff: dict[str, dict[str, dict]]
def _diff(
result: Dict[str, "Result"],
result: dict[str, "Result"],
old_rev: str,
new_rev: str,
**kwargs,
+17 -21
View File
@@ -1,14 +1,10 @@
import os
from collections.abc import Iterable, Iterator
from itertools import chain
from typing import (
TYPE_CHECKING,
Any,
Dict,
Iterable,
Iterator,
List,
Optional,
Tuple,
TypedDict,
Union,
)
@@ -71,7 +67,7 @@ def _read_metric(fs: "FileSystem", path: str, **load_kwargs) -> Any:
def _read_metrics(
fs: "FileSystem", metrics: Iterable[str], **load_kwargs
) -> Iterator[Tuple[str, Union[Exception, Any]]]:
) -> Iterator[tuple[str, Union[Exception, Any]]]:
for metric in metrics:
try:
yield metric, _read_metric(fs, metric, **load_kwargs)
@@ -80,7 +76,7 @@ def _read_metrics(
yield metric, exc
def metrics_from_target(repo: "Repo", targets: List[str]) -> Iterator["Output"]:
def metrics_from_target(repo: "Repo", targets: list[str]) -> Iterator["Output"]:
stages = chain.from_iterable(repo.stage.collect(target) for target in targets)
for stage in stages:
yield from stage.metrics
@@ -88,11 +84,11 @@ def metrics_from_target(repo: "Repo", targets: List[str]) -> Iterator["Output"]:
def _collect_metrics(
repo: "Repo",
targets: Optional[List[str]] = None,
stages: Optional[List[str]] = None,
targets: Optional[list[str]] = None,
stages: Optional[list[str]] = None,
outs_only: bool = False,
) -> List[str]:
metrics: List[str] = []
) -> list[str]:
metrics: list[str] = []
if targets:
# target is a repo-relative path
@@ -122,7 +118,7 @@ class FileResult(TypedDict, total=False):
class Result(TypedDict, total=False):
data: Dict[str, FileResult]
data: dict[str, FileResult]
error: Exception
@@ -139,11 +135,11 @@ def to_relpath(fs: "FileSystem", root_dir: str, d: Result) -> Result:
def _gather_metrics(
repo: "Repo",
targets: Optional[List[str]] = None,
targets: Optional[list[str]] = None,
outs_only: bool = False,
stages: Optional[List[str]] = None,
stages: Optional[list[str]] = None,
on_error: str = "return",
) -> Dict[str, FileResult]:
) -> dict[str, FileResult]:
assert on_error in ("raise", "return", "ignore")
# `files` is a repo-relative posixpath that can be passed to DVCFileSystem
@@ -168,8 +164,8 @@ def _gather_metrics(
def _hide_workspace(
scm: Union["Git", "NoSCM"], res: Dict[str, Result]
) -> Dict[str, Result]:
scm: Union["Git", "NoSCM"], res: dict[str, Result]
) -> dict[str, Result]:
# Hide workspace params if they are the same as in the active branch
try:
active_branch = scm.active_branch()
@@ -186,16 +182,16 @@ def _hide_workspace(
def show(
repo: "Repo",
targets: Optional[List[str]] = None,
stages: Optional[List[str]] = None,
targets: Optional[list[str]] = None,
stages: Optional[list[str]] = None,
outs_only: bool = False,
all_branches: bool = False,
all_tags: bool = False,
revs: Optional[List[str]] = None,
revs: Optional[list[str]] = None,
all_commits: bool = False,
hide_workspace: bool = True,
on_error: str = "return",
) -> Dict[str, Result]:
) -> dict[str, Result]:
assert on_error in ("raise", "return", "ignore")
targets = [os.path.abspath(target) for target in ensure_list(targets)]
+3 -3
View File
@@ -1,7 +1,7 @@
import os
import tempfile
import threading
from typing import TYPE_CHECKING, Dict, Optional, Tuple
from typing import TYPE_CHECKING, Optional
from funcy import retry, wrap_with
@@ -78,8 +78,8 @@ def erepo_factory(url, root_dir, cache_config):
return make_repo
CLONES: Dict[str, Tuple[str, bool]] = {}
CACHE_DIRS: Dict[str, str] = {}
CLONES: dict[str, tuple[str, bool]] = {}
CACHE_DIRS: dict[str, str] = {}
@wrap_with(threading.Lock())
+18 -17
View File
@@ -1,7 +1,8 @@
import os
from collections import defaultdict
from collections.abc import Iterator
from itertools import chain
from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Tuple, Union
from typing import TYPE_CHECKING, Any, Optional, Union
from dvc.dependency.param import ParamsDependency, read_param_file
from dvc.log import logger
@@ -27,7 +28,7 @@ def _collect_top_level_params(repo: "Repo") -> Iterator[str]:
def params_from_target(
repo: "Repo", targets: List[str]
repo: "Repo", targets: list[str]
) -> Iterator["ParamsDependency"]:
stages = chain.from_iterable(repo.stage.collect(target) for target in targets)
for stage in stages:
@@ -36,17 +37,17 @@ def params_from_target(
def _collect_params(
repo: "Repo",
targets: Union[List[str], Dict[str, List[str]], None] = None,
stages: Optional[List[str]] = None,
targets: Union[list[str], dict[str, list[str]], None] = None,
stages: Optional[list[str]] = None,
deps_only: bool = False,
default_file: Optional[str] = None,
) -> Dict[str, List[str]]:
) -> dict[str, list[str]]:
from dvc.dependency import _merge_params
if isinstance(targets, list):
targets = {target: [] for target in targets}
params: List[Dict[str, List[str]]] = []
params: list[dict[str, list[str]]] = []
if targets:
# target is a repo-relative path
@@ -80,8 +81,8 @@ def _collect_params(
return ret
def _collect_vars(repo, params, stages=None) -> Dict:
vars_params: Dict[str, Dict] = defaultdict(dict)
def _collect_vars(repo, params, stages=None) -> dict:
vars_params: dict[str, dict] = defaultdict(dict)
for stage in repo.index.stages:
if isinstance(stage, PipelineStage) and stage.tracked_vars:
@@ -102,8 +103,8 @@ def _collect_vars(repo, params, stages=None) -> Dict:
def _read_params(
fs: "FileSystem", params: Dict[str, List[str]], **load_kwargs
) -> Iterator[Tuple[str, Union[Exception, Any]]]:
fs: "FileSystem", params: dict[str, list[str]], **load_kwargs
) -> Iterator[tuple[str, Union[Exception, Any]]]:
for file_path, key_paths in params.items():
try:
yield file_path, read_param_file(fs, file_path, key_paths, **load_kwargs)
@@ -114,9 +115,9 @@ def _read_params(
def _gather_params(
repo: "Repo",
targets: Union[List[str], Dict[str, List[str]], None] = None,
targets: Union[list[str], dict[str, list[str]], None] = None,
deps_only: bool = False,
stages: Optional[List[str]] = None,
stages: Optional[list[str]] = None,
on_error: str = "return",
):
assert on_error in ("raise", "return", "ignore")
@@ -132,7 +133,7 @@ def _gather_params(
default_file=ParamsDependency.DEFAULT_PARAMS_FILE,
)
data: Dict[str, FileResult] = {}
data: dict[str, FileResult] = {}
fs = repo.dvcfs
for fs_path, result in _read_params(fs, files_keypaths, cache=True):
@@ -159,16 +160,16 @@ def _gather_params(
def show(
repo: "Repo",
targets: Optional[List[str]] = None,
stages: Optional[List[str]] = None,
targets: Optional[list[str]] = None,
stages: Optional[list[str]] = None,
deps_only: bool = False,
all_branches: bool = False,
all_tags: bool = False,
revs: Optional[List[str]] = None,
revs: Optional[list[str]] = None,
all_commits: bool = False,
hide_workspace: bool = True,
on_error: str = "return",
) -> Dict[str, Result]:
) -> dict[str, Result]:
assert on_error in ("raise", "return", "ignore")
res = {}
+30 -33
View File
@@ -2,6 +2,7 @@ import csv
import io
import os
from collections import defaultdict
from collections.abc import Iterator
from copy import deepcopy
from functools import partial
from multiprocessing import cpu_count
@@ -9,11 +10,7 @@ from typing import (
TYPE_CHECKING,
Any,
Callable,
Dict,
Iterator,
List,
Optional,
Set,
Union,
)
@@ -40,7 +37,7 @@ dpath.options.ALLOW_EMPTY_STRING_KEYS = True
logger = logger.getChild(__name__)
def onerror_collect(result: Dict, exception: Exception, *args, **kwargs):
def onerror_collect(result: dict, exception: Exception, *args, **kwargs):
logger.debug("", exc_info=True)
result["error"] = exception
@@ -82,12 +79,12 @@ class Plots:
def collect(
self,
targets: Optional[List[str]] = None,
revs: Optional[List[str]] = None,
targets: Optional[list[str]] = None,
revs: Optional[list[str]] = None,
recursive: bool = False,
onerror: Optional[Callable] = None,
props: Optional[Dict] = None,
) -> Iterator[Dict]:
props: Optional[dict] = None,
) -> Iterator[dict]:
"""Collects plots definitions and data sources.
Generator yielding a structure like:
@@ -142,7 +139,7 @@ class Plots:
revs = ["workspace", *revs]
for rev in revs:
with switch_repo(self.repo, rev) as (repo, _):
res: Dict = {}
res: dict = {}
definitions = _collect_definitions(
repo,
targets=targets,
@@ -168,9 +165,9 @@ class Plots:
def _collect_data_sources(
self,
repo: "Repo",
targets: Optional[List[str]] = None,
targets: Optional[list[str]] = None,
recursive: bool = False,
props: Optional[Dict] = None,
props: Optional[dict] = None,
onerror: Optional[Callable] = None,
):
fs = repo.dvcfs
@@ -178,7 +175,7 @@ class Plots:
props = props or {}
plots = _collect_plots(repo, targets, recursive)
res: Dict[str, Any] = {}
res: dict[str, Any] = {}
for fs_path, rev_props in plots.items():
joined_props = {**rev_props, **props}
res[fs_path] = {"props": joined_props}
@@ -197,7 +194,7 @@ class Plots:
def show(
self,
targets: Optional[List[str]] = None,
targets: Optional[list[str]] = None,
revs=None,
props=None,
recursive=False,
@@ -206,7 +203,7 @@ class Plots:
if onerror is None:
onerror = onerror_collect
result: Dict[str, Dict] = {}
result: dict[str, dict] = {}
for data in self.collect(
targets,
revs,
@@ -286,7 +283,7 @@ def _is_plot(out: "Output") -> bool:
return bool(out.plot)
def _resolve_data_sources(plots_data: Dict, rev: str, cache: bool = False):
def _resolve_data_sources(plots_data: dict, rev: str, cache: bool = False):
from dvc.progress import Tqdm
values = list(plots_data.values())
@@ -325,9 +322,9 @@ def _resolve_data_sources(plots_data: Dict, rev: str, cache: bool = False):
def _collect_plots(
repo: "Repo",
targets: Optional[List[str]] = None,
targets: Optional[list[str]] = None,
recursive: bool = False,
) -> Dict[str, Dict]:
) -> dict[str, dict]:
from dvc.repo.collect import collect
plots, fs_paths = collect(
@@ -344,8 +341,8 @@ def _collect_plots(
return result
def _get_data_targets(definitions: Dict):
result: Set = set()
def _get_data_targets(definitions: dict):
result: set = set()
if "data" in definitions:
for content in definitions["data"].values():
if "data" in content:
@@ -404,7 +401,7 @@ def _relpath(fs, path):
def _collect_output_plots(repo, targets, props, onerror: Optional[Callable] = None):
fs = repo.dvcfs
result: Dict[str, Dict] = {}
result: dict[str, dict] = {}
for plot in repo.index.plots:
plot_props = _plot_props(plot)
dvcfile = plot.stage.dvcfile
@@ -445,15 +442,15 @@ def _adjust_sources(fs, plot_props, config_dir):
def _resolve_definitions(
fs: "FileSystem",
targets: List[str],
props: Dict[str, Any],
targets: list[str],
props: dict[str, Any],
config_path: "StrPath",
definitions: "DictStrAny",
onerror: Optional[Callable[[Any], Any]] = None,
):
config_path = os.fspath(config_path)
config_dir = fs.dirname(config_path)
result: Dict[str, Dict] = {}
result: dict[str, dict] = {}
for plot_id, plot_props in definitions.items():
if plot_props is None:
plot_props = {}
@@ -477,12 +474,12 @@ def _resolve_definitions(
return result
def _collect_pipeline_files(repo, targets: List[str], props, onerror=None):
result: Dict[str, Dict] = {}
def _collect_pipeline_files(repo, targets: list[str], props, onerror=None):
result: dict[str, dict] = {}
top_plots = repo.index._plots
for dvcfile, plots_def in top_plots.items():
dvcfile_path = _relpath(repo.dvcfs, dvcfile)
dvcfile_defs_dict: Dict[str, Union[Dict, None]] = {}
dvcfile_defs_dict: dict[str, Union[dict, None]] = {}
for elem in plots_def:
if isinstance(elem, str):
dvcfile_defs_dict[elem] = None
@@ -510,11 +507,11 @@ def _collect_pipeline_files(repo, targets: List[str], props, onerror=None):
def _collect_definitions(
repo: "Repo",
targets=None,
props: Optional[Dict] = None,
props: Optional[dict] = None,
onerror: Optional[Callable] = None,
**kwargs,
) -> Dict:
result: Dict = defaultdict(dict)
) -> dict:
result: dict = defaultdict(dict)
props = props or {}
fs = repo.dvcfs
@@ -536,8 +533,8 @@ def _collect_definitions(
return dict(result)
def unpack_if_dir(fs, path, props: Dict[str, str], onerror: Optional[Callable] = None):
result: Dict[str, Dict] = defaultdict(dict)
def unpack_if_dir(fs, path, props: dict[str, str], onerror: Optional[Callable] = None):
result: dict[str, dict] = defaultdict(dict)
if fs.isdir(path):
unpacked = _unpack_dir_files(fs, path, onerror=onerror)
else:
@@ -574,7 +571,7 @@ def parse(fs, path, props=None, **fs_kwargs):
return PARSERS[extension](contents, path)
def _plot_props(out: "Output") -> Dict:
def _plot_props(out: "Output") -> dict:
from dvc.schema import PLOT_PROPS
if not (out.plot):
+14 -18
View File
@@ -1,13 +1,9 @@
from collections.abc import Iterable
from typing import (
TYPE_CHECKING,
Callable,
Dict,
Iterable,
List,
NoReturn,
Optional,
Set,
Tuple,
TypeVar,
Union,
cast,
@@ -40,8 +36,8 @@ def collect_stages(
targets: Iterable[str],
recursive: bool = False,
glob: bool = False,
) -> List["Stage"]:
stages: List["Stage"] = []
) -> list["Stage"]:
stages: list["Stage"] = []
for target in targets:
stages.extend(repo.stage.collect(target, recursive=recursive, glob=glob))
return ldistinct(stages)
@@ -49,7 +45,7 @@ def collect_stages(
def get_subgraph(
graph: "DiGraph",
nodes: Optional[List] = None,
nodes: Optional[list] = None,
pipeline: bool = False,
downstream: bool = False,
) -> "DiGraph":
@@ -76,10 +72,10 @@ def get_active_graph(graph: "DiGraph") -> "DiGraph":
def plan_repro(
graph: "DiGraph",
stages: Optional[List["T"]] = None,
stages: Optional[list["T"]] = None,
pipeline: bool = False,
downstream: bool = False,
) -> List["T"]:
) -> list["T"]:
r"""Derive the evaluation of the given node for the given graph.
When you _reproduce a stage_, you want to _evaluate the descendants_
@@ -134,7 +130,7 @@ def _reproduce_stage(stage: "Stage", **kwargs) -> Optional["Stage"]:
def _get_upstream_downstream_nodes(
graph: Optional["DiGraph"], node: T
) -> Tuple[List[T], List[T]]:
) -> tuple[list[T], list[T]]:
succ = list(graph.successors(node)) if graph else []
pre = list(graph.predecessors(node)) if graph else []
return succ, pre
@@ -146,7 +142,7 @@ def _repr(stages: Iterable["Stage"]) -> str:
def handle_error(
graph: Optional["DiGraph"], on_error: str, exc: Exception, stage: "Stage"
) -> Set["Stage"]:
) -> set["Stage"]:
import networkx as nx
logger.warning("%s%s", exc, " (ignored)" if on_error == "ignore" else "")
@@ -168,19 +164,19 @@ def _raise_error(exc: Optional[Exception], *stages: "Stage") -> NoReturn:
def _reproduce(
stages: List["Stage"],
stages: list["Stage"],
graph: Optional["DiGraph"] = None,
force_downstream: bool = False,
on_error: str = "fail",
force: bool = False,
repro_fn: Callable = _reproduce_stage,
**kwargs,
) -> List["Stage"]:
) -> list["Stage"]:
assert on_error in ("fail", "keep-going", "ignore")
result: List["Stage"] = []
failed: List["Stage"] = []
to_skip: Dict["Stage", "Stage"] = {}
result: list["Stage"] = []
failed: list["Stage"] = []
to_skip: dict["Stage", "Stage"] = {}
ret: Optional["Stage"] = None
force_state = dict.fromkeys(stages, force)
@@ -240,7 +236,7 @@ def reproduce(
if not kwargs.get("interactive", False):
kwargs["interactive"] = self.config["core"].get("interactive", False)
stages: List["Stage"] = []
stages: list["Stage"] = []
if not all_pipelines:
targets_list = ensure_list(targets or PROJECT_FILE)
stages = collect_stages(self, targets_list, recursive=recursive, glob=glob)
+4 -8
View File
@@ -1,16 +1,12 @@
import logging
import shlex
from collections.abc import Iterable, Iterator
from contextlib import contextmanager
from functools import wraps
from typing import (
TYPE_CHECKING,
Any,
Dict,
Iterable,
Iterator,
List,
Optional,
Set,
Union,
)
@@ -27,15 +23,15 @@ logger = logger.getChild(__name__)
class SCMContext:
def __init__(self, scm: "Base", config: Optional[Dict[str, Any]] = None) -> None:
def __init__(self, scm: "Base", config: Optional[dict[str, Any]] = None) -> None:
from funcy import get_in
self.scm: "Base" = scm
self.autostage: bool = get_in(
config or {}, ["core", "autostage"], default=False
)
self.ignored_paths: List[str] = []
self.files_to_track: Set[str] = set()
self.ignored_paths: list[str] = []
self.files_to_track: set[str] = set()
self.quiet: bool = False
def track_file(self, paths: Union[str, Iterable[str], None] = None) -> None:
+6 -5
View File
@@ -1,8 +1,9 @@
import fnmatch
import typing
from collections.abc import Iterable
from contextlib import suppress
from functools import wraps
from typing import Iterable, List, NamedTuple, Optional, Set, Tuple, Union
from typing import NamedTuple, Optional, Union
from dvc.exceptions import (
NoOutputOrStageError,
@@ -31,9 +32,9 @@ class StageInfo(NamedTuple):
filter_info: Optional[str] = None
StageList = List["Stage"]
StageList = list["Stage"]
StageIter = Iterable["Stage"]
StageSet = Set["Stage"]
StageSet = set["Stage"]
def _collect_with_deps(stages: StageList, graph: "DiGraph") -> StageSet:
@@ -72,7 +73,7 @@ def _collect_specific_target(
target: str,
with_deps: bool,
recursive: bool,
) -> Tuple[StageIter, Optional[str], Optional[str]]:
) -> tuple[StageIter, Optional[str], Optional[str]]:
from dvc.dvcfile import is_valid_filename
# Optimization: do not collect the graph for a specific target
@@ -363,7 +364,7 @@ class StageLoad:
with_deps: bool = False,
recursive: bool = False,
graph: Optional["DiGraph"] = None,
) -> List[StageInfo]:
) -> list[StageInfo]:
"""Collects a list of (stage, filter_info) from the given target.
Priority is in the order of following in case of ambiguity:
+2 -2
View File
@@ -1,4 +1,4 @@
from typing import TYPE_CHECKING, List
from typing import TYPE_CHECKING
from dvc.exceptions import InvalidArgumentError
@@ -34,7 +34,7 @@ def update( # noqa: C901
raise InvalidArgumentError("--remote can't be used without --to-remote")
import_stages = set()
other_stage_infos: List["StageInfo"] = []
other_stage_infos: list["StageInfo"] = []
for stage_info in self.index.collect_targets(targets, recursive=recursive):
if stage_info.stage.is_import:
+6 -5
View File
@@ -1,5 +1,6 @@
from collections.abc import Iterable
from functools import partial
from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Tuple, Union
from typing import TYPE_CHECKING, Any, Optional, Union
from funcy import first
@@ -35,7 +36,7 @@ def worktree_view_by_remotes(
targets: Optional["TargetType"] = None,
push: bool = False,
**kwargs: Any,
) -> Iterable[Tuple[Optional[str], "IndexView"]]:
) -> Iterable[tuple[Optional[str], "IndexView"]]:
from dvc.repo.index import IndexView
def outs_filter(view: "IndexView", remote: Optional[str]):
@@ -176,7 +177,7 @@ def update_worktree_stages(
outs_filter=outs_filter,
)
local_index = view.data["repo"]
remote_indexes: Dict[str, Tuple["Remote", "DataIndex"]] = {}
remote_indexes: dict[str, tuple["Remote", "DataIndex"]] = {}
for stage in view.stages:
for out in stage.outs:
_update_worktree_out(repo, out, local_index, remote_indexes)
@@ -187,7 +188,7 @@ def _update_worktree_out(
repo: "Repo",
out: "Output",
local_index: Union["DataIndex", "DataIndexView"],
remote_indexes: Dict[str, Tuple["Remote", "DataIndex"]],
remote_indexes: dict[str, tuple["Remote", "DataIndex"]],
):
from dvc_data.index import build
@@ -283,7 +284,7 @@ def _get_diff_indexes(
out: "Output",
local_index: Union["DataIndex", "DataIndexView"],
remote_index: Union["DataIndex", "DataIndexView"],
) -> Tuple["DataIndex", "DataIndex"]:
) -> tuple["DataIndex", "DataIndex"]:
from dvc_data.index import DataIndex
_, key = out.index_key
+2 -2
View File
@@ -1,5 +1,5 @@
from collections.abc import Mapping
from typing import Any, Dict
from typing import Any
import voluptuous as vol
@@ -28,7 +28,7 @@ SINGLE_STAGE_SCHEMA = {
StageParams.PARAM_DESC: str,
}
DATA_SCHEMA: Dict[Any, Any] = {
DATA_SCHEMA: dict[Any, Any] = {
**CHECKSUMS_SCHEMA,
**META_SCHEMA,
vol.Required("path"): str,
+6 -8
View File
@@ -1,13 +1,11 @@
"""Manages source control systems (e.g. Git)."""
import os
from collections.abc import Iterator, Mapping
from contextlib import contextmanager
from functools import partial
from typing import (
TYPE_CHECKING,
Iterator,
List,
Literal,
Mapping,
Optional,
Union,
overload,
@@ -189,7 +187,7 @@ def resolve_rev(scm: Union["Git", "NoSCM"], rev: str) -> str:
raise RevError(str(exc)) # noqa: B904
def _get_n_commits(scm: "Git", revs: List[str], num: int) -> List[str]:
def _get_n_commits(scm: "Git", revs: list[str], num: int) -> list[str]:
results = []
for rev in revs:
if num == 0:
@@ -210,14 +208,14 @@ def _get_n_commits(scm: "Git", revs: List[str], num: int) -> List[str]:
def iter_revs(
scm: "Git",
revs: Optional[List[str]] = None,
revs: Optional[list[str]] = None,
num: int = 1,
all_branches: bool = False,
all_tags: bool = False,
all_commits: bool = False,
all_experiments: bool = False,
commit_date: Optional[str] = None,
) -> Mapping[str, List[str]]:
) -> Mapping[str, list[str]]:
from scmrepo.exceptions import SCMError as _SCMError
from dvc.repo.experiments.utils import exp_commits
@@ -235,7 +233,7 @@ def iter_revs(
return {}
revs = revs or []
results: List[str] = _get_n_commits(scm, revs, num)
results: list[str] = _get_n_commits(scm, revs, num)
if all_commits:
results.extend(scm.list_all_commits())
@@ -266,7 +264,7 @@ def iter_revs(
return group_by(rev_resolver, results)
def lfs_prefetch(fs: "FileSystem", paths: List[str]):
def lfs_prefetch(fs: "FileSystem", paths: list[str]):
from scmrepo.git.lfs import fetch as _lfs_fetch
from dvc.fs.dvc import DVCFileSystem
+21 -26
View File
@@ -1,18 +1,13 @@
import os
import string
from collections import defaultdict
from collections.abc import Iterable
from contextlib import suppress
from dataclasses import dataclass
from typing import (
TYPE_CHECKING,
Any,
Dict,
Iterable,
List,
Optional,
Set,
Tuple,
Type,
TypeVar,
Union,
)
@@ -55,14 +50,14 @@ if TYPE_CHECKING:
logger = logger.getChild(__name__)
# Disallow all punctuation characters except hyphen and underscore
INVALID_STAGENAME_CHARS = set(string.punctuation) - {"_", "-"}
Env = Dict[str, str]
ChangedEntries = Tuple[List[str], List[str], Optional[str]]
Env = dict[str, str]
ChangedEntries = tuple[list[str], list[str], Optional[str]]
_T = TypeVar("_T")
def loads_from(
cls: Type[_T], repo: "Repo", path: str, wdir: str, data: Dict[str, Any]
cls: type[_T], repo: "Repo", path: str, wdir: str, data: dict[str, Any]
) -> _T:
kw = {
"repo": repo,
@@ -91,7 +86,7 @@ class RawData:
generated_from: Optional[str] = None
def create_stage(cls: Type[_T], repo, path, **kwargs) -> _T:
def create_stage(cls: type[_T], repo, path, **kwargs) -> _T:
from dvc.dvcfile import check_dvcfile_path
wdir = os.path.abspath(kwargs.get("wdir", None) or os.curdir)
@@ -205,13 +200,13 @@ class Stage(params.StageParams):
self._dvcfile = dvcfile
@property
def params(self) -> List["ParamsDependency"]:
def params(self) -> list["ParamsDependency"]:
from dvc.dependency import ParamsDependency
return [dep for dep in self.deps if isinstance(dep, ParamsDependency)]
@property
def metrics(self) -> List["Output"]:
def metrics(self) -> list["Output"]:
return [out for out in self.outs if out.metric]
def __repr__(self):
@@ -310,7 +305,7 @@ class Stage(params.StageParams):
return desc
def changed_deps(
self, allow_missing: bool = False, upstream: Optional[List] = None
self, allow_missing: bool = False, upstream: Optional[list] = None
) -> bool:
if self.frozen:
return False
@@ -322,7 +317,7 @@ class Stage(params.StageParams):
@rwlocked(read=["deps"])
def _changed_deps(
self, allow_missing: bool = False, upstream: Optional[List] = None
self, allow_missing: bool = False, upstream: Optional[list] = None
) -> bool:
for dep in self.deps:
status = dep.status()
@@ -370,7 +365,7 @@ class Stage(params.StageParams):
@rwlocked(read=["deps", "outs"])
def changed(
self, allow_missing: bool = False, upstream: Optional[List] = None
self, allow_missing: bool = False, upstream: Optional[list] = None
) -> bool:
is_changed = (
# Short-circuit order: stage md5 is fast,
@@ -477,7 +472,7 @@ class Stage(params.StageParams):
def reload(self) -> "Stage":
return self.dvcfile.stage
def dumpd(self, **kwargs) -> Dict[str, Any]:
def dumpd(self, **kwargs) -> dict[str, Any]:
return get_dump(self, **kwargs)
def compute_md5(self) -> Optional[str]:
@@ -509,7 +504,7 @@ class Stage(params.StageParams):
if not allow_missing:
raise
def get_versioned_outs(self) -> Dict[str, "Output"]:
def get_versioned_outs(self) -> dict[str, "Output"]:
from .exceptions import StageFileDoesNotExistError, StageNotFound
try:
@@ -543,7 +538,7 @@ class Stage(params.StageParams):
out.ignore()
@staticmethod
def _changed_entries(entries) -> List[str]:
def _changed_entries(entries) -> list[str]:
return [str(entry) for entry in entries if entry.workspace_status()]
def _changed_stage_entry(self) -> str:
@@ -663,8 +658,8 @@ class Stage(params.StageParams):
@rwlocked(write=["outs"])
def checkout(
self, allow_missing: bool = False, **kwargs
) -> Dict[str, List["StrPath"]]:
stats: Dict[str, List["StrPath"]] = defaultdict(list)
) -> dict[str, list["StrPath"]]:
stats: dict[str, list["StrPath"]] = defaultdict(list)
if self.is_partial_import:
return stats
@@ -679,7 +674,7 @@ class Stage(params.StageParams):
return stats
@staticmethod
def _checkout(out, **kwargs) -> Tuple[Optional[str], List[str]]:
def _checkout(out, **kwargs) -> tuple[Optional[str], list[str]]:
try:
result = out.checkout(**kwargs)
added, modified = result or (None, None)
@@ -692,8 +687,8 @@ class Stage(params.StageParams):
@rwlocked(read=["deps", "outs"])
def status(
self, check_updates: bool = False, filter_info: Optional[bool] = None
) -> Dict[str, List[Union[str, Dict[str, str]]]]:
ret: List[Union[str, Dict[str, str]]] = []
) -> dict[str, list[Union[str, dict[str, str]]]]:
ret: list[Union[str, dict[str, str]]] = []
show_import = (
self.is_repo_import or self.is_versioned_import
) and check_updates
@@ -706,7 +701,7 @@ class Stage(params.StageParams):
return {self.addressing: ret} if ret else {}
@staticmethod
def _status(entries: Iterable["Output"]) -> Dict[str, str]:
def _status(entries: Iterable["Output"]) -> dict[str, str]:
ret = {}
for entry in entries:
@@ -747,7 +742,7 @@ class Stage(params.StageParams):
def get_used_objs(
self, *args, **kwargs
) -> Dict[Optional["HashFileDB"], Set["HashInfo"]]:
) -> dict[Optional["HashFileDB"], set["HashInfo"]]:
"""Return set of object IDs used by this stage."""
if self.is_partial_import and not self.is_repo_import:
return {}
@@ -802,7 +797,7 @@ class PipelineStage(Stage):
super().__init__(*args, **kwargs)
self.name = name
self.cmd_changed = False
self.tracked_vars: Dict[str, Dict[str, Dict[str, str]]] = {}
self.tracked_vars: dict[str, dict[str, dict[str, str]]] = {}
def __eq__(self, other):
return super().__eq__(other) and self.name == other.name
+2 -2
View File
@@ -1,6 +1,6 @@
import os
from contextlib import contextmanager
from typing import TYPE_CHECKING, List, Optional, Tuple
from typing import TYPE_CHECKING, Optional
from funcy import first
@@ -247,7 +247,7 @@ class StageCache:
message = f"run-cache is not supported for http filesystem: {path}"
raise RunCacheNotSupported(message)
ret: List[Tuple[str, str]] = []
ret: list[tuple[str, str]] = []
if not from_fs.exists(runs):
return ret
+7 -5
View File
@@ -1,7 +1,7 @@
from collections.abc import Mapping
from copy import deepcopy
from itertools import chain
from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Tuple
from typing import TYPE_CHECKING, Any, Optional
from funcy import get_in, lcat, once, project
@@ -17,6 +17,8 @@ from .params import StageParams
from .utils import fill_stage_dependencies, resolve_paths
if TYPE_CHECKING:
from collections.abc import Iterable
from dvc.dvcfile import ProjectFile, SingleStageFile
logger = logger.getChild(__name__)
@@ -39,7 +41,7 @@ class StageLoader(Mapping):
self._lockfile_data = lockfile_data.get("stages", {})
@cached_property
def lockfile_data(self) -> Dict[str, Any]:
def lockfile_data(self) -> dict[str, Any]:
if not self._lockfile_data:
logger.debug("Lockfile for '%s' not found", self.dvcfile.relpath)
return self._lockfile_data
@@ -53,7 +55,7 @@ class StageLoader(Mapping):
from dvc.output import Output, merge_file_meta_from_cloud
assert isinstance(lock_data, dict)
items: Iterable[Tuple[str, "Output"]] = chain(
items: Iterable[tuple[str, "Output"]] = chain(
((StageParams.PARAM_DEPS, dep) for dep in stage.deps),
((StageParams.PARAM_OUTS, out) for out in stage.outs),
)
@@ -174,7 +176,7 @@ class SingleStageLoader(Mapping):
def __init__(
self,
dvcfile: "SingleStageFile",
stage_data: Dict[Any, str],
stage_data: dict[Any, str],
stage_text: Optional[str] = None,
):
self.dvcfile = dvcfile
@@ -196,7 +198,7 @@ class SingleStageLoader(Mapping):
def load_stage(
cls,
dvcfile: "SingleStageFile",
d: Dict[str, Any],
d: dict[str, Any],
stage_text: Optional[str],
) -> Stage:
path, wdir = resolve_paths(
+6 -8
View File
@@ -1,11 +1,9 @@
from collections import OrderedDict
from collections.abc import Iterable
from operator import attrgetter
from typing import (
TYPE_CHECKING,
Any,
Dict,
Iterable,
List,
Optional,
Union,
no_type_check,
@@ -67,7 +65,7 @@ def _serialize_out(out):
@no_type_check
def _serialize_outs(outputs: List[Output]):
def _serialize_outs(outputs: list[Output]):
outs, metrics, plots = [], [], []
for out in sorted(outputs, key=attrgetter("def_path")):
bucket = outs
@@ -88,10 +86,10 @@ def _serialize_params_keys(params: Iterable["ParamsDependency"]):
at the first, and then followed by entry of other files in lexicographic
order. The keys of those custom files are also sorted in the same order.
"""
keys: List[Union[str, Dict[str, Optional[List[str]]]]] = []
keys: list[Union[str, dict[str, Optional[list[str]]]]] = []
for param_dep in sorted(params, key=attrgetter("def_path")):
# when on no_exec, params are not filled and are saved as list
k: List[str] = sorted(param_dep.params)
k: list[str] = sorted(param_dep.params)
if k and param_dep.def_path == DEFAULT_PARAMS_FILE:
keys = k + keys # type: ignore[operator,assignment]
else:
@@ -100,7 +98,7 @@ def _serialize_params_keys(params: Iterable["ParamsDependency"]):
@no_type_check
def _serialize_params_values(params: List[ParamsDependency]):
def _serialize_params_values(params: list[ParamsDependency]):
"""Returns output of following format, used for lockfile:
{'params.yaml': {'lr': '1', 'train': 2}, {'params2.yaml': {'lr': '1'}}
@@ -160,7 +158,7 @@ def to_single_stage_lockfile(stage: "Stage", **kwargs) -> dict:
assert stage.cmd
def _dumpd(item: "Output"):
ret: Dict[str, Any] = {item.PARAM_PATH: item.def_path}
ret: dict[str, Any] = {item.PARAM_PATH: item.def_path}
if item.hash_name not in LEGACY_HASH_NAMES:
ret[item.PARAM_HASH] = "md5"
if item.hash_info.isdir and kwargs.get("with_files"):
+5 -5
View File
@@ -1,6 +1,6 @@
import os
import pathlib
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
from typing import TYPE_CHECKING, Any, Optional, Union
from funcy import concat, first, lsplit, rpartial
@@ -210,7 +210,7 @@ def get_dump(stage: "Stage", **kwargs):
def split_params_deps(
stage: "Stage",
) -> Tuple[List["ParamsDependency"], List["Dependency"]]:
) -> tuple[list["ParamsDependency"], list["Dependency"]]:
from dvc.dependency import ParamsDependency
return lsplit(rpartial(isinstance, ParamsDependency), stage.deps)
@@ -265,7 +265,7 @@ def check_stage_exists(repo: "Repo", stage: Union["Stage", "PipelineStage"], pat
def validate_kwargs(
single_stage: bool = False, fname: Optional[str] = None, **kwargs
) -> Dict[str, Any]:
) -> dict[str, Any]:
"""Prepare, validate and process kwargs passed from cli"""
cmd = kwargs.get("cmd")
if not cmd and not single_stage:
@@ -288,11 +288,11 @@ def validate_kwargs(
return kwargs
def _get_stage_files(stage: "Stage") -> List[str]:
def _get_stage_files(stage: "Stage") -> list[str]:
from dvc.dvcfile import ProjectFile
from dvc.utils import relpath
ret: List[str] = []
ret: list[str] = []
file = stage.dvcfile
ret.append(file.relpath)
if isinstance(file, ProjectFile):
+2 -2
View File
@@ -2,7 +2,7 @@ import os
import shutil
from pathlib import Path
from subprocess import check_output
from typing import Dict, Optional
from typing import Optional
import pytest
from dulwich.porcelain import clone
@@ -27,7 +27,7 @@ class VirtualEnv:
virtualenv.cli_run([os.fspath(self.path)])
def run(self, cmd: str, *args: str, env: Optional[Dict[str, str]] = None) -> None:
def run(self, cmd: str, *args: str, env: Optional[dict[str, str]] = None) -> None:
exe = self.which(cmd)
check_output([exe, *args], env=env) # noqa: S603

Some files were not shown because too many files have changed in this diff Show More