drop Python 3.8 support (#10265)

* drop Python 3.8

Note that this PR only makes typing related changes.
Other changes can be done once this gets merged, they
are relatively few in numbers.

* bump build python version

* add typing_extensions to lint group
This commit is contained in:
skshetry
2024-01-30 11:33:25 +02:00
committed by GitHub
parent 966adec21c
commit b647475a64
126 changed files with 843 additions and 905 deletions
+1 -1
View File
@@ -25,7 +25,7 @@ jobs:
- uses: actions/setup-python@v5 - uses: actions/setup-python@v5
with: with:
python-version: 3.8 python-version: 3.12
- run: python -m pip install --upgrade pip setuptools_scm build twine - run: python -m pip install --upgrade pip setuptools_scm build twine
+1 -1
View File
@@ -23,7 +23,7 @@ jobs:
fail-fast: false fail-fast: false
matrix: matrix:
os: [ubuntu-20.04, windows-latest, macos-latest] os: [ubuntu-20.04, windows-latest, macos-latest]
pyv: ["3.8", "3.9", "3.10", "3.11", "3.12"] pyv: ["3.9", "3.10", "3.11", "3.12"]
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
+8 -8
View File
@@ -1,5 +1,5 @@
from dataclasses import asdict, dataclass, field, fields from dataclasses import asdict, dataclass, field, fields
from typing import Any, ClassVar, Dict, List, Optional from typing import Any, ClassVar, Optional
from funcy import compact from funcy import compact
from voluptuous import Required from voluptuous import Required
@@ -14,10 +14,10 @@ class Annotation:
desc: Optional[str] = None desc: Optional[str] = None
type: Optional[str] = None type: Optional[str] = None
labels: List[str] = field(default_factory=list) labels: list[str] = field(default_factory=list)
meta: Dict[str, Any] = field(default_factory=dict) meta: dict[str, Any] = field(default_factory=dict)
def to_dict(self) -> Dict[str, str]: def to_dict(self) -> dict[str, str]:
return compact(asdict(self)) return compact(asdict(self))
@@ -32,10 +32,10 @@ class Artifact:
path: str path: str
desc: Optional[str] = None desc: Optional[str] = None
type: Optional[str] = None type: Optional[str] = None
labels: List[str] = field(default_factory=list) labels: list[str] = field(default_factory=list)
meta: Dict[str, Any] = field(default_factory=dict) meta: dict[str, Any] = field(default_factory=dict)
def to_dict(self) -> Dict[str, str]: def to_dict(self) -> dict[str, str]:
return compact(asdict(self)) return compact(asdict(self))
@@ -46,7 +46,7 @@ ANNOTATION_SCHEMA = {
Annotation.PARAM_LABELS: [str], Annotation.PARAM_LABELS: [str],
Annotation.PARAM_META: object, Annotation.PARAM_META: object,
} }
ARTIFACT_SCHEMA: Dict[Any, Any] = { ARTIFACT_SCHEMA: dict[Any, Any] = {
Required(Artifact.PARAM_PATH): str, Required(Artifact.PARAM_PATH): str,
**ANNOTATION_SCHEMA, # type: ignore[arg-type] **ANNOTATION_SCHEMA, # type: ignore[arg-type]
} }
+3 -3
View File
@@ -1,4 +1,4 @@
from typing import Any, Dict, Optional from typing import Any, Optional
from dvc.repo import Repo from dvc.repo import Repo
@@ -8,7 +8,7 @@ def artifacts_show(
version: Optional[str] = None, version: Optional[str] = None,
stage: Optional[str] = None, stage: Optional[str] = None,
repo: Optional[str] = None, repo: Optional[str] = None,
) -> Dict[str, str]: ) -> dict[str, str]:
""" """
Return path and Git revision for an artifact in a DVC project. Return path and Git revision for an artifact in a DVC project.
@@ -36,7 +36,7 @@ def artifacts_show(
if version and stage: if version and stage:
raise ValueError("Artifact version and stage are mutually exclusive.") raise ValueError("Artifact version and stage are mutually exclusive.")
repo_kwargs: Dict[str, Any] = { repo_kwargs: dict[str, Any] = {
"subrepos": True, "subrepos": True,
"uninitialized": True, "uninitialized": True,
} }
+5 -5
View File
@@ -1,6 +1,6 @@
from contextlib import _GeneratorContextManager as GCM from contextlib import _GeneratorContextManager as GCM
from contextlib import contextmanager from contextlib import contextmanager
from typing import Any, Dict, Optional from typing import Any, Optional
from funcy import reraise from funcy import reraise
@@ -39,7 +39,7 @@ def get_url(path, repo=None, rev=None, remote=None):
from dvc.config import NoRemoteError from dvc.config import NoRemoteError
from dvc_data.index import StorageKeyError from dvc_data.index import StorageKeyError
repo_kwargs: Dict[str, Any] = {} repo_kwargs: dict[str, Any] = {}
if remote: if remote:
repo_kwargs["config"] = {"core": {"remote": remote}} repo_kwargs["config"] = {"core": {"remote": remote}}
with Repo.open( with Repo.open(
@@ -74,8 +74,8 @@ def open( # noqa: A001
remote: Optional[str] = None, remote: Optional[str] = None,
mode: str = "r", mode: str = "r",
encoding: Optional[str] = None, encoding: Optional[str] = None,
config: Optional[Dict[str, Any]] = None, config: Optional[dict[str, Any]] = None,
remote_config: Optional[Dict[str, Any]] = None, remote_config: Optional[dict[str, Any]] = None,
): ):
""" """
Opens a file tracked in a DVC project. Opens a file tracked in a DVC project.
@@ -233,7 +233,7 @@ def _open(
config=None, config=None,
remote_config=None, remote_config=None,
): ):
repo_kwargs: Dict[str, Any] = { repo_kwargs: dict[str, Any] = {
"subrepos": True, "subrepos": True,
"uninitialized": True, "uninitialized": True,
"remote": remote, "remote": remote,
+5 -5
View File
@@ -1,4 +1,4 @@
from typing import Dict, List, Optional, Union from typing import Optional, Union
from rich.text import Text from rich.text import Text
@@ -9,7 +9,7 @@ from dvc.repo.experiments.show import tabulate
def exp_save( def exp_save(
name: Optional[str] = None, name: Optional[str] = None,
force: bool = False, force: bool = False,
include_untracked: Optional[List[str]] = None, include_untracked: Optional[list[str]] = None,
): ):
""" """
Create a new DVC experiment using `exp save`. Create a new DVC experiment using `exp save`.
@@ -61,12 +61,12 @@ def _postprocess(exp_rows):
def exp_show( def exp_show(
repo: Optional[str] = None, repo: Optional[str] = None,
revs: Optional[Union[str, List[str]]] = None, revs: Optional[Union[str, list[str]]] = None,
num: int = 1, num: int = 1,
param_deps: bool = False, param_deps: bool = False,
force: bool = False, force: bool = False,
config: Optional[Dict] = None, config: Optional[dict] = None,
) -> List[Dict]: ) -> list[dict]:
"""Get DVC experiments tracked in `repo`. """Get DVC experiments tracked in `repo`.
Without arguments, this function will retrieve all experiments derived from Without arguments, this function will retrieve all experiments derived from
+4 -4
View File
@@ -1,9 +1,9 @@
from typing import List, Optional from typing import Optional
from dvc.repo import Repo from dvc.repo import Repo
def all_branches(repo: Optional[str] = None) -> List[str]: def all_branches(repo: Optional[str] = None) -> list[str]:
"""Get all Git branches in a DVC repository. """Get all Git branches in a DVC repository.
Args: Args:
@@ -20,7 +20,7 @@ def all_branches(repo: Optional[str] = None) -> List[str]:
return _repo.scm.list_branches() return _repo.scm.list_branches()
def all_commits(repo: Optional[str] = None) -> List[str]: def all_commits(repo: Optional[str] = None) -> list[str]:
"""Get all Git commits in a DVC repository. """Get all Git commits in a DVC repository.
Args: Args:
@@ -37,7 +37,7 @@ def all_commits(repo: Optional[str] = None) -> List[str]:
return _repo.scm.list_all_commits() return _repo.scm.list_all_commits()
def all_tags(repo: Optional[str] = None) -> List[str]: def all_tags(repo: Optional[str] = None) -> list[str]:
"""Get all Git tags in a DVC repository. """Get all Git tags in a DVC repository.
Args: Args:
+7 -6
View File
@@ -1,6 +1,7 @@
import typing import typing
from collections import Counter from collections import Counter
from typing import Dict, Iterable, Optional, Union from collections.abc import Iterable
from typing import Optional, Union
from funcy import first from funcy import first
@@ -8,7 +9,7 @@ from dvc.repo import Repo
def _postprocess(results): def _postprocess(results):
processed: Dict[str, Dict] = {} processed: dict[str, dict] = {}
for rev, rev_data in results.items(): for rev, rev_data in results.items():
if not rev_data: if not rev_data:
continue continue
@@ -36,8 +37,8 @@ def metrics_show(
*targets: str, *targets: str,
repo: Optional[str] = None, repo: Optional[str] = None,
rev: Optional[str] = None, rev: Optional[str] = None,
config: Optional[Dict] = None, config: Optional[dict] = None,
) -> Dict: ) -> dict:
"""Get metrics tracked in `repo`. """Get metrics tracked in `repo`.
Without arguments, this function will retrieve all metrics from all tracked Without arguments, this function will retrieve all metrics from all tracked
@@ -159,8 +160,8 @@ def params_show(
stages: Optional[Union[str, Iterable[str]]] = None, stages: Optional[Union[str, Iterable[str]]] = None,
rev: Optional[str] = None, rev: Optional[str] = None,
deps: bool = False, deps: bool = False,
config: Optional[Dict] = None, config: Optional[dict] = None,
) -> Dict: ) -> dict:
"""Get parameters tracked in `repo`. """Get parameters tracked in `repo`.
Without arguments, this function will retrieve all params from all tracked Without arguments, this function will retrieve all params from all tracked
+2 -2
View File
@@ -1,5 +1,5 @@
import os import os
from typing import TYPE_CHECKING, Optional, Tuple from typing import TYPE_CHECKING, Optional
from dvc.fs import GitFileSystem, Schemes from dvc.fs import GitFileSystem, Schemes
from dvc_data.hashfile.db import get_odb from dvc_data.hashfile.db import get_odb
@@ -15,7 +15,7 @@ def _get_odb(
repo, repo,
settings, settings,
fs=None, fs=None,
prefix: Optional[Tuple[str, ...]] = None, prefix: Optional[tuple[str, ...]] = None,
hash_name: Optional[str] = None, hash_name: Optional[str] = None,
**kwargs, **kwargs,
): ):
+4 -4
View File
@@ -1,4 +1,4 @@
from typing import TYPE_CHECKING, ClassVar, Dict, Tuple from typing import TYPE_CHECKING, ClassVar
from funcy import chunks, compact, log_durations from funcy import chunks, compact, log_durations
@@ -17,14 +17,14 @@ logger = logger.getChild(__name__)
class CmdDataStatus(CmdBase): class CmdDataStatus(CmdBase):
COLORS: ClassVar[Dict[str, str]] = { COLORS: ClassVar[dict[str, str]] = {
"not_in_remote": "red", "not_in_remote": "red",
"not_in_cache": "red", "not_in_cache": "red",
"committed": "green", "committed": "green",
"uncommitted": "yellow", "uncommitted": "yellow",
"untracked": "cyan", "untracked": "cyan",
} }
LABELS: ClassVar[Dict[str, str]] = { LABELS: ClassVar[dict[str, str]] = {
"not_in_remote": "Not in remote", "not_in_remote": "Not in remote",
"not_in_cache": "Not in cache", "not_in_cache": "Not in cache",
"committed": "DVC committed changes", "committed": "DVC committed changes",
@@ -32,7 +32,7 @@ class CmdDataStatus(CmdBase):
"untracked": "Untracked files", "untracked": "Untracked files",
"unchanged": "DVC unchanged files", "unchanged": "DVC unchanged files",
} }
HINTS: ClassVar[Dict[str, Tuple[str, ...]]] = { HINTS: ClassVar[dict[str, tuple[str, ...]]] = {
"not_in_remote": ('use "dvc push <file>..." to upload files',), "not_in_remote": ('use "dvc push <file>..." to upload files',),
"not_in_cache": ('use "dvc fetch <file>..." to download files',), "not_in_cache": ('use "dvc fetch <file>..." to download files',),
"committed": ("git commit the corresponding dvc files to update the repo",), "committed": ("git commit the corresponding dvc files to update the repo",),
+2 -2
View File
@@ -1,4 +1,4 @@
from typing import Any, Dict from typing import Any
from dvc.cli import completion, formatter from dvc.cli import completion, formatter
from dvc.cli.command import CmdBase from dvc.cli.command import CmdBase
@@ -11,7 +11,7 @@ logger = logger.getChild(__name__)
class CmdExperimentsPush(CmdBase): class CmdExperimentsPush(CmdBase):
@staticmethod @staticmethod
def log_result(result: Dict[str, Any], remote: str): def log_result(result: dict[str, Any], remote: str):
from dvc.utils import humanize from dvc.utils import humanize
def join_exps(exps): def join_exps(exps):
+3 -2
View File
@@ -1,7 +1,8 @@
import argparse import argparse
import re import re
from collections.abc import Iterable
from datetime import date, datetime from datetime import date, datetime
from typing import TYPE_CHECKING, Dict, Iterable from typing import TYPE_CHECKING
from funcy import lmap from funcy import lmap
@@ -57,7 +58,7 @@ def baseline_styler(typ):
def show_experiments( def show_experiments(
td: "TabularData", td: "TabularData",
headers: Dict[str, Iterable[str]], headers: dict[str, Iterable[str]],
keep=None, keep=None,
drop=None, drop=None,
pager=True, pager=True,
+4 -4
View File
@@ -1,6 +1,6 @@
import argparse import argparse
import os import os
from typing import TYPE_CHECKING, Dict, List, Optional from typing import TYPE_CHECKING, Optional
from funcy import compact, first, get_in from funcy import compact, first, get_in
@@ -20,14 +20,14 @@ logger = logger.getChild(__name__)
def _show_json( def _show_json(
renderers_with_errors: List["RendererWithErrors"], renderers_with_errors: list["RendererWithErrors"],
split=False, split=False,
errors: Optional[Dict[str, Exception]] = None, errors: Optional[dict[str, Exception]] = None,
): ):
from dvc.render.convert import to_json from dvc.render.convert import to_json
from dvc.utils.serialize import encode_exception from dvc.utils.serialize import encode_exception
all_errors: List[Dict] = [] all_errors: list[dict] = []
data = {} data = {}
for renderer, src_errors, def_errors in renderers_with_errors: for renderer, src_errors, def_errors in renderers_with_errors:
+6 -5
View File
@@ -1,8 +1,9 @@
import argparse import argparse
import logging import logging
from collections.abc import Iterable
from contextlib import contextmanager from contextlib import contextmanager
from itertools import chain, filterfalse from itertools import chain, filterfalse
from typing import TYPE_CHECKING, Dict, Iterable, List from typing import TYPE_CHECKING
from dvc.cli import completion, formatter from dvc.cli import completion, formatter
from dvc.cli.command import CmdBase from dvc.cli.command import CmdBase
@@ -33,7 +34,7 @@ def generate_description(stage: "Stage") -> str:
def is_plot_or_metric(out: "Output"): def is_plot_or_metric(out: "Output"):
return bool(out.plot) or bool(out.metric) return bool(out.plot) or bool(out.metric)
desc: List[str] = [] desc: list[str] = []
outs = list(filterfalse(is_plot_or_metric, stage.outs)) outs = list(filterfalse(is_plot_or_metric, stage.outs))
if outs: if outs:
@@ -55,7 +56,7 @@ def prepare_stages_data(
stages: Iterable["Stage"], stages: Iterable["Stage"],
description: bool = True, description: bool = True,
max_length: int = MAX_TEXT_LENGTH, max_length: int = MAX_TEXT_LENGTH,
) -> Dict[str, str]: ) -> dict[str, str]:
return { return {
stage.addressing: ( stage.addressing: (
prepare_description(stage, max_length=max_length) if description else "" prepare_description(stage, max_length=max_length) if description else ""
@@ -67,7 +68,7 @@ def prepare_stages_data(
class CmdStageList(CmdBase): class CmdStageList(CmdBase):
def _get_stages(self) -> Iterable["Stage"]: def _get_stages(self) -> Iterable["Stage"]:
if self.args.all: if self.args.all:
stages: List["Stage"] = self.repo.index.stages stages: list["Stage"] = self.repo.index.stages
logger.trace("%d no. of stages found", len(stages)) logger.trace("%d no. of stages found", len(stages))
return stages return stages
@@ -96,7 +97,7 @@ class CmdStageList(CmdBase):
return 0 return 0
def parse_cmd(commands: List[str]) -> str: def parse_cmd(commands: list[str]) -> str:
""" """
We need to take into account two cases: We need to take into account two cases:
+24 -26
View File
@@ -1,20 +1,18 @@
from collections import abc from collections import abc
from collections.abc import (
ItemsView,
Iterable,
Iterator,
Mapping,
MutableSequence,
Sequence,
)
from itertools import chain, repeat, zip_longest from itertools import chain, repeat, zip_longest
from operator import itemgetter from operator import itemgetter
from typing import ( from typing import (
TYPE_CHECKING, TYPE_CHECKING,
Any, Any,
Dict,
ItemsView,
Iterable,
Iterator,
List,
Mapping,
MutableSequence,
Optional, Optional,
Sequence,
Set,
Tuple,
Union, Union,
overload, overload,
) )
@@ -25,7 +23,7 @@ if TYPE_CHECKING:
from dvc.ui.table import CellT from dvc.ui.table import CellT
class Column(List["CellT"]): class Column(list["CellT"]):
pass pass
@@ -35,13 +33,13 @@ def with_value(value, default):
class TabularData(MutableSequence[Sequence["CellT"]]): class TabularData(MutableSequence[Sequence["CellT"]]):
def __init__(self, columns: Sequence[str], fill_value: Optional[str] = ""): def __init__(self, columns: Sequence[str], fill_value: Optional[str] = ""):
self._columns: Dict[str, Column] = {name: Column() for name in columns} self._columns: dict[str, Column] = {name: Column() for name in columns}
self._keys: List[str] = list(columns) self._keys: list[str] = list(columns)
self._fill_value = fill_value self._fill_value = fill_value
self._protected: Set[str] = set() self._protected: set[str] = set()
@property @property
def columns(self) -> List[Column]: def columns(self) -> list[Column]:
return list(map(self.column, self.keys())) return list(map(self.column, self.keys()))
def is_protected(self, col_name) -> bool: def is_protected(self, col_name) -> bool:
@@ -60,10 +58,10 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
projection = {k: self.column(k) for k in self.keys()} projection = {k: self.column(k) for k in self.keys()}
return projection.items() return projection.items()
def keys(self) -> List[str]: def keys(self) -> list[str]:
return self._keys return self._keys
def _iter_col_row(self, row: Sequence["CellT"]) -> Iterator[Tuple["CellT", Column]]: def _iter_col_row(self, row: Sequence["CellT"]) -> Iterator[tuple["CellT", Column]]:
for val, col in zip_longest(row, self.columns): for val, col in zip_longest(row, self.columns):
if col is None: if col is None:
break break
@@ -81,7 +79,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
for val, col in self._iter_col_row(value): for val, col in self._iter_col_row(value):
col.insert(index, val) col.insert(index, val)
def __iter__(self) -> Iterator[List["CellT"]]: def __iter__(self) -> Iterator[list["CellT"]]:
return map(list, zip(*self.columns)) return map(list, zip(*self.columns))
def __getattr__(self, item: str) -> Column: def __getattr__(self, item: str) -> Column:
@@ -127,7 +125,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
return len(self.columns[0]) return len(self.columns[0])
@property @property
def shape(self) -> Tuple[int, int]: def shape(self) -> tuple[int, int]:
return len(self.columns), len(self) return len(self.columns), len(self)
def drop(self, *col_names: str) -> None: def drop(self, *col_names: str) -> None:
@@ -170,7 +168,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
if key not in keys: if key not in keys:
self.add_column(key) self.add_column(key)
row: List["CellT"] = [ row: list["CellT"] = [
with_value(d.get(key), self._fill_value) for key in self.keys() with_value(d.get(key), self._fill_value) for key in self.keys()
] ]
self.append(row) self.append(row)
@@ -185,7 +183,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
def as_dict( def as_dict(
self, cols: Optional[Iterable[str]] = None self, cols: Optional[Iterable[str]] = None
) -> Iterable[Dict[str, "CellT"]]: ) -> Iterable[dict[str, "CellT"]]:
keys = self.keys() if cols is None else set(cols) keys = self.keys() if cols is None else set(cols)
return [{k: self._columns[k][i] for k in keys} for i in range(len(self))] return [{k: self._columns[k][i] for k in keys} for i in range(len(self))]
@@ -202,7 +200,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
if how not in ["any", "all"]: if how not in ["any", "all"]:
raise ValueError(f"Invalid 'how' value {how}. Choose one of ['any', 'all']") raise ValueError(f"Invalid 'how' value {how}. Choose one of ['any', 'all']")
match_line: Set = set() match_line: set = set()
match_any = True match_any = True
if how == "all": if how == "all":
match_any = False match_any = False
@@ -245,7 +243,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
) )
if axis == "cols": if axis == "cols":
cols_to_drop: List[str] = [] cols_to_drop: list[str] = []
for n_col, col in enumerate(self.columns): for n_col, col in enumerate(self.columns):
if subset and self.keys()[n_col] not in subset: if subset and self.keys()[n_col] not in subset:
continue continue
@@ -259,7 +257,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
elif axis == "rows": elif axis == "rows":
unique_rows = [] unique_rows = []
rows_to_drop: List[int] = [] rows_to_drop: list[int] = []
for n_row, row in enumerate(self): for n_row, row in enumerate(self):
if subset: if subset:
row = [ row = [
@@ -319,7 +317,7 @@ def diff_table(
) -> TabularData: ) -> TabularData:
a_rev = a_rev or "HEAD" a_rev = a_rev or "HEAD"
b_rev = b_rev or "workspace" b_rev = b_rev or "workspace"
headers: List[str] = ["Path", title, a_rev, b_rev, "Change"] headers: list[str] = ["Path", title, a_rev, b_rev, "Change"]
fill_value = "-" fill_value = "-"
td = TabularData(headers, fill_value=fill_value) td = TabularData(headers, fill_value=fill_value)
@@ -394,7 +392,7 @@ def metrics_table(
for branch, val in metrics.items(): for branch, val in metrics.items():
for fname, metric in val.get("data", {}).items(): for fname, metric in val.get("data", {}).items():
row_data: Dict[str, str] = {"Revision": branch, "Path": fname} row_data: dict[str, str] = {"Revision": branch, "Path": fname}
metric = metric.get("data", {}) metric = metric.get("data", {})
flattened = ( flattened = (
flatten(format_dict(metric)) flatten(format_dict(metric))
+4 -4
View File
@@ -3,7 +3,7 @@ import os
import re import re
from contextlib import contextmanager from contextlib import contextmanager
from functools import partial from functools import partial
from typing import TYPE_CHECKING, Dict, Optional from typing import TYPE_CHECKING, Optional
from funcy import compact, memoize, re_find from funcy import compact, memoize, re_find
@@ -138,7 +138,7 @@ class Config(dict):
return system_config_dir() return system_config_dir()
@cached_property @cached_property
def files(self) -> Dict[str, str]: def files(self) -> dict[str, str]:
files = { files = {
level: os.path.join(self.get_dir(level), self.CONFIG) level: os.path.join(self.get_dir(level), self.CONFIG)
for level in ("system", "global") for level in ("system", "global")
@@ -344,7 +344,7 @@ class Config(dict):
return Schema(dirs_schema, extra=ALLOW_EXTRA)(conf) return Schema(dirs_schema, extra=ALLOW_EXTRA)(conf)
def load_config_to_level(self, level=None): def load_config_to_level(self, level=None):
merged_conf: Dict = {} merged_conf: dict = {}
for merge_level in self.LEVELS: for merge_level in self.LEVELS:
if merge_level == level: if merge_level == level:
break break
@@ -390,7 +390,7 @@ class Config(dict):
def _parse_named(conf): def _parse_named(conf):
result: Dict[str, Dict] = {"remote": {}, "machine": {}, "db": {}} result: dict[str, dict] = {"remote": {}, "machine": {}, "db": {}}
for section, val in conf.items(): for section, val in conf.items():
match = re_find(r'^\s*(remote|machine|db)\s*"(.*)"\s*$', section) match = re_find(r'^\s*(remote|machine|db)\s*"(.*)"\s*$', section)
+9 -8
View File
@@ -5,13 +5,14 @@ import logging
import os import os
import subprocess import subprocess
import sys import sys
from collections.abc import Mapping, Sequence
from contextlib import nullcontext from contextlib import nullcontext
from typing import TYPE_CHECKING, Any, List, Mapping, Optional, Sequence, Union from typing import TYPE_CHECKING, Any, Optional, Union
from dvc.log import logger from dvc.log import logger
if TYPE_CHECKING: if TYPE_CHECKING:
from typing import ContextManager from contextlib import AbstractContextManager
from dvc.env import DVC_DAEMON, DVC_DAEMON_LOGFILE from dvc.env import DVC_DAEMON, DVC_DAEMON_LOGFILE
from dvc.utils import fix_env, is_binary from dvc.utils import fix_env, is_binary
@@ -55,7 +56,7 @@ def _win_detached_subprocess(args: Sequence[str], **kwargs) -> int:
return popen.pid return popen.pid
def _get_dvc_args() -> List[str]: def _get_dvc_args() -> list[str]:
args = [sys.executable] args = [sys.executable]
if not is_binary(): if not is_binary():
root_dir = os.path.abspath(os.path.dirname(__file__)) root_dir = os.path.abspath(os.path.dirname(__file__))
@@ -136,7 +137,7 @@ def _map_log_level_to_flag() -> Optional[str]:
return flags.get(logger.getEffectiveLevel()) return flags.get(logger.getEffectiveLevel())
def daemon(args: List[str]) -> None: def daemon(args: list[str]) -> None:
"""Launch a `dvc daemon` command in a detached process. """Launch a `dvc daemon` command in a detached process.
Args: Args:
@@ -148,12 +149,12 @@ def daemon(args: List[str]) -> None:
def _spawn( def _spawn(
args: List[str], args: list[str],
executable: Optional[Union[str, List[str]]] = None, executable: Optional[Union[str, list[str]]] = None,
env: Optional[Mapping[str, str]] = None, env: Optional[Mapping[str, str]] = None,
output_file: Optional[str] = None, output_file: Optional[str] = None,
) -> int: ) -> int:
file: "ContextManager[Any]" = nullcontext() file: "AbstractContextManager[Any]" = nullcontext()
kwargs = {} kwargs = {}
if output_file: if output_file:
file = open(output_file, "ab") # noqa: SIM115 file = open(output_file, "ab") # noqa: SIM115
@@ -168,7 +169,7 @@ def _spawn(
return _detached_subprocess(executable + args, env=env, **kwargs) return _detached_subprocess(executable + args, env=env, **kwargs)
def daemonize(args: List[str], executable: Union[None, str, List[str]] = None) -> None: def daemonize(args: list[str], executable: Union[None, str, list[str]] = None) -> None:
if os.name not in ("posix", "nt"): if os.name not in ("posix", "nt"):
return return
+3 -2
View File
@@ -1,6 +1,7 @@
"""Manages dvc remotes that user can use with push/pull/status commands.""" """Manages dvc remotes that user can use with push/pull/status commands."""
from typing import TYPE_CHECKING, Iterable, Optional, Set, Tuple from collections.abc import Iterable
from typing import TYPE_CHECKING, Optional
from dvc.config import NoRemoteError, RemoteConfigError from dvc.config import NoRemoteError, RemoteConfigError
from dvc.log import logger from dvc.log import logger
@@ -50,7 +51,7 @@ class Remote:
def _split_legacy_hash_infos( def _split_legacy_hash_infos(
hash_infos: Iterable["HashInfo"], hash_infos: Iterable["HashInfo"],
) -> Tuple[Set["HashInfo"], Set["HashInfo"]]: ) -> tuple[set["HashInfo"], set["HashInfo"]]:
from dvc.cachemgr import LEGACY_HASH_NAMES from dvc.cachemgr import LEGACY_HASH_NAMES
legacy = set() legacy = set()
+4 -3
View File
@@ -1,8 +1,9 @@
import os import os
from collections.abc import Iterator
from contextlib import contextmanager from contextlib import contextmanager
from dataclasses import dataclass from dataclasses import dataclass
from tempfile import NamedTemporaryFile from tempfile import NamedTemporaryFile
from typing import TYPE_CHECKING, Any, Callable, Dict, Iterator, Optional, Union from typing import TYPE_CHECKING, Any, Callable, Optional, Union
from sqlalchemy import create_engine from sqlalchemy import create_engine
from sqlalchemy.engine import make_url as _make_url from sqlalchemy.engine import make_url as _make_url
@@ -30,7 +31,7 @@ def make_url(url: Union["URL", str], **kwargs: Any) -> "URL":
return _make_url(url).set(**kwargs) return _make_url(url).set(**kwargs)
def url_from_config(config: Union[str, "URL", Dict[str, str]]) -> "URL": def url_from_config(config: Union[str, "URL", dict[str, str]]) -> "URL":
if isinstance(config, dict): if isinstance(config, dict):
return make_url(**config) return make_url(**config)
return make_url(config) return make_url(config)
@@ -118,7 +119,7 @@ def handle_error(url: "URL"):
@contextmanager @contextmanager
def client( def client(
url_or_config: Union[str, "URL", Dict[str, str]], **engine_kwargs: Any url_or_config: Union[str, "URL", dict[str, str]], **engine_kwargs: Any
) -> Iterator[Client]: ) -> Iterator[Client]:
url = url_from_config(url_or_config) url = url_from_config(url_or_config)
echo = env2bool(env.DVC_SQLALCHEMY_ECHO, False) echo = env2bool(env.DVC_SQLALCHEMY_ECHO, False)
+4 -3
View File
@@ -1,5 +1,6 @@
from collections import defaultdict from collections import defaultdict
from typing import Any, Dict, List, Mapping, Set from collections.abc import Mapping
from typing import Any
from dvc.output import ARTIFACT_SCHEMA, DIR_FILES_SCHEMA, Output from dvc.output import ARTIFACT_SCHEMA, DIR_FILES_SCHEMA, Output
@@ -66,12 +67,12 @@ def loads_from(stage, s_list, erepo=None, fs_config=None, db=None):
] ]
def _merge_params(s_list) -> Dict[str, List[str]]: def _merge_params(s_list) -> dict[str, list[str]]:
d = defaultdict(list) d = defaultdict(list)
default_file = ParamsDependency.DEFAULT_PARAMS_FILE default_file = ParamsDependency.DEFAULT_PARAMS_FILE
# figure out completely tracked params file, and ignore specific keys # figure out completely tracked params file, and ignore specific keys
wholly_tracked: Set[str] = set() wholly_tracked: set[str] = set()
for key in s_list: for key in s_list:
if not isinstance(key, dict): if not isinstance(key, dict):
continue continue
+4 -6
View File
@@ -1,5 +1,3 @@
from typing import Dict, Type
from dvc.exceptions import DvcException from dvc.exceptions import DvcException
from dvc.fs import download as fs_download from dvc.fs import download as fs_download
from dvc.output import Output from dvc.output import Output
@@ -25,11 +23,11 @@ class DependencyIsStageFileError(DvcException):
class Dependency(Output): class Dependency(Output):
IS_DEPENDENCY = True IS_DEPENDENCY = True
DoesNotExistError: Type[DvcException] = DependencyDoesNotExistError DoesNotExistError: type[DvcException] = DependencyDoesNotExistError
IsNotFileOrDirError: Type[DvcException] = DependencyIsNotFileOrDirError IsNotFileOrDirError: type[DvcException] = DependencyIsNotFileOrDirError
IsStageFileError: Type[DvcException] = DependencyIsStageFileError IsStageFileError: type[DvcException] = DependencyIsStageFileError
def workspace_status(self) -> Dict[str, str]: def workspace_status(self) -> dict[str, str]:
if self.fs.version_aware: if self.fs.version_aware:
old_fs_path = self.fs_path old_fs_path = self.fs_path
try: try:
+5 -4
View File
@@ -1,5 +1,6 @@
from collections.abc import Iterator
from contextlib import contextmanager from contextlib import contextmanager
from typing import TYPE_CHECKING, Any, Callable, ClassVar, Dict, Iterator, Optional from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional
from funcy import compact, log_durations from funcy import compact, log_durations
@@ -31,7 +32,7 @@ def download_progress(to: "Output") -> Iterator[Callable[[int], Any]]:
class AbstractDependency(Dependency): class AbstractDependency(Dependency):
"""Dependency without workspace/fs/fs_path""" """Dependency without workspace/fs/fs_path"""
def __init__(self, stage: "Stage", info: Dict[str, Any], *args, **kwargs): def __init__(self, stage: "Stage", info: dict[str, Any], *args, **kwargs):
self.repo = stage.repo self.repo = stage.repo
self.stage = stage self.stage = stage
self.fs = None self.fs = None
@@ -46,7 +47,7 @@ class DbDependency(AbstractDependency):
PARAM_QUERY = "query" PARAM_QUERY = "query"
PARAM_TABLE = "table" PARAM_TABLE = "table"
PARAM_FILE_FORMAT = "file_format" PARAM_FILE_FORMAT = "file_format"
DB_SCHEMA: ClassVar[Dict] = { DB_SCHEMA: ClassVar[dict] = {
PARAM_DB: { PARAM_DB: {
PARAM_QUERY: str, PARAM_QUERY: str,
PARAM_CONNECTION: str, PARAM_CONNECTION: str,
@@ -57,7 +58,7 @@ class DbDependency(AbstractDependency):
def __init__(self, stage: "Stage", info, *args, **kwargs): def __init__(self, stage: "Stage", info, *args, **kwargs):
super().__init__(stage, info, *args, **kwargs) super().__init__(stage, info, *args, **kwargs)
self.db_info: Dict[str, str] = self.info.get(self.PARAM_DB, {}) self.db_info: dict[str, str] = self.info.get(self.PARAM_DB, {})
self.connection = self.db_info.get(self.PARAM_CONNECTION) self.connection = self.db_info.get(self.PARAM_CONNECTION)
@property @property
+4 -4
View File
@@ -1,7 +1,7 @@
import os import os
import typing import typing
from collections import defaultdict from collections import defaultdict
from typing import TYPE_CHECKING, Any, Dict, List, Optional from typing import TYPE_CHECKING, Any, Optional
import dpath import dpath
@@ -37,7 +37,7 @@ class BadParamFileError(DvcException):
def read_param_file( def read_param_file(
fs: "FileSystem", fs: "FileSystem",
path: str, path: str,
key_paths: Optional[List[str]] = None, key_paths: Optional[list[str]] = None,
flatten: bool = False, flatten: bool = False,
**load_kwargs, **load_kwargs,
) -> Any: ) -> Any:
@@ -110,7 +110,7 @@ class ParamsDependency(Dependency):
def read_params( def read_params(
self, flatten: bool = True, **kwargs: typing.Any self, flatten: bool = True, **kwargs: typing.Any
) -> Dict[str, typing.Any]: ) -> dict[str, typing.Any]:
try: try:
self.validate_filepath() self.validate_filepath()
except MissingParamsFile: except MissingParamsFile:
@@ -134,7 +134,7 @@ class ParamsDependency(Dependency):
from funcy import ldistinct from funcy import ldistinct
status: Dict[str, Any] = defaultdict(dict) status: dict[str, Any] = defaultdict(dict)
info = self.hash_info.value if self.hash_info else {} info = self.hash_info.value if self.hash_info else {}
assert isinstance(info, dict) assert isinstance(info, dict)
actual = self.read_params() actual = self.read_params()
+5 -5
View File
@@ -1,5 +1,5 @@
from copy import deepcopy from copy import deepcopy
from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union
import voluptuous as vol import voluptuous as vol
@@ -20,7 +20,7 @@ class RepoDependency(Dependency):
PARAM_CONFIG = "config" PARAM_CONFIG = "config"
PARAM_REMOTE = "remote" PARAM_REMOTE = "remote"
REPO_SCHEMA: ClassVar[Dict] = { REPO_SCHEMA: ClassVar[dict] = {
PARAM_REPO: { PARAM_REPO: {
vol.Required(PARAM_URL): str, vol.Required(PARAM_URL): str,
PARAM_REV: str, PARAM_REV: str,
@@ -30,7 +30,7 @@ class RepoDependency(Dependency):
} }
} }
def __init__(self, def_repo: Dict[str, Any], stage: "Stage", *args, **kwargs): def __init__(self, def_repo: dict[str, Any], stage: "Stage", *args, **kwargs):
self.def_repo = def_repo self.def_repo = def_repo
super().__init__(stage, *args, **kwargs) super().__init__(stage, *args, **kwargs)
@@ -65,7 +65,7 @@ class RepoDependency(Dependency):
self.def_repo[self.PARAM_REV_LOCK] = rev self.def_repo[self.PARAM_REV_LOCK] = rev
@classmethod @classmethod
def _dump_def_repo(cls, def_repo) -> Dict[str, str]: def _dump_def_repo(cls, def_repo) -> dict[str, str]:
repo = {cls.PARAM_URL: def_repo[cls.PARAM_URL]} repo = {cls.PARAM_URL: def_repo[cls.PARAM_URL]}
rev = def_repo.get(cls.PARAM_REV) rev = def_repo.get(cls.PARAM_REV)
@@ -85,7 +85,7 @@ class RepoDependency(Dependency):
repo[cls.PARAM_REMOTE] = remote repo[cls.PARAM_REMOTE] = remote
return repo return repo
def dumpd(self, **kwargs) -> Dict[str, Union[str, Dict[str, str]]]: def dumpd(self, **kwargs) -> dict[str, Union[str, dict[str, str]]]:
return { return {
self.PARAM_PATH: self.def_path, self.PARAM_PATH: self.def_path,
self.PARAM_REPO: self._dump_def_repo(self.def_repo), self.PARAM_REPO: self._dump_def_repo(self.def_repo),
+10 -13
View File
@@ -5,10 +5,7 @@ from typing import (
Any, Any,
Callable, Callable,
ClassVar, ClassVar,
Dict,
List,
Optional, Optional,
Tuple,
TypeVar, TypeVar,
Union, Union,
) )
@@ -134,7 +131,7 @@ class FileMixin:
d, _ = self._load(**kwargs) d, _ = self._load(**kwargs)
return d return d
def _load(self, **kwargs: Any) -> Tuple[Any, str]: def _load(self, **kwargs: Any) -> tuple[Any, str]:
# it raises the proper exceptions by priority: # it raises the proper exceptions by priority:
# 1. when the file doesn't exists # 1. when the file doesn't exists
# 2. filename is not a DVC file # 2. filename is not a DVC file
@@ -157,7 +154,7 @@ class FileMixin:
return validate(d, cls.SCHEMA, path=fname) # type: ignore[arg-type] return validate(d, cls.SCHEMA, path=fname) # type: ignore[arg-type]
def _load_yaml(self, **kwargs: Any) -> Tuple[Any, str]: def _load_yaml(self, **kwargs: Any) -> tuple[Any, str]:
from dvc.utils import strictyaml from dvc.utils import strictyaml
return strictyaml.load( return strictyaml.load(
@@ -182,10 +179,10 @@ class SingleStageFile(FileMixin):
from dvc.schema import COMPILED_SINGLE_STAGE_SCHEMA as SCHEMA from dvc.schema import COMPILED_SINGLE_STAGE_SCHEMA as SCHEMA
from dvc.stage.loader import SingleStageLoader as LOADER # noqa: N814 from dvc.stage.loader import SingleStageLoader as LOADER # noqa: N814
metrics: ClassVar[List[str]] = [] metrics: ClassVar[list[str]] = []
plots: ClassVar[Any] = {} plots: ClassVar[Any] = {}
params: ClassVar[List[str]] = [] params: ClassVar[list[str]] = []
artifacts: ClassVar[Dict[str, Optional[Dict[str, Any]]]] = {} artifacts: ClassVar[dict[str, Optional[dict[str, Any]]]] = {}
@property @property
def stage(self) -> "Stage": def stage(self) -> "Stage":
@@ -284,11 +281,11 @@ class ProjectFile(FileMixin):
raise DvcException("ProjectFile has multiple stages. Please specify it's name.") raise DvcException("ProjectFile has multiple stages. Please specify it's name.")
@cached_property @cached_property
def contents(self) -> Dict[str, Any]: def contents(self) -> dict[str, Any]:
return self._load()[0] return self._load()[0]
@cached_property @cached_property
def lockfile_contents(self) -> Dict[str, Any]: def lockfile_contents(self) -> dict[str, Any]:
return self._lockfile.load() return self._lockfile.load()
@cached_property @cached_property
@@ -303,7 +300,7 @@ class ProjectFile(FileMixin):
return self.LOADER(self, self.contents, self.lockfile_contents) return self.LOADER(self, self.contents, self.lockfile_contents)
@property @property
def metrics(self) -> List[str]: def metrics(self) -> list[str]:
return self.contents.get("metrics", []) return self.contents.get("metrics", [])
@property @property
@@ -311,11 +308,11 @@ class ProjectFile(FileMixin):
return self.contents.get("plots", {}) return self.contents.get("plots", {})
@property @property
def params(self) -> List[str]: def params(self) -> list[str]:
return self.contents.get("params", []) return self.contents.get("params", [])
@property @property
def artifacts(self) -> Dict[str, Optional[Dict[str, Any]]]: def artifacts(self) -> dict[str, Optional[dict[str, Any]]]:
return self.contents.get("artifacts", {}) return self.contents.get("artifacts", {})
def remove(self, force=False): def remove(self, force=False):
+3 -3
View File
@@ -1,6 +1,6 @@
"""Exceptions raised by the dvc.""" """Exceptions raised by the dvc."""
import errno import errno
from typing import TYPE_CHECKING, Dict, List, Optional, Set from typing import TYPE_CHECKING, Optional
from dvc.utils import format_link from dvc.utils import format_link
@@ -34,7 +34,7 @@ class OutputDuplicationError(DvcException):
stages (list): list of paths to stages. stages (list): list of paths to stages.
""" """
def __init__(self, output: str, stages: Set["Stage"]): def __init__(self, output: str, stages: set["Stage"]):
from funcy import first from funcy import first
assert isinstance(output, str) assert isinstance(output, str)
@@ -241,7 +241,7 @@ class UploadError(FileTransferError):
class CheckoutError(DvcException): class CheckoutError(DvcException):
def __init__(self, target_infos: List[str], stats: Dict[str, List[str]]): def __init__(self, target_infos: list[str], stats: dict[str, list[str]]):
from dvc.utils import error_link from dvc.utils import error_link
self.target_infos = target_infos self.target_infos = target_infos
+3 -3
View File
@@ -1,5 +1,5 @@
from contextlib import ExitStack from contextlib import ExitStack
from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optional, Union from typing import TYPE_CHECKING, Any, BinaryIO, Optional, Union
from dvc.progress import Tqdm from dvc.progress import Tqdm
from dvc.utils.objects import cached_property from dvc.utils.objects import cached_property
@@ -55,7 +55,7 @@ class TqdmCallback(Callback):
self, self,
path_1: "Union[str, BinaryIO]", path_1: "Union[str, BinaryIO]",
path_2: str, path_2: str,
kwargs: Dict[str, Any], kwargs: dict[str, Any],
child: Optional[Callback] = None, child: Optional[Callback] = None,
): ):
child = child or TqdmCallback( child = child or TqdmCallback(
@@ -130,7 +130,7 @@ class RichCallback(Callback):
self, self,
path_1: Union[str, BinaryIO], path_1: Union[str, BinaryIO],
path_2: str, path_2: str,
kwargs: Dict[str, Any], kwargs: dict[str, Any],
child: Optional["Callback"] = None, child: Optional["Callback"] = None,
): ):
child = child or RichCallback( child = child or RichCallback(
+9 -9
View File
@@ -6,7 +6,7 @@ import posixpath
import threading import threading
from collections import deque from collections import deque
from contextlib import ExitStack, suppress from contextlib import ExitStack, suppress
from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple, Type, Union from typing import TYPE_CHECKING, Any, Callable, Optional, Union
from fsspec.spec import AbstractFileSystem from fsspec.spec import AbstractFileSystem
from funcy import wrap_with from funcy import wrap_with
@@ -22,8 +22,8 @@ if TYPE_CHECKING:
logger = logger.getChild(__name__) logger = logger.getChild(__name__)
RepoFactory = Union[Callable[..., "Repo"], Type["Repo"]] RepoFactory = Union[Callable[..., "Repo"], type["Repo"]]
Key = Tuple[str, ...] Key = tuple[str, ...]
def as_posix(path: str) -> str: def as_posix(path: str) -> str:
@@ -85,7 +85,7 @@ class _DVCFileSystem(AbstractFileSystem):
subrepos: bool = False, subrepos: bool = False,
repo_factory: Optional[RepoFactory] = None, repo_factory: Optional[RepoFactory] = None,
fo: Optional[str] = None, fo: Optional[str] = None,
target_options: Optional[Dict[str, Any]] = None, # noqa: ARG002 target_options: Optional[dict[str, Any]] = None, # noqa: ARG002
target_protocol: Optional[str] = None, # noqa: ARG002 target_protocol: Optional[str] = None, # noqa: ARG002
config: Optional["DictStrAny"] = None, config: Optional["DictStrAny"] = None,
remote: Optional[str] = None, remote: Optional[str] = None,
@@ -145,7 +145,7 @@ class _DVCFileSystem(AbstractFileSystem):
} }
def getcwd(self): def getcwd(self):
relparts: Tuple[str, ...] = () relparts: tuple[str, ...] = ()
assert self.repo is not None assert self.repo is not None
if self.repo.fs.isin(self.repo.fs.getcwd(), self.repo.root_dir): if self.repo.fs.isin(self.repo.fs.getcwd(), self.repo.root_dir):
relparts = self.repo.fs.relparts(self.repo.fs.getcwd(), self.repo.root_dir) relparts = self.repo.fs.relparts(self.repo.fs.getcwd(), self.repo.root_dir)
@@ -156,7 +156,7 @@ class _DVCFileSystem(AbstractFileSystem):
return posixpath.join(*parts) return posixpath.join(*parts)
@classmethod @classmethod
def parts(cls, path: str) -> Tuple[str, ...]: def parts(cls, path: str) -> tuple[str, ...]:
ret = [] ret = []
while True: while True:
path, part = posixpath.split(path) path, part = posixpath.split(path)
@@ -187,7 +187,7 @@ class _DVCFileSystem(AbstractFileSystem):
start = "." start = "."
return posixpath.relpath(self.abspath(path), start=self.abspath(start)) return posixpath.relpath(self.abspath(path), start=self.abspath(start))
def relparts(self, path: str, start: Optional[str] = None) -> Tuple[str, ...]: def relparts(self, path: str, start: Optional[str] = None) -> tuple[str, ...]:
return self.parts(self.relpath(path, start=start)) return self.parts(self.relpath(path, start=start))
@functools.cached_property @functools.cached_property
@@ -321,7 +321,7 @@ class _DVCFileSystem(AbstractFileSystem):
def _get_subrepo_info( def _get_subrepo_info(
self, key: Key self, key: Key
) -> Tuple["Repo", Optional[DataFileSystem], Key]: ) -> tuple["Repo", Optional[DataFileSystem], Key]:
""" """
Returns information about the subrepo the key is part of. Returns information about the subrepo the key is part of.
""" """
@@ -525,7 +525,7 @@ class DVCFileSystem(FileSystem):
protocol = "local" protocol = "local"
PARAM_CHECKSUM = "md5" PARAM_CHECKSUM = "md5"
def _prepare_credentials(self, **config) -> Dict[str, Any]: def _prepare_credentials(self, **config) -> dict[str, Any]:
return config return config
@functools.cached_property @functools.cached_property
+4 -4
View File
@@ -2,7 +2,7 @@ import os
import re import re
from collections import namedtuple from collections import namedtuple
from itertools import chain, groupby, takewhile from itertools import chain, groupby, takewhile
from typing import TYPE_CHECKING, List, Optional from typing import TYPE_CHECKING, Optional
from pathspec.patterns import GitWildMatchPattern from pathspec.patterns import GitWildMatchPattern
from pathspec.util import normalize_file from pathspec.util import normalize_file
@@ -62,7 +62,7 @@ class DvcIgnorePatterns(DvcIgnore):
return cls(path_spec_lines, dirname, fs.sep) return cls(path_spec_lines, dirname, fs.sep)
def __call__(self, root: List[str], dirs: List[str], files: List[str]): def __call__(self, root: list[str], dirs: list[str], files: list[str]):
files = [f for f in files if not self.matches(root, f)] files = [f for f in files if not self.matches(root, f)]
dirs = [d for d in dirs if not self.matches(root, d, True)] dirs = [d for d in dirs if not self.matches(root, d, True)]
@@ -226,7 +226,7 @@ class DvcIgnoreFilter:
self, self,
dirname: str, dirname: str,
ignore_trie: Trie, ignore_trie: Trie,
dnames: Optional["List"], dnames: Optional["list"],
ignore_subrepos: bool, ignore_subrepos: bool,
) -> None: ) -> None:
self._update_trie(dirname, ignore_trie) self._update_trie(dirname, ignore_trie)
@@ -331,7 +331,7 @@ class DvcIgnoreFilter:
yield from fs.find(path) yield from fs.find(path)
def _get_trie_pattern( def _get_trie_pattern(
self, dirname, dnames: Optional["List"] = None, ignore_subrepos=True self, dirname, dnames: Optional["list"] = None, ignore_subrepos=True
) -> Optional["DvcIgnorePatterns"]: ) -> Optional["DvcIgnorePatterns"]:
if ignore_subrepos: if ignore_subrepos:
ignores_trie = self.ignores_trie_fs ignores_trie = self.ignores_trie_fs
+2 -2
View File
@@ -5,7 +5,7 @@ import logging.config
import logging.handlers import logging.handlers
import os import os
import sys import sys
from typing import ClassVar, Dict from typing import ClassVar
import colorama import colorama
@@ -79,7 +79,7 @@ class ColorFormatter(logging.Formatter):
""" """
reset = colorama.Fore.RESET reset = colorama.Fore.RESET
color_codes: ClassVar[Dict[str, str]] = { color_codes: ClassVar[dict[str, str]] = {
"TRACE": colorama.Fore.GREEN, "TRACE": colorama.Fore.GREEN,
"DEBUG": colorama.Fore.BLUE, "DEBUG": colorama.Fore.BLUE,
"WARNING": colorama.Fore.YELLOW, "WARNING": colorama.Fore.YELLOW,
+23 -23
View File
@@ -4,7 +4,7 @@ import posixpath
from collections import defaultdict from collections import defaultdict
from contextlib import suppress from contextlib import suppress
from operator import itemgetter from operator import itemgetter
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Type, Union from typing import TYPE_CHECKING, Any, Optional, Union
from urllib.parse import urlparse from urllib.parse import urlparse
import voluptuous as vol import voluptuous as vol
@@ -148,7 +148,7 @@ def _split_dict(d, keys):
def _merge_data(s_list): def _merge_data(s_list):
d: Dict[str, Dict] = defaultdict(dict) d: dict[str, dict] = defaultdict(dict)
for key in s_list: for key in s_list:
if isinstance(key, str): if isinstance(key, str):
d[key].update({}) d[key].update({})
@@ -207,7 +207,7 @@ def load_from_pipeline(stage, data, typ="outs"):
) )
def split_file_meta_from_cloud(entry: Dict) -> Dict: def split_file_meta_from_cloud(entry: dict) -> dict:
if remote_name := entry.pop(Meta.PARAM_REMOTE, None): if remote_name := entry.pop(Meta.PARAM_REMOTE, None):
remote_meta = {} remote_meta = {}
for key in ( for key in (
@@ -223,7 +223,7 @@ def split_file_meta_from_cloud(entry: Dict) -> Dict:
return entry return entry
def merge_file_meta_from_cloud(entry: Dict) -> Dict: def merge_file_meta_from_cloud(entry: dict) -> dict:
cloud_meta = entry.pop(Output.PARAM_CLOUD, {}) cloud_meta = entry.pop(Output.PARAM_CLOUD, {})
if remote_name := first(cloud_meta): if remote_name := first(cloud_meta):
entry.update(cloud_meta[remote_name]) entry.update(cloud_meta[remote_name])
@@ -231,7 +231,7 @@ def merge_file_meta_from_cloud(entry: Dict) -> Dict:
return entry return entry
def _serialize_tree_obj_to_files(obj: Tree) -> List[Dict[str, Any]]: def _serialize_tree_obj_to_files(obj: Tree) -> list[dict[str, Any]]:
key = obj.PARAM_RELPATH key = obj.PARAM_RELPATH
return sorted( return sorted(
( (
@@ -246,7 +246,7 @@ def _serialize_tree_obj_to_files(obj: Tree) -> List[Dict[str, Any]]:
) )
def _serialize_hi_to_dict(hash_info: Optional[HashInfo]) -> Dict[str, Any]: def _serialize_hi_to_dict(hash_info: Optional[HashInfo]) -> dict[str, Any]:
if hash_info: if hash_info:
if hash_info.name == "md5-dos2unix": if hash_info.name == "md5-dos2unix":
return {"md5": hash_info.value} return {"md5": hash_info.value}
@@ -316,10 +316,10 @@ class Output:
PARAM_HASH = "hash" PARAM_HASH = "hash"
PARAM_FS_CONFIG = "fs_config" PARAM_FS_CONFIG = "fs_config"
DoesNotExistError: Type[DvcException] = OutputDoesNotExistError DoesNotExistError: type[DvcException] = OutputDoesNotExistError
IsNotFileOrDirError: Type[DvcException] = OutputIsNotFileOrDirError IsNotFileOrDirError: type[DvcException] = OutputIsNotFileOrDirError
IsStageFileError: Type[DvcException] = OutputIsStageFileError IsStageFileError: type[DvcException] = OutputIsStageFileError
IsIgnoredError: Type[DvcException] = OutputIsIgnoredError IsIgnoredError: type[DvcException] = OutputIsIgnoredError
def __init__( # noqa: PLR0913 def __init__( # noqa: PLR0913
self, self,
@@ -337,7 +337,7 @@ class Output:
remote=None, remote=None,
repo=None, repo=None,
fs_config=None, fs_config=None,
files: Optional[List[Dict[str, Any]]] = None, files: Optional[list[dict[str, Any]]] = None,
push: bool = True, push: bool = True,
hash_name: Optional[str] = DEFAULT_ALGORITHM, hash_name: Optional[str] = DEFAULT_ALGORITHM,
): ):
@@ -420,7 +420,7 @@ class Output:
def _compute_hash_info_from_meta( def _compute_hash_info_from_meta(
self, hash_name: Optional[str] self, hash_name: Optional[str]
) -> Tuple[str, HashInfo]: ) -> tuple[str, HashInfo]:
if self.is_in_repo: if self.is_in_repo:
if hash_name is None: if hash_name is None:
# Legacy 2.x output, use "md5-dos2unix" but read "md5" from # Legacy 2.x output, use "md5-dos2unix" but read "md5" from
@@ -555,7 +555,7 @@ class Output:
def _build( def _build(
self, *args, no_progress_bar=False, **kwargs self, *args, no_progress_bar=False, **kwargs
) -> Tuple["HashFileDB", "Meta", "HashFile"]: ) -> tuple["HashFileDB", "Meta", "HashFile"]:
from dvc.ui import ui from dvc.ui import ui
with ui.progress( with ui.progress(
@@ -601,7 +601,7 @@ class Output:
return self.fs.exists(self.fs_path) return self.fs.exists(self.fs_path)
@cached_property @cached_property
def index_key(self) -> Tuple[str, "DataIndexKey"]: def index_key(self) -> tuple[str, "DataIndexKey"]:
if self.is_in_repo: if self.is_in_repo:
workspace = "repo" workspace = "repo"
key = self.repo.fs.relparts(self.fs_path, self.repo.root_dir) key = self.repo.fs.relparts(self.fs_path, self.repo.root_dir)
@@ -633,7 +633,7 @@ class Output:
return self.meta.version_id != self.get_meta().version_id return self.meta.version_id != self.get_meta().version_id
return False return False
def workspace_status(self) -> Dict[str, str]: def workspace_status(self) -> dict[str, str]:
if not self.exists: if not self.exists:
return {str(self): "deleted"} return {str(self): "deleted"}
@@ -645,7 +645,7 @@ class Output:
return {} return {}
def status(self) -> Dict[str, str]: def status(self) -> dict[str, str]:
if self.hash_info and self.use_cache and self.changed_cache(): if self.hash_info and self.use_cache and self.changed_cache():
return {str(self): "not in cache"} return {str(self): "not in cache"}
@@ -840,7 +840,7 @@ class Output:
def dumpd(self, **kwargs): # noqa: C901, PLR0912 def dumpd(self, **kwargs): # noqa: C901, PLR0912
from dvc.cachemgr import LEGACY_HASH_NAMES from dvc.cachemgr import LEGACY_HASH_NAMES
ret: Dict[str, Any] = {} ret: dict[str, Any] = {}
with_files = ( with_files = (
(not self.IS_DEPENDENCY or self.stage.is_import) (not self.IS_DEPENDENCY or self.stage.is_import)
and self.hash_info.isdir and self.hash_info.isdir
@@ -962,7 +962,7 @@ class Output:
filter_info: Optional[str] = None, filter_info: Optional[str] = None,
allow_missing: bool = False, allow_missing: bool = False,
**kwargs, **kwargs,
) -> Optional[Tuple[bool, Optional[bool]]]: ) -> Optional[tuple[bool, Optional[bool]]]:
# callback passed act as a aggregate callback. # callback passed act as a aggregate callback.
# do not let checkout to call set_size and change progressbar. # do not let checkout to call set_size and change progressbar.
class CallbackProxy(Callback): class CallbackProxy(Callback):
@@ -1157,7 +1157,7 @@ class Output:
def get_used_objs( # noqa: PLR0911 def get_used_objs( # noqa: PLR0911
self, **kwargs self, **kwargs
) -> Dict[Optional["HashFileDB"], Set["HashInfo"]]: ) -> dict[Optional["HashFileDB"], set["HashInfo"]]:
"""Return filtered set of used object IDs for this out.""" """Return filtered set of used object IDs for this out."""
from dvc.cachemgr import LEGACY_HASH_NAMES from dvc.cachemgr import LEGACY_HASH_NAMES
@@ -1291,7 +1291,7 @@ class Output:
nfiles=len(merged), nfiles=len(merged),
) )
def unstage(self, path: str) -> Tuple["Meta", "Tree"]: def unstage(self, path: str) -> tuple["Meta", "Tree"]:
from pygtrie import Trie from pygtrie import Trie
rel_key = tuple(self.fs.parts(self.fs.relpath(path, self.fs_path))) rel_key = tuple(self.fs.parts(self.fs.relpath(path, self.fs_path)))
@@ -1324,7 +1324,7 @@ class Output:
path: str, path: str,
obj: Union["Tree", "HashFile"], obj: Union["Tree", "HashFile"],
meta: "Meta", meta: "Meta",
) -> Tuple["Meta", "Tree"]: ) -> tuple["Meta", "Tree"]:
from pygtrie import Trie from pygtrie import Trie
append_only = True append_only = True
@@ -1514,7 +1514,7 @@ META_SCHEMA = {
CLOUD_SCHEMA = vol.All({str: {**META_SCHEMA, **CHECKSUMS_SCHEMA}}, vol.Length(max=1)) CLOUD_SCHEMA = vol.All({str: {**META_SCHEMA, **CHECKSUMS_SCHEMA}}, vol.Length(max=1))
ARTIFACT_SCHEMA: Dict[Any, Any] = { ARTIFACT_SCHEMA: dict[Any, Any] = {
**CHECKSUMS_SCHEMA, **CHECKSUMS_SCHEMA,
**META_SCHEMA, **META_SCHEMA,
Output.PARAM_PATH: str, Output.PARAM_PATH: str,
@@ -1523,7 +1523,7 @@ ARTIFACT_SCHEMA: Dict[Any, Any] = {
Output.PARAM_HASH: str, Output.PARAM_HASH: str,
} }
DIR_FILES_SCHEMA: Dict[Any, Any] = { DIR_FILES_SCHEMA: dict[Any, Any] = {
**CHECKSUMS_SCHEMA, **CHECKSUMS_SCHEMA,
**META_SCHEMA, **META_SCHEMA,
vol.Required(Tree.PARAM_RELPATH): str, vol.Required(Tree.PARAM_RELPATH): str,
+13 -17
View File
@@ -6,12 +6,8 @@ from itertools import product
from typing import ( from typing import (
TYPE_CHECKING, TYPE_CHECKING,
Any, Any,
Dict,
List,
NamedTuple, NamedTuple,
Optional, Optional,
Tuple,
Type,
Union, Union,
) )
@@ -85,7 +81,7 @@ def format_and_raise(exc: Exception, msg: str, path: str) -> "NoReturn":
def _reraise_err( def _reraise_err(
exc_cls: Type[Exception], *args, from_exc: Optional[Exception] = None exc_cls: type[Exception], *args, from_exc: Optional[Exception] = None
) -> "NoReturn": ) -> "NoReturn":
err = exc_cls(*args) err = exc_cls(*args)
if from_exc and logger.isEnabledFor(logging.DEBUG): if from_exc and logger.isEnabledFor(logging.DEBUG):
@@ -108,7 +104,7 @@ def is_map_or_seq(data: Any) -> bool:
return not isinstance(data, str) and _is_map_or_seq(data) return not isinstance(data, str) and _is_map_or_seq(data)
def split_group_name(name: str) -> Tuple[str, Optional[str]]: def split_group_name(name: str) -> tuple[str, Optional[str]]:
group, *keys = name.rsplit(JOIN, maxsplit=1) group, *keys = name.rsplit(JOIN, maxsplit=1)
return group, first(keys) return group, first(keys)
@@ -161,14 +157,14 @@ class DataResolver:
# we use `tracked_vars` to keep a dictionary of used variables # we use `tracked_vars` to keep a dictionary of used variables
# by the interpolated entries. # by the interpolated entries.
self.tracked_vars: Dict[str, Mapping] = {} self.tracked_vars: dict[str, Mapping] = {}
stages_data = d.get(STAGES_KWD, {}) stages_data = d.get(STAGES_KWD, {})
# we wrap the definitions into: # we wrap the definitions into:
# ForeachDefinition, MatrixDefinition, and EntryDefinition # ForeachDefinition, MatrixDefinition, and EntryDefinition
# that helps us to optimize, cache and selectively load each one of # that helps us to optimize, cache and selectively load each one of
# them as we need, and simplify all of this DSL/parsing logic. # them as we need, and simplify all of this DSL/parsing logic.
self.definitions: Dict[str, Definition] = { self.definitions: dict[str, Definition] = {
name: make_definition(self, name, definition) name: make_definition(self, name, definition)
for name, definition in stages_data.items() for name, definition in stages_data.items()
} }
@@ -376,7 +372,7 @@ class ForeachDefinition:
f" in '{self.relpath}': expected list/dictionary, got " + typ f" in '{self.relpath}': expected list/dictionary, got " + typ
) )
def _warn_if_overwriting(self, keys: List[str]): def _warn_if_overwriting(self, keys: list[str]):
warn_for = [k for k in keys if k in self.context] warn_for = [k for k in keys if k in self.context]
if warn_for: if warn_for:
linking_verb = "is" if len(warn_for) == 1 else "are" linking_verb = "is" if len(warn_for) == 1 else "are"
@@ -390,7 +386,7 @@ class ForeachDefinition:
self.name, self.name,
) )
def _inserted_keys(self, iterable) -> List[str]: def _inserted_keys(self, iterable) -> list[str]:
keys = [self.pair.value] keys = [self.pair.value]
if isinstance(iterable, Mapping): if isinstance(iterable, Mapping):
keys.append(self.pair.key) keys.append(self.pair.key)
@@ -490,10 +486,10 @@ class MatrixDefinition:
return self._template return self._template
@cached_property @cached_property
def resolved_iterable(self) -> Dict[str, List]: def resolved_iterable(self) -> dict[str, list]:
return self._resolve_matrix_data() return self._resolve_matrix_data()
def _resolve_matrix_data(self) -> Dict[str, List]: def _resolve_matrix_data(self) -> dict[str, list]:
try: try:
iterable = self.context.resolve(self.matrix_data, unwrap=False) iterable = self.context.resolve(self.matrix_data, unwrap=False)
except (ContextError, ParseError) as exc: except (ContextError, ParseError) as exc:
@@ -504,7 +500,7 @@ class MatrixDefinition:
self._warn_if_overwriting([self.pair.key, self.pair.value]) self._warn_if_overwriting([self.pair.key, self.pair.value])
return iterable return iterable
def _warn_if_overwriting(self, keys: List[str]): def _warn_if_overwriting(self, keys: list[str]):
warn_for = [k for k in keys if k in self.context] warn_for = [k for k in keys if k in self.context]
if warn_for: if warn_for:
linking_verb = "is" if len(warn_for) == 1 else "are" linking_verb = "is" if len(warn_for) == 1 else "are"
@@ -519,16 +515,16 @@ class MatrixDefinition:
) )
@cached_property @cached_property
def normalized_iterable(self) -> Dict[str, "DictStrAny"]: def normalized_iterable(self) -> dict[str, "DictStrAny"]:
"""Convert sequence to Mapping with keys normalized.""" """Convert sequence to Mapping with keys normalized."""
iterable = self.resolved_iterable iterable = self.resolved_iterable
assert isinstance(iterable, Mapping) assert isinstance(iterable, Mapping)
ret: Dict[str, "DictStrAny"] = {} ret: dict[str, "DictStrAny"] = {}
matrix = {key: enumerate(v) for key, v in iterable.items()} matrix = {key: enumerate(v) for key, v in iterable.items()}
for combination in product(*matrix.values()): for combination in product(*matrix.values()):
d: "DictStrAny" = {} d: "DictStrAny" = {}
fragments: List[str] = [] fragments: list[str] = []
for k, (i, v) in zip(matrix.keys(), combination): for k, (i, v) in zip(matrix.keys(), combination):
d[k] = v d[k] = v
fragments.append(f"{k}{i}" if is_map_or_seq(v) else to_str(v)) fragments.append(f"{k}{i}" if is_map_or_seq(v) else to_str(v))
@@ -540,7 +536,7 @@ class MatrixDefinition:
def has_member(self, key: str) -> bool: def has_member(self, key: str) -> bool:
return key in self.normalized_iterable return key in self.normalized_iterable
def get_generated_names(self) -> List[str]: def get_generated_names(self) -> list[str]:
return list(map(self._generate_name, self.normalized_iterable)) return list(map(self._generate_name, self.normalized_iterable))
def _generate_name(self, key: str) -> str: def _generate_name(self, key: str) -> str:
+6 -6
View File
@@ -4,7 +4,7 @@ from collections.abc import Mapping, MutableMapping, MutableSequence, Sequence
from contextlib import contextmanager from contextlib import contextmanager
from copy import deepcopy from copy import deepcopy
from dataclasses import dataclass, field, replace from dataclasses import dataclass, field, replace
from typing import Any, Dict, List, Optional, Union from typing import Any, Optional, Union
from funcy import identity, lfilter, nullcontext, select from funcy import identity, lfilter, nullcontext, select
@@ -22,7 +22,7 @@ from dvc.parsing.interpolate import (
logger = logger.getChild(__name__) logger = logger.getChild(__name__)
SeqOrMap = Union[Sequence, Mapping] SeqOrMap = Union[Sequence, Mapping]
DictStr = Dict[str, Any] DictStr = dict[str, Any]
class ContextError(DvcException): class ContextError(DvcException):
@@ -100,7 +100,7 @@ def recurse_not_a_node(data: dict):
@dataclass @dataclass
class Meta: class Meta:
source: Optional[str] = None source: Optional[str] = None
dpaths: List[str] = field(default_factory=list) dpaths: list[str] = field(default_factory=list)
local: bool = True local: bool = True
@staticmethod @staticmethod
@@ -297,7 +297,7 @@ class Context(CtxDict):
""" """
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
self._track = False self._track = False
self._tracked_data: Dict[str, Dict] = defaultdict(dict) self._tracked_data: dict[str, dict] = defaultdict(dict)
self.imports = {} self.imports = {}
self._reserved_keys = {} self._reserved_keys = {}
@@ -348,7 +348,7 @@ class Context(CtxDict):
@classmethod @classmethod
def load_from( def load_from(
cls, fs, path: str, select_keys: Optional[List[str]] = None cls, fs, path: str, select_keys: Optional[list[str]] = None
) -> "Context": ) -> "Context":
from dvc.utils.serialize import load_path from dvc.utils.serialize import load_path
@@ -423,7 +423,7 @@ class Context(CtxDict):
def load_from_vars( def load_from_vars(
self, self,
fs, fs,
vars_: List, vars_: list,
wdir: str, wdir: str,
stage_name: Optional[str] = None, stage_name: Optional[str] = None,
default: Optional[str] = None, default: Optional[str] = None,
+4 -3
View File
@@ -10,7 +10,8 @@ from dvc.exceptions import DvcException
from dvc.utils.flatten import flatten from dvc.utils.flatten import flatten
if typing.TYPE_CHECKING: if typing.TYPE_CHECKING:
from typing import List, Match, NoReturn from re import Match
from typing import NoReturn
from pyparsing import ParseException from pyparsing import ParseException
@@ -205,7 +206,7 @@ def validate_value(value, key):
def str_interpolate( def str_interpolate(
template: str, template: str,
matches: "List[Match]", matches: "list[Match]",
context: "Context", context: "Context",
skip_checks: bool = False, skip_checks: bool = False,
key=None, key=None,
@@ -225,5 +226,5 @@ def str_interpolate(
return buf.replace(r"\${", BRACE_OPEN) return buf.replace(r"\${", BRACE_OPEN)
def is_exact_string(src: str, matches: "List[Match]"): def is_exact_string(src: str, matches: "list[Match]"):
return len(matches) == 1 and src == matches[0].group(0) return len(matches) == 1 and src == matches[0].group(0)
+2 -2
View File
@@ -2,7 +2,7 @@
import logging import logging
import sys import sys
from threading import RLock from threading import RLock
from typing import TYPE_CHECKING, Any, ClassVar, Dict from typing import TYPE_CHECKING, Any, ClassVar
from tqdm import tqdm from tqdm import tqdm
@@ -33,7 +33,7 @@ class Tqdm(tqdm):
" [{elapsed}<{remaining}, {rate_fmt:>11}]" " [{elapsed}<{remaining}, {rate_fmt:>11}]"
) )
BAR_FMT_NOTOTAL = "{desc}{bar:b}|{postfix[info]}{n_fmt} [{elapsed}, {rate_fmt:>11}]" BAR_FMT_NOTOTAL = "{desc}{bar:b}|{postfix[info]}{n_fmt} [{elapsed}, {rate_fmt:>11}]"
BYTES_DEFAULTS: ClassVar[Dict[str, Any]] = { BYTES_DEFAULTS: ClassVar[dict[str, Any]] = {
"unit": "B", "unit": "B",
"unit_scale": True, "unit_scale": True,
"unit_divisor": 1024, "unit_divisor": 1024,
+2 -1
View File
@@ -1,7 +1,8 @@
"""Manages user prompts.""" """Manages user prompts."""
from collections.abc import Collection
from getpass import getpass from getpass import getpass
from typing import Collection, Optional from typing import Optional
from dvc.log import logger from dvc.log import logger
+2 -2
View File
@@ -1,4 +1,4 @@
from typing import Dict, List, Union from typing import Union
from dvc.render import REVISION, REVISIONS, SRC, TYPE_KEY from dvc.render import REVISION, REVISIONS, SRC, TYPE_KEY
from dvc.render.converter.image import ImageConverter from dvc.render.converter.image import ImageConverter
@@ -18,7 +18,7 @@ def _get_converter(
raise ValueError(f"Invalid renderer class {renderer_class}") raise ValueError(f"Invalid renderer class {renderer_class}")
def to_json(renderer, split: bool = False) -> List[Dict]: def to_json(renderer, split: bool = False) -> list[dict]:
if renderer.TYPE == "vega": if renderer.TYPE == "vega":
if not renderer.datapoints: if not renderer.datapoints:
return [] return []
+5 -5
View File
@@ -1,19 +1,19 @@
from typing import Any, Dict, List, Optional, Tuple from typing import Any, Optional
class Converter: class Converter:
def __init__( def __init__(
self, self,
plot_id: str, plot_id: str,
data: Optional[Dict[str, Any]] = None, data: Optional[dict[str, Any]] = None,
properties: Optional[Dict] = None, properties: Optional[dict] = None,
): ):
self.plot_id = plot_id self.plot_id = plot_id
self.properties = properties or {} self.properties = properties or {}
self.data = data or {} self.data = data or {}
def convert(self) -> Tuple[List[Tuple[str, str, Any]], Dict]: def convert(self) -> tuple[list[tuple[str, str, Any]], dict]:
raise NotImplementedError raise NotImplementedError
def flat_datapoints(self, revision: str) -> Tuple[List[Dict], Dict]: def flat_datapoints(self, revision: str) -> tuple[list[dict], dict]:
raise NotImplementedError raise NotImplementedError
+3 -3
View File
@@ -1,6 +1,6 @@
import base64 import base64
import os import os
from typing import TYPE_CHECKING, Any, Dict, List, Tuple from typing import TYPE_CHECKING, Any
from dvc.render import FILENAME, REVISION, SRC from dvc.render import FILENAME, REVISION, SRC
@@ -34,13 +34,13 @@ class ImageConverter(Converter):
base64_str = base64.b64encode(image_data).decode() base64_str = base64.b64encode(image_data).decode()
return f"data:image;base64,{base64_str}" return f"data:image;base64,{base64_str}"
def convert(self) -> Tuple[List[Tuple[str, str, Any]], Dict]: def convert(self) -> tuple[list[tuple[str, str, Any]], dict]:
datas = [] datas = []
for filename, image_data in self.data.items(): for filename, image_data in self.data.items():
datas.append((filename, "", image_data)) datas.append((filename, "", image_data))
return datas, self.properties return datas, self.properties
def flat_datapoints(self, revision: str) -> Tuple[List[Dict], Dict]: def flat_datapoints(self, revision: str) -> tuple[list[dict], dict]:
""" """
Convert the DVC Plots content to DVC Render datapoints. Convert the DVC Plots content to DVC Render datapoints.
Return both generated datapoints and updated properties. Return both generated datapoints and updated properties.
+18 -17
View File
@@ -1,5 +1,6 @@
import os import os
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union from collections.abc import Iterable
from typing import Any, Optional, Union
from funcy import first, last from funcy import first, last
@@ -18,7 +19,7 @@ class FieldNotFoundError(DvcException):
) )
def _lists(blob: Union[Dict, List]) -> Iterable[List]: def _lists(blob: Union[dict, list]) -> Iterable[list]:
if isinstance(blob, list): if isinstance(blob, list):
yield blob yield blob
else: else:
@@ -43,7 +44,7 @@ def _file_field(*args):
def _find( def _find(
filename: str, filename: str,
field: str, field: str,
data_series: List[Tuple[str, str, Any]], data_series: list[tuple[str, str, Any]],
): ):
for data_file, data_field, data in data_series: for data_file, data_field, data in data_series:
if data_file == filename and data_field == field: if data_file == filename and data_field == field:
@@ -51,14 +52,14 @@ def _find(
return None return None
def _verify_field(file2datapoints: Dict[str, List], filename: str, field: str): def _verify_field(file2datapoints: dict[str, list], filename: str, field: str):
if filename in file2datapoints: if filename in file2datapoints:
datapoint = first(file2datapoints[filename]) datapoint = first(file2datapoints[filename])
if field not in datapoint: if field not in datapoint:
raise FieldNotFoundError(field, datapoint.keys()) raise FieldNotFoundError(field, datapoint.keys())
def _get_xs(properties: Dict, file2datapoints: Dict[str, List[Dict]]): def _get_xs(properties: dict, file2datapoints: dict[str, list[dict]]):
x = properties.get("x") x = properties.get("x")
if x is not None and isinstance(x, dict): if x is not None and isinstance(x, dict):
for filename, field in _file_field(x): for filename, field in _file_field(x):
@@ -66,7 +67,7 @@ def _get_xs(properties: Dict, file2datapoints: Dict[str, List[Dict]]):
yield filename, field yield filename, field
def _get_ys(properties, file2datapoints: Dict[str, List[Dict]]): def _get_ys(properties, file2datapoints: dict[str, list[dict]]):
y = properties.get("y", None) y = properties.get("y", None)
if y is not None: if y is not None:
for filename, field in _file_field(y): for filename, field in _file_field(y):
@@ -74,7 +75,7 @@ def _get_ys(properties, file2datapoints: Dict[str, List[Dict]]):
yield filename, field yield filename, field
def _is_datapoints(lst: List[Dict]): def _is_datapoints(lst: list[dict]):
""" """
check if dict keys match, datapoints with different keys mgiht lead check if dict keys match, datapoints with different keys mgiht lead
to unexpected behavior to unexpected behavior
@@ -85,8 +86,8 @@ def _is_datapoints(lst: List[Dict]):
} }
def get_datapoints(file_content: Dict): def get_datapoints(file_content: dict):
result: List[Dict[str, Any]] = [] result: list[dict[str, Any]] = []
for lst in _lists(file_content): for lst in _lists(file_content):
if _is_datapoints(lst): if _is_datapoints(lst):
for index, datapoint in enumerate(lst): for index, datapoint in enumerate(lst):
@@ -107,8 +108,8 @@ class VegaConverter(Converter):
def __init__( def __init__(
self, self,
plot_id: str, plot_id: str,
data: Optional[Dict] = None, data: Optional[dict] = None,
properties: Optional[Dict] = None, properties: Optional[dict] = None,
): ):
super().__init__(plot_id, data, properties) super().__init__(plot_id, data, properties)
self.plot_id = plot_id self.plot_id = plot_id
@@ -126,7 +127,7 @@ class VegaConverter(Converter):
x = self.properties.get("x", None) x = self.properties.get("x", None)
y = self.properties.get("y", None) y = self.properties.get("y", None)
inferred_properties: Dict = {} inferred_properties: dict = {}
# Infer x. # Infer x.
if isinstance(x, str): if isinstance(x, str):
@@ -195,7 +196,7 @@ class VegaConverter(Converter):
def flat_datapoints(self, revision): # noqa: C901, PLR0912 def flat_datapoints(self, revision): # noqa: C901, PLR0912
file2datapoints, properties = self.convert() file2datapoints, properties = self.convert()
props_update: Dict[str, Union[str, List[Dict[str, str]]]] = {} props_update: dict[str, Union[str, list[dict[str, str]]]] = {}
xs = list(_get_xs(properties, file2datapoints)) xs = list(_get_xs(properties, file2datapoints))
@@ -322,9 +323,9 @@ def _get_short_y_file(y_file, common_prefix_len):
def _update_from_field( def _update_from_field(
target_datapoints: List[Dict], target_datapoints: list[dict],
field: str, field: str,
source_datapoints: Optional[List[Dict]] = None, source_datapoints: Optional[list[dict]] = None,
source_field: Optional[str] = None, source_field: Optional[str] = None,
): ):
if source_datapoints is None: if source_datapoints is None:
@@ -341,11 +342,11 @@ def _update_from_field(
datapoint[field] = source_datapoint[source_field] datapoint[field] = source_datapoint[source_field]
def _update_from_index(datapoints: List[Dict], new_field: str): def _update_from_index(datapoints: list[dict], new_field: str):
for index, datapoint in enumerate(datapoints): for index, datapoint in enumerate(datapoints):
datapoint[new_field] = index datapoint[new_field] = index
def _update_all(datapoints: List[Dict], update_dict: Dict): def _update_all(datapoints: list[dict], update_dict: dict):
for datapoint in datapoints: for datapoint in datapoints:
datapoint.update(update_dict) datapoint.update(update_dict)
+11 -11
View File
@@ -1,6 +1,6 @@
import os import os
from collections import defaultdict from collections import defaultdict
from typing import TYPE_CHECKING, DefaultDict, Dict, List, NamedTuple, Optional from typing import TYPE_CHECKING, NamedTuple, Optional
import dpath import dpath
import dpath.options import dpath.options
@@ -21,16 +21,16 @@ dpath.options.ALLOW_EMPTY_STRING_KEYS = True
logger = logger.getChild(__name__) logger = logger.getChild(__name__)
def _squash_plots_properties(data: List) -> Dict: def _squash_plots_properties(data: list) -> dict:
configs = [last(group) for group in data] configs = [last(group) for group in data]
resolved: Dict = {} resolved: dict = {}
for config in reversed(configs): for config in reversed(configs):
resolved = {**resolved, **config} resolved = {**resolved, **config}
return resolved return resolved
class PlotsData: class PlotsData:
def __init__(self, data: Dict): def __init__(self, data: dict):
self.data = data self.data = data
def group_definitions(self): def group_definitions(self):
@@ -62,15 +62,15 @@ class PlotsData:
class RendererWithErrors(NamedTuple): class RendererWithErrors(NamedTuple):
renderer: "Renderer" renderer: "Renderer"
source_errors: Dict[str, Dict[str, Exception]] source_errors: dict[str, dict[str, Exception]]
definition_errors: Dict[str, Exception] definition_errors: dict[str, Exception]
def match_defs_renderers( # noqa: C901, PLR0912 def match_defs_renderers( # noqa: C901, PLR0912
data, data,
out=None, out=None,
templates_dir: Optional["StrPath"] = None, templates_dir: Optional["StrPath"] = None,
) -> List[RendererWithErrors]: ) -> list[RendererWithErrors]:
from dvc_render import ImageRenderer, VegaRenderer from dvc_render import ImageRenderer, VegaRenderer
plots_data = PlotsData(data) plots_data = PlotsData(data)
@@ -78,12 +78,12 @@ def match_defs_renderers( # noqa: C901, PLR0912
renderer_cls = None renderer_cls = None
for plot_id, group in plots_data.group_definitions().items(): for plot_id, group in plots_data.group_definitions().items():
plot_datapoints: List[Dict] = [] plot_datapoints: list[dict] = []
props = _squash_plots_properties(group) props = _squash_plots_properties(group)
first_props: Dict = {} first_props: dict = {}
def_errors: Dict[str, Exception] = {} def_errors: dict[str, Exception] = {}
src_errors: DefaultDict[str, Dict[str, Exception]] = defaultdict(dict) src_errors: defaultdict[str, dict[str, Exception]] = defaultdict(dict)
if out is not None: if out is not None:
props["out"] = out props["out"] = out
+7 -9
View File
@@ -1,14 +1,12 @@
import os import os
from collections import defaultdict from collections import defaultdict
from contextlib import contextmanager from collections.abc import Iterable
from contextlib import AbstractContextManager, contextmanager
from functools import wraps from functools import wraps
from typing import ( from typing import (
TYPE_CHECKING, TYPE_CHECKING,
Callable, Callable,
ContextManager,
Iterable,
Optional, Optional,
Tuple,
Union, Union,
) )
@@ -111,7 +109,7 @@ class Repo:
fs: Optional["FileSystem"] = None, fs: Optional["FileSystem"] = None,
uninitialized: bool = False, uninitialized: bool = False,
scm: Optional[Union["Git", "NoSCM"]] = None, scm: Optional[Union["Git", "NoSCM"]] = None,
) -> Tuple[str, Optional[str]]: ) -> tuple[str, Optional[str]]:
from dvc.fs import localfs from dvc.fs import localfs
from dvc.scm import SCM, SCMError from dvc.scm import SCM, SCMError
@@ -268,7 +266,7 @@ class Repo:
if not isinstance(self.fs, GitFileSystem): if not isinstance(self.fs, GitFileSystem):
return None return None
relparts: Tuple[str, ...] = () relparts: tuple[str, ...] = ()
if self.root_dir != "/": if self.root_dir != "/":
# subrepo # subrepo
relparts = self.fs.relparts(self.root_dir, "/") relparts = self.fs.relparts(self.root_dir, "/")
@@ -392,7 +390,7 @@ class Repo:
self, self,
path: str, path: str,
workspace: str = "repo", workspace: str = "repo",
) -> Tuple["DataIndex", "DataIndexEntry"]: ) -> tuple["DataIndex", "DataIndexEntry"]:
if self.subrepos: if self.subrepos:
fs_path = self.dvcfs.from_os_path(path) fs_path = self.dvcfs.from_os_path(path)
fs = self.dvcfs.fs fs = self.dvcfs.fs
@@ -476,7 +474,7 @@ class Repo:
return brancher(self, *args, **kwargs) return brancher(self, *args, **kwargs)
def switch(self, rev: str) -> ContextManager[str]: def switch(self, rev: str) -> AbstractContextManager[str]:
from dvc.repo.brancher import switch from dvc.repo.brancher import switch
return switch(self, rev) return switch(self, rev)
@@ -626,7 +624,7 @@ class Repo:
cache_dir = self.config["core"].get("site_cache_dir") or site_cache_dir() cache_dir = self.config["core"].get("site_cache_dir") or site_cache_dir()
if isinstance(self.fs, GitFileSystem): if isinstance(self.fs, GitFileSystem):
relparts: Tuple[str, ...] = () relparts: tuple[str, ...] = ()
if self.root_dir != "/": if self.root_dir != "/":
# subrepo # subrepo
relparts = self.fs.relparts(self.root_dir, "/") relparts = self.fs.relparts(self.root_dir, "/")
+7 -10
View File
@@ -1,13 +1,10 @@
import os import os
from collections.abc import Iterator
from contextlib import contextmanager from contextlib import contextmanager
from typing import ( from typing import (
TYPE_CHECKING, TYPE_CHECKING,
Dict,
Iterator,
List,
NamedTuple, NamedTuple,
Optional, Optional,
Tuple,
Union, Union,
) )
@@ -37,7 +34,7 @@ class StageInfo(NamedTuple):
def find_targets( def find_targets(
targets: Union["StrOrBytesPath", Iterator["StrOrBytesPath"]], glob: bool = False targets: Union["StrOrBytesPath", Iterator["StrOrBytesPath"]], glob: bool = False
) -> List[str]: ) -> list[str]:
if isinstance(targets, (str, bytes, os.PathLike)): if isinstance(targets, (str, bytes, os.PathLike)):
targets_list = [os.fsdecode(targets)] targets_list = [os.fsdecode(targets)]
else: else:
@@ -99,7 +96,7 @@ OVERLAPPING_PARENT_FMT = (
@contextmanager @contextmanager
def translate_graph_error(stages: List["Stage"]) -> Iterator[None]: def translate_graph_error(stages: list["Stage"]) -> Iterator[None]:
try: try:
yield yield
except OverlappingOutputPathsError as exc: except OverlappingOutputPathsError as exc:
@@ -124,7 +121,7 @@ def translate_graph_error(stages: List["Stage"]) -> Iterator[None]:
) )
def progress_iter(stages: Dict[str, StageInfo]) -> Iterator[Tuple[str, StageInfo]]: def progress_iter(stages: dict[str, StageInfo]) -> Iterator[tuple[str, StageInfo]]:
total = len(stages) total = len(stages)
desc = "Adding..." desc = "Adding..."
with ui.progress( with ui.progress(
@@ -151,8 +148,8 @@ LINK_FAILURE_MESSAGE = (
@contextmanager @contextmanager
def warn_link_failures() -> Iterator[List[str]]: def warn_link_failures() -> Iterator[list[str]]:
link_failures: List[str] = [] link_failures: list[str] = []
try: try:
yield link_failures yield link_failures
finally: finally:
@@ -202,7 +199,7 @@ def add(
to_remote: bool = False, to_remote: bool = False,
remote_jobs: Optional[int] = None, remote_jobs: Optional[int] = None,
force: bool = False, force: bool = False,
) -> List["Stage"]: ) -> list["Stage"]:
add_targets = find_targets(targets, glob=glob) add_targets = find_targets(targets, glob=glob)
if not add_targets: if not add_targets:
return [] return []
+12 -12
View File
@@ -1,7 +1,7 @@
import os import os
import posixpath import posixpath
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union from typing import TYPE_CHECKING, Any, Optional, Union
from dvc.annotations import Artifact from dvc.annotations import Artifact
from dvc.dvcfile import PROJECT_FILE from dvc.dvcfile import PROJECT_FILE
@@ -95,9 +95,9 @@ class Artifacts:
return self.repo.scm return self.repo.scm
return None return None
def read(self) -> Dict[str, Dict[str, Artifact]]: def read(self) -> dict[str, dict[str, Artifact]]:
"""Read artifacts from dvc.yaml.""" """Read artifacts from dvc.yaml."""
artifacts: Dict[str, Dict[str, Artifact]] = {} artifacts: dict[str, dict[str, Artifact]] = {}
for ( for (
dvcfile, dvcfile,
dvcfile_artifacts, dvcfile_artifacts,
@@ -140,14 +140,14 @@ class Artifacts:
assert not (version and stage) assert not (version and stage)
name = _reformat_name(name) name = _reformat_name(name)
tags: List["GitTag"] = find( tags: list["GitTag"] = find(
name=name, version=version, stage=stage, scm=self.scm name=name, version=version, stage=stage, scm=self.scm
) )
if not tags: if not tags:
raise ArtifactNotFoundError(name, version=version, stage=stage) raise ArtifactNotFoundError(name, version=version, stage=stage)
if version or stage: if version or stage:
return tags[-1].target return tags[-1].target
gto_tags: List["GTOTag"] = sort_versions(parse_tag(tag) for tag in tags) gto_tags: list["GTOTag"] = sort_versions(parse_tag(tag) for tag in tags)
return gto_tags[0].tag.target return gto_tags[0].tag.target
def get_path(self, name: str): def get_path(self, name: str):
@@ -177,7 +177,7 @@ class Artifacts:
out: Optional[str] = None, out: Optional[str] = None,
force: bool = False, force: bool = False,
jobs: Optional[int] = None, jobs: Optional[int] = None,
) -> Tuple[int, str]: ) -> tuple[int, str]:
"""Download the specified artifact.""" """Download the specified artifact."""
from dvc.fs import download as fs_download from dvc.fs import download as fs_download
@@ -205,9 +205,9 @@ class Artifacts:
out: Optional[str] = None, out: Optional[str] = None,
force: bool = False, force: bool = False,
jobs: Optional[int] = None, jobs: Optional[int] = None,
dvc_studio_config: Optional[Dict[str, Any]] = None, dvc_studio_config: Optional[dict[str, Any]] = None,
**kwargs, **kwargs,
) -> Tuple[int, str]: ) -> tuple[int, str]:
from dvc_studio_client.model_registry import get_download_uris from dvc_studio_client.model_registry import get_download_uris
from dvc.fs import HTTPFileSystem, generic, localfs from dvc.fs import HTTPFileSystem, generic, localfs
@@ -215,8 +215,8 @@ class Artifacts:
logger.debug("Trying to download artifact '%s' via studio", name) logger.debug("Trying to download artifact '%s' via studio", name)
out = out or os.getcwd() out = out or os.getcwd()
to_infos: List[str] = [] to_infos: list[str] = []
from_infos: List[str] = [] from_infos: list[str] = []
if dvc_studio_config is None: if dvc_studio_config is None:
dvc_studio_config = {} dvc_studio_config = {}
dvc_studio_config["repo_url"] = repo_url dvc_studio_config["repo_url"] = repo_url
@@ -263,9 +263,9 @@ class Artifacts:
name: str, name: str,
version: Optional[str] = None, version: Optional[str] = None,
stage: Optional[str] = None, stage: Optional[str] = None,
config: Optional[Union[str, Dict[str, Any]]] = None, config: Optional[Union[str, dict[str, Any]]] = None,
remote: Optional[str] = None, remote: Optional[str] = None,
remote_config: Optional[Union[str, Dict[str, Any]]] = None, remote_config: Optional[Union[str, dict[str, Any]]] = None,
out: Optional[str] = None, out: Optional[str] = None,
force: bool = False, force: bool = False,
jobs: Optional[int] = None, jobs: Optional[int] = None,
+8 -7
View File
@@ -1,5 +1,6 @@
from collections.abc import Iterator
from contextlib import contextmanager from contextlib import contextmanager
from typing import TYPE_CHECKING, Iterator, Optional, Tuple from typing import TYPE_CHECKING, Optional
from scmrepo.git import Git from scmrepo.git import Git
@@ -59,11 +60,11 @@ def brancher(
from dvc.fs import LocalFileSystem from dvc.fs import LocalFileSystem
repo_root_parts: Tuple[str, ...] = () repo_root_parts: tuple[str, ...] = ()
if self.fs.isin(self.root_dir, self.scm.root_dir): if self.fs.isin(self.root_dir, self.scm.root_dir):
repo_root_parts = self.fs.relparts(self.root_dir, self.scm.root_dir) repo_root_parts = self.fs.relparts(self.root_dir, self.scm.root_dir)
cwd_parts: Tuple[str, ...] = () cwd_parts: tuple[str, ...] = ()
if self.fs.isin(self.fs.getcwd(), self.scm.root_dir): if self.fs.isin(self.fs.getcwd(), self.scm.root_dir):
cwd_parts = self.fs.relparts(self.fs.getcwd(), self.scm.root_dir) cwd_parts = self.fs.relparts(self.fs.getcwd(), self.scm.root_dir)
@@ -112,8 +113,8 @@ def brancher(
def _switch_fs( def _switch_fs(
repo: "Repo", repo: "Repo",
rev: str, rev: str,
repo_root_parts: Tuple[str, ...], repo_root_parts: tuple[str, ...],
cwd_parts: Tuple[str, ...], cwd_parts: tuple[str, ...],
): ):
from dvc.fs import GitFileSystem, LocalFileSystem from dvc.fs import GitFileSystem, LocalFileSystem
@@ -147,11 +148,11 @@ def switch(repo: "Repo", rev: str) -> Iterator[str]:
if rev != "workspace": if rev != "workspace":
rev = resolve_rev(repo.scm, rev) rev = resolve_rev(repo.scm, rev)
repo_root_parts: Tuple[str, ...] = () repo_root_parts: tuple[str, ...] = ()
if repo.fs.isin(repo.root_dir, repo.scm.root_dir): if repo.fs.isin(repo.root_dir, repo.scm.root_dir):
repo_root_parts = repo.fs.relparts(repo.root_dir, repo.scm.root_dir) repo_root_parts = repo.fs.relparts(repo.root_dir, repo.scm.root_dir)
cwd_parts: Tuple[str, ...] = () cwd_parts: tuple[str, ...] = ()
if repo.fs.isin(repo.fs.getcwd(), repo.scm.root_dir): if repo.fs.isin(repo.fs.getcwd(), repo.scm.root_dir):
cwd_parts = repo.fs.relparts(repo.fs.getcwd(), repo.scm.root_dir) cwd_parts = repo.fs.relparts(repo.fs.getcwd(), repo.scm.root_dir)
+3 -3
View File
@@ -1,5 +1,5 @@
import os import os
from typing import TYPE_CHECKING, Dict, List from typing import TYPE_CHECKING
from dvc.exceptions import ( from dvc.exceptions import (
CheckoutError, CheckoutError,
@@ -66,7 +66,7 @@ def _build_out_changes(index, changes):
def _check_can_delete( def _check_can_delete(
entries: List["DataIndexEntry"], entries: list["DataIndexEntry"],
index: "BaseDataIndex", index: "BaseDataIndex",
path: str, path: str,
fs: "FileSystem", fs: "FileSystem",
@@ -107,7 +107,7 @@ def checkout( # noqa: C901
from dvc.stage.exceptions import StageFileBadNameError, StageFileDoesNotExistError from dvc.stage.exceptions import StageFileBadNameError, StageFileDoesNotExistError
from dvc_data.index.checkout import ADD, DELETE, MODIFY, apply, compare from dvc_data.index.checkout import ADD, DELETE, MODIFY, apply, compare
stats: Dict[str, List[str]] = { stats: dict[str, list[str]] = {
"added": [], "added": [],
"deleted": [], "deleted": [],
"modified": [], "modified": [],
+6 -5
View File
@@ -1,4 +1,5 @@
from typing import TYPE_CHECKING, Callable, Iterable, List, Optional, Tuple from collections.abc import Iterable
from typing import TYPE_CHECKING, Callable, Optional
from dvc.log import logger from dvc.log import logger
@@ -10,8 +11,8 @@ logger = logger.getChild(__name__)
FilterFn = Callable[["Output"], bool] FilterFn = Callable[["Output"], bool]
Outputs = List["Output"] Outputs = list["Output"]
StrPaths = List[str] StrPaths = list[str]
def _collect_outs( def _collect_outs(
@@ -43,7 +44,7 @@ def _collect_paths(
def _filter_outs( def _filter_outs(
outs: Outputs, fs_paths: StrPaths, duplicates=False outs: Outputs, fs_paths: StrPaths, duplicates=False
) -> Tuple[Outputs, StrPaths]: ) -> tuple[Outputs, StrPaths]:
res_outs: Outputs = [] res_outs: Outputs = []
fs_res_paths = fs_paths fs_res_paths = fs_paths
@@ -65,7 +66,7 @@ def collect(
output_filter: Optional[FilterFn] = None, output_filter: Optional[FilterFn] = None,
recursive: bool = False, recursive: bool = False,
duplicates: bool = False, duplicates: bool = False,
) -> Tuple[Outputs, StrPaths]: ) -> tuple[Outputs, StrPaths]:
assert targets or output_filter assert targets or output_filter
outs: Outputs = _collect_outs(repo, output_filter=output_filter, deps=deps) outs: Outputs = _collect_outs(repo, output_filter=output_filter, deps=deps)
+2 -2
View File
@@ -1,4 +1,4 @@
from typing import TYPE_CHECKING, Set from typing import TYPE_CHECKING
from dvc import prompt from dvc import prompt
@@ -122,7 +122,7 @@ def commit_2_to_3(repo: "Repo", dry: bool = False):
stage.dump(update_pipeline=False) stage.dump(update_pipeline=False)
def _migrateable_dvcfiles(view: "IndexView") -> Set[str]: def _migrateable_dvcfiles(view: "IndexView") -> set[str]:
from dvc.dvcfile import ProjectFile from dvc.dvcfile import ProjectFile
migrated = set() migrated = set()
+16 -15
View File
@@ -1,6 +1,7 @@
import os import os
import posixpath import posixpath
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, TypedDict, Union from collections.abc import Iterable
from typing import TYPE_CHECKING, Any, TypedDict, Union
from dvc.fs.callbacks import DEFAULT_CALLBACK from dvc.fs.callbacks import DEFAULT_CALLBACK
from dvc.ui import ui from dvc.ui import ui
@@ -53,11 +54,11 @@ def _diff(
not_in_remote: bool = False, not_in_remote: bool = False,
remote_refresh: bool = False, remote_refresh: bool = False,
callback: "Callback" = DEFAULT_CALLBACK, callback: "Callback" = DEFAULT_CALLBACK,
) -> Dict[str, List[str]]: ) -> dict[str, list[str]]:
from dvc_data.index import StorageError from dvc_data.index import StorageError
from dvc_data.index.diff import UNCHANGED, UNKNOWN, diff from dvc_data.index.diff import UNCHANGED, UNKNOWN, diff
ret: Dict[str, List[str]] = {} ret: dict[str, list[str]] = {}
def _add_change(typ, change): def _add_change(typ, change):
typ = _adapt_typ(typ) typ = _adapt_typ(typ)
@@ -115,9 +116,9 @@ def _diff(
class GitInfo(TypedDict, total=False): class GitInfo(TypedDict, total=False):
staged: Dict[str, List[str]] staged: dict[str, list[str]]
unstaged: Dict[str, List[str]] unstaged: dict[str, list[str]]
untracked: List[str] untracked: list[str]
is_empty: bool is_empty: bool
is_dirty: bool is_dirty: bool
@@ -150,7 +151,7 @@ def _git_info(scm: Union["Git", "NoSCM"], untracked_files: str = "all") -> GitIn
) )
def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> Dict[str, List[str]]: def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> dict[str, list[str]]:
from .index import build_data_index from .index import build_data_index
with ui.progress( with ui.progress(
@@ -180,7 +181,7 @@ def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> Dict[str, List[str]]:
def _diff_head_to_index( def _diff_head_to_index(
repo: "Repo", head: str = "HEAD", **kwargs: Any repo: "Repo", head: str = "HEAD", **kwargs: Any
) -> Dict[str, List[str]]: ) -> dict[str, list[str]]:
index = repo.index.data["repo"] index = repo.index.data["repo"]
with repo.switch(head): with repo.switch(head):
@@ -194,16 +195,16 @@ def _diff_head_to_index(
class Status(TypedDict): class Status(TypedDict):
not_in_cache: List[str] not_in_cache: list[str]
not_in_remote: List[str] not_in_remote: list[str]
committed: Dict[str, List[str]] committed: dict[str, list[str]]
uncommitted: Dict[str, List[str]] uncommitted: dict[str, list[str]]
untracked: List[str] untracked: list[str]
unchanged: List[str] unchanged: list[str]
git: GitInfo git: GitInfo
def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> List[str]: def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> list[str]:
"""Transform files rel. to Git root to DVC root, and drop outside files.""" """Transform files rel. to Git root to DVC root, and drop outside files."""
rel = repo.fs.relpath(repo.root_dir, repo.scm.root_dir).rstrip("/") rel = repo.fs.relpath(repo.root_dir, repo.scm.root_dir).rstrip("/")
+3 -3
View File
@@ -1,7 +1,7 @@
import errno import errno
import os import os
from collections import defaultdict from collections import defaultdict
from typing import Dict, List, Optional from typing import Optional
from dvc.log import logger from dvc.log import logger
from dvc.repo import locked from dvc.repo import locked
@@ -26,7 +26,7 @@ def _diff(old, new, data_keys, with_missing=False):
from dvc_data.index.diff import ADD, DELETE, MODIFY, RENAME from dvc_data.index.diff import ADD, DELETE, MODIFY, RENAME
from dvc_data.index.diff import diff as idiff from dvc_data.index.diff import diff as idiff
ret: "Dict[str, List[Dict]]" = { ret: "dict[str, list[dict]]" = {
"added": [], "added": [],
"deleted": [], "deleted": [],
"modified": [], "modified": [],
@@ -107,7 +107,7 @@ def diff(
self, self,
a_rev: str = "HEAD", a_rev: str = "HEAD",
b_rev: Optional[str] = None, b_rev: Optional[str] = None,
targets: Optional[List[str]] = None, targets: Optional[list[str]] = None,
recursive: bool = False, recursive: bool = False,
): ):
""" """
+2 -2
View File
@@ -1,4 +1,4 @@
from typing import Any, Dict, Optional, Union from typing import Any, Optional, Union
def du( def du(
@@ -6,7 +6,7 @@ def du(
path: Optional[str] = None, path: Optional[str] = None,
rev: Optional[str] = None, rev: Optional[str] = None,
summarize: bool = False, summarize: bool = False,
config: Union[None, Dict[str, Any], str] = None, config: Union[None, dict[str, Any], str] = None,
remote: Optional[str] = None, remote: Optional[str] = None,
remote_config: Optional[dict] = None, remote_config: Optional[dict] = None,
): ):
+11 -10
View File
@@ -1,6 +1,7 @@
import os import os
import re import re
from typing import TYPE_CHECKING, Dict, Iterable, List, Optional from collections.abc import Iterable
from typing import TYPE_CHECKING, Optional
from funcy import chain, first from funcy import chain, first
@@ -104,7 +105,7 @@ class Experiments:
return ExpCache(self.repo) return ExpCache(self.repo)
@property @property
def stash_revs(self) -> Dict[str, "ExpStashEntry"]: def stash_revs(self) -> dict[str, "ExpStashEntry"]:
revs = {} revs = {}
for queue in (self.workspace_queue, self.celery_queue): for queue in (self.workspace_queue, self.celery_queue):
revs.update(queue.stash.stash_revs) revs.update(queue.stash.stash_revs)
@@ -113,7 +114,7 @@ class Experiments:
def reproduce_one( def reproduce_one(
self, self,
tmp_dir: bool = False, tmp_dir: bool = False,
copy_paths: Optional[List[str]] = None, copy_paths: Optional[list[str]] = None,
message: Optional[str] = None, message: Optional[str] = None,
**kwargs, **kwargs,
): ):
@@ -143,8 +144,8 @@ class Experiments:
def reproduce_celery( def reproduce_celery(
self, entries: Optional[Iterable["QueueEntry"]] = None, **kwargs self, entries: Optional[Iterable["QueueEntry"]] = None, **kwargs
) -> Dict[str, str]: ) -> dict[str, str]:
results: Dict[str, str] = {} results: dict[str, str] = {}
if entries is None: if entries is None:
entries = list( entries = list(
chain( chain(
@@ -250,10 +251,10 @@ class Experiments:
def _reproduce_queue( def _reproduce_queue(
self, self,
queue: "BaseStashQueue", queue: "BaseStashQueue",
copy_paths: Optional[List[str]] = None, copy_paths: Optional[list[str]] = None,
message: Optional[str] = None, message: Optional[str] = None,
**kwargs, **kwargs,
) -> Dict[str, str]: ) -> dict[str, str]:
"""Reproduce queued experiments. """Reproduce queued experiments.
Arguments: Arguments:
@@ -265,7 +266,7 @@ class Experiments:
""" """
exec_results = queue.reproduce(copy_paths=copy_paths, message=message) exec_results = queue.reproduce(copy_paths=copy_paths, message=message)
results: Dict[str, str] = {} results: dict[str, str] = {}
for _, exp_result in exec_results.items(): for _, exp_result in exec_results.items():
results.update(exp_result) results.update(exp_result)
return results return results
@@ -319,12 +320,12 @@ class Experiments:
raise MultipleBranchError(rev, ref_infos) raise MultipleBranchError(rev, ref_infos)
return str(ref_infos[0]) return str(ref_infos[0])
def get_exact_name(self, revs: Iterable[str]) -> Dict[str, Optional[str]]: def get_exact_name(self, revs: Iterable[str]) -> dict[str, Optional[str]]:
"""Returns preferred name for the specified revision. """Returns preferred name for the specified revision.
Prefers tags, branches (heads), experiments in that order. Prefers tags, branches (heads), experiments in that order.
""" """
result: Dict[str, Optional[str]] = {} result: dict[str, Optional[str]] = {}
exclude = f"{EXEC_NAMESPACE}/*" exclude = f"{EXEC_NAMESPACE}/*"
ref_dict = self.scm.describe(revs, base=EXPS_NAMESPACE, exclude=exclude) ref_dict = self.scm.describe(revs, base=EXPS_NAMESPACE, exclude=exclude)
for rev in revs: for rev in revs:
+3 -2
View File
@@ -1,5 +1,6 @@
from collections.abc import Iterator
from contextlib import ExitStack, contextmanager from contextlib import ExitStack, contextmanager
from typing import TYPE_CHECKING, Iterator, Tuple from typing import TYPE_CHECKING
from dvc.repo.experiments.exceptions import InvalidExpRevError from dvc.repo.experiments.exceptions import InvalidExpRevError
from dvc.scm import RevError from dvc.scm import RevError
@@ -12,7 +13,7 @@ if TYPE_CHECKING:
def switch_repo( def switch_repo(
repo: "Repo", repo: "Repo",
rev: str, rev: str,
) -> Iterator[Tuple["Repo", str]]: ) -> Iterator[tuple["Repo", str]]:
"""Return a repo instance (brancher) switched to rev. """Return a repo instance (brancher) switched to rev.
If rev is the name of a running experiment, the returned instance will be If rev is the name of a running experiment, the returned instance will be
+13 -18
View File
@@ -1,16 +1,11 @@
import itertools import itertools
import os import os
from collections.abc import Collection, Iterable, Iterator
from dataclasses import fields from dataclasses import fields
from datetime import datetime from datetime import datetime
from typing import ( from typing import (
TYPE_CHECKING, TYPE_CHECKING,
Collection,
Dict,
Iterable,
Iterator,
List,
Optional, Optional,
Tuple,
Union, Union,
) )
@@ -142,7 +137,7 @@ def collect_queued(
repo: "Repo", repo: "Repo",
baseline_revs: Collection[str], baseline_revs: Collection[str],
**kwargs, **kwargs,
) -> Dict[str, List["ExpRange"]]: ) -> dict[str, list["ExpRange"]]:
"""Collect queued experiments derived from the specified revisions. """Collect queued experiments derived from the specified revisions.
Args: Args:
@@ -177,7 +172,7 @@ def collect_active(
repo: "Repo", repo: "Repo",
baseline_revs: Collection[str], baseline_revs: Collection[str],
**kwargs, **kwargs,
) -> Dict[str, List["ExpRange"]]: ) -> dict[str, list["ExpRange"]]:
"""Collect active (running) experiments derived from the specified revisions. """Collect active (running) experiments derived from the specified revisions.
Args: Args:
@@ -189,7 +184,7 @@ def collect_active(
""" """
if not baseline_revs: if not baseline_revs:
return {} return {}
result: Dict[str, List["ExpRange"]] = {} result: dict[str, list["ExpRange"]] = {}
exps = repo.experiments exps = repo.experiments
for queue in (exps.workspace_queue, exps.tempdir_queue, exps.celery_queue): for queue in (exps.workspace_queue, exps.tempdir_queue, exps.celery_queue):
for baseline, active_exps in queue.collect_active_data( for baseline, active_exps in queue.collect_active_data(
@@ -206,7 +201,7 @@ def collect_failed(
repo: "Repo", repo: "Repo",
baseline_revs: Collection[str], baseline_revs: Collection[str],
**kwargs, **kwargs,
) -> Dict[str, List["ExpRange"]]: ) -> dict[str, list["ExpRange"]]:
"""Collect failed experiments derived from the specified revisions. """Collect failed experiments derived from the specified revisions.
Args: Args:
@@ -225,7 +220,7 @@ def collect_successful(
repo: "Repo", repo: "Repo",
baseline_revs: Collection[str], baseline_revs: Collection[str],
**kwargs, **kwargs,
) -> Dict[str, List["ExpRange"]]: ) -> dict[str, list["ExpRange"]]:
"""Collect successful experiments derived from the specified revisions. """Collect successful experiments derived from the specified revisions.
Args: Args:
@@ -235,7 +230,7 @@ def collect_successful(
Returns: Returns:
Dict mapping baseline revision to successful experiments. Dict mapping baseline revision to successful experiments.
""" """
result: Dict[str, List["ExpRange"]] = {} result: dict[str, list["ExpRange"]] = {}
for baseline_rev in baseline_revs: for baseline_rev in baseline_revs:
result[baseline_rev] = list(_collect_baseline(repo, baseline_rev, **kwargs)) result[baseline_rev] = list(_collect_baseline(repo, baseline_rev, **kwargs))
return result return result
@@ -282,7 +277,7 @@ def _collect_baseline(
def collect( def collect(
repo: "Repo", repo: "Repo",
revs: Union[List[str], str, None] = None, revs: Union[list[str], str, None] = None,
all_branches: bool = False, all_branches: bool = False,
all_tags: bool = False, all_tags: bool = False,
all_commits: bool = False, all_commits: bool = False,
@@ -291,7 +286,7 @@ def collect(
hide_failed: bool = False, hide_failed: bool = False,
sha_only: bool = False, sha_only: bool = False,
**kwargs, **kwargs,
) -> List["ExpState"]: ) -> list["ExpState"]:
"""Collect baseline revisions and derived experiments.""" """Collect baseline revisions and derived experiments."""
assert isinstance(repo.scm, Git) assert isinstance(repo.scm, Git)
if repo.scm.no_commits: if repo.scm.no_commits:
@@ -312,14 +307,14 @@ def collect(
) )
) )
if sha_only: if sha_only:
baseline_names: Dict[str, Optional[str]] = {} baseline_names: dict[str, Optional[str]] = {}
else: else:
baseline_names = describe( baseline_names = describe(
repo.scm, baseline_revs, refs=cached_refs, logger=logger repo.scm, baseline_revs, refs=cached_refs, logger=logger
) )
workspace_data = collect_rev(repo, "workspace", **kwargs) workspace_data = collect_rev(repo, "workspace", **kwargs)
result: List["ExpState"] = [workspace_data] result: list["ExpState"] = [workspace_data]
queued = collect_queued(repo, baseline_revs, **kwargs) if not hide_queued else {} queued = collect_queued(repo, baseline_revs, **kwargs) if not hide_queued else {}
active = collect_active(repo, baseline_revs, **kwargs) active = collect_active(repo, baseline_revs, **kwargs)
failed = collect_failed(repo, baseline_revs, **kwargs) if not hide_failed else {} failed = collect_failed(repo, baseline_revs, **kwargs) if not hide_failed else {}
@@ -345,10 +340,10 @@ def collect(
return result return result
def _sorted_ranges(exp_ranges: Iterable["ExpRange"]) -> List["ExpRange"]: def _sorted_ranges(exp_ranges: Iterable["ExpRange"]) -> list["ExpRange"]:
"""Return list of ExpRange sorted by (timestamp, rev).""" """Return list of ExpRange sorted by (timestamp, rev)."""
def _head_timestamp(exp_range: "ExpRange") -> Tuple[datetime, str]: def _head_timestamp(exp_range: "ExpRange") -> tuple[datetime, str]:
head_exp = first(exp_range.revs) head_exp = first(exp_range.revs)
if head_exp and head_exp.data and head_exp.data.timestamp: if head_exp and head_exp.data and head_exp.data.timestamp:
return head_exp.data.timestamp, head_exp.rev return head_exp.data.timestamp, head_exp.rev
+2 -1
View File
@@ -1,4 +1,5 @@
from typing import TYPE_CHECKING, Collection, Iterable, Optional from collections.abc import Collection, Iterable
from typing import TYPE_CHECKING, Optional
from dvc.exceptions import DvcException, InvalidArgumentError from dvc.exceptions import DvcException, InvalidArgumentError
+16 -23
View File
@@ -3,6 +3,7 @@ import os
import pickle import pickle
import shutil import shutil
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from collections.abc import Iterable, Iterator
from contextlib import contextmanager from contextlib import contextmanager
from dataclasses import asdict, dataclass from dataclasses import asdict, dataclass
from enum import IntEnum from enum import IntEnum
@@ -11,15 +12,8 @@ from typing import (
TYPE_CHECKING, TYPE_CHECKING,
Any, Any,
Callable, Callable,
Dict,
Iterable,
Iterator,
List,
NamedTuple, NamedTuple,
Optional, Optional,
Tuple,
Type,
TypeVar,
Union, Union,
) )
@@ -41,6 +35,8 @@ from dvc.utils.studio import env_to_config
if TYPE_CHECKING: if TYPE_CHECKING:
from queue import Queue from queue import Queue
from typing_extensions import Self
from dvc.repo import Repo from dvc.repo import Repo
from dvc.repo.experiments.stash import ExpStashEntry from dvc.repo.experiments.stash import ExpStashEntry
from dvc.scm import Git from dvc.scm import Git
@@ -112,9 +108,6 @@ class ExecutorInfo:
return cls.from_dict(load_json(filename)) return cls.from_dict(load_json(filename))
_T = TypeVar("_T", bound="BaseExecutor")
class BaseExecutor(ABC): class BaseExecutor(ABC):
"""Base class for executing experiments in parallel. """Base class for executing experiments in parallel.
@@ -185,7 +178,7 @@ class BaseExecutor(ABC):
@property @property
def info(self) -> "ExecutorInfo": def info(self) -> "ExecutorInfo":
if self.result is not None: if self.result is not None:
result_dict: Dict[str, Any] = { result_dict: dict[str, Any] = {
"result_hash": self.result.exp_hash, "result_hash": self.result.exp_hash,
"result_ref": ( "result_ref": (
str(self.result.ref_info) if self.result.ref_info else None str(self.result.ref_info) if self.result.ref_info else None
@@ -207,7 +200,7 @@ class BaseExecutor(ABC):
) )
@classmethod @classmethod
def from_info(cls: Type[_T], info: "ExecutorInfo") -> _T: def from_info(cls, info: "ExecutorInfo") -> "Self":
if info.result_hash: if info.result_hash:
result: Optional["ExecutorResult"] = ExecutorResult( result: Optional["ExecutorResult"] = ExecutorResult(
info.result_hash, info.result_hash,
@@ -229,21 +222,21 @@ class BaseExecutor(ABC):
@classmethod @classmethod
@abstractmethod @abstractmethod
def from_stash_entry( def from_stash_entry(
cls: Type[_T], cls,
repo: "Repo", repo: "Repo",
entry: "ExpStashEntry", entry: "ExpStashEntry",
**kwargs, **kwargs,
) -> _T: ) -> "Self":
pass pass
@classmethod @classmethod
def _from_stash_entry( def _from_stash_entry(
cls: Type[_T], cls,
repo: "Repo", repo: "Repo",
entry: "ExpStashEntry", entry: "ExpStashEntry",
root_dir: str, root_dir: str,
**kwargs, **kwargs,
) -> _T: ) -> "Self":
return cls( return cls(
root_dir=root_dir, root_dir=root_dir,
dvc_dir=relpath(repo.dvc_dir, repo.scm.root_dir), dvc_dir=relpath(repo.dvc_dir, repo.scm.root_dir),
@@ -255,7 +248,7 @@ class BaseExecutor(ABC):
) )
@classmethod @classmethod
def _get_top_level_paths(cls, repo: "Repo") -> List["str"]: def _get_top_level_paths(cls, repo: "Repo") -> list["str"]:
return list( return list(
chain( chain(
_collect_top_level_metrics(repo), _collect_top_level_metrics(repo),
@@ -271,7 +264,7 @@ class BaseExecutor(ABC):
targets: Optional[Iterable[str]] = None, targets: Optional[Iterable[str]] = None,
recursive: bool = False, recursive: bool = False,
force: bool = False, force: bool = False,
include_untracked: Optional[List[str]] = None, include_untracked: Optional[list[str]] = None,
message: Optional[str] = None, message: Optional[str] = None,
) -> ExecutorResult: ) -> ExecutorResult:
from dvc.dvcfile import LOCK_FILE from dvc.dvcfile import LOCK_FILE
@@ -382,7 +375,7 @@ class BaseExecutor(ABC):
def fetch_exps( def fetch_exps(
self, self,
dest_scm: "Git", dest_scm: "Git",
refs: List[str], refs: list[str],
force: bool = False, force: bool = False,
on_diverged: Optional[Callable[[str], None]] = None, on_diverged: Optional[Callable[[str], None]] = None,
**kwargs, **kwargs,
@@ -457,7 +450,7 @@ class BaseExecutor(ABC):
infofile: Optional[str] = None, infofile: Optional[str] = None,
log_errors: bool = True, log_errors: bool = True,
log_level: Optional[int] = None, log_level: Optional[int] = None,
copy_paths: Optional[List[str]] = None, copy_paths: Optional[list[str]] = None,
message: Optional[str] = None, message: Optional[str] = None,
**kwargs, **kwargs,
) -> "ExecutorResult": ) -> "ExecutorResult":
@@ -565,7 +558,7 @@ class BaseExecutor(ABC):
git_remote, git_remote,
repro_force, repro_force,
message: Optional[str] = None, message: Optional[str] = None,
) -> Tuple[Optional[str], Optional["ExpRefInfo"], bool]: ) -> tuple[Optional[str], Optional["ExpRefInfo"], bool]:
cls.commit( cls.commit(
dvc.scm, dvc.scm,
exp_hash, exp_hash,
@@ -598,7 +591,7 @@ class BaseExecutor(ABC):
info: "ExecutorInfo", info: "ExecutorInfo",
infofile: Optional[str] = None, infofile: Optional[str] = None,
log_errors: bool = True, log_errors: bool = True,
copy_paths: Optional[List[str]] = None, copy_paths: Optional[list[str]] = None,
message: Optional[str] = None, message: Optional[str] = None,
**kwargs, **kwargs,
) -> Iterator["Repo"]: ) -> Iterator["Repo"]:
@@ -790,7 +783,7 @@ class BaseExecutor(ABC):
raise DvcException(f"Unable to copy '{src}' to '{dst}'.") from exc raise DvcException(f"Unable to copy '{src}' to '{dst}'.") from exc
@contextmanager @contextmanager
def set_temp_refs(self, scm: "Git", temp_dict: Dict[str, str]): def set_temp_refs(self, scm: "Git", temp_dict: dict[str, str]):
try: try:
for ref, rev in temp_dict.items(): for ref, rev in temp_dict.items():
scm.set_ref(ref, rev) scm.set_ref(ref, rev)
+3 -3
View File
@@ -1,5 +1,5 @@
from collections import defaultdict from collections import defaultdict
from typing import Dict, List, Optional, Tuple, Union from typing import Optional, Union
from dvc.log import logger from dvc.log import logger
from dvc.repo import locked from dvc.repo import locked
@@ -15,11 +15,11 @@ logger = logger.getChild(__name__)
@scm_context @scm_context
def ls( def ls(
repo, repo,
rev: Optional[Union[List[str], str]] = None, rev: Optional[Union[list[str], str]] = None,
all_commits: bool = False, all_commits: bool = False,
num: int = 1, num: int = 1,
git_remote: Optional[str] = None, git_remote: Optional[str] = None,
) -> Dict[str, List[Tuple[str, Optional[str]]]]: ) -> dict[str, list[tuple[str, Optional[str]]]]:
"""List experiments. """List experiments.
Returns a dict mapping baseline revs to a list of (exp_name, exp_sha) tuples. Returns a dict mapping baseline revs to a list of (exp_name, exp_sha) tuples.
+6 -5
View File
@@ -1,4 +1,5 @@
from typing import Iterable, List, Mapping, Optional, Set, Union from collections.abc import Iterable, Mapping
from typing import Optional, Union
from funcy import group_by from funcy import group_by
from scmrepo.git.backend.base import SyncStatus from scmrepo.git.backend.base import SyncStatus
@@ -23,13 +24,13 @@ def pull( # noqa: C901
git_remote: str, git_remote: str,
exp_names: Optional[Union[Iterable[str], str]] = None, exp_names: Optional[Union[Iterable[str], str]] = None,
all_commits=False, all_commits=False,
rev: Optional[Union[List[str], str]] = None, rev: Optional[Union[list[str], str]] = None,
num=1, num=1,
force: bool = False, force: bool = False,
pull_cache: bool = False, pull_cache: bool = False,
**kwargs, **kwargs,
) -> Iterable[str]: ) -> Iterable[str]:
exp_ref_set: Set["ExpRefInfo"] = set() exp_ref_set: set["ExpRefInfo"] = set()
if all_commits: if all_commits:
exp_ref_set.update(exp_refs(repo.scm, git_remote)) exp_ref_set.update(exp_refs(repo.scm, git_remote))
elif exp_names: elif exp_names:
@@ -81,7 +82,7 @@ def _pull(
git_remote: str, git_remote: str,
refs: Iterable["ExpRefInfo"], refs: Iterable["ExpRefInfo"],
force: bool, force: bool,
) -> Mapping[SyncStatus, List["ExpRefInfo"]]: ) -> Mapping[SyncStatus, list["ExpRefInfo"]]:
refspec_list = [f"{exp_ref}:{exp_ref}" for exp_ref in refs] refspec_list = [f"{exp_ref}:{exp_ref}" for exp_ref in refs]
logger.debug("git pull experiment '%s' -> '%s'", git_remote, refspec_list) logger.debug("git pull experiment '%s' -> '%s'", git_remote, refspec_list)
@@ -96,7 +97,7 @@ def _pull(
def group_result(refspec): def group_result(refspec):
return results[str(refspec)] return results[str(refspec)]
pull_result: Mapping[SyncStatus, List["ExpRefInfo"]] = group_by(group_result, refs) pull_result: Mapping[SyncStatus, list["ExpRefInfo"]] = group_by(group_result, refs)
return pull_result return pull_result
+11 -15
View File
@@ -1,12 +1,8 @@
from collections.abc import Iterable, Mapping
from typing import ( from typing import (
TYPE_CHECKING, TYPE_CHECKING,
Any, Any,
Dict,
Iterable,
List,
Mapping,
Optional, Optional,
Set,
Union, Union,
) )
@@ -39,7 +35,7 @@ class UploadError(DvcException):
def notify_refs_to_studio( def notify_refs_to_studio(
repo: "Repo", git_remote: str, **refs: List[str] repo: "Repo", git_remote: str, **refs: list[str]
) -> Optional[str]: ) -> Optional[str]:
import os import os
@@ -67,7 +63,7 @@ def notify_refs_to_studio(
return d.get("url") return d.get("url")
def exp_refs_from_names(scm: "Git", exp_names: List[str]) -> Set["ExpRefInfo"]: def exp_refs_from_names(scm: "Git", exp_names: list[str]) -> set["ExpRefInfo"]:
exp_ref_set = set() exp_ref_set = set()
exp_ref_dict = resolve_name(scm, exp_names) exp_ref_dict = resolve_name(scm, exp_names)
unresolved_exp_names = [] unresolved_exp_names = []
@@ -82,7 +78,7 @@ def exp_refs_from_names(scm: "Git", exp_names: List[str]) -> Set["ExpRefInfo"]:
return exp_ref_set return exp_ref_set
def exp_refs_from_rev(scm: "Git", rev: List[str], num: int = 1) -> Set["ExpRefInfo"]: def exp_refs_from_rev(scm: "Git", rev: list[str], num: int = 1) -> set["ExpRefInfo"]:
exp_ref_set = set() exp_ref_set = set()
rev_dict = iter_revs(scm, rev, num) rev_dict = iter_revs(scm, rev, num)
rev_set = set(rev_dict.keys()) rev_set = set(rev_dict.keys())
@@ -97,15 +93,15 @@ def exp_refs_from_rev(scm: "Git", rev: List[str], num: int = 1) -> Set["ExpRefIn
def push( def push(
repo: "Repo", repo: "Repo",
git_remote: str, git_remote: str,
exp_names: Optional[Union[List[str], str]] = None, exp_names: Optional[Union[list[str], str]] = None,
all_commits: bool = False, all_commits: bool = False,
rev: Optional[Union[List[str], str]] = None, rev: Optional[Union[list[str], str]] = None,
num: int = 1, num: int = 1,
force: bool = False, force: bool = False,
push_cache: bool = False, push_cache: bool = False,
**kwargs: Any, **kwargs: Any,
) -> Dict[str, Any]: ) -> dict[str, Any]:
exp_ref_set: Set["ExpRefInfo"] = set() exp_ref_set: set["ExpRefInfo"] = set()
assert isinstance(repo.scm, Git) assert isinstance(repo.scm, Git)
if all_commits: if all_commits:
exp_ref_set.update(exp_refs(repo.scm)) exp_ref_set.update(exp_refs(repo.scm))
@@ -123,7 +119,7 @@ def push(
status.name.lower(): [ref.name for ref in ref_list] status.name.lower(): [ref.name for ref in ref_list]
for status, ref_list in push_result.items() for status, ref_list in push_result.items()
} }
result: Dict[str, Any] = {**refs, "uploaded": 0} result: dict[str, Any] = {**refs, "uploaded": 0}
pushed_refs_info = ( pushed_refs_info = (
push_result[SyncStatus.UP_TO_DATE] + push_result[SyncStatus.SUCCESS] push_result[SyncStatus.UP_TO_DATE] + push_result[SyncStatus.SUCCESS]
@@ -149,7 +145,7 @@ def _push(
git_remote: str, git_remote: str,
refs: Iterable["ExpRefInfo"], refs: Iterable["ExpRefInfo"],
force: bool, force: bool,
) -> Mapping[SyncStatus, List["ExpRefInfo"]]: ) -> Mapping[SyncStatus, list["ExpRefInfo"]]:
from scmrepo.exceptions import AuthError from scmrepo.exceptions import AuthError
from dvc.scm import GitAuthError from dvc.scm import GitAuthError
@@ -171,7 +167,7 @@ def _push(
def group_result(refspec): def group_result(refspec):
return results[str(refspec)] return results[str(refspec)]
pull_result: Mapping[SyncStatus, List["ExpRefInfo"]] = group_by(group_result, refs) pull_result: Mapping[SyncStatus, list["ExpRefInfo"]] = group_by(group_result, refs)
return pull_result return pull_result
+23 -29
View File
@@ -1,18 +1,12 @@
import os import os
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from collections.abc import Collection, Generator, Iterable, Mapping
from dataclasses import asdict, dataclass from dataclasses import asdict, dataclass
from typing import ( from typing import (
TYPE_CHECKING, TYPE_CHECKING,
Any, Any,
Collection,
Dict,
Generator,
Iterable,
List,
Mapping,
NamedTuple, NamedTuple,
Optional, Optional,
Type,
Union, Union,
) )
@@ -69,11 +63,11 @@ class QueueEntry:
and self.stash_rev == other.stash_rev and self.stash_rev == other.stash_rev
) )
def asdict(self) -> Dict[str, Any]: def asdict(self) -> dict[str, Any]:
return asdict(self) return asdict(self)
@classmethod @classmethod
def from_dict(cls, d: Dict[str, Any]) -> "QueueEntry": def from_dict(cls, d: dict[str, Any]) -> "QueueEntry":
return cls(**d) return cls(**d)
@@ -150,7 +144,7 @@ class BaseStashQueue(ABC):
all_: bool = False, all_: bool = False,
queued: bool = False, queued: bool = False,
**kwargs, **kwargs,
) -> List[str]: ) -> list[str]:
"""Remove the specified entries from the queue. """Remove the specified entries from the queue.
Arguments: Arguments:
@@ -165,8 +159,8 @@ class BaseStashQueue(ABC):
if all_ or queued: if all_ or queued:
return self.clear() return self.clear()
name_to_remove: List[str] = [] name_to_remove: list[str] = []
entry_to_remove: List[ExpStashEntry] = [] entry_to_remove: list[ExpStashEntry] = []
queue_entries = self.match_queue_entry_by_name(revs, self.iter_queued()) queue_entries = self.match_queue_entry_by_name(revs, self.iter_queued())
for name, entry in queue_entries.items(): for name, entry in queue_entries.items():
if entry: if entry:
@@ -176,7 +170,7 @@ class BaseStashQueue(ABC):
self.stash.remove_revs(entry_to_remove) self.stash.remove_revs(entry_to_remove)
return name_to_remove return name_to_remove
def clear(self, **kwargs) -> List[str]: def clear(self, **kwargs) -> list[str]:
"""Remove all entries from the queue.""" """Remove all entries from the queue."""
stash_revs = self.stash.stash_revs stash_revs = self.stash.stash_revs
name_to_remove = list(stash_revs) name_to_remove = list(stash_revs)
@@ -184,11 +178,11 @@ class BaseStashQueue(ABC):
return name_to_remove return name_to_remove
def status(self) -> List[Dict[str, Any]]: def status(self) -> list[dict[str, Any]]:
"""Show the status of exp tasks in queue""" """Show the status of exp tasks in queue"""
from datetime import datetime from datetime import datetime
result: List[Dict[str, Optional[str]]] = [] result: list[dict[str, Optional[str]]] = []
def _get_timestamp(rev: str) -> datetime: def _get_timestamp(rev: str) -> datetime:
commit = self.scm.resolve_commit(rev) commit = self.scm.resolve_commit(rev)
@@ -198,7 +192,7 @@ class BaseStashQueue(ABC):
entry: QueueEntry, entry: QueueEntry,
exp_result: Optional["ExecutorResult"] = None, exp_result: Optional["ExecutorResult"] = None,
status: str = "Unknown", status: str = "Unknown",
) -> Dict[str, Any]: ) -> dict[str, Any]:
name = entry.name name = entry.name
if not name and exp_result and exp_result.ref_info: if not name and exp_result and exp_result.ref_info:
name = exp_result.ref_info.name name = exp_result.ref_info.name
@@ -251,7 +245,7 @@ class BaseStashQueue(ABC):
@abstractmethod @abstractmethod
def reproduce( def reproduce(
self, copy_paths: Optional[List[str]] = None, message: Optional[str] = None self, copy_paths: Optional[list[str]] = None, message: Optional[str] = None
) -> Mapping[str, Mapping[str, str]]: ) -> Mapping[str, Mapping[str, str]]:
"""Reproduce queued experiments sequentially.""" """Reproduce queued experiments sequentially."""
@@ -300,7 +294,7 @@ class BaseStashQueue(ABC):
def _stash_exp( def _stash_exp(
self, self,
*args, *args,
params: Optional[Dict[str, List[str]]] = None, params: Optional[dict[str, list[str]]] = None,
baseline_rev: Optional[str] = None, baseline_rev: Optional[str] = None,
branch: Optional[str] = None, branch: Optional[str] = None,
name: Optional[str] = None, name: Optional[str] = None,
@@ -460,7 +454,7 @@ class BaseStashQueue(ABC):
f"from '{config_path}': {param_list}" f"from '{config_path}': {param_list}"
) )
def _update_params(self, params: Dict[str, List[str]]): def _update_params(self, params: dict[str, list[str]]):
"""Update param files with the provided `Hydra Override`_ patterns. """Update param files with the provided `Hydra Override`_ patterns.
Args: Args:
@@ -528,7 +522,7 @@ class BaseStashQueue(ABC):
cls, cls,
exp: "Experiments", exp: "Experiments",
queue_entry: QueueEntry, queue_entry: QueueEntry,
executor_cls: Type[BaseExecutor] = WorkspaceExecutor, executor_cls: type[BaseExecutor] = WorkspaceExecutor,
**kwargs, **kwargs,
) -> BaseExecutor: ) -> BaseExecutor:
stash_entry = cls.get_stash_entry(exp, queue_entry) stash_entry = cls.get_stash_entry(exp, queue_entry)
@@ -563,7 +557,7 @@ class BaseStashQueue(ABC):
exp: "Experiments", exp: "Experiments",
executor: BaseExecutor, executor: BaseExecutor,
exec_result: "ExecutorResult", exec_result: "ExecutorResult",
) -> Dict[str, str]: ) -> dict[str, str]:
results = {} results = {}
def on_diverged(ref: str): def on_diverged(ref: str):
@@ -593,7 +587,7 @@ class BaseStashQueue(ABC):
exp: "Experiments", exp: "Experiments",
executor: BaseExecutor, executor: BaseExecutor,
exec_result: "ExecutorResult", exec_result: "ExecutorResult",
) -> Dict[str, str]: ) -> dict[str, str]:
results = cls.collect_git(exp, executor, exec_result) results = cls.collect_git(exp, executor, exec_result)
if exec_result.ref_info is not None: if exec_result.ref_info is not None:
@@ -605,11 +599,11 @@ class BaseStashQueue(ABC):
self, self,
exp_names: Collection[str], exp_names: Collection[str],
*entries: Iterable[Union[QueueEntry, QueueDoneResult]], *entries: Iterable[Union[QueueEntry, QueueDoneResult]],
) -> Dict[str, Optional[QueueEntry]]: ) -> dict[str, Optional[QueueEntry]]:
from funcy import concat from funcy import concat
entry_name_dict: Dict[str, QueueEntry] = {} entry_name_dict: dict[str, QueueEntry] = {}
entry_rev_dict: Dict[str, QueueEntry] = {} entry_rev_dict: dict[str, QueueEntry] = {}
for entry in concat(*entries): for entry in concat(*entries):
if isinstance(entry, QueueDoneResult): if isinstance(entry, QueueDoneResult):
queue_entry: QueueEntry = entry.entry queue_entry: QueueEntry = entry.entry
@@ -624,7 +618,7 @@ class BaseStashQueue(ABC):
entry_name_dict[name] = queue_entry entry_name_dict[name] = queue_entry
entry_rev_dict[queue_entry.stash_rev] = queue_entry entry_rev_dict[queue_entry.stash_rev] = queue_entry
result: Dict[str, Optional[QueueEntry]] = {} result: dict[str, Optional[QueueEntry]] = {}
for exp_name in exp_names: for exp_name in exp_names:
result[exp_name] = None result[exp_name] = None
if exp_name in entry_name_dict: if exp_name in entry_name_dict:
@@ -666,7 +660,7 @@ class BaseStashQueue(ABC):
baseline_revs: Optional[Collection[str]], baseline_revs: Optional[Collection[str]],
fetch_refs: bool = False, fetch_refs: bool = False,
**kwargs, **kwargs,
) -> Dict[str, List["ExpRange"]]: ) -> dict[str, list["ExpRange"]]:
"""Collect data for active (running) experiments. """Collect data for active (running) experiments.
Args: Args:
@@ -685,7 +679,7 @@ class BaseStashQueue(ABC):
self, self,
baseline_revs: Optional[Collection[str]], baseline_revs: Optional[Collection[str]],
**kwargs, **kwargs,
) -> Dict[str, List["ExpRange"]]: ) -> dict[str, list["ExpRange"]]:
"""Collect data for queued experiments. """Collect data for queued experiments.
Args: Args:
@@ -702,7 +696,7 @@ class BaseStashQueue(ABC):
self, self,
baseline_revs: Optional[Collection[str]], baseline_revs: Optional[Collection[str]],
**kwargs, **kwargs,
) -> Dict[str, List["ExpRange"]]: ) -> dict[str, list["ExpRange"]]:
"""Collect data for failed experiments. """Collect data for failed experiments.
Args: Args:
+31 -36
View File
@@ -3,16 +3,11 @@ import locale
import logging import logging
import os import os
from collections import defaultdict from collections import defaultdict
from collections.abc import Collection, Generator, Mapping
from typing import ( from typing import (
TYPE_CHECKING, TYPE_CHECKING,
Collection,
Dict,
Generator,
List,
Mapping,
NamedTuple, NamedTuple,
Optional, Optional,
Set,
Union, Union,
) )
@@ -166,7 +161,7 @@ class LocalCeleryQueue(BaseStashQueue):
""" """
logger.debug("Spawning %s exp queue workers", count) logger.debug("Spawning %s exp queue workers", count)
active_worker: Dict = self.worker_status() active_worker: dict = self.worker_status()
started = 0 started = 0
for num in range(1, 1 + count): for num in range(1, 1 + count):
@@ -183,7 +178,7 @@ class LocalCeleryQueue(BaseStashQueue):
def put( def put(
self, self,
*args, *args,
copy_paths: Optional[List[str]] = None, copy_paths: Optional[list[str]] = None,
message: Optional[str] = None, message: Optional[str] = None,
**kwargs, **kwargs,
) -> QueueEntry: ) -> QueueEntry:
@@ -265,7 +260,7 @@ class LocalCeleryQueue(BaseStashQueue):
yield QueueDoneResult(queue_entry, exp_result) yield QueueDoneResult(queue_entry, exp_result)
def reproduce( def reproduce(
self, copy_paths: Optional[List[str]] = None, message: Optional[str] = None self, copy_paths: Optional[list[str]] = None, message: Optional[str] = None
) -> Mapping[str, Mapping[str, str]]: ) -> Mapping[str, Mapping[str, str]]:
raise NotImplementedError raise NotImplementedError
@@ -333,8 +328,8 @@ class LocalCeleryQueue(BaseStashQueue):
while not self.proc.get(entry.stash_rev): while not self.proc.get(entry.stash_rev):
time.sleep(sleep_interval) time.sleep(sleep_interval)
def _get_running_task_ids(self) -> Set[str]: def _get_running_task_ids(self) -> set[str]:
running_task_ids: Set[str] = set() running_task_ids: set[str] = set()
active_workers = self.worker_status() active_workers = self.worker_status()
for _, tasks in active_workers.items(): for _, tasks in active_workers.items():
task = first(tasks) task = first(tasks)
@@ -343,9 +338,9 @@ class LocalCeleryQueue(BaseStashQueue):
return running_task_ids return running_task_ids
def _try_to_kill_tasks( def _try_to_kill_tasks(
self, to_kill: Dict[QueueEntry, str], force: bool self, to_kill: dict[QueueEntry, str], force: bool
) -> Dict[QueueEntry, str]: ) -> dict[QueueEntry, str]:
fail_to_kill_entries: Dict[QueueEntry, str] = {} fail_to_kill_entries: dict[QueueEntry, str] = {}
for queue_entry, rev in to_kill.items(): for queue_entry, rev in to_kill.items():
try: try:
if force: if force:
@@ -358,9 +353,9 @@ class LocalCeleryQueue(BaseStashQueue):
return fail_to_kill_entries return fail_to_kill_entries
def _mark_inactive_tasks_failure( def _mark_inactive_tasks_failure(
self, remained_entries: Dict[QueueEntry, str] self, remained_entries: dict[QueueEntry, str]
) -> None: ) -> None:
remained_revs: List[str] = [] remained_revs: list[str] = []
running_ids = self._get_running_task_ids() running_ids = self._get_running_task_ids()
logger.debug("Current running tasks ids: %s.", running_ids) logger.debug("Current running tasks ids: %s.", running_ids)
for msg, entry in self._iter_processed(): for msg, entry in self._iter_processed():
@@ -383,12 +378,12 @@ class LocalCeleryQueue(BaseStashQueue):
if remained_revs: if remained_revs:
raise CannotKillTasksError(remained_revs) raise CannotKillTasksError(remained_revs)
def _kill_entries(self, entries: Dict[QueueEntry, str], force: bool) -> None: def _kill_entries(self, entries: dict[QueueEntry, str], force: bool) -> None:
logger.debug( logger.debug(
"Found active tasks: '%s' to kill", "Found active tasks: '%s' to kill",
list(entries.values()), list(entries.values()),
) )
inactive_entries: Dict[QueueEntry, str] = self._try_to_kill_tasks( inactive_entries: dict[QueueEntry, str] = self._try_to_kill_tasks(
entries, force entries, force
) )
@@ -396,12 +391,12 @@ class LocalCeleryQueue(BaseStashQueue):
self._mark_inactive_tasks_failure(inactive_entries) self._mark_inactive_tasks_failure(inactive_entries)
def kill(self, revs: Collection[str], force: bool = False) -> None: def kill(self, revs: Collection[str], force: bool = False) -> None:
name_dict: Dict[str, Optional[QueueEntry]] = self.match_queue_entry_by_name( name_dict: dict[str, Optional[QueueEntry]] = self.match_queue_entry_by_name(
set(revs), self.iter_active() set(revs), self.iter_active()
) )
missing_revs: List[str] = [] missing_revs: list[str] = []
to_kill: Dict[QueueEntry, str] = {} to_kill: dict[QueueEntry, str] = {}
for rev, queue_entry in name_dict.items(): for rev, queue_entry in name_dict.items():
if queue_entry is None: if queue_entry is None:
missing_revs.append(rev) missing_revs.append(rev)
@@ -417,7 +412,7 @@ class LocalCeleryQueue(BaseStashQueue):
def shutdown(self, kill: bool = False): def shutdown(self, kill: bool = False):
self.celery.control.shutdown() self.celery.control.shutdown()
if kill: if kill:
to_kill: Dict[QueueEntry, str] = {} to_kill: dict[QueueEntry, str] = {}
for entry in self.iter_active(): for entry in self.iter_active():
to_kill[entry] = entry.name or entry.stash_rev to_kill[entry] = entry.name or entry.stash_rev
if to_kill: if to_kill:
@@ -468,7 +463,7 @@ class LocalCeleryQueue(BaseStashQueue):
) as fobj: ) as fobj:
ui.write(fobj.read()) ui.write(fobj.read())
def worker_status(self) -> Dict[str, List[Dict]]: def worker_status(self) -> dict[str, list[dict]]:
"""Return the current active celery worker""" """Return the current active celery worker"""
status = self.celery.control.inspect().active() or {} status = self.celery.control.inspect().active() or {}
logger.debug("Worker status: %s", status) logger.debug("Worker status: %s", status)
@@ -486,21 +481,21 @@ class LocalCeleryQueue(BaseStashQueue):
def get_ref_and_entry_by_names( def get_ref_and_entry_by_names(
self, self,
exp_names: Union[str, List[str]], exp_names: Union[str, list[str]],
git_remote: Optional[str] = None, git_remote: Optional[str] = None,
) -> Dict[str, ExpRefAndQueueEntry]: ) -> dict[str, ExpRefAndQueueEntry]:
"""Find finished ExpRefInfo or queued or failed QueueEntry by name""" """Find finished ExpRefInfo or queued or failed QueueEntry by name"""
from dvc.repo.experiments.utils import resolve_name from dvc.repo.experiments.utils import resolve_name
if isinstance(exp_names, str): if isinstance(exp_names, str):
exp_names = [exp_names] exp_names = [exp_names]
results: Dict[str, ExpRefAndQueueEntry] = {} results: dict[str, ExpRefAndQueueEntry] = {}
exp_ref_match: Dict[str, Optional["ExpRefInfo"]] = resolve_name( exp_ref_match: dict[str, Optional["ExpRefInfo"]] = resolve_name(
self.scm, exp_names, git_remote self.scm, exp_names, git_remote
) )
if not git_remote: if not git_remote:
queue_entry_match: Dict[ queue_entry_match: dict[
str, Optional["QueueEntry"] str, Optional["QueueEntry"]
] = self.match_queue_entry_by_name( ] = self.match_queue_entry_by_name(
exp_names, self.iter_queued(), self.iter_done() exp_names, self.iter_queued(), self.iter_done()
@@ -517,7 +512,7 @@ class LocalCeleryQueue(BaseStashQueue):
baseline_revs: Optional[Collection[str]], baseline_revs: Optional[Collection[str]],
fetch_refs: bool = False, fetch_refs: bool = False,
**kwargs, **kwargs,
) -> Dict[str, List["ExpRange"]]: ) -> dict[str, list["ExpRange"]]:
from dvc.repo import Repo from dvc.repo import Repo
from dvc.repo.experiments.collect import collect_exec_branch from dvc.repo.experiments.collect import collect_exec_branch
from dvc.repo.experiments.serialize import ( from dvc.repo.experiments.serialize import (
@@ -526,7 +521,7 @@ class LocalCeleryQueue(BaseStashQueue):
LocalExpExecutor, LocalExpExecutor,
) )
result: Dict[str, List[ExpRange]] = defaultdict(list) result: dict[str, list[ExpRange]] = defaultdict(list)
for entry in self.iter_active(): for entry in self.iter_active():
if baseline_revs and entry.baseline_rev not in baseline_revs: if baseline_revs and entry.baseline_rev not in baseline_revs:
continue continue
@@ -568,7 +563,7 @@ class LocalCeleryQueue(BaseStashQueue):
self, self,
baseline_revs: Optional[Collection[str]], baseline_revs: Optional[Collection[str]],
**kwargs, **kwargs,
) -> Dict[str, List["ExpRange"]]: ) -> dict[str, list["ExpRange"]]:
from dvc.repo.experiments.collect import collect_rev from dvc.repo.experiments.collect import collect_rev
from dvc.repo.experiments.serialize import ( from dvc.repo.experiments.serialize import (
ExpExecutor, ExpExecutor,
@@ -576,7 +571,7 @@ class LocalCeleryQueue(BaseStashQueue):
LocalExpExecutor, LocalExpExecutor,
) )
result: Dict[str, List[ExpRange]] = defaultdict(list) result: dict[str, list[ExpRange]] = defaultdict(list)
for entry in self.iter_queued(): for entry in self.iter_queued():
if baseline_revs and entry.baseline_rev not in baseline_revs: if baseline_revs and entry.baseline_rev not in baseline_revs:
continue continue
@@ -598,7 +593,7 @@ class LocalCeleryQueue(BaseStashQueue):
self, self,
baseline_revs: Optional[Collection[str]], baseline_revs: Optional[Collection[str]],
**kwargs, **kwargs,
) -> Dict[str, List["ExpRange"]]: ) -> dict[str, list["ExpRange"]]:
from dvc.repo.experiments.collect import collect_rev from dvc.repo.experiments.collect import collect_rev
from dvc.repo.experiments.serialize import ( from dvc.repo.experiments.serialize import (
ExpExecutor, ExpExecutor,
@@ -607,7 +602,7 @@ class LocalCeleryQueue(BaseStashQueue):
SerializableError, SerializableError,
) )
result: Dict[str, List[ExpRange]] = defaultdict(list) result: dict[str, list[ExpRange]] = defaultdict(list)
for entry, _ in self.iter_failed(): for entry, _ in self.iter_failed():
if baseline_revs and entry.baseline_rev not in baseline_revs: if baseline_revs and entry.baseline_rev not in baseline_revs:
continue continue
@@ -637,11 +632,11 @@ class LocalCeleryQueue(BaseStashQueue):
self, self,
baseline_revs: Optional[Collection[str]], baseline_revs: Optional[Collection[str]],
**kwargs, **kwargs,
) -> Dict[str, "ExpExecutor"]: ) -> dict[str, "ExpExecutor"]:
"""Map exp refs to any available successful executors.""" """Map exp refs to any available successful executors."""
from dvc.repo.experiments.serialize import ExpExecutor, LocalExpExecutor from dvc.repo.experiments.serialize import ExpExecutor, LocalExpExecutor
result: Dict[str, "ExpExecutor"] = {} result: dict[str, "ExpExecutor"] = {}
for entry, exec_result in self.iter_success(): for entry, exec_result in self.iter_success():
if baseline_revs and entry.baseline_rev not in baseline_revs: if baseline_revs and entry.baseline_rev not in baseline_revs:
continue continue
+1 -1
View File
@@ -1,4 +1,4 @@
from typing import Collection from collections.abc import Collection
from dvc.exceptions import DvcException from dvc.exceptions import DvcException
+17 -16
View File
@@ -1,4 +1,5 @@
from typing import TYPE_CHECKING, Collection, Dict, Iterable, List, Set, Union from collections.abc import Collection, Iterable
from typing import TYPE_CHECKING, Union
from dvc.repo.experiments.exceptions import UnresolvedExpNamesError from dvc.repo.experiments.exceptions import UnresolvedExpNamesError
from dvc.repo.experiments.queue.base import QueueDoneResult from dvc.repo.experiments.queue.base import QueueDoneResult
@@ -20,11 +21,11 @@ def remove_tasks( # noqa: C901, PLR0912
""" """
from celery.result import AsyncResult from celery.result import AsyncResult
stash_revs: Dict[str, "ExpStashEntry"] = {} stash_revs: dict[str, "ExpStashEntry"] = {}
failed_stash_revs: List["ExpStashEntry"] = [] failed_stash_revs: list["ExpStashEntry"] = []
done_entry_set: Set["QueueEntry"] = set() done_entry_set: set["QueueEntry"] = set()
stash_rev_all = celery_queue.stash.stash_revs stash_rev_all = celery_queue.stash.stash_revs
failed_rev_all: Dict[str, "ExpStashEntry"] = {} failed_rev_all: dict[str, "ExpStashEntry"] = {}
if celery_queue.failed_stash: if celery_queue.failed_stash:
failed_rev_all = celery_queue.failed_stash.stash_revs failed_rev_all = celery_queue.failed_stash.stash_revs
for entry in queue_entries: for entry in queue_entries:
@@ -64,7 +65,7 @@ def remove_tasks( # noqa: C901, PLR0912
def _get_names(entries: Iterable[Union["QueueEntry", "QueueDoneResult"]]): def _get_names(entries: Iterable[Union["QueueEntry", "QueueDoneResult"]]):
names: List[str] = [] names: list[str] = []
for entry in entries: for entry in entries:
if isinstance(entry, QueueDoneResult): if isinstance(entry, QueueDoneResult):
if entry.result and entry.result.ref_info: if entry.result and entry.result.ref_info:
@@ -82,7 +83,7 @@ def celery_clear(
queued: bool = False, queued: bool = False,
failed: bool = False, failed: bool = False,
success: bool = False, success: bool = False,
) -> List[str]: ) -> list[str]:
"""Remove entries from the queue. """Remove entries from the queue.
Arguments: Arguments:
@@ -94,18 +95,18 @@ def celery_clear(
Revisions which were removed. Revisions which were removed.
""" """
removed: List[str] = [] removed: list[str] = []
entry_list: List["QueueEntry"] = [] entry_list: list["QueueEntry"] = []
if queued: if queued:
queue_entries: List["QueueEntry"] = list(self.iter_queued()) queue_entries: list["QueueEntry"] = list(self.iter_queued())
entry_list.extend(queue_entries) entry_list.extend(queue_entries)
removed.extend(_get_names(queue_entries)) removed.extend(_get_names(queue_entries))
if failed: if failed:
failed_tasks: List["QueueDoneResult"] = list(self.iter_failed()) failed_tasks: list["QueueDoneResult"] = list(self.iter_failed())
entry_list.extend([result.entry for result in failed_tasks]) entry_list.extend([result.entry for result in failed_tasks])
removed.extend(_get_names(failed_tasks)) removed.extend(_get_names(failed_tasks))
if success: if success:
success_tasks: List["QueueDoneResult"] = list(self.iter_success()) success_tasks: list["QueueDoneResult"] = list(self.iter_success())
entry_list.extend([result.entry for result in success_tasks]) entry_list.extend([result.entry for result in success_tasks])
removed.extend(_get_names(success_tasks)) removed.extend(_get_names(success_tasks))
@@ -117,7 +118,7 @@ def celery_clear(
def celery_remove( def celery_remove(
self: "LocalCeleryQueue", self: "LocalCeleryQueue",
revs: Collection[str], revs: Collection[str],
) -> List[str]: ) -> list[str]:
"""Remove the specified entries from the queue. """Remove the specified entries from the queue.
Arguments: Arguments:
@@ -131,9 +132,9 @@ def celery_remove(
revs, self.iter_queued(), self.iter_done() revs, self.iter_queued(), self.iter_done()
) )
remained: List[str] = [] remained: list[str] = []
removed: List[str] = [] removed: list[str] = []
entry_to_remove: List["QueueEntry"] = [] entry_to_remove: list["QueueEntry"] = []
for name, entry in match_results.items(): for name, entry in match_results.items():
if entry: if entry:
entry_to_remove.append(entry) entry_to_remove.append(entry)
+6 -6
View File
@@ -1,4 +1,4 @@
from typing import TYPE_CHECKING, Any, Dict, List, Optional from typing import TYPE_CHECKING, Any, Optional
from celery import shared_task from celery import shared_task
from celery.utils.log import get_task_logger from celery.utils.log import get_task_logger
@@ -16,7 +16,7 @@ logger = get_task_logger(__name__)
@shared_task @shared_task
def setup_exp(entry_dict: Dict[str, Any]) -> "BaseExecutor": def setup_exp(entry_dict: dict[str, Any]) -> "BaseExecutor":
"""Setup an experiment. """Setup an experiment.
Arguments: Arguments:
@@ -44,8 +44,8 @@ def setup_exp(entry_dict: Dict[str, Any]) -> "BaseExecutor":
@shared_task @shared_task
def collect_exp( def collect_exp(
proc_dict: Dict[str, Any], # noqa: ARG001 proc_dict: dict[str, Any], # noqa: ARG001
entry_dict: Dict[str, Any], entry_dict: dict[str, Any],
) -> str: ) -> str:
"""Collect results for an experiment. """Collect results for an experiment.
@@ -92,8 +92,8 @@ def cleanup_exp(executor: TempDirExecutor, infofile: str) -> None:
@shared_task @shared_task
def run_exp( def run_exp(
entry_dict: Dict[str, Any], entry_dict: dict[str, Any],
copy_paths: Optional[List[str]] = None, copy_paths: Optional[list[str]] = None,
message: Optional[str] = None, message: Optional[str] = None,
) -> None: ) -> None:
"""Run a full experiment. """Run a full experiment.
+8 -7
View File
@@ -1,6 +1,7 @@
import os import os
from collections import defaultdict from collections import defaultdict
from typing import TYPE_CHECKING, Collection, Dict, Generator, List, Optional from collections.abc import Collection, Generator
from typing import TYPE_CHECKING, Optional
from funcy import first from funcy import first
@@ -96,11 +97,11 @@ class TempDirQueue(WorkspaceQueue):
self, self,
entry: QueueEntry, entry: QueueEntry,
executor: "BaseExecutor", executor: "BaseExecutor",
copy_paths: Optional[List[str]] = None, copy_paths: Optional[list[str]] = None,
message: Optional[str] = None, message: Optional[str] = None,
**kwargs, **kwargs,
) -> Dict[str, Dict[str, str]]: ) -> dict[str, dict[str, str]]:
results: Dict[str, Dict[str, str]] = defaultdict(dict) results: dict[str, dict[str, str]] = defaultdict(dict)
exec_name = self._EXEC_NAME or entry.stash_rev exec_name = self._EXEC_NAME or entry.stash_rev
infofile = self.get_infofile_path(exec_name) infofile = self.get_infofile_path(exec_name)
try: try:
@@ -135,7 +136,7 @@ class TempDirQueue(WorkspaceQueue):
exp: "Experiments", exp: "Experiments",
executor: "BaseExecutor", executor: "BaseExecutor",
exec_result: "ExecutorResult", exec_result: "ExecutorResult",
) -> Dict[str, str]: ) -> dict[str, str]:
return BaseStashQueue.collect_executor(exp, executor, exec_result) return BaseStashQueue.collect_executor(exp, executor, exec_result)
def collect_active_data( def collect_active_data(
@@ -143,7 +144,7 @@ class TempDirQueue(WorkspaceQueue):
baseline_revs: Optional[Collection[str]], baseline_revs: Optional[Collection[str]],
fetch_refs: bool = False, fetch_refs: bool = False,
**kwargs, **kwargs,
) -> Dict[str, List["ExpRange"]]: ) -> dict[str, list["ExpRange"]]:
from dvc.repo import Repo from dvc.repo import Repo
from dvc.repo.experiments.collect import collect_exec_branch from dvc.repo.experiments.collect import collect_exec_branch
from dvc.repo.experiments.serialize import ( from dvc.repo.experiments.serialize import (
@@ -152,7 +153,7 @@ class TempDirQueue(WorkspaceQueue):
LocalExpExecutor, LocalExpExecutor,
) )
result: Dict[str, List[ExpRange]] = defaultdict(list) result: dict[str, list[ExpRange]] = defaultdict(list)
for entry in self.iter_active(): for entry in self.iter_active():
if baseline_revs and entry.baseline_rev not in baseline_revs: if baseline_revs and entry.baseline_rev not in baseline_revs:
continue continue
+4 -4
View File
@@ -1,4 +1,4 @@
from typing import TYPE_CHECKING, Dict, List from typing import TYPE_CHECKING
from scmrepo.exceptions import SCMError from scmrepo.exceptions import SCMError
@@ -16,7 +16,7 @@ if TYPE_CHECKING:
from .base import BaseStashQueue from .base import BaseStashQueue
def get_remote_executor_refs(scm: "Git", remote_url: str) -> List[str]: def get_remote_executor_refs(scm: "Git", remote_url: str) -> list[str]:
"""Get result list refs from a remote repository """Get result list refs from a remote repository
Args: Args:
@@ -35,7 +35,7 @@ def get_remote_executor_refs(scm: "Git", remote_url: str) -> List[str]:
def fetch_running_exp_from_temp_dir( def fetch_running_exp_from_temp_dir(
queue: "BaseStashQueue", rev: str, fetch_refs: bool queue: "BaseStashQueue", rev: str, fetch_refs: bool
) -> Dict[str, Dict]: ) -> dict[str, dict]:
"""Fetch status of running exps out of current working directory """Fetch status of running exps out of current working directory
Args: Args:
@@ -50,7 +50,7 @@ def fetch_running_exp_from_temp_dir(
from dvc.scm import InvalidRemoteSCMRepo from dvc.scm import InvalidRemoteSCMRepo
from dvc.utils.serialize import load_json from dvc.utils.serialize import load_json
result: Dict[str, Dict] = {} result: dict[str, dict] = {}
infofile = queue.get_infofile_path(rev) infofile = queue.get_infofile_path(rev)
try: try:
info = ExecutorInfo.from_dict(load_json(infofile)) info = ExecutorInfo.from_dict(load_json(infofile))
+13 -12
View File
@@ -1,7 +1,8 @@
import json import json
import os import os
from collections import defaultdict from collections import defaultdict
from typing import TYPE_CHECKING, Collection, Dict, Generator, List, Optional from collections.abc import Collection, Generator
from typing import TYPE_CHECKING, Optional
import psutil import psutil
from funcy import first from funcy import first
@@ -82,9 +83,9 @@ class WorkspaceQueue(BaseStashQueue):
raise NotImplementedError raise NotImplementedError
def reproduce( def reproduce(
self, copy_paths: Optional[List[str]] = None, message: Optional[str] = None self, copy_paths: Optional[list[str]] = None, message: Optional[str] = None
) -> Dict[str, Dict[str, str]]: ) -> dict[str, dict[str, str]]:
results: Dict[str, Dict[str, str]] = defaultdict(dict) results: dict[str, dict[str, str]] = defaultdict(dict)
try: try:
while True: while True:
entry, executor = self.get() entry, executor = self.get()
@@ -99,11 +100,11 @@ class WorkspaceQueue(BaseStashQueue):
def _reproduce_entry( def _reproduce_entry(
self, entry: QueueEntry, executor: "BaseExecutor", **kwargs self, entry: QueueEntry, executor: "BaseExecutor", **kwargs
) -> Dict[str, Dict[str, str]]: ) -> dict[str, dict[str, str]]:
kwargs.pop("copy_paths", None) kwargs.pop("copy_paths", None)
from dvc_task.proc.process import ProcessInfo from dvc_task.proc.process import ProcessInfo
results: Dict[str, Dict[str, str]] = defaultdict(dict) results: dict[str, dict[str, str]] = defaultdict(dict)
exec_name = self._EXEC_NAME or entry.stash_rev exec_name = self._EXEC_NAME or entry.stash_rev
proc_info = ProcessInfo(os.getpid(), None, None, None, None) proc_info = ProcessInfo(os.getpid(), None, None, None, None)
proc_info_path = self._proc_info_path(exec_name) proc_info_path = self._proc_info_path(exec_name)
@@ -162,8 +163,8 @@ class WorkspaceQueue(BaseStashQueue):
exp: "Experiments", exp: "Experiments",
executor: "BaseExecutor", # noqa: ARG004 executor: "BaseExecutor", # noqa: ARG004
exec_result: "ExecutorResult", exec_result: "ExecutorResult",
) -> Dict[str, str]: ) -> dict[str, str]:
results: Dict[str, str] = {} results: dict[str, str] = {}
exp_rev = exp.scm.get_ref(EXEC_BRANCH) exp_rev = exp.scm.get_ref(EXEC_BRANCH)
if exp_rev: if exp_rev:
assert exec_result.exp_hash assert exec_result.exp_hash
@@ -207,7 +208,7 @@ class WorkspaceQueue(BaseStashQueue):
baseline_revs: Optional[Collection[str]], baseline_revs: Optional[Collection[str]],
fetch_refs: bool = False, # noqa: ARG002 fetch_refs: bool = False, # noqa: ARG002
**kwargs, **kwargs,
) -> Dict[str, List["ExpRange"]]: ) -> dict[str, list["ExpRange"]]:
from dvc.repo.experiments.collect import collect_exec_branch from dvc.repo.experiments.collect import collect_exec_branch
from dvc.repo.experiments.serialize import ( from dvc.repo.experiments.serialize import (
ExpExecutor, ExpExecutor,
@@ -215,7 +216,7 @@ class WorkspaceQueue(BaseStashQueue):
LocalExpExecutor, LocalExpExecutor,
) )
result: Dict[str, List[ExpRange]] = defaultdict(list) result: dict[str, list[ExpRange]] = defaultdict(list)
pid = self._active_pid pid = self._active_pid
if pid is None: if pid is None:
return result return result
@@ -248,12 +249,12 @@ class WorkspaceQueue(BaseStashQueue):
self, self,
baseline_revs: Optional[Collection[str]], baseline_revs: Optional[Collection[str]],
**kwargs, **kwargs,
) -> Dict[str, List["ExpRange"]]: ) -> dict[str, list["ExpRange"]]:
raise NotImplementedError raise NotImplementedError
def collect_failed_data( def collect_failed_data(
self, self,
baseline_revs: Optional[Collection[str]], baseline_revs: Optional[Collection[str]],
**kwargs, **kwargs,
) -> Dict[str, List["ExpRange"]]: ) -> dict[str, list["ExpRange"]]:
raise NotImplementedError raise NotImplementedError
+14 -13
View File
@@ -1,4 +1,5 @@
from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Union from collections.abc import Iterable
from typing import TYPE_CHECKING, Optional, Union
from dvc.log import logger from dvc.log import logger
from dvc.repo import locked from dvc.repo import locked
@@ -23,14 +24,14 @@ logger = logger.getChild(__name__)
@scm_context @scm_context
def remove( # noqa: C901, PLR0912 def remove( # noqa: C901, PLR0912
repo: "Repo", repo: "Repo",
exp_names: Union[None, str, List[str]] = None, exp_names: Union[None, str, list[str]] = None,
rev: Optional[Union[List[str], str]] = None, rev: Optional[Union[list[str], str]] = None,
all_commits: bool = False, all_commits: bool = False,
num: int = 1, num: int = 1,
queue: bool = False, queue: bool = False,
git_remote: Optional[str] = None, git_remote: Optional[str] = None,
) -> List[str]: ) -> list[str]:
removed: List[str] = [] removed: list[str] = []
if not any([exp_names, queue, all_commits, rev]): if not any([exp_names, queue, all_commits, rev]):
return removed return removed
celery_queue: "LocalCeleryQueue" = repo.experiments.celery_queue celery_queue: "LocalCeleryQueue" = repo.experiments.celery_queue
@@ -40,13 +41,13 @@ def remove( # noqa: C901, PLR0912
assert isinstance(repo.scm, Git) assert isinstance(repo.scm, Git)
exp_ref_list: List["ExpRefInfo"] = [] exp_ref_list: list["ExpRefInfo"] = []
queue_entry_list: List["QueueEntry"] = [] queue_entry_list: list["QueueEntry"] = []
if exp_names: if exp_names:
results: Dict[ results: dict[
str, "ExpRefAndQueueEntry" str, "ExpRefAndQueueEntry"
] = celery_queue.get_ref_and_entry_by_names(exp_names, git_remote) ] = celery_queue.get_ref_and_entry_by_names(exp_names, git_remote)
remained: List[str] = [] remained: list[str] = []
for name, result in results.items(): for name, result in results.items():
if not result.exp_ref_info and not result.queue_entry: if not result.exp_ref_info and not result.queue_entry:
remained.append(name) remained.append(name)
@@ -87,13 +88,13 @@ def remove( # noqa: C901, PLR0912
def _resolve_exp_by_baseline( def _resolve_exp_by_baseline(
repo: "Repo", repo: "Repo",
rev: List[str], rev: list[str],
num: int, num: int,
git_remote: Optional[str] = None, git_remote: Optional[str] = None,
) -> Dict[str, "ExpRefInfo"]: ) -> dict[str, "ExpRefInfo"]:
assert isinstance(repo.scm, Git) assert isinstance(repo.scm, Git)
commit_ref_dict: Dict[str, "ExpRefInfo"] = {} commit_ref_dict: dict[str, "ExpRefInfo"] = {}
rev_dict = iter_revs(repo.scm, rev, num) rev_dict = iter_revs(repo.scm, rev, num)
rev_set = set(rev_dict.keys()) rev_set = set(rev_dict.keys())
ref_info_dict = exp_refs_by_baseline(repo.scm, rev_set, git_remote) ref_info_dict = exp_refs_by_baseline(repo.scm, rev_set, git_remote)
@@ -105,7 +106,7 @@ def _resolve_exp_by_baseline(
def _remove_commited_exps( def _remove_commited_exps(
scm: "Git", exp_refs_list: Iterable["ExpRefInfo"], remote: Optional[str] scm: "Git", exp_refs_list: Iterable["ExpRefInfo"], remote: Optional[str]
) -> List[str]: ) -> list[str]:
if remote: if remote:
from dvc.scm import TqdmGit from dvc.scm import TqdmGit
+5 -5
View File
@@ -1,4 +1,4 @@
from typing import TYPE_CHECKING, Dict, List, Optional, Union from typing import TYPE_CHECKING, Optional, Union
from dvc.log import logger from dvc.log import logger
from dvc.repo.experiments.exceptions import ( from dvc.repo.experiments.exceptions import (
@@ -22,16 +22,16 @@ def rename(
exp_name: Union[str, None] = None, exp_name: Union[str, None] = None,
git_remote: Optional[str] = None, git_remote: Optional[str] = None,
force: bool = False, force: bool = False,
) -> Union[List[str], None]: ) -> Union[list[str], None]:
renamed: List[str] = [] renamed: list[str] = []
remained: List[str] = [] remained: list[str] = []
assert isinstance(repo.scm, Git) assert isinstance(repo.scm, Git)
if exp_name == new_name: if exp_name == new_name:
return None return None
if exp_name: if exp_name:
results: Dict[str, Union[ExpRefInfo, None]] = resolve_name( results: dict[str, Union[ExpRefInfo, None]] = resolve_name(
scm=repo.scm, exp_names=exp_name, git_remote=git_remote scm=repo.scm, exp_names=exp_name, git_remote=git_remote
) )
for name, result in results.items(): for name, result in results.items():
+3 -2
View File
@@ -1,4 +1,5 @@
from typing import Dict, Iterable, Optional from collections.abc import Iterable
from typing import Optional
from dvc.dependency.param import ParamsDependency from dvc.dependency.param import ParamsDependency
from dvc.exceptions import InvalidArgumentError from dvc.exceptions import InvalidArgumentError
@@ -22,7 +23,7 @@ def run( # noqa: C901, PLR0912
copy_paths: Optional[Iterable[str]] = None, copy_paths: Optional[Iterable[str]] = None,
message: Optional[str] = None, message: Optional[str] = None,
**kwargs, **kwargs,
) -> Dict[str, str]: ) -> dict[str, str]:
"""Reproduce the specified targets as an experiment. """Reproduce the specified targets as an experiment.
Accepts the same additional kwargs as Repo.reproduce. Accepts the same additional kwargs as Repo.reproduce.
+3 -2
View File
@@ -1,5 +1,6 @@
import os import os
from typing import TYPE_CHECKING, Iterable, List, Optional from collections.abc import Iterable
from typing import TYPE_CHECKING, Optional
from funcy import first from funcy import first
@@ -18,7 +19,7 @@ def save(
name: Optional[str] = None, name: Optional[str] = None,
recursive: bool = False, recursive: bool = False,
force: bool = False, force: bool = False,
include_untracked: Optional[List[str]] = None, include_untracked: Optional[list[str]] = None,
message: Optional[str] = None, message: Optional[str] = None,
) -> Optional[str]: ) -> Optional[str]:
"""Save the current workspace status as an experiment. """Save the current workspace status as an experiment.
+13 -12
View File
@@ -1,7 +1,8 @@
import json import json
from collections.abc import Iterator
from dataclasses import asdict, dataclass, field from dataclasses import asdict, dataclass, field
from datetime import datetime from datetime import datetime
from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Literal, Optional from typing import TYPE_CHECKING, Any, Literal, Optional
from dvc.exceptions import DvcException from dvc.exceptions import DvcException
from dvc.repo.metrics.show import _gather_metrics from dvc.repo.metrics.show import _gather_metrics
@@ -30,11 +31,11 @@ class SerializableExp:
rev: str rev: str
timestamp: Optional[datetime] = None timestamp: Optional[datetime] = None
params: Dict[str, "FileResult"] = field(default_factory=dict) params: dict[str, "FileResult"] = field(default_factory=dict)
metrics: Dict[str, "FileResult"] = field(default_factory=dict) metrics: dict[str, "FileResult"] = field(default_factory=dict)
deps: Dict[str, "ExpDep"] = field(default_factory=dict) deps: dict[str, "ExpDep"] = field(default_factory=dict)
outs: Dict[str, "ExpOut"] = field(default_factory=dict) outs: dict[str, "ExpOut"] = field(default_factory=dict)
meta: Dict[str, Any] = field(default_factory=dict) meta: dict[str, Any] = field(default_factory=dict)
@classmethod @classmethod
def from_repo( def from_repo(
@@ -83,7 +84,7 @@ class SerializableExp:
**kwargs, **kwargs,
) )
def dumpd(self) -> Dict[str, Any]: def dumpd(self) -> dict[str, Any]:
return asdict(self) return asdict(self)
def as_bytes(self) -> bytes: def as_bytes(self) -> bytes:
@@ -131,7 +132,7 @@ class SerializableError:
msg: str msg: str
type: str = "" type: str = ""
def dumpd(self) -> Dict[str, Any]: def dumpd(self) -> dict[str, Any]:
return asdict(self) return asdict(self)
def as_bytes(self) -> bytes: def as_bytes(self) -> bytes:
@@ -154,15 +155,15 @@ class ExpState:
name: Optional[str] = None name: Optional[str] = None
data: Optional[SerializableExp] = None data: Optional[SerializableExp] = None
error: Optional[SerializableError] = None error: Optional[SerializableError] = None
experiments: Optional[List["ExpRange"]] = None experiments: Optional[list["ExpRange"]] = None
def dumpd(self) -> Dict[str, Any]: def dumpd(self) -> dict[str, Any]:
return asdict(self) return asdict(self)
@dataclass @dataclass
class ExpRange: class ExpRange:
revs: List["ExpState"] revs: list["ExpState"]
executor: Optional["ExpExecutor"] = None executor: Optional["ExpExecutor"] = None
name: Optional[str] = None name: Optional[str] = None
@@ -175,7 +176,7 @@ class ExpRange:
def __getitem__(self, index: int) -> "ExpState": def __getitem__(self, index: int) -> "ExpState":
return self.revs[index] return self.revs[index]
def dumpd(self) -> Dict[str, Any]: def dumpd(self) -> dict[str, Any]:
return asdict(self) return asdict(self)
+20 -26
View File
@@ -1,18 +1,12 @@
from collections import Counter, defaultdict from collections import Counter, defaultdict
from collections.abc import Iterable, Iterator, Mapping
from datetime import date, datetime from datetime import date, datetime
from typing import ( from typing import (
TYPE_CHECKING, TYPE_CHECKING,
Any, Any,
Dict,
Iterable,
Iterator,
List,
Literal, Literal,
Mapping,
NamedTuple, NamedTuple,
Optional, Optional,
Set,
Tuple,
Union, Union,
) )
@@ -36,7 +30,7 @@ logger = logger.getChild(__name__)
def show( def show(
repo: "Repo", repo: "Repo",
revs: Union[List[str], str, None] = None, revs: Union[list[str], str, None] = None,
all_branches: bool = False, all_branches: bool = False,
all_tags: bool = False, all_tags: bool = False,
all_commits: bool = False, all_commits: bool = False,
@@ -45,7 +39,7 @@ def show(
hide_failed: bool = False, hide_failed: bool = False,
sha_only: bool = False, sha_only: bool = False,
**kwargs, **kwargs,
) -> List["ExpState"]: ) -> list["ExpState"]:
return collect( return collect(
repo, repo,
revs=revs, revs=revs,
@@ -65,7 +59,7 @@ def tabulate(
fill_value: Optional[str] = "-", fill_value: Optional[str] = "-",
error_value: str = "!", error_value: str = "!",
**kwargs, **kwargs,
) -> Tuple["TabularData", Dict[str, Iterable[str]]]: ) -> tuple["TabularData", dict[str, Iterable[str]]]:
"""Return table data for experiments. """Return table data for experiments.
Returns: Returns:
@@ -112,7 +106,7 @@ def tabulate(
**kwargs, **kwargs,
) )
) )
data_headers: Dict[str, Iterable[str]] = { data_headers: dict[str, Iterable[str]] = {
"metrics": metrics_headers, "metrics": metrics_headers,
"params": params_headers, "params": params_headers,
"deps": deps_names, "deps": deps_names,
@@ -128,9 +122,9 @@ def _build_rows(
sort_by: Optional[str] = None, sort_by: Optional[str] = None,
sort_order: Optional[Literal["asc", "desc"]] = None, sort_order: Optional[Literal["asc", "desc"]] = None,
**kwargs, **kwargs,
) -> Iterator[Tuple["CellT", ...]]: ) -> Iterator[tuple["CellT", ...]]:
for baseline in baseline_states: for baseline in baseline_states:
row: Dict[str, "CellT"] = dict.fromkeys(all_headers, fill_value) row: dict[str, "CellT"] = dict.fromkeys(all_headers, fill_value)
row["Experiment"] = "" row["Experiment"] = ""
if baseline.name: if baseline.name:
row["rev"] = baseline.name row["rev"] = baseline.name
@@ -177,10 +171,10 @@ def _sort_column( # noqa: C901
sort_by: str, sort_by: str,
metric_names: Mapping[str, Iterable[str]], metric_names: Mapping[str, Iterable[str]],
param_names: Mapping[str, Iterable[str]], param_names: Mapping[str, Iterable[str]],
) -> Tuple[str, str, str]: ) -> tuple[str, str, str]:
sep = ":" sep = ":"
parts = sort_by.split(sep) parts = sort_by.split(sep)
matches: Set[Tuple[str, str, str]] = set() matches: set[tuple[str, str, str]] = set()
for split_num in range(len(parts)): for split_num in range(len(parts)):
path = sep.join(parts[:split_num]) path = sep.join(parts[:split_num])
@@ -217,7 +211,7 @@ def _sort_exp(
sort_name: str, sort_name: str,
typ: str, typ: str,
reverse: bool, reverse: bool,
) -> List["ExpRange"]: ) -> list["ExpRange"]:
from funcy import first from funcy import first
def _sort(exp_range: "ExpRange"): def _sort(exp_range: "ExpRange"):
@@ -238,14 +232,14 @@ def _exp_range_rows(
fill_value: Optional[str], fill_value: Optional[str],
is_base: bool = False, is_base: bool = False,
**kwargs, **kwargs,
) -> Iterator[Tuple["CellT", ...]]: ) -> Iterator[tuple["CellT", ...]]:
from funcy import first from funcy import first
if len(exp_range.revs) > 1: if len(exp_range.revs) > 1:
logger.debug("Returning tip commit for legacy checkpoint exp") logger.debug("Returning tip commit for legacy checkpoint exp")
exp = first(exp_range.revs) exp = first(exp_range.revs)
if exp: if exp:
row: Dict[str, "CellT"] = dict.fromkeys(all_headers, fill_value) row: dict[str, "CellT"] = dict.fromkeys(all_headers, fill_value)
row["Experiment"] = exp.name or "" row["Experiment"] = exp.name or ""
row["rev"] = exp.rev[:7] if Git.is_sha(exp.rev) else exp.rev row["rev"] = exp.rev[:7] if Git.is_sha(exp.rev) else exp.rev
row["typ"] = "branch_base" if is_base else "branch_commit" row["typ"] = "branch_base" if is_base else "branch_commit"
@@ -274,12 +268,12 @@ def _data_cells(
error_value: str = "!", error_value: str = "!",
precision: Optional[int] = None, precision: Optional[int] = None,
**kwargs, **kwargs,
) -> Iterator[Tuple[str, "CellT"]]: ) -> Iterator[tuple[str, "CellT"]]:
def _d_cells( def _d_cells(
d: Mapping[str, Any], d: Mapping[str, Any],
names: Mapping[str, Iterable[str]], names: Mapping[str, Iterable[str]],
headers: Iterable[str], headers: Iterable[str],
) -> Iterator[Tuple[str, "CellT"]]: ) -> Iterator[tuple[str, "CellT"]]:
from dvc.compare import _format_field, with_value from dvc.compare import _format_field, with_value
for fname, data in d.items(): for fname, data in d.items():
@@ -329,9 +323,9 @@ def format_time(
class _DataNames(NamedTuple): class _DataNames(NamedTuple):
# NOTE: we use nested dict instead of set for metrics/params names to # NOTE: we use nested dict instead of set for metrics/params names to
# preserve key ordering # preserve key ordering
metrics: Dict[str, Dict[str, Any]] metrics: dict[str, dict[str, Any]]
params: Dict[str, Dict[str, Any]] params: dict[str, dict[str, Any]]
deps: Set[str] deps: set[str]
@property @property
def sorted_deps(self): def sorted_deps(self):
@@ -339,7 +333,7 @@ class _DataNames(NamedTuple):
def update(self, other: "_DataNames"): def update(self, other: "_DataNames"):
def _update_d( def _update_d(
d: Dict[str, Dict[str, Any]], other_d: Mapping[str, Mapping[str, Any]] d: dict[str, dict[str, Any]], other_d: Mapping[str, Mapping[str, Any]]
): ):
for k, v in other_d.items(): for k, v in other_d.items():
if k in d: if k in d:
@@ -355,7 +349,7 @@ class _DataNames(NamedTuple):
def _collect_names(exp_states: Iterable["ExpState"]) -> _DataNames: def _collect_names(exp_states: Iterable["ExpState"]) -> _DataNames:
result = _DataNames(defaultdict(dict), defaultdict(dict), set()) result = _DataNames(defaultdict(dict), defaultdict(dict), set())
def _collect_d(result_d: Dict[str, Dict[str, Any]], data_d: Dict[str, Any]): def _collect_d(result_d: dict[str, dict[str, Any]], data_d: dict[str, Any]):
for path, item in data_d.items(): for path, item in data_d.items():
item = item.get("data", {}) item = item.get("data", {})
if isinstance(item, dict): if isinstance(item, dict):
@@ -376,7 +370,7 @@ def _collect_names(exp_states: Iterable["ExpState"]) -> _DataNames:
def _normalize_headers( def _normalize_headers(
names: Mapping[str, Mapping[str, Any]], count: Mapping[str, int] names: Mapping[str, Mapping[str, Any]], count: Mapping[str, int]
) -> List[str]: ) -> list[str]:
return [ return [
name if count[name] == 1 else f"{path}:{name}" name if count[name] == 1 else f"{path}:{name}"
for path in names for path in names
+4 -3
View File
@@ -1,6 +1,7 @@
import re import re
from collections.abc import Iterable, Iterator
from contextlib import contextmanager from contextlib import contextmanager
from typing import Dict, Iterable, Iterator, NamedTuple, Optional from typing import NamedTuple, Optional
from scmrepo.git import Stash from scmrepo.git import Stash
@@ -42,7 +43,7 @@ class ExpStash(Stash):
) )
@property @property
def stash_revs(self) -> Dict[str, ExpStashEntry]: def stash_revs(self) -> dict[str, ExpStashEntry]:
revs = {} revs = {}
for i, entry in enumerate(self): for i, entry in enumerate(self):
msg = entry.message.decode("utf-8").strip() msg = entry.message.decode("utf-8").strip()
@@ -110,7 +111,7 @@ class ApplyStash(Stash):
) )
@property @property
def stash_revs(self) -> Dict[str, ApplyStashEntry]: def stash_revs(self) -> dict[str, ApplyStashEntry]:
revs = {} revs = {}
for i, entry in enumerate(self): for i, entry in enumerate(self):
msg = entry.message.decode("utf-8").strip() msg = entry.message.decode("utf-8").strip()
+13 -19
View File
@@ -2,18 +2,12 @@ import os
import random import random
import sys import sys
from collections import defaultdict from collections import defaultdict
from collections.abc import Generator, Iterable, Mapping
from functools import wraps from functools import wraps
from typing import ( from typing import (
TYPE_CHECKING, TYPE_CHECKING,
Callable, Callable,
Dict,
Generator,
Iterable,
List,
Mapping,
Optional, Optional,
Set,
Tuple,
Union, Union,
) )
@@ -43,8 +37,8 @@ EXEC_PID_DIR = "run"
def get_exp_rwlock( def get_exp_rwlock(
repo: "Repo", repo: "Repo",
reads: Optional[List[str]] = None, reads: Optional[list[str]] = None,
writes: Optional[List[str]] = None, writes: Optional[list[str]] = None,
): ):
reads = reads or [] reads = reads or []
writes = writes or [] writes = writes or []
@@ -109,9 +103,9 @@ def exp_refs_by_rev(scm: "Git", rev: str) -> Generator[ExpRefInfo, None, None]:
def exp_refs_by_baseline( def exp_refs_by_baseline(
scm: "Git", scm: "Git",
revs: Optional[Set[str]] = None, revs: Optional[set[str]] = None,
url: Optional[str] = None, url: Optional[str] = None,
) -> Mapping[str, List[ExpRefInfo]]: ) -> Mapping[str, list[ExpRefInfo]]:
"""Iterate over all experiment refs with the specified baseline.""" """Iterate over all experiment refs with the specified baseline."""
all_exp_refs = exp_refs(scm, url) all_exp_refs = exp_refs(scm, url)
result = defaultdict(list) result = defaultdict(list)
@@ -137,7 +131,7 @@ def iter_remote_refs(scm: "Git", url: str, base: Optional[str] = None, **kwargs)
def push_refspec( def push_refspec(
scm: "Git", scm: "Git",
url: str, url: str,
push_list=List[Tuple[Optional[str], str]], push_list=list[tuple[Optional[str], str]],
force: bool = False, force: bool = False,
on_diverged: Optional[Callable[[str, str], bool]] = None, on_diverged: Optional[Callable[[str, str], bool]] = None,
**kwargs, **kwargs,
@@ -183,8 +177,8 @@ def remote_exp_refs(scm: "Git", url: str) -> Generator[ExpRefInfo, None, None]:
def exp_refs_by_names( def exp_refs_by_names(
scm: "Git", names: Set[str], url: Optional[str] = None scm: "Git", names: set[str], url: Optional[str] = None
) -> Dict[str, List[ExpRefInfo]]: ) -> dict[str, list[ExpRefInfo]]:
"""Iterate over all experiment refs matching the specified names.""" """Iterate over all experiment refs matching the specified names."""
resolve_results = defaultdict(list) resolve_results = defaultdict(list)
ref_info_gen = exp_refs(scm, url) ref_info_gen = exp_refs(scm, url)
@@ -210,7 +204,7 @@ def exp_commits(
scm: "Git", ref_infos: Optional[Iterable[ExpRefInfo]] = None scm: "Git", ref_infos: Optional[Iterable[ExpRefInfo]] = None
) -> Iterable[str]: ) -> Iterable[str]:
"""Iterate over all experiment commits.""" """Iterate over all experiment commits."""
shas: Set["str"] = set() shas: set["str"] = set()
refs = ref_infos if ref_infos else exp_refs(scm) refs = ref_infos if ref_infos else exp_refs(scm)
for ref_info in refs: for ref_info in refs:
shas.update(scm.branch_revs(str(ref_info), ref_info.baseline_sha)) shas.update(scm.branch_revs(str(ref_info), ref_info.baseline_sha))
@@ -242,7 +236,7 @@ def resolve_name(
scm: "Git", scm: "Git",
exp_names: Union[Iterable[str], str], exp_names: Union[Iterable[str], str],
git_remote: Optional[str] = None, git_remote: Optional[str] = None,
) -> Dict[str, Optional[ExpRefInfo]]: ) -> dict[str, Optional[ExpRefInfo]]:
"""find the ref_info of specified names.""" """find the ref_info of specified names."""
if isinstance(exp_names, str): if isinstance(exp_names, str):
exp_names = [exp_names] exp_names = [exp_names]
@@ -338,7 +332,7 @@ def to_studio_params(dvc_params):
"params.yaml": {"foo": 1} "params.yaml": {"foo": 1}
} }
""" """
result: Dict = {} result: dict = {}
if not dvc_params: if not dvc_params:
return result return result
for rev_data in dvc_params.values(): for rev_data in dvc_params.values():
@@ -353,7 +347,7 @@ def describe(
revs: Iterable[str], revs: Iterable[str],
logger, logger,
refs: Optional[Iterable[str]] = None, refs: Optional[Iterable[str]] = None,
) -> Dict[str, Optional[str]]: ) -> dict[str, Optional[str]]:
"""Describe revisions using a tag, branch. """Describe revisions using a tag, branch.
The first matching name will be returned for each rev. Names are preferred in this The first matching name will be returned for each rev. Names are preferred in this
@@ -390,7 +384,7 @@ def describe(
if is_branch and rev not in branches: if is_branch and rev not in branches:
branches[rev] = ref[len("refs/heads/") :] branches[rev] = ref[len("refs/heads/") :]
names: Dict[str, Optional[str]] = {} names: dict[str, Optional[str]] = {}
for rev in revs: for rev in revs:
if rev == head_rev and head_branch: if rev == head_rev and head_branch:
names[rev] = head_branch names[rev] = head_branch
+4 -4
View File
@@ -1,4 +1,4 @@
from typing import TYPE_CHECKING, List, Tuple from typing import TYPE_CHECKING
from dvc.exceptions import DownloadError from dvc.exceptions import DownloadError
from dvc.log import logger from dvc.log import logger
@@ -205,9 +205,9 @@ def fetch( # noqa: PLR0913
return transferred_count return transferred_count
def _log_unversioned(data: List["DataIndex"]) -> Tuple[List["DataIndex"], int]: def _log_unversioned(data: list["DataIndex"]) -> tuple[list["DataIndex"], int]:
ret: List["DataIndex"] = [] ret: list["DataIndex"] = []
unversioned: List[str] = [] unversioned: list[str] = []
for fs_index in data: for fs_index in data:
remote = fs_index.storage_map[()].remote remote = fs_index.storage_map[()].remote
if not isinstance(remote, FileStorage) or not remote.fs.version_aware: if not isinstance(remote, FileStorage) or not remote.fs.version_aware:
+2 -2
View File
@@ -1,4 +1,4 @@
from typing import TYPE_CHECKING, List, Optional from typing import TYPE_CHECKING, Optional
from dvc.exceptions import InvalidArgumentError from dvc.exceptions import InvalidArgumentError
from dvc.log import logger from dvc.log import logger
@@ -61,7 +61,7 @@ def gc( # noqa: PLR0913, C901
all_experiments: bool = False, all_experiments: bool = False,
force: bool = False, force: bool = False,
jobs: Optional[int] = None, jobs: Optional[int] = None,
repos: Optional[List[str]] = None, repos: Optional[list[str]] = None,
workspace: bool = False, workspace: bool = False,
commit_date: Optional[str] = None, commit_date: Optional[str] = None,
rev: Optional[str] = None, rev: Optional[str] = None,
+5 -4
View File
@@ -1,4 +1,5 @@
from typing import TYPE_CHECKING, Any, Iterator, List, Optional, Set, TypeVar from collections.abc import Iterator
from typing import TYPE_CHECKING, Any, Optional, TypeVar
from dvc.fs import localfs from dvc.fs import localfs
from dvc.utils.fs import path_isin from dvc.utils.fs import path_isin
@@ -21,7 +22,7 @@ def check_acyclic(graph: "DiGraph") -> None:
except nx.NetworkXNoCycle: except nx.NetworkXNoCycle:
return return
stages: Set["Stage"] = set() stages: set["Stage"] = set()
for from_node, to_node, _ in edges: for from_node, to_node, _ in edges:
stages.add(from_node) stages.add(from_node)
stages.add(to_node) stages.add(to_node)
@@ -45,7 +46,7 @@ def get_pipelines(graph: "DiGraph"):
def get_subgraph_of_nodes( def get_subgraph_of_nodes(
graph: "DiGraph", sources: Optional[List[Any]] = None, downstream: bool = False graph: "DiGraph", sources: Optional[list[Any]] = None, downstream: bool = False
) -> "DiGraph": ) -> "DiGraph":
from networkx import dfs_postorder_nodes, reverse_view from networkx import dfs_postorder_nodes, reverse_view
@@ -69,7 +70,7 @@ def collect_pipeline(stage: "Stage", graph: "DiGraph") -> Iterator["Stage"]:
return nx.dfs_postorder_nodes(pipeline, stage) return nx.dfs_postorder_nodes(pipeline, stage)
def collect_inside_path(path: str, graph: "DiGraph") -> List["Stage"]: def collect_inside_path(path: str, graph: "DiGraph") -> list["Stage"]:
import networkx as nx import networkx as nx
stages = nx.dfs_postorder_nodes(graph) stages = nx.dfs_postorder_nodes(graph)
+2 -2
View File
@@ -1,4 +1,4 @@
from typing import TYPE_CHECKING, Dict, Optional from typing import TYPE_CHECKING, Optional
from funcy import compact from funcy import compact
@@ -33,7 +33,7 @@ def imp_db(
assert sql or table assert sql or table
assert output_format in ("csv", "json") assert output_format in ("csv", "json")
db: Dict[str, str] = compact( db: dict[str, str] = compact(
{ {
"connection": connection, "connection": connection,
"file_format": output_format, "file_format": output_format,
+5 -5
View File
@@ -1,7 +1,7 @@
import os import os
from functools import partial from functools import partial
from tempfile import TemporaryDirectory from tempfile import TemporaryDirectory
from typing import TYPE_CHECKING, List, Set, Tuple, Union from typing import TYPE_CHECKING, Union
from dvc.log import logger from dvc.log import logger
@@ -19,7 +19,7 @@ logger = logger.getChild(__name__)
def unfetched_view( def unfetched_view(
index: "Index", targets: "TargetType", unpartial: bool = False, **kwargs index: "Index", targets: "TargetType", unpartial: bool = False, **kwargs
) -> Tuple["IndexView", "IndexView", List["Dependency"]]: ) -> tuple["IndexView", "IndexView", list["Dependency"]]:
"""Return index view of imports which have not been fetched. """Return index view of imports which have not been fetched.
Returns: Returns:
@@ -28,7 +28,7 @@ def unfetched_view(
""" """
from dvc.cachemgr import LEGACY_HASH_NAMES from dvc.cachemgr import LEGACY_HASH_NAMES
changed_deps: List["Dependency"] = [] changed_deps: list["Dependency"] = []
def need_fetch(stage: "Stage", legacy: bool = False) -> bool: def need_fetch(stage: "Stage", legacy: bool = False) -> bool:
if not stage.is_import or (stage.is_partial_import and not unpartial): if not stage.is_import or (stage.is_partial_import and not unpartial):
@@ -92,7 +92,7 @@ def unpartial_imports(index: Union["Index", "IndexView"]) -> int:
def save_imports( def save_imports(
repo: "Repo", targets: "TargetType", unpartial: bool = False, **kwargs repo: "Repo", targets: "TargetType", unpartial: bool = False, **kwargs
) -> Set["HashInfo"]: ) -> set["HashInfo"]:
"""Save (download) imports from their original source location. """Save (download) imports from their original source location.
Imports which are already cached will not be downloaded. Imports which are already cached will not be downloaded.
@@ -105,7 +105,7 @@ def save_imports(
from dvc_data.index import md5, save from dvc_data.index import md5, save
from dvc_data.index.checkout import apply, compare from dvc_data.index.checkout import apply, compare
downloaded: Set["HashInfo"] = set() downloaded: set["HashInfo"] = set()
legacy_unfetched, unfetched, changed = unfetched_view( legacy_unfetched, unfetched, changed = unfetched_view(
repo.index, targets, unpartial=unpartial, **kwargs repo.index, targets, unpartial=unpartial, **kwargs
+35 -40
View File
@@ -1,20 +1,15 @@
import logging import logging
import time import time
from collections import defaultdict from collections import defaultdict
from collections.abc import Iterable, Iterator
from functools import partial from functools import partial
from itertools import chain from itertools import chain
from typing import ( from typing import (
TYPE_CHECKING, TYPE_CHECKING,
Any, Any,
Callable, Callable,
Dict,
Iterable,
Iterator,
List,
NamedTuple, NamedTuple,
Optional, Optional,
Set,
Tuple,
Union, Union,
) )
@@ -44,7 +39,7 @@ if TYPE_CHECKING:
logger = logger.getChild(__name__) logger = logger.getChild(__name__)
ObjectContainer = Dict[Optional["HashFileDB"], Set["HashInfo"]] ObjectContainer = dict[Optional["HashFileDB"], set["HashInfo"]]
def log_walk(seq): def log_walk(seq):
@@ -74,7 +69,7 @@ def collect_files(
scm = repo.scm scm = repo.scm
fs = repo.fs fs = repo.fs
sep = fs.sep sep = fs.sep
outs: Set[str] = set() outs: set[str] = set()
is_local_fs = isinstance(fs, LocalFileSystem) is_local_fs = isinstance(fs, LocalFileSystem)
@@ -288,11 +283,11 @@ class Index:
def __init__( def __init__(
self, self,
repo: "Repo", repo: "Repo",
stages: Optional[List["Stage"]] = None, stages: Optional[list["Stage"]] = None,
metrics: Optional[Dict[str, List[str]]] = None, metrics: Optional[dict[str, list[str]]] = None,
plots: Optional[Dict[str, List[str]]] = None, plots: Optional[dict[str, list[str]]] = None,
params: Optional[Dict[str, Any]] = None, params: Optional[dict[str, Any]] = None,
artifacts: Optional[Dict[str, Any]] = None, artifacts: Optional[dict[str, Any]] = None,
) -> None: ) -> None:
self.repo = repo self.repo = repo
self.stages = stages or [] self.stages = stages or []
@@ -300,7 +295,7 @@ class Index:
self._plots = plots or {} self._plots = plots or {}
self._params = params or {} self._params = params or {}
self._artifacts = artifacts or {} self._artifacts = artifacts or {}
self._collected_targets: Dict[int, List["StageInfo"]] = {} self._collected_targets: dict[int, list["StageInfo"]] = {}
@cached_property @cached_property
def rev(self) -> Optional[str]: def rev(self) -> Optional[str]:
@@ -404,8 +399,8 @@ class Index:
yield from stage.outs yield from stage.outs
@cached_property @cached_property
def out_data_keys(self) -> Dict[str, Set["DataIndexKey"]]: def out_data_keys(self) -> dict[str, set["DataIndexKey"]]:
by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set) by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set)
by_workspace["repo"] = set() by_workspace["repo"] = set()
by_workspace["local"] = set() by_workspace["local"] = set()
@@ -443,10 +438,10 @@ class Index:
yield from stage.deps yield from stage.deps
@cached_property @cached_property
def _plot_sources(self) -> List[str]: def _plot_sources(self) -> list[str]:
from dvc.repo.plots import _collect_pipeline_files from dvc.repo.plots import _collect_pipeline_files
sources: List[str] = [] sources: list[str] = []
for data in _collect_pipeline_files(self.repo, [], {}).values(): for data in _collect_pipeline_files(self.repo, [], {}).values():
for plot_id, props in data.get("data", {}).items(): for plot_id, props in data.get("data", {}).items():
if isinstance(props.get("y"), dict): if isinstance(props.get("y"), dict):
@@ -458,8 +453,8 @@ class Index:
return sources return sources
@cached_property @cached_property
def data_keys(self) -> Dict[str, Set["DataIndexKey"]]: def data_keys(self) -> dict[str, set["DataIndexKey"]]:
by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set) by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set)
by_workspace["repo"] = set() by_workspace["repo"] = set()
by_workspace["local"] = set() by_workspace["local"] = set()
@@ -474,10 +469,10 @@ class Index:
return dict(by_workspace) return dict(by_workspace)
@cached_property @cached_property
def metric_keys(self) -> Dict[str, Set["DataIndexKey"]]: def metric_keys(self) -> dict[str, set["DataIndexKey"]]:
from .metrics.show import _collect_top_level_metrics from .metrics.show import _collect_top_level_metrics
by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set) by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set)
by_workspace["repo"] = set() by_workspace["repo"] = set()
@@ -495,10 +490,10 @@ class Index:
return dict(by_workspace) return dict(by_workspace)
@cached_property @cached_property
def param_keys(self) -> Dict[str, Set["DataIndexKey"]]: def param_keys(self) -> dict[str, set["DataIndexKey"]]:
from .params.show import _collect_top_level_params from .params.show import _collect_top_level_params
by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set) by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set)
by_workspace["repo"] = set() by_workspace["repo"] = set()
param_paths = _collect_top_level_params(self.repo) param_paths = _collect_top_level_params(self.repo)
@@ -513,8 +508,8 @@ class Index:
return dict(by_workspace) return dict(by_workspace)
@cached_property @cached_property
def plot_keys(self) -> Dict[str, Set["DataIndexKey"]]: def plot_keys(self) -> dict[str, set["DataIndexKey"]]:
by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set) by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set)
by_workspace["repo"] = set() by_workspace["repo"] = set()
@@ -536,7 +531,7 @@ class Index:
return _build_tree_from_outs(self.outs) return _build_tree_from_outs(self.outs)
@cached_property @cached_property
def data(self) -> "Dict[str, DataIndex]": def data(self) -> "dict[str, DataIndex]":
prefix: "DataIndexKey" prefix: "DataIndexKey"
loaded = False loaded = False
@@ -584,7 +579,7 @@ class Index:
def collect_targets( def collect_targets(
self, targets: Optional["TargetType"], *, onerror=None, **kwargs: Any self, targets: Optional["TargetType"], *, onerror=None, **kwargs: Any
) -> List["StageInfo"]: ) -> list["StageInfo"]:
from dvc.exceptions import DvcException from dvc.exceptions import DvcException
from dvc.repo.stage import StageInfo from dvc.repo.stage import StageInfo
from dvc.utils.collections import ensure_list from dvc.utils.collections import ensure_list
@@ -658,7 +653,7 @@ class Index:
stage_filter: Optional[Callable[["Stage"], bool]] = None, stage_filter: Optional[Callable[["Stage"], bool]] = None,
outs_filter: Optional[Callable[["Output"], bool]] = None, outs_filter: Optional[Callable[["Output"], bool]] = None,
max_size: Optional[int] = None, max_size: Optional[int] = None,
types: Optional[List[str]] = None, types: Optional[list[str]] = None,
**kwargs: Any, **kwargs: Any,
) -> "IndexView": ) -> "IndexView":
"""Return read-only view of index for the specified targets. """Return read-only view of index for the specified targets.
@@ -698,8 +693,8 @@ class Index:
class _DataPrefixes(NamedTuple): class _DataPrefixes(NamedTuple):
explicit: Set["DataIndexKey"] explicit: set["DataIndexKey"]
recursive: Set["DataIndexKey"] recursive: set["DataIndexKey"]
class IndexView: class IndexView:
@@ -728,7 +723,7 @@ class IndexView:
yield from stage.deps yield from stage.deps
@property @property
def _filtered_outs(self) -> Iterator[Tuple["Output", Optional[str]]]: def _filtered_outs(self) -> Iterator[tuple["Output", Optional[str]]]:
for stage, filter_info in self._stage_infos: for stage, filter_info in self._stage_infos:
for out in stage.filter_outs(filter_info): for out in stage.filter_outs(filter_info):
if not self._outs_filter or self._outs_filter(out): if not self._outs_filter or self._outs_filter(out):
@@ -739,8 +734,8 @@ class IndexView:
yield from {out for (out, _) in self._filtered_outs} yield from {out for (out, _) in self._filtered_outs}
@cached_property @cached_property
def out_data_keys(self) -> Dict[str, Set["DataIndexKey"]]: def out_data_keys(self) -> dict[str, set["DataIndexKey"]]:
by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set) by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set)
by_workspace["repo"] = set() by_workspace["repo"] = set()
by_workspace["local"] = set() by_workspace["local"] = set()
@@ -755,8 +750,8 @@ class IndexView:
return dict(by_workspace) return dict(by_workspace)
@cached_property @cached_property
def _data_prefixes(self) -> Dict[str, "_DataPrefixes"]: def _data_prefixes(self) -> dict[str, "_DataPrefixes"]:
prefixes: Dict[str, "_DataPrefixes"] = defaultdict( prefixes: dict[str, "_DataPrefixes"] = defaultdict(
lambda: _DataPrefixes(set(), set()) lambda: _DataPrefixes(set(), set())
) )
for out, filter_info in self._filtered_outs: for out, filter_info in self._filtered_outs:
@@ -772,8 +767,8 @@ class IndexView:
return prefixes return prefixes
@cached_property @cached_property
def data_keys(self) -> Dict[str, Set["DataIndexKey"]]: def data_keys(self) -> dict[str, set["DataIndexKey"]]:
ret: Dict[str, Set["DataIndexKey"]] = defaultdict(set) ret: dict[str, set["DataIndexKey"]] = defaultdict(set)
for out, filter_info in self._filtered_outs: for out, filter_info in self._filtered_outs:
if not out.use_cache: if not out.use_cache:
@@ -791,7 +786,7 @@ class IndexView:
return _build_tree_from_outs(self.outs) return _build_tree_from_outs(self.outs)
@cached_property @cached_property
def data(self) -> Dict[str, Union["DataIndex", "DataIndexView"]]: def data(self) -> dict[str, Union["DataIndex", "DataIndexView"]]:
from dvc_data.index import DataIndex, view from dvc_data.index import DataIndex, view
def key_filter(workspace: str, key: "DataIndexKey"): def key_filter(workspace: str, key: "DataIndexKey"):
@@ -803,7 +798,7 @@ class IndexView:
except KeyError: except KeyError:
return False return False
data: Dict[str, Union["DataIndex", "DataIndexView"]] = {} data: dict[str, Union["DataIndex", "DataIndexView"]] = {}
for workspace, data_index in self._index.data.items(): for workspace, data_index in self._index.data.items():
if self.stages: if self.stages:
data[workspace] = view(data_index, partial(key_filter, workspace)) data[workspace] = view(data_index, partial(key_filter, workspace))
+2 -2
View File
@@ -1,5 +1,5 @@
import os import os
from typing import TYPE_CHECKING, Any, Dict, Optional, Union from typing import TYPE_CHECKING, Any, Optional, Union
if TYPE_CHECKING: if TYPE_CHECKING:
from dvc.fs.dvc import DVCFileSystem from dvc.fs.dvc import DVCFileSystem
@@ -13,7 +13,7 @@ def ls(
rev: Optional[str] = None, rev: Optional[str] = None,
recursive: Optional[bool] = None, recursive: Optional[bool] = None,
dvc_only: bool = False, dvc_only: bool = False,
config: Union[None, Dict[str, Any], str] = None, config: Union[None, dict[str, Any], str] = None,
remote: Optional[str] = None, remote: Optional[str] = None,
remote_config: Optional[dict] = None, remote_config: Optional[dict] = None,
): ):
+4 -4
View File
@@ -1,4 +1,4 @@
from typing import TYPE_CHECKING, Dict, TypedDict, Union from typing import TYPE_CHECKING, TypedDict, Union
from funcy import compact from funcy import compact
@@ -12,12 +12,12 @@ if TYPE_CHECKING:
class DiffResult(TypedDict, total=False): class DiffResult(TypedDict, total=False):
errors: Dict[str, Union[Exception, Dict[str, Exception]]] errors: dict[str, Union[Exception, dict[str, Exception]]]
diff: Dict[str, Dict[str, Dict]] diff: dict[str, dict[str, dict]]
def _diff( def _diff(
result: Dict[str, "Result"], result: dict[str, "Result"],
old_rev: str, old_rev: str,
new_rev: str, new_rev: str,
**kwargs, **kwargs,
+17 -21
View File
@@ -1,14 +1,10 @@
import os import os
from collections.abc import Iterable, Iterator
from itertools import chain from itertools import chain
from typing import ( from typing import (
TYPE_CHECKING, TYPE_CHECKING,
Any, Any,
Dict,
Iterable,
Iterator,
List,
Optional, Optional,
Tuple,
TypedDict, TypedDict,
Union, Union,
) )
@@ -71,7 +67,7 @@ def _read_metric(fs: "FileSystem", path: str, **load_kwargs) -> Any:
def _read_metrics( def _read_metrics(
fs: "FileSystem", metrics: Iterable[str], **load_kwargs fs: "FileSystem", metrics: Iterable[str], **load_kwargs
) -> Iterator[Tuple[str, Union[Exception, Any]]]: ) -> Iterator[tuple[str, Union[Exception, Any]]]:
for metric in metrics: for metric in metrics:
try: try:
yield metric, _read_metric(fs, metric, **load_kwargs) yield metric, _read_metric(fs, metric, **load_kwargs)
@@ -80,7 +76,7 @@ def _read_metrics(
yield metric, exc yield metric, exc
def metrics_from_target(repo: "Repo", targets: List[str]) -> Iterator["Output"]: def metrics_from_target(repo: "Repo", targets: list[str]) -> Iterator["Output"]:
stages = chain.from_iterable(repo.stage.collect(target) for target in targets) stages = chain.from_iterable(repo.stage.collect(target) for target in targets)
for stage in stages: for stage in stages:
yield from stage.metrics yield from stage.metrics
@@ -88,11 +84,11 @@ def metrics_from_target(repo: "Repo", targets: List[str]) -> Iterator["Output"]:
def _collect_metrics( def _collect_metrics(
repo: "Repo", repo: "Repo",
targets: Optional[List[str]] = None, targets: Optional[list[str]] = None,
stages: Optional[List[str]] = None, stages: Optional[list[str]] = None,
outs_only: bool = False, outs_only: bool = False,
) -> List[str]: ) -> list[str]:
metrics: List[str] = [] metrics: list[str] = []
if targets: if targets:
# target is a repo-relative path # target is a repo-relative path
@@ -122,7 +118,7 @@ class FileResult(TypedDict, total=False):
class Result(TypedDict, total=False): class Result(TypedDict, total=False):
data: Dict[str, FileResult] data: dict[str, FileResult]
error: Exception error: Exception
@@ -139,11 +135,11 @@ def to_relpath(fs: "FileSystem", root_dir: str, d: Result) -> Result:
def _gather_metrics( def _gather_metrics(
repo: "Repo", repo: "Repo",
targets: Optional[List[str]] = None, targets: Optional[list[str]] = None,
outs_only: bool = False, outs_only: bool = False,
stages: Optional[List[str]] = None, stages: Optional[list[str]] = None,
on_error: str = "return", on_error: str = "return",
) -> Dict[str, FileResult]: ) -> dict[str, FileResult]:
assert on_error in ("raise", "return", "ignore") assert on_error in ("raise", "return", "ignore")
# `files` is a repo-relative posixpath that can be passed to DVCFileSystem # `files` is a repo-relative posixpath that can be passed to DVCFileSystem
@@ -168,8 +164,8 @@ def _gather_metrics(
def _hide_workspace( def _hide_workspace(
scm: Union["Git", "NoSCM"], res: Dict[str, Result] scm: Union["Git", "NoSCM"], res: dict[str, Result]
) -> Dict[str, Result]: ) -> dict[str, Result]:
# Hide workspace params if they are the same as in the active branch # Hide workspace params if they are the same as in the active branch
try: try:
active_branch = scm.active_branch() active_branch = scm.active_branch()
@@ -186,16 +182,16 @@ def _hide_workspace(
def show( def show(
repo: "Repo", repo: "Repo",
targets: Optional[List[str]] = None, targets: Optional[list[str]] = None,
stages: Optional[List[str]] = None, stages: Optional[list[str]] = None,
outs_only: bool = False, outs_only: bool = False,
all_branches: bool = False, all_branches: bool = False,
all_tags: bool = False, all_tags: bool = False,
revs: Optional[List[str]] = None, revs: Optional[list[str]] = None,
all_commits: bool = False, all_commits: bool = False,
hide_workspace: bool = True, hide_workspace: bool = True,
on_error: str = "return", on_error: str = "return",
) -> Dict[str, Result]: ) -> dict[str, Result]:
assert on_error in ("raise", "return", "ignore") assert on_error in ("raise", "return", "ignore")
targets = [os.path.abspath(target) for target in ensure_list(targets)] targets = [os.path.abspath(target) for target in ensure_list(targets)]
+3 -3
View File
@@ -1,7 +1,7 @@
import os import os
import tempfile import tempfile
import threading import threading
from typing import TYPE_CHECKING, Dict, Optional, Tuple from typing import TYPE_CHECKING, Optional
from funcy import retry, wrap_with from funcy import retry, wrap_with
@@ -78,8 +78,8 @@ def erepo_factory(url, root_dir, cache_config):
return make_repo return make_repo
CLONES: Dict[str, Tuple[str, bool]] = {} CLONES: dict[str, tuple[str, bool]] = {}
CACHE_DIRS: Dict[str, str] = {} CACHE_DIRS: dict[str, str] = {}
@wrap_with(threading.Lock()) @wrap_with(threading.Lock())
+18 -17
View File
@@ -1,7 +1,8 @@
import os import os
from collections import defaultdict from collections import defaultdict
from collections.abc import Iterator
from itertools import chain from itertools import chain
from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Tuple, Union from typing import TYPE_CHECKING, Any, Optional, Union
from dvc.dependency.param import ParamsDependency, read_param_file from dvc.dependency.param import ParamsDependency, read_param_file
from dvc.log import logger from dvc.log import logger
@@ -27,7 +28,7 @@ def _collect_top_level_params(repo: "Repo") -> Iterator[str]:
def params_from_target( def params_from_target(
repo: "Repo", targets: List[str] repo: "Repo", targets: list[str]
) -> Iterator["ParamsDependency"]: ) -> Iterator["ParamsDependency"]:
stages = chain.from_iterable(repo.stage.collect(target) for target in targets) stages = chain.from_iterable(repo.stage.collect(target) for target in targets)
for stage in stages: for stage in stages:
@@ -36,17 +37,17 @@ def params_from_target(
def _collect_params( def _collect_params(
repo: "Repo", repo: "Repo",
targets: Union[List[str], Dict[str, List[str]], None] = None, targets: Union[list[str], dict[str, list[str]], None] = None,
stages: Optional[List[str]] = None, stages: Optional[list[str]] = None,
deps_only: bool = False, deps_only: bool = False,
default_file: Optional[str] = None, default_file: Optional[str] = None,
) -> Dict[str, List[str]]: ) -> dict[str, list[str]]:
from dvc.dependency import _merge_params from dvc.dependency import _merge_params
if isinstance(targets, list): if isinstance(targets, list):
targets = {target: [] for target in targets} targets = {target: [] for target in targets}
params: List[Dict[str, List[str]]] = [] params: list[dict[str, list[str]]] = []
if targets: if targets:
# target is a repo-relative path # target is a repo-relative path
@@ -80,8 +81,8 @@ def _collect_params(
return ret return ret
def _collect_vars(repo, params, stages=None) -> Dict: def _collect_vars(repo, params, stages=None) -> dict:
vars_params: Dict[str, Dict] = defaultdict(dict) vars_params: dict[str, dict] = defaultdict(dict)
for stage in repo.index.stages: for stage in repo.index.stages:
if isinstance(stage, PipelineStage) and stage.tracked_vars: if isinstance(stage, PipelineStage) and stage.tracked_vars:
@@ -102,8 +103,8 @@ def _collect_vars(repo, params, stages=None) -> Dict:
def _read_params( def _read_params(
fs: "FileSystem", params: Dict[str, List[str]], **load_kwargs fs: "FileSystem", params: dict[str, list[str]], **load_kwargs
) -> Iterator[Tuple[str, Union[Exception, Any]]]: ) -> Iterator[tuple[str, Union[Exception, Any]]]:
for file_path, key_paths in params.items(): for file_path, key_paths in params.items():
try: try:
yield file_path, read_param_file(fs, file_path, key_paths, **load_kwargs) yield file_path, read_param_file(fs, file_path, key_paths, **load_kwargs)
@@ -114,9 +115,9 @@ def _read_params(
def _gather_params( def _gather_params(
repo: "Repo", repo: "Repo",
targets: Union[List[str], Dict[str, List[str]], None] = None, targets: Union[list[str], dict[str, list[str]], None] = None,
deps_only: bool = False, deps_only: bool = False,
stages: Optional[List[str]] = None, stages: Optional[list[str]] = None,
on_error: str = "return", on_error: str = "return",
): ):
assert on_error in ("raise", "return", "ignore") assert on_error in ("raise", "return", "ignore")
@@ -132,7 +133,7 @@ def _gather_params(
default_file=ParamsDependency.DEFAULT_PARAMS_FILE, default_file=ParamsDependency.DEFAULT_PARAMS_FILE,
) )
data: Dict[str, FileResult] = {} data: dict[str, FileResult] = {}
fs = repo.dvcfs fs = repo.dvcfs
for fs_path, result in _read_params(fs, files_keypaths, cache=True): for fs_path, result in _read_params(fs, files_keypaths, cache=True):
@@ -159,16 +160,16 @@ def _gather_params(
def show( def show(
repo: "Repo", repo: "Repo",
targets: Optional[List[str]] = None, targets: Optional[list[str]] = None,
stages: Optional[List[str]] = None, stages: Optional[list[str]] = None,
deps_only: bool = False, deps_only: bool = False,
all_branches: bool = False, all_branches: bool = False,
all_tags: bool = False, all_tags: bool = False,
revs: Optional[List[str]] = None, revs: Optional[list[str]] = None,
all_commits: bool = False, all_commits: bool = False,
hide_workspace: bool = True, hide_workspace: bool = True,
on_error: str = "return", on_error: str = "return",
) -> Dict[str, Result]: ) -> dict[str, Result]:
assert on_error in ("raise", "return", "ignore") assert on_error in ("raise", "return", "ignore")
res = {} res = {}
+30 -33
View File
@@ -2,6 +2,7 @@ import csv
import io import io
import os import os
from collections import defaultdict from collections import defaultdict
from collections.abc import Iterator
from copy import deepcopy from copy import deepcopy
from functools import partial from functools import partial
from multiprocessing import cpu_count from multiprocessing import cpu_count
@@ -9,11 +10,7 @@ from typing import (
TYPE_CHECKING, TYPE_CHECKING,
Any, Any,
Callable, Callable,
Dict,
Iterator,
List,
Optional, Optional,
Set,
Union, Union,
) )
@@ -40,7 +37,7 @@ dpath.options.ALLOW_EMPTY_STRING_KEYS = True
logger = logger.getChild(__name__) logger = logger.getChild(__name__)
def onerror_collect(result: Dict, exception: Exception, *args, **kwargs): def onerror_collect(result: dict, exception: Exception, *args, **kwargs):
logger.debug("", exc_info=True) logger.debug("", exc_info=True)
result["error"] = exception result["error"] = exception
@@ -82,12 +79,12 @@ class Plots:
def collect( def collect(
self, self,
targets: Optional[List[str]] = None, targets: Optional[list[str]] = None,
revs: Optional[List[str]] = None, revs: Optional[list[str]] = None,
recursive: bool = False, recursive: bool = False,
onerror: Optional[Callable] = None, onerror: Optional[Callable] = None,
props: Optional[Dict] = None, props: Optional[dict] = None,
) -> Iterator[Dict]: ) -> Iterator[dict]:
"""Collects plots definitions and data sources. """Collects plots definitions and data sources.
Generator yielding a structure like: Generator yielding a structure like:
@@ -142,7 +139,7 @@ class Plots:
revs = ["workspace", *revs] revs = ["workspace", *revs]
for rev in revs: for rev in revs:
with switch_repo(self.repo, rev) as (repo, _): with switch_repo(self.repo, rev) as (repo, _):
res: Dict = {} res: dict = {}
definitions = _collect_definitions( definitions = _collect_definitions(
repo, repo,
targets=targets, targets=targets,
@@ -168,9 +165,9 @@ class Plots:
def _collect_data_sources( def _collect_data_sources(
self, self,
repo: "Repo", repo: "Repo",
targets: Optional[List[str]] = None, targets: Optional[list[str]] = None,
recursive: bool = False, recursive: bool = False,
props: Optional[Dict] = None, props: Optional[dict] = None,
onerror: Optional[Callable] = None, onerror: Optional[Callable] = None,
): ):
fs = repo.dvcfs fs = repo.dvcfs
@@ -178,7 +175,7 @@ class Plots:
props = props or {} props = props or {}
plots = _collect_plots(repo, targets, recursive) plots = _collect_plots(repo, targets, recursive)
res: Dict[str, Any] = {} res: dict[str, Any] = {}
for fs_path, rev_props in plots.items(): for fs_path, rev_props in plots.items():
joined_props = {**rev_props, **props} joined_props = {**rev_props, **props}
res[fs_path] = {"props": joined_props} res[fs_path] = {"props": joined_props}
@@ -197,7 +194,7 @@ class Plots:
def show( def show(
self, self,
targets: Optional[List[str]] = None, targets: Optional[list[str]] = None,
revs=None, revs=None,
props=None, props=None,
recursive=False, recursive=False,
@@ -206,7 +203,7 @@ class Plots:
if onerror is None: if onerror is None:
onerror = onerror_collect onerror = onerror_collect
result: Dict[str, Dict] = {} result: dict[str, dict] = {}
for data in self.collect( for data in self.collect(
targets, targets,
revs, revs,
@@ -286,7 +283,7 @@ def _is_plot(out: "Output") -> bool:
return bool(out.plot) return bool(out.plot)
def _resolve_data_sources(plots_data: Dict, rev: str, cache: bool = False): def _resolve_data_sources(plots_data: dict, rev: str, cache: bool = False):
from dvc.progress import Tqdm from dvc.progress import Tqdm
values = list(plots_data.values()) values = list(plots_data.values())
@@ -325,9 +322,9 @@ def _resolve_data_sources(plots_data: Dict, rev: str, cache: bool = False):
def _collect_plots( def _collect_plots(
repo: "Repo", repo: "Repo",
targets: Optional[List[str]] = None, targets: Optional[list[str]] = None,
recursive: bool = False, recursive: bool = False,
) -> Dict[str, Dict]: ) -> dict[str, dict]:
from dvc.repo.collect import collect from dvc.repo.collect import collect
plots, fs_paths = collect( plots, fs_paths = collect(
@@ -344,8 +341,8 @@ def _collect_plots(
return result return result
def _get_data_targets(definitions: Dict): def _get_data_targets(definitions: dict):
result: Set = set() result: set = set()
if "data" in definitions: if "data" in definitions:
for content in definitions["data"].values(): for content in definitions["data"].values():
if "data" in content: if "data" in content:
@@ -404,7 +401,7 @@ def _relpath(fs, path):
def _collect_output_plots(repo, targets, props, onerror: Optional[Callable] = None): def _collect_output_plots(repo, targets, props, onerror: Optional[Callable] = None):
fs = repo.dvcfs fs = repo.dvcfs
result: Dict[str, Dict] = {} result: dict[str, dict] = {}
for plot in repo.index.plots: for plot in repo.index.plots:
plot_props = _plot_props(plot) plot_props = _plot_props(plot)
dvcfile = plot.stage.dvcfile dvcfile = plot.stage.dvcfile
@@ -445,15 +442,15 @@ def _adjust_sources(fs, plot_props, config_dir):
def _resolve_definitions( def _resolve_definitions(
fs: "FileSystem", fs: "FileSystem",
targets: List[str], targets: list[str],
props: Dict[str, Any], props: dict[str, Any],
config_path: "StrPath", config_path: "StrPath",
definitions: "DictStrAny", definitions: "DictStrAny",
onerror: Optional[Callable[[Any], Any]] = None, onerror: Optional[Callable[[Any], Any]] = None,
): ):
config_path = os.fspath(config_path) config_path = os.fspath(config_path)
config_dir = fs.dirname(config_path) config_dir = fs.dirname(config_path)
result: Dict[str, Dict] = {} result: dict[str, dict] = {}
for plot_id, plot_props in definitions.items(): for plot_id, plot_props in definitions.items():
if plot_props is None: if plot_props is None:
plot_props = {} plot_props = {}
@@ -477,12 +474,12 @@ def _resolve_definitions(
return result return result
def _collect_pipeline_files(repo, targets: List[str], props, onerror=None): def _collect_pipeline_files(repo, targets: list[str], props, onerror=None):
result: Dict[str, Dict] = {} result: dict[str, dict] = {}
top_plots = repo.index._plots top_plots = repo.index._plots
for dvcfile, plots_def in top_plots.items(): for dvcfile, plots_def in top_plots.items():
dvcfile_path = _relpath(repo.dvcfs, dvcfile) dvcfile_path = _relpath(repo.dvcfs, dvcfile)
dvcfile_defs_dict: Dict[str, Union[Dict, None]] = {} dvcfile_defs_dict: dict[str, Union[dict, None]] = {}
for elem in plots_def: for elem in plots_def:
if isinstance(elem, str): if isinstance(elem, str):
dvcfile_defs_dict[elem] = None dvcfile_defs_dict[elem] = None
@@ -510,11 +507,11 @@ def _collect_pipeline_files(repo, targets: List[str], props, onerror=None):
def _collect_definitions( def _collect_definitions(
repo: "Repo", repo: "Repo",
targets=None, targets=None,
props: Optional[Dict] = None, props: Optional[dict] = None,
onerror: Optional[Callable] = None, onerror: Optional[Callable] = None,
**kwargs, **kwargs,
) -> Dict: ) -> dict:
result: Dict = defaultdict(dict) result: dict = defaultdict(dict)
props = props or {} props = props or {}
fs = repo.dvcfs fs = repo.dvcfs
@@ -536,8 +533,8 @@ def _collect_definitions(
return dict(result) return dict(result)
def unpack_if_dir(fs, path, props: Dict[str, str], onerror: Optional[Callable] = None): def unpack_if_dir(fs, path, props: dict[str, str], onerror: Optional[Callable] = None):
result: Dict[str, Dict] = defaultdict(dict) result: dict[str, dict] = defaultdict(dict)
if fs.isdir(path): if fs.isdir(path):
unpacked = _unpack_dir_files(fs, path, onerror=onerror) unpacked = _unpack_dir_files(fs, path, onerror=onerror)
else: else:
@@ -574,7 +571,7 @@ def parse(fs, path, props=None, **fs_kwargs):
return PARSERS[extension](contents, path) return PARSERS[extension](contents, path)
def _plot_props(out: "Output") -> Dict: def _plot_props(out: "Output") -> dict:
from dvc.schema import PLOT_PROPS from dvc.schema import PLOT_PROPS
if not (out.plot): if not (out.plot):
+14 -18
View File
@@ -1,13 +1,9 @@
from collections.abc import Iterable
from typing import ( from typing import (
TYPE_CHECKING, TYPE_CHECKING,
Callable, Callable,
Dict,
Iterable,
List,
NoReturn, NoReturn,
Optional, Optional,
Set,
Tuple,
TypeVar, TypeVar,
Union, Union,
cast, cast,
@@ -40,8 +36,8 @@ def collect_stages(
targets: Iterable[str], targets: Iterable[str],
recursive: bool = False, recursive: bool = False,
glob: bool = False, glob: bool = False,
) -> List["Stage"]: ) -> list["Stage"]:
stages: List["Stage"] = [] stages: list["Stage"] = []
for target in targets: for target in targets:
stages.extend(repo.stage.collect(target, recursive=recursive, glob=glob)) stages.extend(repo.stage.collect(target, recursive=recursive, glob=glob))
return ldistinct(stages) return ldistinct(stages)
@@ -49,7 +45,7 @@ def collect_stages(
def get_subgraph( def get_subgraph(
graph: "DiGraph", graph: "DiGraph",
nodes: Optional[List] = None, nodes: Optional[list] = None,
pipeline: bool = False, pipeline: bool = False,
downstream: bool = False, downstream: bool = False,
) -> "DiGraph": ) -> "DiGraph":
@@ -76,10 +72,10 @@ def get_active_graph(graph: "DiGraph") -> "DiGraph":
def plan_repro( def plan_repro(
graph: "DiGraph", graph: "DiGraph",
stages: Optional[List["T"]] = None, stages: Optional[list["T"]] = None,
pipeline: bool = False, pipeline: bool = False,
downstream: bool = False, downstream: bool = False,
) -> List["T"]: ) -> list["T"]:
r"""Derive the evaluation of the given node for the given graph. r"""Derive the evaluation of the given node for the given graph.
When you _reproduce a stage_, you want to _evaluate the descendants_ When you _reproduce a stage_, you want to _evaluate the descendants_
@@ -134,7 +130,7 @@ def _reproduce_stage(stage: "Stage", **kwargs) -> Optional["Stage"]:
def _get_upstream_downstream_nodes( def _get_upstream_downstream_nodes(
graph: Optional["DiGraph"], node: T graph: Optional["DiGraph"], node: T
) -> Tuple[List[T], List[T]]: ) -> tuple[list[T], list[T]]:
succ = list(graph.successors(node)) if graph else [] succ = list(graph.successors(node)) if graph else []
pre = list(graph.predecessors(node)) if graph else [] pre = list(graph.predecessors(node)) if graph else []
return succ, pre return succ, pre
@@ -146,7 +142,7 @@ def _repr(stages: Iterable["Stage"]) -> str:
def handle_error( def handle_error(
graph: Optional["DiGraph"], on_error: str, exc: Exception, stage: "Stage" graph: Optional["DiGraph"], on_error: str, exc: Exception, stage: "Stage"
) -> Set["Stage"]: ) -> set["Stage"]:
import networkx as nx import networkx as nx
logger.warning("%s%s", exc, " (ignored)" if on_error == "ignore" else "") logger.warning("%s%s", exc, " (ignored)" if on_error == "ignore" else "")
@@ -168,19 +164,19 @@ def _raise_error(exc: Optional[Exception], *stages: "Stage") -> NoReturn:
def _reproduce( def _reproduce(
stages: List["Stage"], stages: list["Stage"],
graph: Optional["DiGraph"] = None, graph: Optional["DiGraph"] = None,
force_downstream: bool = False, force_downstream: bool = False,
on_error: str = "fail", on_error: str = "fail",
force: bool = False, force: bool = False,
repro_fn: Callable = _reproduce_stage, repro_fn: Callable = _reproduce_stage,
**kwargs, **kwargs,
) -> List["Stage"]: ) -> list["Stage"]:
assert on_error in ("fail", "keep-going", "ignore") assert on_error in ("fail", "keep-going", "ignore")
result: List["Stage"] = [] result: list["Stage"] = []
failed: List["Stage"] = [] failed: list["Stage"] = []
to_skip: Dict["Stage", "Stage"] = {} to_skip: dict["Stage", "Stage"] = {}
ret: Optional["Stage"] = None ret: Optional["Stage"] = None
force_state = dict.fromkeys(stages, force) force_state = dict.fromkeys(stages, force)
@@ -240,7 +236,7 @@ def reproduce(
if not kwargs.get("interactive", False): if not kwargs.get("interactive", False):
kwargs["interactive"] = self.config["core"].get("interactive", False) kwargs["interactive"] = self.config["core"].get("interactive", False)
stages: List["Stage"] = [] stages: list["Stage"] = []
if not all_pipelines: if not all_pipelines:
targets_list = ensure_list(targets or PROJECT_FILE) targets_list = ensure_list(targets or PROJECT_FILE)
stages = collect_stages(self, targets_list, recursive=recursive, glob=glob) stages = collect_stages(self, targets_list, recursive=recursive, glob=glob)
+4 -8
View File
@@ -1,16 +1,12 @@
import logging import logging
import shlex import shlex
from collections.abc import Iterable, Iterator
from contextlib import contextmanager from contextlib import contextmanager
from functools import wraps from functools import wraps
from typing import ( from typing import (
TYPE_CHECKING, TYPE_CHECKING,
Any, Any,
Dict,
Iterable,
Iterator,
List,
Optional, Optional,
Set,
Union, Union,
) )
@@ -27,15 +23,15 @@ logger = logger.getChild(__name__)
class SCMContext: class SCMContext:
def __init__(self, scm: "Base", config: Optional[Dict[str, Any]] = None) -> None: def __init__(self, scm: "Base", config: Optional[dict[str, Any]] = None) -> None:
from funcy import get_in from funcy import get_in
self.scm: "Base" = scm self.scm: "Base" = scm
self.autostage: bool = get_in( self.autostage: bool = get_in(
config or {}, ["core", "autostage"], default=False config or {}, ["core", "autostage"], default=False
) )
self.ignored_paths: List[str] = [] self.ignored_paths: list[str] = []
self.files_to_track: Set[str] = set() self.files_to_track: set[str] = set()
self.quiet: bool = False self.quiet: bool = False
def track_file(self, paths: Union[str, Iterable[str], None] = None) -> None: def track_file(self, paths: Union[str, Iterable[str], None] = None) -> None:
+6 -5
View File
@@ -1,8 +1,9 @@
import fnmatch import fnmatch
import typing import typing
from collections.abc import Iterable
from contextlib import suppress from contextlib import suppress
from functools import wraps from functools import wraps
from typing import Iterable, List, NamedTuple, Optional, Set, Tuple, Union from typing import NamedTuple, Optional, Union
from dvc.exceptions import ( from dvc.exceptions import (
NoOutputOrStageError, NoOutputOrStageError,
@@ -31,9 +32,9 @@ class StageInfo(NamedTuple):
filter_info: Optional[str] = None filter_info: Optional[str] = None
StageList = List["Stage"] StageList = list["Stage"]
StageIter = Iterable["Stage"] StageIter = Iterable["Stage"]
StageSet = Set["Stage"] StageSet = set["Stage"]
def _collect_with_deps(stages: StageList, graph: "DiGraph") -> StageSet: def _collect_with_deps(stages: StageList, graph: "DiGraph") -> StageSet:
@@ -72,7 +73,7 @@ def _collect_specific_target(
target: str, target: str,
with_deps: bool, with_deps: bool,
recursive: bool, recursive: bool,
) -> Tuple[StageIter, Optional[str], Optional[str]]: ) -> tuple[StageIter, Optional[str], Optional[str]]:
from dvc.dvcfile import is_valid_filename from dvc.dvcfile import is_valid_filename
# Optimization: do not collect the graph for a specific target # Optimization: do not collect the graph for a specific target
@@ -363,7 +364,7 @@ class StageLoad:
with_deps: bool = False, with_deps: bool = False,
recursive: bool = False, recursive: bool = False,
graph: Optional["DiGraph"] = None, graph: Optional["DiGraph"] = None,
) -> List[StageInfo]: ) -> list[StageInfo]:
"""Collects a list of (stage, filter_info) from the given target. """Collects a list of (stage, filter_info) from the given target.
Priority is in the order of following in case of ambiguity: Priority is in the order of following in case of ambiguity:
+2 -2
View File
@@ -1,4 +1,4 @@
from typing import TYPE_CHECKING, List from typing import TYPE_CHECKING
from dvc.exceptions import InvalidArgumentError from dvc.exceptions import InvalidArgumentError
@@ -34,7 +34,7 @@ def update( # noqa: C901
raise InvalidArgumentError("--remote can't be used without --to-remote") raise InvalidArgumentError("--remote can't be used without --to-remote")
import_stages = set() import_stages = set()
other_stage_infos: List["StageInfo"] = [] other_stage_infos: list["StageInfo"] = []
for stage_info in self.index.collect_targets(targets, recursive=recursive): for stage_info in self.index.collect_targets(targets, recursive=recursive):
if stage_info.stage.is_import: if stage_info.stage.is_import:
+6 -5
View File
@@ -1,5 +1,6 @@
from collections.abc import Iterable
from functools import partial from functools import partial
from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Tuple, Union from typing import TYPE_CHECKING, Any, Optional, Union
from funcy import first from funcy import first
@@ -35,7 +36,7 @@ def worktree_view_by_remotes(
targets: Optional["TargetType"] = None, targets: Optional["TargetType"] = None,
push: bool = False, push: bool = False,
**kwargs: Any, **kwargs: Any,
) -> Iterable[Tuple[Optional[str], "IndexView"]]: ) -> Iterable[tuple[Optional[str], "IndexView"]]:
from dvc.repo.index import IndexView from dvc.repo.index import IndexView
def outs_filter(view: "IndexView", remote: Optional[str]): def outs_filter(view: "IndexView", remote: Optional[str]):
@@ -176,7 +177,7 @@ def update_worktree_stages(
outs_filter=outs_filter, outs_filter=outs_filter,
) )
local_index = view.data["repo"] local_index = view.data["repo"]
remote_indexes: Dict[str, Tuple["Remote", "DataIndex"]] = {} remote_indexes: dict[str, tuple["Remote", "DataIndex"]] = {}
for stage in view.stages: for stage in view.stages:
for out in stage.outs: for out in stage.outs:
_update_worktree_out(repo, out, local_index, remote_indexes) _update_worktree_out(repo, out, local_index, remote_indexes)
@@ -187,7 +188,7 @@ def _update_worktree_out(
repo: "Repo", repo: "Repo",
out: "Output", out: "Output",
local_index: Union["DataIndex", "DataIndexView"], local_index: Union["DataIndex", "DataIndexView"],
remote_indexes: Dict[str, Tuple["Remote", "DataIndex"]], remote_indexes: dict[str, tuple["Remote", "DataIndex"]],
): ):
from dvc_data.index import build from dvc_data.index import build
@@ -283,7 +284,7 @@ def _get_diff_indexes(
out: "Output", out: "Output",
local_index: Union["DataIndex", "DataIndexView"], local_index: Union["DataIndex", "DataIndexView"],
remote_index: Union["DataIndex", "DataIndexView"], remote_index: Union["DataIndex", "DataIndexView"],
) -> Tuple["DataIndex", "DataIndex"]: ) -> tuple["DataIndex", "DataIndex"]:
from dvc_data.index import DataIndex from dvc_data.index import DataIndex
_, key = out.index_key _, key = out.index_key
+2 -2
View File
@@ -1,5 +1,5 @@
from collections.abc import Mapping from collections.abc import Mapping
from typing import Any, Dict from typing import Any
import voluptuous as vol import voluptuous as vol
@@ -28,7 +28,7 @@ SINGLE_STAGE_SCHEMA = {
StageParams.PARAM_DESC: str, StageParams.PARAM_DESC: str,
} }
DATA_SCHEMA: Dict[Any, Any] = { DATA_SCHEMA: dict[Any, Any] = {
**CHECKSUMS_SCHEMA, **CHECKSUMS_SCHEMA,
**META_SCHEMA, **META_SCHEMA,
vol.Required("path"): str, vol.Required("path"): str,
+6 -8
View File
@@ -1,13 +1,11 @@
"""Manages source control systems (e.g. Git).""" """Manages source control systems (e.g. Git)."""
import os import os
from collections.abc import Iterator, Mapping
from contextlib import contextmanager from contextlib import contextmanager
from functools import partial from functools import partial
from typing import ( from typing import (
TYPE_CHECKING, TYPE_CHECKING,
Iterator,
List,
Literal, Literal,
Mapping,
Optional, Optional,
Union, Union,
overload, overload,
@@ -189,7 +187,7 @@ def resolve_rev(scm: Union["Git", "NoSCM"], rev: str) -> str:
raise RevError(str(exc)) # noqa: B904 raise RevError(str(exc)) # noqa: B904
def _get_n_commits(scm: "Git", revs: List[str], num: int) -> List[str]: def _get_n_commits(scm: "Git", revs: list[str], num: int) -> list[str]:
results = [] results = []
for rev in revs: for rev in revs:
if num == 0: if num == 0:
@@ -210,14 +208,14 @@ def _get_n_commits(scm: "Git", revs: List[str], num: int) -> List[str]:
def iter_revs( def iter_revs(
scm: "Git", scm: "Git",
revs: Optional[List[str]] = None, revs: Optional[list[str]] = None,
num: int = 1, num: int = 1,
all_branches: bool = False, all_branches: bool = False,
all_tags: bool = False, all_tags: bool = False,
all_commits: bool = False, all_commits: bool = False,
all_experiments: bool = False, all_experiments: bool = False,
commit_date: Optional[str] = None, commit_date: Optional[str] = None,
) -> Mapping[str, List[str]]: ) -> Mapping[str, list[str]]:
from scmrepo.exceptions import SCMError as _SCMError from scmrepo.exceptions import SCMError as _SCMError
from dvc.repo.experiments.utils import exp_commits from dvc.repo.experiments.utils import exp_commits
@@ -235,7 +233,7 @@ def iter_revs(
return {} return {}
revs = revs or [] revs = revs or []
results: List[str] = _get_n_commits(scm, revs, num) results: list[str] = _get_n_commits(scm, revs, num)
if all_commits: if all_commits:
results.extend(scm.list_all_commits()) results.extend(scm.list_all_commits())
@@ -266,7 +264,7 @@ def iter_revs(
return group_by(rev_resolver, results) return group_by(rev_resolver, results)
def lfs_prefetch(fs: "FileSystem", paths: List[str]): def lfs_prefetch(fs: "FileSystem", paths: list[str]):
from scmrepo.git.lfs import fetch as _lfs_fetch from scmrepo.git.lfs import fetch as _lfs_fetch
from dvc.fs.dvc import DVCFileSystem from dvc.fs.dvc import DVCFileSystem
+21 -26
View File
@@ -1,18 +1,13 @@
import os import os
import string import string
from collections import defaultdict from collections import defaultdict
from collections.abc import Iterable
from contextlib import suppress from contextlib import suppress
from dataclasses import dataclass from dataclasses import dataclass
from typing import ( from typing import (
TYPE_CHECKING, TYPE_CHECKING,
Any, Any,
Dict,
Iterable,
List,
Optional, Optional,
Set,
Tuple,
Type,
TypeVar, TypeVar,
Union, Union,
) )
@@ -55,14 +50,14 @@ if TYPE_CHECKING:
logger = logger.getChild(__name__) logger = logger.getChild(__name__)
# Disallow all punctuation characters except hyphen and underscore # Disallow all punctuation characters except hyphen and underscore
INVALID_STAGENAME_CHARS = set(string.punctuation) - {"_", "-"} INVALID_STAGENAME_CHARS = set(string.punctuation) - {"_", "-"}
Env = Dict[str, str] Env = dict[str, str]
ChangedEntries = Tuple[List[str], List[str], Optional[str]] ChangedEntries = tuple[list[str], list[str], Optional[str]]
_T = TypeVar("_T") _T = TypeVar("_T")
def loads_from( def loads_from(
cls: Type[_T], repo: "Repo", path: str, wdir: str, data: Dict[str, Any] cls: type[_T], repo: "Repo", path: str, wdir: str, data: dict[str, Any]
) -> _T: ) -> _T:
kw = { kw = {
"repo": repo, "repo": repo,
@@ -91,7 +86,7 @@ class RawData:
generated_from: Optional[str] = None generated_from: Optional[str] = None
def create_stage(cls: Type[_T], repo, path, **kwargs) -> _T: def create_stage(cls: type[_T], repo, path, **kwargs) -> _T:
from dvc.dvcfile import check_dvcfile_path from dvc.dvcfile import check_dvcfile_path
wdir = os.path.abspath(kwargs.get("wdir", None) or os.curdir) wdir = os.path.abspath(kwargs.get("wdir", None) or os.curdir)
@@ -205,13 +200,13 @@ class Stage(params.StageParams):
self._dvcfile = dvcfile self._dvcfile = dvcfile
@property @property
def params(self) -> List["ParamsDependency"]: def params(self) -> list["ParamsDependency"]:
from dvc.dependency import ParamsDependency from dvc.dependency import ParamsDependency
return [dep for dep in self.deps if isinstance(dep, ParamsDependency)] return [dep for dep in self.deps if isinstance(dep, ParamsDependency)]
@property @property
def metrics(self) -> List["Output"]: def metrics(self) -> list["Output"]:
return [out for out in self.outs if out.metric] return [out for out in self.outs if out.metric]
def __repr__(self): def __repr__(self):
@@ -310,7 +305,7 @@ class Stage(params.StageParams):
return desc return desc
def changed_deps( def changed_deps(
self, allow_missing: bool = False, upstream: Optional[List] = None self, allow_missing: bool = False, upstream: Optional[list] = None
) -> bool: ) -> bool:
if self.frozen: if self.frozen:
return False return False
@@ -322,7 +317,7 @@ class Stage(params.StageParams):
@rwlocked(read=["deps"]) @rwlocked(read=["deps"])
def _changed_deps( def _changed_deps(
self, allow_missing: bool = False, upstream: Optional[List] = None self, allow_missing: bool = False, upstream: Optional[list] = None
) -> bool: ) -> bool:
for dep in self.deps: for dep in self.deps:
status = dep.status() status = dep.status()
@@ -370,7 +365,7 @@ class Stage(params.StageParams):
@rwlocked(read=["deps", "outs"]) @rwlocked(read=["deps", "outs"])
def changed( def changed(
self, allow_missing: bool = False, upstream: Optional[List] = None self, allow_missing: bool = False, upstream: Optional[list] = None
) -> bool: ) -> bool:
is_changed = ( is_changed = (
# Short-circuit order: stage md5 is fast, # Short-circuit order: stage md5 is fast,
@@ -477,7 +472,7 @@ class Stage(params.StageParams):
def reload(self) -> "Stage": def reload(self) -> "Stage":
return self.dvcfile.stage return self.dvcfile.stage
def dumpd(self, **kwargs) -> Dict[str, Any]: def dumpd(self, **kwargs) -> dict[str, Any]:
return get_dump(self, **kwargs) return get_dump(self, **kwargs)
def compute_md5(self) -> Optional[str]: def compute_md5(self) -> Optional[str]:
@@ -509,7 +504,7 @@ class Stage(params.StageParams):
if not allow_missing: if not allow_missing:
raise raise
def get_versioned_outs(self) -> Dict[str, "Output"]: def get_versioned_outs(self) -> dict[str, "Output"]:
from .exceptions import StageFileDoesNotExistError, StageNotFound from .exceptions import StageFileDoesNotExistError, StageNotFound
try: try:
@@ -543,7 +538,7 @@ class Stage(params.StageParams):
out.ignore() out.ignore()
@staticmethod @staticmethod
def _changed_entries(entries) -> List[str]: def _changed_entries(entries) -> list[str]:
return [str(entry) for entry in entries if entry.workspace_status()] return [str(entry) for entry in entries if entry.workspace_status()]
def _changed_stage_entry(self) -> str: def _changed_stage_entry(self) -> str:
@@ -663,8 +658,8 @@ class Stage(params.StageParams):
@rwlocked(write=["outs"]) @rwlocked(write=["outs"])
def checkout( def checkout(
self, allow_missing: bool = False, **kwargs self, allow_missing: bool = False, **kwargs
) -> Dict[str, List["StrPath"]]: ) -> dict[str, list["StrPath"]]:
stats: Dict[str, List["StrPath"]] = defaultdict(list) stats: dict[str, list["StrPath"]] = defaultdict(list)
if self.is_partial_import: if self.is_partial_import:
return stats return stats
@@ -679,7 +674,7 @@ class Stage(params.StageParams):
return stats return stats
@staticmethod @staticmethod
def _checkout(out, **kwargs) -> Tuple[Optional[str], List[str]]: def _checkout(out, **kwargs) -> tuple[Optional[str], list[str]]:
try: try:
result = out.checkout(**kwargs) result = out.checkout(**kwargs)
added, modified = result or (None, None) added, modified = result or (None, None)
@@ -692,8 +687,8 @@ class Stage(params.StageParams):
@rwlocked(read=["deps", "outs"]) @rwlocked(read=["deps", "outs"])
def status( def status(
self, check_updates: bool = False, filter_info: Optional[bool] = None self, check_updates: bool = False, filter_info: Optional[bool] = None
) -> Dict[str, List[Union[str, Dict[str, str]]]]: ) -> dict[str, list[Union[str, dict[str, str]]]]:
ret: List[Union[str, Dict[str, str]]] = [] ret: list[Union[str, dict[str, str]]] = []
show_import = ( show_import = (
self.is_repo_import or self.is_versioned_import self.is_repo_import or self.is_versioned_import
) and check_updates ) and check_updates
@@ -706,7 +701,7 @@ class Stage(params.StageParams):
return {self.addressing: ret} if ret else {} return {self.addressing: ret} if ret else {}
@staticmethod @staticmethod
def _status(entries: Iterable["Output"]) -> Dict[str, str]: def _status(entries: Iterable["Output"]) -> dict[str, str]:
ret = {} ret = {}
for entry in entries: for entry in entries:
@@ -747,7 +742,7 @@ class Stage(params.StageParams):
def get_used_objs( def get_used_objs(
self, *args, **kwargs self, *args, **kwargs
) -> Dict[Optional["HashFileDB"], Set["HashInfo"]]: ) -> dict[Optional["HashFileDB"], set["HashInfo"]]:
"""Return set of object IDs used by this stage.""" """Return set of object IDs used by this stage."""
if self.is_partial_import and not self.is_repo_import: if self.is_partial_import and not self.is_repo_import:
return {} return {}
@@ -802,7 +797,7 @@ class PipelineStage(Stage):
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
self.name = name self.name = name
self.cmd_changed = False self.cmd_changed = False
self.tracked_vars: Dict[str, Dict[str, Dict[str, str]]] = {} self.tracked_vars: dict[str, dict[str, dict[str, str]]] = {}
def __eq__(self, other): def __eq__(self, other):
return super().__eq__(other) and self.name == other.name return super().__eq__(other) and self.name == other.name
+2 -2
View File
@@ -1,6 +1,6 @@
import os import os
from contextlib import contextmanager from contextlib import contextmanager
from typing import TYPE_CHECKING, List, Optional, Tuple from typing import TYPE_CHECKING, Optional
from funcy import first from funcy import first
@@ -247,7 +247,7 @@ class StageCache:
message = f"run-cache is not supported for http filesystem: {path}" message = f"run-cache is not supported for http filesystem: {path}"
raise RunCacheNotSupported(message) raise RunCacheNotSupported(message)
ret: List[Tuple[str, str]] = [] ret: list[tuple[str, str]] = []
if not from_fs.exists(runs): if not from_fs.exists(runs):
return ret return ret
+7 -5
View File
@@ -1,7 +1,7 @@
from collections.abc import Mapping from collections.abc import Mapping
from copy import deepcopy from copy import deepcopy
from itertools import chain from itertools import chain
from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Tuple from typing import TYPE_CHECKING, Any, Optional
from funcy import get_in, lcat, once, project from funcy import get_in, lcat, once, project
@@ -17,6 +17,8 @@ from .params import StageParams
from .utils import fill_stage_dependencies, resolve_paths from .utils import fill_stage_dependencies, resolve_paths
if TYPE_CHECKING: if TYPE_CHECKING:
from collections.abc import Iterable
from dvc.dvcfile import ProjectFile, SingleStageFile from dvc.dvcfile import ProjectFile, SingleStageFile
logger = logger.getChild(__name__) logger = logger.getChild(__name__)
@@ -39,7 +41,7 @@ class StageLoader(Mapping):
self._lockfile_data = lockfile_data.get("stages", {}) self._lockfile_data = lockfile_data.get("stages", {})
@cached_property @cached_property
def lockfile_data(self) -> Dict[str, Any]: def lockfile_data(self) -> dict[str, Any]:
if not self._lockfile_data: if not self._lockfile_data:
logger.debug("Lockfile for '%s' not found", self.dvcfile.relpath) logger.debug("Lockfile for '%s' not found", self.dvcfile.relpath)
return self._lockfile_data return self._lockfile_data
@@ -53,7 +55,7 @@ class StageLoader(Mapping):
from dvc.output import Output, merge_file_meta_from_cloud from dvc.output import Output, merge_file_meta_from_cloud
assert isinstance(lock_data, dict) assert isinstance(lock_data, dict)
items: Iterable[Tuple[str, "Output"]] = chain( items: Iterable[tuple[str, "Output"]] = chain(
((StageParams.PARAM_DEPS, dep) for dep in stage.deps), ((StageParams.PARAM_DEPS, dep) for dep in stage.deps),
((StageParams.PARAM_OUTS, out) for out in stage.outs), ((StageParams.PARAM_OUTS, out) for out in stage.outs),
) )
@@ -174,7 +176,7 @@ class SingleStageLoader(Mapping):
def __init__( def __init__(
self, self,
dvcfile: "SingleStageFile", dvcfile: "SingleStageFile",
stage_data: Dict[Any, str], stage_data: dict[Any, str],
stage_text: Optional[str] = None, stage_text: Optional[str] = None,
): ):
self.dvcfile = dvcfile self.dvcfile = dvcfile
@@ -196,7 +198,7 @@ class SingleStageLoader(Mapping):
def load_stage( def load_stage(
cls, cls,
dvcfile: "SingleStageFile", dvcfile: "SingleStageFile",
d: Dict[str, Any], d: dict[str, Any],
stage_text: Optional[str], stage_text: Optional[str],
) -> Stage: ) -> Stage:
path, wdir = resolve_paths( path, wdir = resolve_paths(
+6 -8
View File
@@ -1,11 +1,9 @@
from collections import OrderedDict from collections import OrderedDict
from collections.abc import Iterable
from operator import attrgetter from operator import attrgetter
from typing import ( from typing import (
TYPE_CHECKING, TYPE_CHECKING,
Any, Any,
Dict,
Iterable,
List,
Optional, Optional,
Union, Union,
no_type_check, no_type_check,
@@ -67,7 +65,7 @@ def _serialize_out(out):
@no_type_check @no_type_check
def _serialize_outs(outputs: List[Output]): def _serialize_outs(outputs: list[Output]):
outs, metrics, plots = [], [], [] outs, metrics, plots = [], [], []
for out in sorted(outputs, key=attrgetter("def_path")): for out in sorted(outputs, key=attrgetter("def_path")):
bucket = outs bucket = outs
@@ -88,10 +86,10 @@ def _serialize_params_keys(params: Iterable["ParamsDependency"]):
at the first, and then followed by entry of other files in lexicographic at the first, and then followed by entry of other files in lexicographic
order. The keys of those custom files are also sorted in the same order. order. The keys of those custom files are also sorted in the same order.
""" """
keys: List[Union[str, Dict[str, Optional[List[str]]]]] = [] keys: list[Union[str, dict[str, Optional[list[str]]]]] = []
for param_dep in sorted(params, key=attrgetter("def_path")): for param_dep in sorted(params, key=attrgetter("def_path")):
# when on no_exec, params are not filled and are saved as list # when on no_exec, params are not filled and are saved as list
k: List[str] = sorted(param_dep.params) k: list[str] = sorted(param_dep.params)
if k and param_dep.def_path == DEFAULT_PARAMS_FILE: if k and param_dep.def_path == DEFAULT_PARAMS_FILE:
keys = k + keys # type: ignore[operator,assignment] keys = k + keys # type: ignore[operator,assignment]
else: else:
@@ -100,7 +98,7 @@ def _serialize_params_keys(params: Iterable["ParamsDependency"]):
@no_type_check @no_type_check
def _serialize_params_values(params: List[ParamsDependency]): def _serialize_params_values(params: list[ParamsDependency]):
"""Returns output of following format, used for lockfile: """Returns output of following format, used for lockfile:
{'params.yaml': {'lr': '1', 'train': 2}, {'params2.yaml': {'lr': '1'}} {'params.yaml': {'lr': '1', 'train': 2}, {'params2.yaml': {'lr': '1'}}
@@ -160,7 +158,7 @@ def to_single_stage_lockfile(stage: "Stage", **kwargs) -> dict:
assert stage.cmd assert stage.cmd
def _dumpd(item: "Output"): def _dumpd(item: "Output"):
ret: Dict[str, Any] = {item.PARAM_PATH: item.def_path} ret: dict[str, Any] = {item.PARAM_PATH: item.def_path}
if item.hash_name not in LEGACY_HASH_NAMES: if item.hash_name not in LEGACY_HASH_NAMES:
ret[item.PARAM_HASH] = "md5" ret[item.PARAM_HASH] = "md5"
if item.hash_info.isdir and kwargs.get("with_files"): if item.hash_info.isdir and kwargs.get("with_files"):
+5 -5
View File
@@ -1,6 +1,6 @@
import os import os
import pathlib import pathlib
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union from typing import TYPE_CHECKING, Any, Optional, Union
from funcy import concat, first, lsplit, rpartial from funcy import concat, first, lsplit, rpartial
@@ -210,7 +210,7 @@ def get_dump(stage: "Stage", **kwargs):
def split_params_deps( def split_params_deps(
stage: "Stage", stage: "Stage",
) -> Tuple[List["ParamsDependency"], List["Dependency"]]: ) -> tuple[list["ParamsDependency"], list["Dependency"]]:
from dvc.dependency import ParamsDependency from dvc.dependency import ParamsDependency
return lsplit(rpartial(isinstance, ParamsDependency), stage.deps) return lsplit(rpartial(isinstance, ParamsDependency), stage.deps)
@@ -265,7 +265,7 @@ def check_stage_exists(repo: "Repo", stage: Union["Stage", "PipelineStage"], pat
def validate_kwargs( def validate_kwargs(
single_stage: bool = False, fname: Optional[str] = None, **kwargs single_stage: bool = False, fname: Optional[str] = None, **kwargs
) -> Dict[str, Any]: ) -> dict[str, Any]:
"""Prepare, validate and process kwargs passed from cli""" """Prepare, validate and process kwargs passed from cli"""
cmd = kwargs.get("cmd") cmd = kwargs.get("cmd")
if not cmd and not single_stage: if not cmd and not single_stage:
@@ -288,11 +288,11 @@ def validate_kwargs(
return kwargs return kwargs
def _get_stage_files(stage: "Stage") -> List[str]: def _get_stage_files(stage: "Stage") -> list[str]:
from dvc.dvcfile import ProjectFile from dvc.dvcfile import ProjectFile
from dvc.utils import relpath from dvc.utils import relpath
ret: List[str] = [] ret: list[str] = []
file = stage.dvcfile file = stage.dvcfile
ret.append(file.relpath) ret.append(file.relpath)
if isinstance(file, ProjectFile): if isinstance(file, ProjectFile):
+2 -2
View File
@@ -2,7 +2,7 @@ import os
import shutil import shutil
from pathlib import Path from pathlib import Path
from subprocess import check_output from subprocess import check_output
from typing import Dict, Optional from typing import Optional
import pytest import pytest
from dulwich.porcelain import clone from dulwich.porcelain import clone
@@ -27,7 +27,7 @@ class VirtualEnv:
virtualenv.cli_run([os.fspath(self.path)]) virtualenv.cli_run([os.fspath(self.path)])
def run(self, cmd: str, *args: str, env: Optional[Dict[str, str]] = None) -> None: def run(self, cmd: str, *args: str, env: Optional[dict[str, str]] = None) -> None:
exe = self.which(cmd) exe = self.which(cmd)
check_output([exe, *args], env=env) # noqa: S603 check_output([exe, *args], env=env) # noqa: S603

Some files were not shown because too many files have changed in this diff Show More