mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
drop Python 3.8 support (#10265)
* drop Python 3.8 Note that this PR only makes typing related changes. Other changes can be done once this gets merged, they are relatively few in numbers. * bump build python version * add typing_extensions to lint group
This commit is contained in:
@@ -25,7 +25,7 @@ jobs:
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: 3.8
|
||||
python-version: 3.12
|
||||
|
||||
- run: python -m pip install --upgrade pip setuptools_scm build twine
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-20.04, windows-latest, macos-latest]
|
||||
pyv: ["3.8", "3.9", "3.10", "3.11", "3.12"]
|
||||
pyv: ["3.9", "3.10", "3.11", "3.12"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
+8
-8
@@ -1,5 +1,5 @@
|
||||
from dataclasses import asdict, dataclass, field, fields
|
||||
from typing import Any, ClassVar, Dict, List, Optional
|
||||
from typing import Any, ClassVar, Optional
|
||||
|
||||
from funcy import compact
|
||||
from voluptuous import Required
|
||||
@@ -14,10 +14,10 @@ class Annotation:
|
||||
|
||||
desc: Optional[str] = None
|
||||
type: Optional[str] = None
|
||||
labels: List[str] = field(default_factory=list)
|
||||
meta: Dict[str, Any] = field(default_factory=dict)
|
||||
labels: list[str] = field(default_factory=list)
|
||||
meta: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> Dict[str, str]:
|
||||
def to_dict(self) -> dict[str, str]:
|
||||
return compact(asdict(self))
|
||||
|
||||
|
||||
@@ -32,10 +32,10 @@ class Artifact:
|
||||
path: str
|
||||
desc: Optional[str] = None
|
||||
type: Optional[str] = None
|
||||
labels: List[str] = field(default_factory=list)
|
||||
meta: Dict[str, Any] = field(default_factory=dict)
|
||||
labels: list[str] = field(default_factory=list)
|
||||
meta: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> Dict[str, str]:
|
||||
def to_dict(self) -> dict[str, str]:
|
||||
return compact(asdict(self))
|
||||
|
||||
|
||||
@@ -46,7 +46,7 @@ ANNOTATION_SCHEMA = {
|
||||
Annotation.PARAM_LABELS: [str],
|
||||
Annotation.PARAM_META: object,
|
||||
}
|
||||
ARTIFACT_SCHEMA: Dict[Any, Any] = {
|
||||
ARTIFACT_SCHEMA: dict[Any, Any] = {
|
||||
Required(Artifact.PARAM_PATH): str,
|
||||
**ANNOTATION_SCHEMA, # type: ignore[arg-type]
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from typing import Any, Dict, Optional
|
||||
from typing import Any, Optional
|
||||
|
||||
from dvc.repo import Repo
|
||||
|
||||
@@ -8,7 +8,7 @@ def artifacts_show(
|
||||
version: Optional[str] = None,
|
||||
stage: Optional[str] = None,
|
||||
repo: Optional[str] = None,
|
||||
) -> Dict[str, str]:
|
||||
) -> dict[str, str]:
|
||||
"""
|
||||
Return path and Git revision for an artifact in a DVC project.
|
||||
|
||||
@@ -36,7 +36,7 @@ def artifacts_show(
|
||||
if version and stage:
|
||||
raise ValueError("Artifact version and stage are mutually exclusive.")
|
||||
|
||||
repo_kwargs: Dict[str, Any] = {
|
||||
repo_kwargs: dict[str, Any] = {
|
||||
"subrepos": True,
|
||||
"uninitialized": True,
|
||||
}
|
||||
|
||||
+5
-5
@@ -1,6 +1,6 @@
|
||||
from contextlib import _GeneratorContextManager as GCM
|
||||
from contextlib import contextmanager
|
||||
from typing import Any, Dict, Optional
|
||||
from typing import Any, Optional
|
||||
|
||||
from funcy import reraise
|
||||
|
||||
@@ -39,7 +39,7 @@ def get_url(path, repo=None, rev=None, remote=None):
|
||||
from dvc.config import NoRemoteError
|
||||
from dvc_data.index import StorageKeyError
|
||||
|
||||
repo_kwargs: Dict[str, Any] = {}
|
||||
repo_kwargs: dict[str, Any] = {}
|
||||
if remote:
|
||||
repo_kwargs["config"] = {"core": {"remote": remote}}
|
||||
with Repo.open(
|
||||
@@ -74,8 +74,8 @@ def open( # noqa: A001
|
||||
remote: Optional[str] = None,
|
||||
mode: str = "r",
|
||||
encoding: Optional[str] = None,
|
||||
config: Optional[Dict[str, Any]] = None,
|
||||
remote_config: Optional[Dict[str, Any]] = None,
|
||||
config: Optional[dict[str, Any]] = None,
|
||||
remote_config: Optional[dict[str, Any]] = None,
|
||||
):
|
||||
"""
|
||||
Opens a file tracked in a DVC project.
|
||||
@@ -233,7 +233,7 @@ def _open(
|
||||
config=None,
|
||||
remote_config=None,
|
||||
):
|
||||
repo_kwargs: Dict[str, Any] = {
|
||||
repo_kwargs: dict[str, Any] = {
|
||||
"subrepos": True,
|
||||
"uninitialized": True,
|
||||
"remote": remote,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from typing import Dict, List, Optional, Union
|
||||
from typing import Optional, Union
|
||||
|
||||
from rich.text import Text
|
||||
|
||||
@@ -9,7 +9,7 @@ from dvc.repo.experiments.show import tabulate
|
||||
def exp_save(
|
||||
name: Optional[str] = None,
|
||||
force: bool = False,
|
||||
include_untracked: Optional[List[str]] = None,
|
||||
include_untracked: Optional[list[str]] = None,
|
||||
):
|
||||
"""
|
||||
Create a new DVC experiment using `exp save`.
|
||||
@@ -61,12 +61,12 @@ def _postprocess(exp_rows):
|
||||
|
||||
def exp_show(
|
||||
repo: Optional[str] = None,
|
||||
revs: Optional[Union[str, List[str]]] = None,
|
||||
revs: Optional[Union[str, list[str]]] = None,
|
||||
num: int = 1,
|
||||
param_deps: bool = False,
|
||||
force: bool = False,
|
||||
config: Optional[Dict] = None,
|
||||
) -> List[Dict]:
|
||||
config: Optional[dict] = None,
|
||||
) -> list[dict]:
|
||||
"""Get DVC experiments tracked in `repo`.
|
||||
|
||||
Without arguments, this function will retrieve all experiments derived from
|
||||
|
||||
+4
-4
@@ -1,9 +1,9 @@
|
||||
from typing import List, Optional
|
||||
from typing import Optional
|
||||
|
||||
from dvc.repo import Repo
|
||||
|
||||
|
||||
def all_branches(repo: Optional[str] = None) -> List[str]:
|
||||
def all_branches(repo: Optional[str] = None) -> list[str]:
|
||||
"""Get all Git branches in a DVC repository.
|
||||
|
||||
Args:
|
||||
@@ -20,7 +20,7 @@ def all_branches(repo: Optional[str] = None) -> List[str]:
|
||||
return _repo.scm.list_branches()
|
||||
|
||||
|
||||
def all_commits(repo: Optional[str] = None) -> List[str]:
|
||||
def all_commits(repo: Optional[str] = None) -> list[str]:
|
||||
"""Get all Git commits in a DVC repository.
|
||||
|
||||
Args:
|
||||
@@ -37,7 +37,7 @@ def all_commits(repo: Optional[str] = None) -> List[str]:
|
||||
return _repo.scm.list_all_commits()
|
||||
|
||||
|
||||
def all_tags(repo: Optional[str] = None) -> List[str]:
|
||||
def all_tags(repo: Optional[str] = None) -> list[str]:
|
||||
"""Get all Git tags in a DVC repository.
|
||||
|
||||
Args:
|
||||
|
||||
+7
-6
@@ -1,6 +1,7 @@
|
||||
import typing
|
||||
from collections import Counter
|
||||
from typing import Dict, Iterable, Optional, Union
|
||||
from collections.abc import Iterable
|
||||
from typing import Optional, Union
|
||||
|
||||
from funcy import first
|
||||
|
||||
@@ -8,7 +9,7 @@ from dvc.repo import Repo
|
||||
|
||||
|
||||
def _postprocess(results):
|
||||
processed: Dict[str, Dict] = {}
|
||||
processed: dict[str, dict] = {}
|
||||
for rev, rev_data in results.items():
|
||||
if not rev_data:
|
||||
continue
|
||||
@@ -36,8 +37,8 @@ def metrics_show(
|
||||
*targets: str,
|
||||
repo: Optional[str] = None,
|
||||
rev: Optional[str] = None,
|
||||
config: Optional[Dict] = None,
|
||||
) -> Dict:
|
||||
config: Optional[dict] = None,
|
||||
) -> dict:
|
||||
"""Get metrics tracked in `repo`.
|
||||
|
||||
Without arguments, this function will retrieve all metrics from all tracked
|
||||
@@ -159,8 +160,8 @@ def params_show(
|
||||
stages: Optional[Union[str, Iterable[str]]] = None,
|
||||
rev: Optional[str] = None,
|
||||
deps: bool = False,
|
||||
config: Optional[Dict] = None,
|
||||
) -> Dict:
|
||||
config: Optional[dict] = None,
|
||||
) -> dict:
|
||||
"""Get parameters tracked in `repo`.
|
||||
|
||||
Without arguments, this function will retrieve all params from all tracked
|
||||
|
||||
+2
-2
@@ -1,5 +1,5 @@
|
||||
import os
|
||||
from typing import TYPE_CHECKING, Optional, Tuple
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from dvc.fs import GitFileSystem, Schemes
|
||||
from dvc_data.hashfile.db import get_odb
|
||||
@@ -15,7 +15,7 @@ def _get_odb(
|
||||
repo,
|
||||
settings,
|
||||
fs=None,
|
||||
prefix: Optional[Tuple[str, ...]] = None,
|
||||
prefix: Optional[tuple[str, ...]] = None,
|
||||
hash_name: Optional[str] = None,
|
||||
**kwargs,
|
||||
):
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from typing import TYPE_CHECKING, ClassVar, Dict, Tuple
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
from funcy import chunks, compact, log_durations
|
||||
|
||||
@@ -17,14 +17,14 @@ logger = logger.getChild(__name__)
|
||||
|
||||
|
||||
class CmdDataStatus(CmdBase):
|
||||
COLORS: ClassVar[Dict[str, str]] = {
|
||||
COLORS: ClassVar[dict[str, str]] = {
|
||||
"not_in_remote": "red",
|
||||
"not_in_cache": "red",
|
||||
"committed": "green",
|
||||
"uncommitted": "yellow",
|
||||
"untracked": "cyan",
|
||||
}
|
||||
LABELS: ClassVar[Dict[str, str]] = {
|
||||
LABELS: ClassVar[dict[str, str]] = {
|
||||
"not_in_remote": "Not in remote",
|
||||
"not_in_cache": "Not in cache",
|
||||
"committed": "DVC committed changes",
|
||||
@@ -32,7 +32,7 @@ class CmdDataStatus(CmdBase):
|
||||
"untracked": "Untracked files",
|
||||
"unchanged": "DVC unchanged files",
|
||||
}
|
||||
HINTS: ClassVar[Dict[str, Tuple[str, ...]]] = {
|
||||
HINTS: ClassVar[dict[str, tuple[str, ...]]] = {
|
||||
"not_in_remote": ('use "dvc push <file>..." to upload files',),
|
||||
"not_in_cache": ('use "dvc fetch <file>..." to download files',),
|
||||
"committed": ("git commit the corresponding dvc files to update the repo",),
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from typing import Any, Dict
|
||||
from typing import Any
|
||||
|
||||
from dvc.cli import completion, formatter
|
||||
from dvc.cli.command import CmdBase
|
||||
@@ -11,7 +11,7 @@ logger = logger.getChild(__name__)
|
||||
|
||||
class CmdExperimentsPush(CmdBase):
|
||||
@staticmethod
|
||||
def log_result(result: Dict[str, Any], remote: str):
|
||||
def log_result(result: dict[str, Any], remote: str):
|
||||
from dvc.utils import humanize
|
||||
|
||||
def join_exps(exps):
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import argparse
|
||||
import re
|
||||
from collections.abc import Iterable
|
||||
from datetime import date, datetime
|
||||
from typing import TYPE_CHECKING, Dict, Iterable
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from funcy import lmap
|
||||
|
||||
@@ -57,7 +58,7 @@ def baseline_styler(typ):
|
||||
|
||||
def show_experiments(
|
||||
td: "TabularData",
|
||||
headers: Dict[str, Iterable[str]],
|
||||
headers: dict[str, Iterable[str]],
|
||||
keep=None,
|
||||
drop=None,
|
||||
pager=True,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import argparse
|
||||
import os
|
||||
from typing import TYPE_CHECKING, Dict, List, Optional
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from funcy import compact, first, get_in
|
||||
|
||||
@@ -20,14 +20,14 @@ logger = logger.getChild(__name__)
|
||||
|
||||
|
||||
def _show_json(
|
||||
renderers_with_errors: List["RendererWithErrors"],
|
||||
renderers_with_errors: list["RendererWithErrors"],
|
||||
split=False,
|
||||
errors: Optional[Dict[str, Exception]] = None,
|
||||
errors: Optional[dict[str, Exception]] = None,
|
||||
):
|
||||
from dvc.render.convert import to_json
|
||||
from dvc.utils.serialize import encode_exception
|
||||
|
||||
all_errors: List[Dict] = []
|
||||
all_errors: list[dict] = []
|
||||
data = {}
|
||||
|
||||
for renderer, src_errors, def_errors in renderers_with_errors:
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import argparse
|
||||
import logging
|
||||
from collections.abc import Iterable
|
||||
from contextlib import contextmanager
|
||||
from itertools import chain, filterfalse
|
||||
from typing import TYPE_CHECKING, Dict, Iterable, List
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from dvc.cli import completion, formatter
|
||||
from dvc.cli.command import CmdBase
|
||||
@@ -33,7 +34,7 @@ def generate_description(stage: "Stage") -> str:
|
||||
def is_plot_or_metric(out: "Output"):
|
||||
return bool(out.plot) or bool(out.metric)
|
||||
|
||||
desc: List[str] = []
|
||||
desc: list[str] = []
|
||||
|
||||
outs = list(filterfalse(is_plot_or_metric, stage.outs))
|
||||
if outs:
|
||||
@@ -55,7 +56,7 @@ def prepare_stages_data(
|
||||
stages: Iterable["Stage"],
|
||||
description: bool = True,
|
||||
max_length: int = MAX_TEXT_LENGTH,
|
||||
) -> Dict[str, str]:
|
||||
) -> dict[str, str]:
|
||||
return {
|
||||
stage.addressing: (
|
||||
prepare_description(stage, max_length=max_length) if description else ""
|
||||
@@ -67,7 +68,7 @@ def prepare_stages_data(
|
||||
class CmdStageList(CmdBase):
|
||||
def _get_stages(self) -> Iterable["Stage"]:
|
||||
if self.args.all:
|
||||
stages: List["Stage"] = self.repo.index.stages
|
||||
stages: list["Stage"] = self.repo.index.stages
|
||||
logger.trace("%d no. of stages found", len(stages))
|
||||
return stages
|
||||
|
||||
@@ -96,7 +97,7 @@ class CmdStageList(CmdBase):
|
||||
return 0
|
||||
|
||||
|
||||
def parse_cmd(commands: List[str]) -> str:
|
||||
def parse_cmd(commands: list[str]) -> str:
|
||||
"""
|
||||
We need to take into account two cases:
|
||||
|
||||
|
||||
+24
-26
@@ -1,20 +1,18 @@
|
||||
from collections import abc
|
||||
from collections.abc import (
|
||||
ItemsView,
|
||||
Iterable,
|
||||
Iterator,
|
||||
Mapping,
|
||||
MutableSequence,
|
||||
Sequence,
|
||||
)
|
||||
from itertools import chain, repeat, zip_longest
|
||||
from operator import itemgetter
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Dict,
|
||||
ItemsView,
|
||||
Iterable,
|
||||
Iterator,
|
||||
List,
|
||||
Mapping,
|
||||
MutableSequence,
|
||||
Optional,
|
||||
Sequence,
|
||||
Set,
|
||||
Tuple,
|
||||
Union,
|
||||
overload,
|
||||
)
|
||||
@@ -25,7 +23,7 @@ if TYPE_CHECKING:
|
||||
from dvc.ui.table import CellT
|
||||
|
||||
|
||||
class Column(List["CellT"]):
|
||||
class Column(list["CellT"]):
|
||||
pass
|
||||
|
||||
|
||||
@@ -35,13 +33,13 @@ def with_value(value, default):
|
||||
|
||||
class TabularData(MutableSequence[Sequence["CellT"]]):
|
||||
def __init__(self, columns: Sequence[str], fill_value: Optional[str] = ""):
|
||||
self._columns: Dict[str, Column] = {name: Column() for name in columns}
|
||||
self._keys: List[str] = list(columns)
|
||||
self._columns: dict[str, Column] = {name: Column() for name in columns}
|
||||
self._keys: list[str] = list(columns)
|
||||
self._fill_value = fill_value
|
||||
self._protected: Set[str] = set()
|
||||
self._protected: set[str] = set()
|
||||
|
||||
@property
|
||||
def columns(self) -> List[Column]:
|
||||
def columns(self) -> list[Column]:
|
||||
return list(map(self.column, self.keys()))
|
||||
|
||||
def is_protected(self, col_name) -> bool:
|
||||
@@ -60,10 +58,10 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
|
||||
projection = {k: self.column(k) for k in self.keys()}
|
||||
return projection.items()
|
||||
|
||||
def keys(self) -> List[str]:
|
||||
def keys(self) -> list[str]:
|
||||
return self._keys
|
||||
|
||||
def _iter_col_row(self, row: Sequence["CellT"]) -> Iterator[Tuple["CellT", Column]]:
|
||||
def _iter_col_row(self, row: Sequence["CellT"]) -> Iterator[tuple["CellT", Column]]:
|
||||
for val, col in zip_longest(row, self.columns):
|
||||
if col is None:
|
||||
break
|
||||
@@ -81,7 +79,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
|
||||
for val, col in self._iter_col_row(value):
|
||||
col.insert(index, val)
|
||||
|
||||
def __iter__(self) -> Iterator[List["CellT"]]:
|
||||
def __iter__(self) -> Iterator[list["CellT"]]:
|
||||
return map(list, zip(*self.columns))
|
||||
|
||||
def __getattr__(self, item: str) -> Column:
|
||||
@@ -127,7 +125,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
|
||||
return len(self.columns[0])
|
||||
|
||||
@property
|
||||
def shape(self) -> Tuple[int, int]:
|
||||
def shape(self) -> tuple[int, int]:
|
||||
return len(self.columns), len(self)
|
||||
|
||||
def drop(self, *col_names: str) -> None:
|
||||
@@ -170,7 +168,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
|
||||
if key not in keys:
|
||||
self.add_column(key)
|
||||
|
||||
row: List["CellT"] = [
|
||||
row: list["CellT"] = [
|
||||
with_value(d.get(key), self._fill_value) for key in self.keys()
|
||||
]
|
||||
self.append(row)
|
||||
@@ -185,7 +183,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
|
||||
|
||||
def as_dict(
|
||||
self, cols: Optional[Iterable[str]] = None
|
||||
) -> Iterable[Dict[str, "CellT"]]:
|
||||
) -> Iterable[dict[str, "CellT"]]:
|
||||
keys = self.keys() if cols is None else set(cols)
|
||||
return [{k: self._columns[k][i] for k in keys} for i in range(len(self))]
|
||||
|
||||
@@ -202,7 +200,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
|
||||
if how not in ["any", "all"]:
|
||||
raise ValueError(f"Invalid 'how' value {how}. Choose one of ['any', 'all']")
|
||||
|
||||
match_line: Set = set()
|
||||
match_line: set = set()
|
||||
match_any = True
|
||||
if how == "all":
|
||||
match_any = False
|
||||
@@ -245,7 +243,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
|
||||
)
|
||||
|
||||
if axis == "cols":
|
||||
cols_to_drop: List[str] = []
|
||||
cols_to_drop: list[str] = []
|
||||
for n_col, col in enumerate(self.columns):
|
||||
if subset and self.keys()[n_col] not in subset:
|
||||
continue
|
||||
@@ -259,7 +257,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
|
||||
|
||||
elif axis == "rows":
|
||||
unique_rows = []
|
||||
rows_to_drop: List[int] = []
|
||||
rows_to_drop: list[int] = []
|
||||
for n_row, row in enumerate(self):
|
||||
if subset:
|
||||
row = [
|
||||
@@ -319,7 +317,7 @@ def diff_table(
|
||||
) -> TabularData:
|
||||
a_rev = a_rev or "HEAD"
|
||||
b_rev = b_rev or "workspace"
|
||||
headers: List[str] = ["Path", title, a_rev, b_rev, "Change"]
|
||||
headers: list[str] = ["Path", title, a_rev, b_rev, "Change"]
|
||||
fill_value = "-"
|
||||
td = TabularData(headers, fill_value=fill_value)
|
||||
|
||||
@@ -394,7 +392,7 @@ def metrics_table(
|
||||
|
||||
for branch, val in metrics.items():
|
||||
for fname, metric in val.get("data", {}).items():
|
||||
row_data: Dict[str, str] = {"Revision": branch, "Path": fname}
|
||||
row_data: dict[str, str] = {"Revision": branch, "Path": fname}
|
||||
metric = metric.get("data", {})
|
||||
flattened = (
|
||||
flatten(format_dict(metric))
|
||||
|
||||
+4
-4
@@ -3,7 +3,7 @@ import os
|
||||
import re
|
||||
from contextlib import contextmanager
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, Dict, Optional
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from funcy import compact, memoize, re_find
|
||||
|
||||
@@ -138,7 +138,7 @@ class Config(dict):
|
||||
return system_config_dir()
|
||||
|
||||
@cached_property
|
||||
def files(self) -> Dict[str, str]:
|
||||
def files(self) -> dict[str, str]:
|
||||
files = {
|
||||
level: os.path.join(self.get_dir(level), self.CONFIG)
|
||||
for level in ("system", "global")
|
||||
@@ -344,7 +344,7 @@ class Config(dict):
|
||||
return Schema(dirs_schema, extra=ALLOW_EXTRA)(conf)
|
||||
|
||||
def load_config_to_level(self, level=None):
|
||||
merged_conf: Dict = {}
|
||||
merged_conf: dict = {}
|
||||
for merge_level in self.LEVELS:
|
||||
if merge_level == level:
|
||||
break
|
||||
@@ -390,7 +390,7 @@ class Config(dict):
|
||||
|
||||
|
||||
def _parse_named(conf):
|
||||
result: Dict[str, Dict] = {"remote": {}, "machine": {}, "db": {}}
|
||||
result: dict[str, dict] = {"remote": {}, "machine": {}, "db": {}}
|
||||
|
||||
for section, val in conf.items():
|
||||
match = re_find(r'^\s*(remote|machine|db)\s*"(.*)"\s*$', section)
|
||||
|
||||
+9
-8
@@ -5,13 +5,14 @@ import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from collections.abc import Mapping, Sequence
|
||||
from contextlib import nullcontext
|
||||
from typing import TYPE_CHECKING, Any, List, Mapping, Optional, Sequence, Union
|
||||
from typing import TYPE_CHECKING, Any, Optional, Union
|
||||
|
||||
from dvc.log import logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from typing import ContextManager
|
||||
from contextlib import AbstractContextManager
|
||||
|
||||
from dvc.env import DVC_DAEMON, DVC_DAEMON_LOGFILE
|
||||
from dvc.utils import fix_env, is_binary
|
||||
@@ -55,7 +56,7 @@ def _win_detached_subprocess(args: Sequence[str], **kwargs) -> int:
|
||||
return popen.pid
|
||||
|
||||
|
||||
def _get_dvc_args() -> List[str]:
|
||||
def _get_dvc_args() -> list[str]:
|
||||
args = [sys.executable]
|
||||
if not is_binary():
|
||||
root_dir = os.path.abspath(os.path.dirname(__file__))
|
||||
@@ -136,7 +137,7 @@ def _map_log_level_to_flag() -> Optional[str]:
|
||||
return flags.get(logger.getEffectiveLevel())
|
||||
|
||||
|
||||
def daemon(args: List[str]) -> None:
|
||||
def daemon(args: list[str]) -> None:
|
||||
"""Launch a `dvc daemon` command in a detached process.
|
||||
|
||||
Args:
|
||||
@@ -148,12 +149,12 @@ def daemon(args: List[str]) -> None:
|
||||
|
||||
|
||||
def _spawn(
|
||||
args: List[str],
|
||||
executable: Optional[Union[str, List[str]]] = None,
|
||||
args: list[str],
|
||||
executable: Optional[Union[str, list[str]]] = None,
|
||||
env: Optional[Mapping[str, str]] = None,
|
||||
output_file: Optional[str] = None,
|
||||
) -> int:
|
||||
file: "ContextManager[Any]" = nullcontext()
|
||||
file: "AbstractContextManager[Any]" = nullcontext()
|
||||
kwargs = {}
|
||||
if output_file:
|
||||
file = open(output_file, "ab") # noqa: SIM115
|
||||
@@ -168,7 +169,7 @@ def _spawn(
|
||||
return _detached_subprocess(executable + args, env=env, **kwargs)
|
||||
|
||||
|
||||
def daemonize(args: List[str], executable: Union[None, str, List[str]] = None) -> None:
|
||||
def daemonize(args: list[str], executable: Union[None, str, list[str]] = None) -> None:
|
||||
if os.name not in ("posix", "nt"):
|
||||
return
|
||||
|
||||
|
||||
+3
-2
@@ -1,6 +1,7 @@
|
||||
"""Manages dvc remotes that user can use with push/pull/status commands."""
|
||||
|
||||
from typing import TYPE_CHECKING, Iterable, Optional, Set, Tuple
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from dvc.config import NoRemoteError, RemoteConfigError
|
||||
from dvc.log import logger
|
||||
@@ -50,7 +51,7 @@ class Remote:
|
||||
|
||||
def _split_legacy_hash_infos(
|
||||
hash_infos: Iterable["HashInfo"],
|
||||
) -> Tuple[Set["HashInfo"], Set["HashInfo"]]:
|
||||
) -> tuple[set["HashInfo"], set["HashInfo"]]:
|
||||
from dvc.cachemgr import LEGACY_HASH_NAMES
|
||||
|
||||
legacy = set()
|
||||
|
||||
+4
-3
@@ -1,8 +1,9 @@
|
||||
import os
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from tempfile import NamedTemporaryFile
|
||||
from typing import TYPE_CHECKING, Any, Callable, Dict, Iterator, Optional, Union
|
||||
from typing import TYPE_CHECKING, Any, Callable, Optional, Union
|
||||
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.engine import make_url as _make_url
|
||||
@@ -30,7 +31,7 @@ def make_url(url: Union["URL", str], **kwargs: Any) -> "URL":
|
||||
return _make_url(url).set(**kwargs)
|
||||
|
||||
|
||||
def url_from_config(config: Union[str, "URL", Dict[str, str]]) -> "URL":
|
||||
def url_from_config(config: Union[str, "URL", dict[str, str]]) -> "URL":
|
||||
if isinstance(config, dict):
|
||||
return make_url(**config)
|
||||
return make_url(config)
|
||||
@@ -118,7 +119,7 @@ def handle_error(url: "URL"):
|
||||
|
||||
@contextmanager
|
||||
def client(
|
||||
url_or_config: Union[str, "URL", Dict[str, str]], **engine_kwargs: Any
|
||||
url_or_config: Union[str, "URL", dict[str, str]], **engine_kwargs: Any
|
||||
) -> Iterator[Client]:
|
||||
url = url_from_config(url_or_config)
|
||||
echo = env2bool(env.DVC_SQLALCHEMY_ECHO, False)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from collections import defaultdict
|
||||
from typing import Any, Dict, List, Mapping, Set
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
from dvc.output import ARTIFACT_SCHEMA, DIR_FILES_SCHEMA, Output
|
||||
|
||||
@@ -66,12 +67,12 @@ def loads_from(stage, s_list, erepo=None, fs_config=None, db=None):
|
||||
]
|
||||
|
||||
|
||||
def _merge_params(s_list) -> Dict[str, List[str]]:
|
||||
def _merge_params(s_list) -> dict[str, list[str]]:
|
||||
d = defaultdict(list)
|
||||
default_file = ParamsDependency.DEFAULT_PARAMS_FILE
|
||||
|
||||
# figure out completely tracked params file, and ignore specific keys
|
||||
wholly_tracked: Set[str] = set()
|
||||
wholly_tracked: set[str] = set()
|
||||
for key in s_list:
|
||||
if not isinstance(key, dict):
|
||||
continue
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
from typing import Dict, Type
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.fs import download as fs_download
|
||||
from dvc.output import Output
|
||||
@@ -25,11 +23,11 @@ class DependencyIsStageFileError(DvcException):
|
||||
class Dependency(Output):
|
||||
IS_DEPENDENCY = True
|
||||
|
||||
DoesNotExistError: Type[DvcException] = DependencyDoesNotExistError
|
||||
IsNotFileOrDirError: Type[DvcException] = DependencyIsNotFileOrDirError
|
||||
IsStageFileError: Type[DvcException] = DependencyIsStageFileError
|
||||
DoesNotExistError: type[DvcException] = DependencyDoesNotExistError
|
||||
IsNotFileOrDirError: type[DvcException] = DependencyIsNotFileOrDirError
|
||||
IsStageFileError: type[DvcException] = DependencyIsStageFileError
|
||||
|
||||
def workspace_status(self) -> Dict[str, str]:
|
||||
def workspace_status(self) -> dict[str, str]:
|
||||
if self.fs.version_aware:
|
||||
old_fs_path = self.fs_path
|
||||
try:
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from typing import TYPE_CHECKING, Any, Callable, ClassVar, Dict, Iterator, Optional
|
||||
from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional
|
||||
|
||||
from funcy import compact, log_durations
|
||||
|
||||
@@ -31,7 +32,7 @@ def download_progress(to: "Output") -> Iterator[Callable[[int], Any]]:
|
||||
class AbstractDependency(Dependency):
|
||||
"""Dependency without workspace/fs/fs_path"""
|
||||
|
||||
def __init__(self, stage: "Stage", info: Dict[str, Any], *args, **kwargs):
|
||||
def __init__(self, stage: "Stage", info: dict[str, Any], *args, **kwargs):
|
||||
self.repo = stage.repo
|
||||
self.stage = stage
|
||||
self.fs = None
|
||||
@@ -46,7 +47,7 @@ class DbDependency(AbstractDependency):
|
||||
PARAM_QUERY = "query"
|
||||
PARAM_TABLE = "table"
|
||||
PARAM_FILE_FORMAT = "file_format"
|
||||
DB_SCHEMA: ClassVar[Dict] = {
|
||||
DB_SCHEMA: ClassVar[dict] = {
|
||||
PARAM_DB: {
|
||||
PARAM_QUERY: str,
|
||||
PARAM_CONNECTION: str,
|
||||
@@ -57,7 +58,7 @@ class DbDependency(AbstractDependency):
|
||||
|
||||
def __init__(self, stage: "Stage", info, *args, **kwargs):
|
||||
super().__init__(stage, info, *args, **kwargs)
|
||||
self.db_info: Dict[str, str] = self.info.get(self.PARAM_DB, {})
|
||||
self.db_info: dict[str, str] = self.info.get(self.PARAM_DB, {})
|
||||
self.connection = self.db_info.get(self.PARAM_CONNECTION)
|
||||
|
||||
@property
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import os
|
||||
import typing
|
||||
from collections import defaultdict
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional
|
||||
from typing import TYPE_CHECKING, Any, Optional
|
||||
|
||||
import dpath
|
||||
|
||||
@@ -37,7 +37,7 @@ class BadParamFileError(DvcException):
|
||||
def read_param_file(
|
||||
fs: "FileSystem",
|
||||
path: str,
|
||||
key_paths: Optional[List[str]] = None,
|
||||
key_paths: Optional[list[str]] = None,
|
||||
flatten: bool = False,
|
||||
**load_kwargs,
|
||||
) -> Any:
|
||||
@@ -110,7 +110,7 @@ class ParamsDependency(Dependency):
|
||||
|
||||
def read_params(
|
||||
self, flatten: bool = True, **kwargs: typing.Any
|
||||
) -> Dict[str, typing.Any]:
|
||||
) -> dict[str, typing.Any]:
|
||||
try:
|
||||
self.validate_filepath()
|
||||
except MissingParamsFile:
|
||||
@@ -134,7 +134,7 @@ class ParamsDependency(Dependency):
|
||||
|
||||
from funcy import ldistinct
|
||||
|
||||
status: Dict[str, Any] = defaultdict(dict)
|
||||
status: dict[str, Any] = defaultdict(dict)
|
||||
info = self.hash_info.value if self.hash_info else {}
|
||||
assert isinstance(info, dict)
|
||||
actual = self.read_params()
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from copy import deepcopy
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union
|
||||
|
||||
import voluptuous as vol
|
||||
|
||||
@@ -20,7 +20,7 @@ class RepoDependency(Dependency):
|
||||
PARAM_CONFIG = "config"
|
||||
PARAM_REMOTE = "remote"
|
||||
|
||||
REPO_SCHEMA: ClassVar[Dict] = {
|
||||
REPO_SCHEMA: ClassVar[dict] = {
|
||||
PARAM_REPO: {
|
||||
vol.Required(PARAM_URL): str,
|
||||
PARAM_REV: str,
|
||||
@@ -30,7 +30,7 @@ class RepoDependency(Dependency):
|
||||
}
|
||||
}
|
||||
|
||||
def __init__(self, def_repo: Dict[str, Any], stage: "Stage", *args, **kwargs):
|
||||
def __init__(self, def_repo: dict[str, Any], stage: "Stage", *args, **kwargs):
|
||||
self.def_repo = def_repo
|
||||
super().__init__(stage, *args, **kwargs)
|
||||
|
||||
@@ -65,7 +65,7 @@ class RepoDependency(Dependency):
|
||||
self.def_repo[self.PARAM_REV_LOCK] = rev
|
||||
|
||||
@classmethod
|
||||
def _dump_def_repo(cls, def_repo) -> Dict[str, str]:
|
||||
def _dump_def_repo(cls, def_repo) -> dict[str, str]:
|
||||
repo = {cls.PARAM_URL: def_repo[cls.PARAM_URL]}
|
||||
|
||||
rev = def_repo.get(cls.PARAM_REV)
|
||||
@@ -85,7 +85,7 @@ class RepoDependency(Dependency):
|
||||
repo[cls.PARAM_REMOTE] = remote
|
||||
return repo
|
||||
|
||||
def dumpd(self, **kwargs) -> Dict[str, Union[str, Dict[str, str]]]:
|
||||
def dumpd(self, **kwargs) -> dict[str, Union[str, dict[str, str]]]:
|
||||
return {
|
||||
self.PARAM_PATH: self.def_path,
|
||||
self.PARAM_REPO: self._dump_def_repo(self.def_repo),
|
||||
|
||||
+10
-13
@@ -5,10 +5,7 @@ from typing import (
|
||||
Any,
|
||||
Callable,
|
||||
ClassVar,
|
||||
Dict,
|
||||
List,
|
||||
Optional,
|
||||
Tuple,
|
||||
TypeVar,
|
||||
Union,
|
||||
)
|
||||
@@ -134,7 +131,7 @@ class FileMixin:
|
||||
d, _ = self._load(**kwargs)
|
||||
return d
|
||||
|
||||
def _load(self, **kwargs: Any) -> Tuple[Any, str]:
|
||||
def _load(self, **kwargs: Any) -> tuple[Any, str]:
|
||||
# it raises the proper exceptions by priority:
|
||||
# 1. when the file doesn't exists
|
||||
# 2. filename is not a DVC file
|
||||
@@ -157,7 +154,7 @@ class FileMixin:
|
||||
|
||||
return validate(d, cls.SCHEMA, path=fname) # type: ignore[arg-type]
|
||||
|
||||
def _load_yaml(self, **kwargs: Any) -> Tuple[Any, str]:
|
||||
def _load_yaml(self, **kwargs: Any) -> tuple[Any, str]:
|
||||
from dvc.utils import strictyaml
|
||||
|
||||
return strictyaml.load(
|
||||
@@ -182,10 +179,10 @@ class SingleStageFile(FileMixin):
|
||||
from dvc.schema import COMPILED_SINGLE_STAGE_SCHEMA as SCHEMA
|
||||
from dvc.stage.loader import SingleStageLoader as LOADER # noqa: N814
|
||||
|
||||
metrics: ClassVar[List[str]] = []
|
||||
metrics: ClassVar[list[str]] = []
|
||||
plots: ClassVar[Any] = {}
|
||||
params: ClassVar[List[str]] = []
|
||||
artifacts: ClassVar[Dict[str, Optional[Dict[str, Any]]]] = {}
|
||||
params: ClassVar[list[str]] = []
|
||||
artifacts: ClassVar[dict[str, Optional[dict[str, Any]]]] = {}
|
||||
|
||||
@property
|
||||
def stage(self) -> "Stage":
|
||||
@@ -284,11 +281,11 @@ class ProjectFile(FileMixin):
|
||||
raise DvcException("ProjectFile has multiple stages. Please specify it's name.")
|
||||
|
||||
@cached_property
|
||||
def contents(self) -> Dict[str, Any]:
|
||||
def contents(self) -> dict[str, Any]:
|
||||
return self._load()[0]
|
||||
|
||||
@cached_property
|
||||
def lockfile_contents(self) -> Dict[str, Any]:
|
||||
def lockfile_contents(self) -> dict[str, Any]:
|
||||
return self._lockfile.load()
|
||||
|
||||
@cached_property
|
||||
@@ -303,7 +300,7 @@ class ProjectFile(FileMixin):
|
||||
return self.LOADER(self, self.contents, self.lockfile_contents)
|
||||
|
||||
@property
|
||||
def metrics(self) -> List[str]:
|
||||
def metrics(self) -> list[str]:
|
||||
return self.contents.get("metrics", [])
|
||||
|
||||
@property
|
||||
@@ -311,11 +308,11 @@ class ProjectFile(FileMixin):
|
||||
return self.contents.get("plots", {})
|
||||
|
||||
@property
|
||||
def params(self) -> List[str]:
|
||||
def params(self) -> list[str]:
|
||||
return self.contents.get("params", [])
|
||||
|
||||
@property
|
||||
def artifacts(self) -> Dict[str, Optional[Dict[str, Any]]]:
|
||||
def artifacts(self) -> dict[str, Optional[dict[str, Any]]]:
|
||||
return self.contents.get("artifacts", {})
|
||||
|
||||
def remove(self, force=False):
|
||||
|
||||
+3
-3
@@ -1,6 +1,6 @@
|
||||
"""Exceptions raised by the dvc."""
|
||||
import errno
|
||||
from typing import TYPE_CHECKING, Dict, List, Optional, Set
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from dvc.utils import format_link
|
||||
|
||||
@@ -34,7 +34,7 @@ class OutputDuplicationError(DvcException):
|
||||
stages (list): list of paths to stages.
|
||||
"""
|
||||
|
||||
def __init__(self, output: str, stages: Set["Stage"]):
|
||||
def __init__(self, output: str, stages: set["Stage"]):
|
||||
from funcy import first
|
||||
|
||||
assert isinstance(output, str)
|
||||
@@ -241,7 +241,7 @@ class UploadError(FileTransferError):
|
||||
|
||||
|
||||
class CheckoutError(DvcException):
|
||||
def __init__(self, target_infos: List[str], stats: Dict[str, List[str]]):
|
||||
def __init__(self, target_infos: list[str], stats: dict[str, list[str]]):
|
||||
from dvc.utils import error_link
|
||||
|
||||
self.target_infos = target_infos
|
||||
|
||||
+3
-3
@@ -1,5 +1,5 @@
|
||||
from contextlib import ExitStack
|
||||
from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optional, Union
|
||||
from typing import TYPE_CHECKING, Any, BinaryIO, Optional, Union
|
||||
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.utils.objects import cached_property
|
||||
@@ -55,7 +55,7 @@ class TqdmCallback(Callback):
|
||||
self,
|
||||
path_1: "Union[str, BinaryIO]",
|
||||
path_2: str,
|
||||
kwargs: Dict[str, Any],
|
||||
kwargs: dict[str, Any],
|
||||
child: Optional[Callback] = None,
|
||||
):
|
||||
child = child or TqdmCallback(
|
||||
@@ -130,7 +130,7 @@ class RichCallback(Callback):
|
||||
self,
|
||||
path_1: Union[str, BinaryIO],
|
||||
path_2: str,
|
||||
kwargs: Dict[str, Any],
|
||||
kwargs: dict[str, Any],
|
||||
child: Optional["Callback"] = None,
|
||||
):
|
||||
child = child or RichCallback(
|
||||
|
||||
+9
-9
@@ -6,7 +6,7 @@ import posixpath
|
||||
import threading
|
||||
from collections import deque
|
||||
from contextlib import ExitStack, suppress
|
||||
from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple, Type, Union
|
||||
from typing import TYPE_CHECKING, Any, Callable, Optional, Union
|
||||
|
||||
from fsspec.spec import AbstractFileSystem
|
||||
from funcy import wrap_with
|
||||
@@ -22,8 +22,8 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
RepoFactory = Union[Callable[..., "Repo"], Type["Repo"]]
|
||||
Key = Tuple[str, ...]
|
||||
RepoFactory = Union[Callable[..., "Repo"], type["Repo"]]
|
||||
Key = tuple[str, ...]
|
||||
|
||||
|
||||
def as_posix(path: str) -> str:
|
||||
@@ -85,7 +85,7 @@ class _DVCFileSystem(AbstractFileSystem):
|
||||
subrepos: bool = False,
|
||||
repo_factory: Optional[RepoFactory] = None,
|
||||
fo: Optional[str] = None,
|
||||
target_options: Optional[Dict[str, Any]] = None, # noqa: ARG002
|
||||
target_options: Optional[dict[str, Any]] = None, # noqa: ARG002
|
||||
target_protocol: Optional[str] = None, # noqa: ARG002
|
||||
config: Optional["DictStrAny"] = None,
|
||||
remote: Optional[str] = None,
|
||||
@@ -145,7 +145,7 @@ class _DVCFileSystem(AbstractFileSystem):
|
||||
}
|
||||
|
||||
def getcwd(self):
|
||||
relparts: Tuple[str, ...] = ()
|
||||
relparts: tuple[str, ...] = ()
|
||||
assert self.repo is not None
|
||||
if self.repo.fs.isin(self.repo.fs.getcwd(), self.repo.root_dir):
|
||||
relparts = self.repo.fs.relparts(self.repo.fs.getcwd(), self.repo.root_dir)
|
||||
@@ -156,7 +156,7 @@ class _DVCFileSystem(AbstractFileSystem):
|
||||
return posixpath.join(*parts)
|
||||
|
||||
@classmethod
|
||||
def parts(cls, path: str) -> Tuple[str, ...]:
|
||||
def parts(cls, path: str) -> tuple[str, ...]:
|
||||
ret = []
|
||||
while True:
|
||||
path, part = posixpath.split(path)
|
||||
@@ -187,7 +187,7 @@ class _DVCFileSystem(AbstractFileSystem):
|
||||
start = "."
|
||||
return posixpath.relpath(self.abspath(path), start=self.abspath(start))
|
||||
|
||||
def relparts(self, path: str, start: Optional[str] = None) -> Tuple[str, ...]:
|
||||
def relparts(self, path: str, start: Optional[str] = None) -> tuple[str, ...]:
|
||||
return self.parts(self.relpath(path, start=start))
|
||||
|
||||
@functools.cached_property
|
||||
@@ -321,7 +321,7 @@ class _DVCFileSystem(AbstractFileSystem):
|
||||
|
||||
def _get_subrepo_info(
|
||||
self, key: Key
|
||||
) -> Tuple["Repo", Optional[DataFileSystem], Key]:
|
||||
) -> tuple["Repo", Optional[DataFileSystem], Key]:
|
||||
"""
|
||||
Returns information about the subrepo the key is part of.
|
||||
"""
|
||||
@@ -525,7 +525,7 @@ class DVCFileSystem(FileSystem):
|
||||
protocol = "local"
|
||||
PARAM_CHECKSUM = "md5"
|
||||
|
||||
def _prepare_credentials(self, **config) -> Dict[str, Any]:
|
||||
def _prepare_credentials(self, **config) -> dict[str, Any]:
|
||||
return config
|
||||
|
||||
@functools.cached_property
|
||||
|
||||
+4
-4
@@ -2,7 +2,7 @@ import os
|
||||
import re
|
||||
from collections import namedtuple
|
||||
from itertools import chain, groupby, takewhile
|
||||
from typing import TYPE_CHECKING, List, Optional
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from pathspec.patterns import GitWildMatchPattern
|
||||
from pathspec.util import normalize_file
|
||||
@@ -62,7 +62,7 @@ class DvcIgnorePatterns(DvcIgnore):
|
||||
|
||||
return cls(path_spec_lines, dirname, fs.sep)
|
||||
|
||||
def __call__(self, root: List[str], dirs: List[str], files: List[str]):
|
||||
def __call__(self, root: list[str], dirs: list[str], files: list[str]):
|
||||
files = [f for f in files if not self.matches(root, f)]
|
||||
dirs = [d for d in dirs if not self.matches(root, d, True)]
|
||||
|
||||
@@ -226,7 +226,7 @@ class DvcIgnoreFilter:
|
||||
self,
|
||||
dirname: str,
|
||||
ignore_trie: Trie,
|
||||
dnames: Optional["List"],
|
||||
dnames: Optional["list"],
|
||||
ignore_subrepos: bool,
|
||||
) -> None:
|
||||
self._update_trie(dirname, ignore_trie)
|
||||
@@ -331,7 +331,7 @@ class DvcIgnoreFilter:
|
||||
yield from fs.find(path)
|
||||
|
||||
def _get_trie_pattern(
|
||||
self, dirname, dnames: Optional["List"] = None, ignore_subrepos=True
|
||||
self, dirname, dnames: Optional["list"] = None, ignore_subrepos=True
|
||||
) -> Optional["DvcIgnorePatterns"]:
|
||||
if ignore_subrepos:
|
||||
ignores_trie = self.ignores_trie_fs
|
||||
|
||||
+2
-2
@@ -5,7 +5,7 @@ import logging.config
|
||||
import logging.handlers
|
||||
import os
|
||||
import sys
|
||||
from typing import ClassVar, Dict
|
||||
from typing import ClassVar
|
||||
|
||||
import colorama
|
||||
|
||||
@@ -79,7 +79,7 @@ class ColorFormatter(logging.Formatter):
|
||||
"""
|
||||
|
||||
reset = colorama.Fore.RESET
|
||||
color_codes: ClassVar[Dict[str, str]] = {
|
||||
color_codes: ClassVar[dict[str, str]] = {
|
||||
"TRACE": colorama.Fore.GREEN,
|
||||
"DEBUG": colorama.Fore.BLUE,
|
||||
"WARNING": colorama.Fore.YELLOW,
|
||||
|
||||
+23
-23
@@ -4,7 +4,7 @@ import posixpath
|
||||
from collections import defaultdict
|
||||
from contextlib import suppress
|
||||
from operator import itemgetter
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Type, Union
|
||||
from typing import TYPE_CHECKING, Any, Optional, Union
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import voluptuous as vol
|
||||
@@ -148,7 +148,7 @@ def _split_dict(d, keys):
|
||||
|
||||
|
||||
def _merge_data(s_list):
|
||||
d: Dict[str, Dict] = defaultdict(dict)
|
||||
d: dict[str, dict] = defaultdict(dict)
|
||||
for key in s_list:
|
||||
if isinstance(key, str):
|
||||
d[key].update({})
|
||||
@@ -207,7 +207,7 @@ def load_from_pipeline(stage, data, typ="outs"):
|
||||
)
|
||||
|
||||
|
||||
def split_file_meta_from_cloud(entry: Dict) -> Dict:
|
||||
def split_file_meta_from_cloud(entry: dict) -> dict:
|
||||
if remote_name := entry.pop(Meta.PARAM_REMOTE, None):
|
||||
remote_meta = {}
|
||||
for key in (
|
||||
@@ -223,7 +223,7 @@ def split_file_meta_from_cloud(entry: Dict) -> Dict:
|
||||
return entry
|
||||
|
||||
|
||||
def merge_file_meta_from_cloud(entry: Dict) -> Dict:
|
||||
def merge_file_meta_from_cloud(entry: dict) -> dict:
|
||||
cloud_meta = entry.pop(Output.PARAM_CLOUD, {})
|
||||
if remote_name := first(cloud_meta):
|
||||
entry.update(cloud_meta[remote_name])
|
||||
@@ -231,7 +231,7 @@ def merge_file_meta_from_cloud(entry: Dict) -> Dict:
|
||||
return entry
|
||||
|
||||
|
||||
def _serialize_tree_obj_to_files(obj: Tree) -> List[Dict[str, Any]]:
|
||||
def _serialize_tree_obj_to_files(obj: Tree) -> list[dict[str, Any]]:
|
||||
key = obj.PARAM_RELPATH
|
||||
return sorted(
|
||||
(
|
||||
@@ -246,7 +246,7 @@ def _serialize_tree_obj_to_files(obj: Tree) -> List[Dict[str, Any]]:
|
||||
)
|
||||
|
||||
|
||||
def _serialize_hi_to_dict(hash_info: Optional[HashInfo]) -> Dict[str, Any]:
|
||||
def _serialize_hi_to_dict(hash_info: Optional[HashInfo]) -> dict[str, Any]:
|
||||
if hash_info:
|
||||
if hash_info.name == "md5-dos2unix":
|
||||
return {"md5": hash_info.value}
|
||||
@@ -316,10 +316,10 @@ class Output:
|
||||
PARAM_HASH = "hash"
|
||||
PARAM_FS_CONFIG = "fs_config"
|
||||
|
||||
DoesNotExistError: Type[DvcException] = OutputDoesNotExistError
|
||||
IsNotFileOrDirError: Type[DvcException] = OutputIsNotFileOrDirError
|
||||
IsStageFileError: Type[DvcException] = OutputIsStageFileError
|
||||
IsIgnoredError: Type[DvcException] = OutputIsIgnoredError
|
||||
DoesNotExistError: type[DvcException] = OutputDoesNotExistError
|
||||
IsNotFileOrDirError: type[DvcException] = OutputIsNotFileOrDirError
|
||||
IsStageFileError: type[DvcException] = OutputIsStageFileError
|
||||
IsIgnoredError: type[DvcException] = OutputIsIgnoredError
|
||||
|
||||
def __init__( # noqa: PLR0913
|
||||
self,
|
||||
@@ -337,7 +337,7 @@ class Output:
|
||||
remote=None,
|
||||
repo=None,
|
||||
fs_config=None,
|
||||
files: Optional[List[Dict[str, Any]]] = None,
|
||||
files: Optional[list[dict[str, Any]]] = None,
|
||||
push: bool = True,
|
||||
hash_name: Optional[str] = DEFAULT_ALGORITHM,
|
||||
):
|
||||
@@ -420,7 +420,7 @@ class Output:
|
||||
|
||||
def _compute_hash_info_from_meta(
|
||||
self, hash_name: Optional[str]
|
||||
) -> Tuple[str, HashInfo]:
|
||||
) -> tuple[str, HashInfo]:
|
||||
if self.is_in_repo:
|
||||
if hash_name is None:
|
||||
# Legacy 2.x output, use "md5-dos2unix" but read "md5" from
|
||||
@@ -555,7 +555,7 @@ class Output:
|
||||
|
||||
def _build(
|
||||
self, *args, no_progress_bar=False, **kwargs
|
||||
) -> Tuple["HashFileDB", "Meta", "HashFile"]:
|
||||
) -> tuple["HashFileDB", "Meta", "HashFile"]:
|
||||
from dvc.ui import ui
|
||||
|
||||
with ui.progress(
|
||||
@@ -601,7 +601,7 @@ class Output:
|
||||
return self.fs.exists(self.fs_path)
|
||||
|
||||
@cached_property
|
||||
def index_key(self) -> Tuple[str, "DataIndexKey"]:
|
||||
def index_key(self) -> tuple[str, "DataIndexKey"]:
|
||||
if self.is_in_repo:
|
||||
workspace = "repo"
|
||||
key = self.repo.fs.relparts(self.fs_path, self.repo.root_dir)
|
||||
@@ -633,7 +633,7 @@ class Output:
|
||||
return self.meta.version_id != self.get_meta().version_id
|
||||
return False
|
||||
|
||||
def workspace_status(self) -> Dict[str, str]:
|
||||
def workspace_status(self) -> dict[str, str]:
|
||||
if not self.exists:
|
||||
return {str(self): "deleted"}
|
||||
|
||||
@@ -645,7 +645,7 @@ class Output:
|
||||
|
||||
return {}
|
||||
|
||||
def status(self) -> Dict[str, str]:
|
||||
def status(self) -> dict[str, str]:
|
||||
if self.hash_info and self.use_cache and self.changed_cache():
|
||||
return {str(self): "not in cache"}
|
||||
|
||||
@@ -840,7 +840,7 @@ class Output:
|
||||
def dumpd(self, **kwargs): # noqa: C901, PLR0912
|
||||
from dvc.cachemgr import LEGACY_HASH_NAMES
|
||||
|
||||
ret: Dict[str, Any] = {}
|
||||
ret: dict[str, Any] = {}
|
||||
with_files = (
|
||||
(not self.IS_DEPENDENCY or self.stage.is_import)
|
||||
and self.hash_info.isdir
|
||||
@@ -962,7 +962,7 @@ class Output:
|
||||
filter_info: Optional[str] = None,
|
||||
allow_missing: bool = False,
|
||||
**kwargs,
|
||||
) -> Optional[Tuple[bool, Optional[bool]]]:
|
||||
) -> Optional[tuple[bool, Optional[bool]]]:
|
||||
# callback passed act as a aggregate callback.
|
||||
# do not let checkout to call set_size and change progressbar.
|
||||
class CallbackProxy(Callback):
|
||||
@@ -1157,7 +1157,7 @@ class Output:
|
||||
|
||||
def get_used_objs( # noqa: PLR0911
|
||||
self, **kwargs
|
||||
) -> Dict[Optional["HashFileDB"], Set["HashInfo"]]:
|
||||
) -> dict[Optional["HashFileDB"], set["HashInfo"]]:
|
||||
"""Return filtered set of used object IDs for this out."""
|
||||
from dvc.cachemgr import LEGACY_HASH_NAMES
|
||||
|
||||
@@ -1291,7 +1291,7 @@ class Output:
|
||||
nfiles=len(merged),
|
||||
)
|
||||
|
||||
def unstage(self, path: str) -> Tuple["Meta", "Tree"]:
|
||||
def unstage(self, path: str) -> tuple["Meta", "Tree"]:
|
||||
from pygtrie import Trie
|
||||
|
||||
rel_key = tuple(self.fs.parts(self.fs.relpath(path, self.fs_path)))
|
||||
@@ -1324,7 +1324,7 @@ class Output:
|
||||
path: str,
|
||||
obj: Union["Tree", "HashFile"],
|
||||
meta: "Meta",
|
||||
) -> Tuple["Meta", "Tree"]:
|
||||
) -> tuple["Meta", "Tree"]:
|
||||
from pygtrie import Trie
|
||||
|
||||
append_only = True
|
||||
@@ -1514,7 +1514,7 @@ META_SCHEMA = {
|
||||
|
||||
CLOUD_SCHEMA = vol.All({str: {**META_SCHEMA, **CHECKSUMS_SCHEMA}}, vol.Length(max=1))
|
||||
|
||||
ARTIFACT_SCHEMA: Dict[Any, Any] = {
|
||||
ARTIFACT_SCHEMA: dict[Any, Any] = {
|
||||
**CHECKSUMS_SCHEMA,
|
||||
**META_SCHEMA,
|
||||
Output.PARAM_PATH: str,
|
||||
@@ -1523,7 +1523,7 @@ ARTIFACT_SCHEMA: Dict[Any, Any] = {
|
||||
Output.PARAM_HASH: str,
|
||||
}
|
||||
|
||||
DIR_FILES_SCHEMA: Dict[Any, Any] = {
|
||||
DIR_FILES_SCHEMA: dict[Any, Any] = {
|
||||
**CHECKSUMS_SCHEMA,
|
||||
**META_SCHEMA,
|
||||
vol.Required(Tree.PARAM_RELPATH): str,
|
||||
|
||||
+13
-17
@@ -6,12 +6,8 @@ from itertools import product
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Dict,
|
||||
List,
|
||||
NamedTuple,
|
||||
Optional,
|
||||
Tuple,
|
||||
Type,
|
||||
Union,
|
||||
)
|
||||
|
||||
@@ -85,7 +81,7 @@ def format_and_raise(exc: Exception, msg: str, path: str) -> "NoReturn":
|
||||
|
||||
|
||||
def _reraise_err(
|
||||
exc_cls: Type[Exception], *args, from_exc: Optional[Exception] = None
|
||||
exc_cls: type[Exception], *args, from_exc: Optional[Exception] = None
|
||||
) -> "NoReturn":
|
||||
err = exc_cls(*args)
|
||||
if from_exc and logger.isEnabledFor(logging.DEBUG):
|
||||
@@ -108,7 +104,7 @@ def is_map_or_seq(data: Any) -> bool:
|
||||
return not isinstance(data, str) and _is_map_or_seq(data)
|
||||
|
||||
|
||||
def split_group_name(name: str) -> Tuple[str, Optional[str]]:
|
||||
def split_group_name(name: str) -> tuple[str, Optional[str]]:
|
||||
group, *keys = name.rsplit(JOIN, maxsplit=1)
|
||||
return group, first(keys)
|
||||
|
||||
@@ -161,14 +157,14 @@ class DataResolver:
|
||||
|
||||
# we use `tracked_vars` to keep a dictionary of used variables
|
||||
# by the interpolated entries.
|
||||
self.tracked_vars: Dict[str, Mapping] = {}
|
||||
self.tracked_vars: dict[str, Mapping] = {}
|
||||
|
||||
stages_data = d.get(STAGES_KWD, {})
|
||||
# we wrap the definitions into:
|
||||
# ForeachDefinition, MatrixDefinition, and EntryDefinition
|
||||
# that helps us to optimize, cache and selectively load each one of
|
||||
# them as we need, and simplify all of this DSL/parsing logic.
|
||||
self.definitions: Dict[str, Definition] = {
|
||||
self.definitions: dict[str, Definition] = {
|
||||
name: make_definition(self, name, definition)
|
||||
for name, definition in stages_data.items()
|
||||
}
|
||||
@@ -376,7 +372,7 @@ class ForeachDefinition:
|
||||
f" in '{self.relpath}': expected list/dictionary, got " + typ
|
||||
)
|
||||
|
||||
def _warn_if_overwriting(self, keys: List[str]):
|
||||
def _warn_if_overwriting(self, keys: list[str]):
|
||||
warn_for = [k for k in keys if k in self.context]
|
||||
if warn_for:
|
||||
linking_verb = "is" if len(warn_for) == 1 else "are"
|
||||
@@ -390,7 +386,7 @@ class ForeachDefinition:
|
||||
self.name,
|
||||
)
|
||||
|
||||
def _inserted_keys(self, iterable) -> List[str]:
|
||||
def _inserted_keys(self, iterable) -> list[str]:
|
||||
keys = [self.pair.value]
|
||||
if isinstance(iterable, Mapping):
|
||||
keys.append(self.pair.key)
|
||||
@@ -490,10 +486,10 @@ class MatrixDefinition:
|
||||
return self._template
|
||||
|
||||
@cached_property
|
||||
def resolved_iterable(self) -> Dict[str, List]:
|
||||
def resolved_iterable(self) -> dict[str, list]:
|
||||
return self._resolve_matrix_data()
|
||||
|
||||
def _resolve_matrix_data(self) -> Dict[str, List]:
|
||||
def _resolve_matrix_data(self) -> dict[str, list]:
|
||||
try:
|
||||
iterable = self.context.resolve(self.matrix_data, unwrap=False)
|
||||
except (ContextError, ParseError) as exc:
|
||||
@@ -504,7 +500,7 @@ class MatrixDefinition:
|
||||
self._warn_if_overwriting([self.pair.key, self.pair.value])
|
||||
return iterable
|
||||
|
||||
def _warn_if_overwriting(self, keys: List[str]):
|
||||
def _warn_if_overwriting(self, keys: list[str]):
|
||||
warn_for = [k for k in keys if k in self.context]
|
||||
if warn_for:
|
||||
linking_verb = "is" if len(warn_for) == 1 else "are"
|
||||
@@ -519,16 +515,16 @@ class MatrixDefinition:
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def normalized_iterable(self) -> Dict[str, "DictStrAny"]:
|
||||
def normalized_iterable(self) -> dict[str, "DictStrAny"]:
|
||||
"""Convert sequence to Mapping with keys normalized."""
|
||||
iterable = self.resolved_iterable
|
||||
assert isinstance(iterable, Mapping)
|
||||
|
||||
ret: Dict[str, "DictStrAny"] = {}
|
||||
ret: dict[str, "DictStrAny"] = {}
|
||||
matrix = {key: enumerate(v) for key, v in iterable.items()}
|
||||
for combination in product(*matrix.values()):
|
||||
d: "DictStrAny" = {}
|
||||
fragments: List[str] = []
|
||||
fragments: list[str] = []
|
||||
for k, (i, v) in zip(matrix.keys(), combination):
|
||||
d[k] = v
|
||||
fragments.append(f"{k}{i}" if is_map_or_seq(v) else to_str(v))
|
||||
@@ -540,7 +536,7 @@ class MatrixDefinition:
|
||||
def has_member(self, key: str) -> bool:
|
||||
return key in self.normalized_iterable
|
||||
|
||||
def get_generated_names(self) -> List[str]:
|
||||
def get_generated_names(self) -> list[str]:
|
||||
return list(map(self._generate_name, self.normalized_iterable))
|
||||
|
||||
def _generate_name(self, key: str) -> str:
|
||||
|
||||
@@ -4,7 +4,7 @@ from collections.abc import Mapping, MutableMapping, MutableSequence, Sequence
|
||||
from contextlib import contextmanager
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass, field, replace
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from funcy import identity, lfilter, nullcontext, select
|
||||
|
||||
@@ -22,7 +22,7 @@ from dvc.parsing.interpolate import (
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
SeqOrMap = Union[Sequence, Mapping]
|
||||
DictStr = Dict[str, Any]
|
||||
DictStr = dict[str, Any]
|
||||
|
||||
|
||||
class ContextError(DvcException):
|
||||
@@ -100,7 +100,7 @@ def recurse_not_a_node(data: dict):
|
||||
@dataclass
|
||||
class Meta:
|
||||
source: Optional[str] = None
|
||||
dpaths: List[str] = field(default_factory=list)
|
||||
dpaths: list[str] = field(default_factory=list)
|
||||
local: bool = True
|
||||
|
||||
@staticmethod
|
||||
@@ -297,7 +297,7 @@ class Context(CtxDict):
|
||||
"""
|
||||
super().__init__(*args, **kwargs)
|
||||
self._track = False
|
||||
self._tracked_data: Dict[str, Dict] = defaultdict(dict)
|
||||
self._tracked_data: dict[str, dict] = defaultdict(dict)
|
||||
self.imports = {}
|
||||
self._reserved_keys = {}
|
||||
|
||||
@@ -348,7 +348,7 @@ class Context(CtxDict):
|
||||
|
||||
@classmethod
|
||||
def load_from(
|
||||
cls, fs, path: str, select_keys: Optional[List[str]] = None
|
||||
cls, fs, path: str, select_keys: Optional[list[str]] = None
|
||||
) -> "Context":
|
||||
from dvc.utils.serialize import load_path
|
||||
|
||||
@@ -423,7 +423,7 @@ class Context(CtxDict):
|
||||
def load_from_vars(
|
||||
self,
|
||||
fs,
|
||||
vars_: List,
|
||||
vars_: list,
|
||||
wdir: str,
|
||||
stage_name: Optional[str] = None,
|
||||
default: Optional[str] = None,
|
||||
|
||||
@@ -10,7 +10,8 @@ from dvc.exceptions import DvcException
|
||||
from dvc.utils.flatten import flatten
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from typing import List, Match, NoReturn
|
||||
from re import Match
|
||||
from typing import NoReturn
|
||||
|
||||
from pyparsing import ParseException
|
||||
|
||||
@@ -205,7 +206,7 @@ def validate_value(value, key):
|
||||
|
||||
def str_interpolate(
|
||||
template: str,
|
||||
matches: "List[Match]",
|
||||
matches: "list[Match]",
|
||||
context: "Context",
|
||||
skip_checks: bool = False,
|
||||
key=None,
|
||||
@@ -225,5 +226,5 @@ def str_interpolate(
|
||||
return buf.replace(r"\${", BRACE_OPEN)
|
||||
|
||||
|
||||
def is_exact_string(src: str, matches: "List[Match]"):
|
||||
def is_exact_string(src: str, matches: "list[Match]"):
|
||||
return len(matches) == 1 and src == matches[0].group(0)
|
||||
|
||||
+2
-2
@@ -2,7 +2,7 @@
|
||||
import logging
|
||||
import sys
|
||||
from threading import RLock
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, Dict
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
|
||||
from tqdm import tqdm
|
||||
|
||||
@@ -33,7 +33,7 @@ class Tqdm(tqdm):
|
||||
" [{elapsed}<{remaining}, {rate_fmt:>11}]"
|
||||
)
|
||||
BAR_FMT_NOTOTAL = "{desc}{bar:b}|{postfix[info]}{n_fmt} [{elapsed}, {rate_fmt:>11}]"
|
||||
BYTES_DEFAULTS: ClassVar[Dict[str, Any]] = {
|
||||
BYTES_DEFAULTS: ClassVar[dict[str, Any]] = {
|
||||
"unit": "B",
|
||||
"unit_scale": True,
|
||||
"unit_divisor": 1024,
|
||||
|
||||
+2
-1
@@ -1,7 +1,8 @@
|
||||
"""Manages user prompts."""
|
||||
|
||||
from collections.abc import Collection
|
||||
from getpass import getpass
|
||||
from typing import Collection, Optional
|
||||
from typing import Optional
|
||||
|
||||
from dvc.log import logger
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from typing import Dict, List, Union
|
||||
from typing import Union
|
||||
|
||||
from dvc.render import REVISION, REVISIONS, SRC, TYPE_KEY
|
||||
from dvc.render.converter.image import ImageConverter
|
||||
@@ -18,7 +18,7 @@ def _get_converter(
|
||||
raise ValueError(f"Invalid renderer class {renderer_class}")
|
||||
|
||||
|
||||
def to_json(renderer, split: bool = False) -> List[Dict]:
|
||||
def to_json(renderer, split: bool = False) -> list[dict]:
|
||||
if renderer.TYPE == "vega":
|
||||
if not renderer.datapoints:
|
||||
return []
|
||||
|
||||
@@ -1,19 +1,19 @@
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from typing import Any, Optional
|
||||
|
||||
|
||||
class Converter:
|
||||
def __init__(
|
||||
self,
|
||||
plot_id: str,
|
||||
data: Optional[Dict[str, Any]] = None,
|
||||
properties: Optional[Dict] = None,
|
||||
data: Optional[dict[str, Any]] = None,
|
||||
properties: Optional[dict] = None,
|
||||
):
|
||||
self.plot_id = plot_id
|
||||
self.properties = properties or {}
|
||||
self.data = data or {}
|
||||
|
||||
def convert(self) -> Tuple[List[Tuple[str, str, Any]], Dict]:
|
||||
def convert(self) -> tuple[list[tuple[str, str, Any]], dict]:
|
||||
raise NotImplementedError
|
||||
|
||||
def flat_datapoints(self, revision: str) -> Tuple[List[Dict], Dict]:
|
||||
def flat_datapoints(self, revision: str) -> tuple[list[dict], dict]:
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import base64
|
||||
import os
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Tuple
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from dvc.render import FILENAME, REVISION, SRC
|
||||
|
||||
@@ -34,13 +34,13 @@ class ImageConverter(Converter):
|
||||
base64_str = base64.b64encode(image_data).decode()
|
||||
return f"data:image;base64,{base64_str}"
|
||||
|
||||
def convert(self) -> Tuple[List[Tuple[str, str, Any]], Dict]:
|
||||
def convert(self) -> tuple[list[tuple[str, str, Any]], dict]:
|
||||
datas = []
|
||||
for filename, image_data in self.data.items():
|
||||
datas.append((filename, "", image_data))
|
||||
return datas, self.properties
|
||||
|
||||
def flat_datapoints(self, revision: str) -> Tuple[List[Dict], Dict]:
|
||||
def flat_datapoints(self, revision: str) -> tuple[list[dict], dict]:
|
||||
"""
|
||||
Convert the DVC Plots content to DVC Render datapoints.
|
||||
Return both generated datapoints and updated properties.
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import os
|
||||
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
|
||||
from collections.abc import Iterable
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from funcy import first, last
|
||||
|
||||
@@ -18,7 +19,7 @@ class FieldNotFoundError(DvcException):
|
||||
)
|
||||
|
||||
|
||||
def _lists(blob: Union[Dict, List]) -> Iterable[List]:
|
||||
def _lists(blob: Union[dict, list]) -> Iterable[list]:
|
||||
if isinstance(blob, list):
|
||||
yield blob
|
||||
else:
|
||||
@@ -43,7 +44,7 @@ def _file_field(*args):
|
||||
def _find(
|
||||
filename: str,
|
||||
field: str,
|
||||
data_series: List[Tuple[str, str, Any]],
|
||||
data_series: list[tuple[str, str, Any]],
|
||||
):
|
||||
for data_file, data_field, data in data_series:
|
||||
if data_file == filename and data_field == field:
|
||||
@@ -51,14 +52,14 @@ def _find(
|
||||
return None
|
||||
|
||||
|
||||
def _verify_field(file2datapoints: Dict[str, List], filename: str, field: str):
|
||||
def _verify_field(file2datapoints: dict[str, list], filename: str, field: str):
|
||||
if filename in file2datapoints:
|
||||
datapoint = first(file2datapoints[filename])
|
||||
if field not in datapoint:
|
||||
raise FieldNotFoundError(field, datapoint.keys())
|
||||
|
||||
|
||||
def _get_xs(properties: Dict, file2datapoints: Dict[str, List[Dict]]):
|
||||
def _get_xs(properties: dict, file2datapoints: dict[str, list[dict]]):
|
||||
x = properties.get("x")
|
||||
if x is not None and isinstance(x, dict):
|
||||
for filename, field in _file_field(x):
|
||||
@@ -66,7 +67,7 @@ def _get_xs(properties: Dict, file2datapoints: Dict[str, List[Dict]]):
|
||||
yield filename, field
|
||||
|
||||
|
||||
def _get_ys(properties, file2datapoints: Dict[str, List[Dict]]):
|
||||
def _get_ys(properties, file2datapoints: dict[str, list[dict]]):
|
||||
y = properties.get("y", None)
|
||||
if y is not None:
|
||||
for filename, field in _file_field(y):
|
||||
@@ -74,7 +75,7 @@ def _get_ys(properties, file2datapoints: Dict[str, List[Dict]]):
|
||||
yield filename, field
|
||||
|
||||
|
||||
def _is_datapoints(lst: List[Dict]):
|
||||
def _is_datapoints(lst: list[dict]):
|
||||
"""
|
||||
check if dict keys match, datapoints with different keys mgiht lead
|
||||
to unexpected behavior
|
||||
@@ -85,8 +86,8 @@ def _is_datapoints(lst: List[Dict]):
|
||||
}
|
||||
|
||||
|
||||
def get_datapoints(file_content: Dict):
|
||||
result: List[Dict[str, Any]] = []
|
||||
def get_datapoints(file_content: dict):
|
||||
result: list[dict[str, Any]] = []
|
||||
for lst in _lists(file_content):
|
||||
if _is_datapoints(lst):
|
||||
for index, datapoint in enumerate(lst):
|
||||
@@ -107,8 +108,8 @@ class VegaConverter(Converter):
|
||||
def __init__(
|
||||
self,
|
||||
plot_id: str,
|
||||
data: Optional[Dict] = None,
|
||||
properties: Optional[Dict] = None,
|
||||
data: Optional[dict] = None,
|
||||
properties: Optional[dict] = None,
|
||||
):
|
||||
super().__init__(plot_id, data, properties)
|
||||
self.plot_id = plot_id
|
||||
@@ -126,7 +127,7 @@ class VegaConverter(Converter):
|
||||
x = self.properties.get("x", None)
|
||||
y = self.properties.get("y", None)
|
||||
|
||||
inferred_properties: Dict = {}
|
||||
inferred_properties: dict = {}
|
||||
|
||||
# Infer x.
|
||||
if isinstance(x, str):
|
||||
@@ -195,7 +196,7 @@ class VegaConverter(Converter):
|
||||
def flat_datapoints(self, revision): # noqa: C901, PLR0912
|
||||
file2datapoints, properties = self.convert()
|
||||
|
||||
props_update: Dict[str, Union[str, List[Dict[str, str]]]] = {}
|
||||
props_update: dict[str, Union[str, list[dict[str, str]]]] = {}
|
||||
|
||||
xs = list(_get_xs(properties, file2datapoints))
|
||||
|
||||
@@ -322,9 +323,9 @@ def _get_short_y_file(y_file, common_prefix_len):
|
||||
|
||||
|
||||
def _update_from_field(
|
||||
target_datapoints: List[Dict],
|
||||
target_datapoints: list[dict],
|
||||
field: str,
|
||||
source_datapoints: Optional[List[Dict]] = None,
|
||||
source_datapoints: Optional[list[dict]] = None,
|
||||
source_field: Optional[str] = None,
|
||||
):
|
||||
if source_datapoints is None:
|
||||
@@ -341,11 +342,11 @@ def _update_from_field(
|
||||
datapoint[field] = source_datapoint[source_field]
|
||||
|
||||
|
||||
def _update_from_index(datapoints: List[Dict], new_field: str):
|
||||
def _update_from_index(datapoints: list[dict], new_field: str):
|
||||
for index, datapoint in enumerate(datapoints):
|
||||
datapoint[new_field] = index
|
||||
|
||||
|
||||
def _update_all(datapoints: List[Dict], update_dict: Dict):
|
||||
def _update_all(datapoints: list[dict], update_dict: dict):
|
||||
for datapoint in datapoints:
|
||||
datapoint.update(update_dict)
|
||||
|
||||
+11
-11
@@ -1,6 +1,6 @@
|
||||
import os
|
||||
from collections import defaultdict
|
||||
from typing import TYPE_CHECKING, DefaultDict, Dict, List, NamedTuple, Optional
|
||||
from typing import TYPE_CHECKING, NamedTuple, Optional
|
||||
|
||||
import dpath
|
||||
import dpath.options
|
||||
@@ -21,16 +21,16 @@ dpath.options.ALLOW_EMPTY_STRING_KEYS = True
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
|
||||
def _squash_plots_properties(data: List) -> Dict:
|
||||
def _squash_plots_properties(data: list) -> dict:
|
||||
configs = [last(group) for group in data]
|
||||
resolved: Dict = {}
|
||||
resolved: dict = {}
|
||||
for config in reversed(configs):
|
||||
resolved = {**resolved, **config}
|
||||
return resolved
|
||||
|
||||
|
||||
class PlotsData:
|
||||
def __init__(self, data: Dict):
|
||||
def __init__(self, data: dict):
|
||||
self.data = data
|
||||
|
||||
def group_definitions(self):
|
||||
@@ -62,15 +62,15 @@ class PlotsData:
|
||||
|
||||
class RendererWithErrors(NamedTuple):
|
||||
renderer: "Renderer"
|
||||
source_errors: Dict[str, Dict[str, Exception]]
|
||||
definition_errors: Dict[str, Exception]
|
||||
source_errors: dict[str, dict[str, Exception]]
|
||||
definition_errors: dict[str, Exception]
|
||||
|
||||
|
||||
def match_defs_renderers( # noqa: C901, PLR0912
|
||||
data,
|
||||
out=None,
|
||||
templates_dir: Optional["StrPath"] = None,
|
||||
) -> List[RendererWithErrors]:
|
||||
) -> list[RendererWithErrors]:
|
||||
from dvc_render import ImageRenderer, VegaRenderer
|
||||
|
||||
plots_data = PlotsData(data)
|
||||
@@ -78,12 +78,12 @@ def match_defs_renderers( # noqa: C901, PLR0912
|
||||
renderer_cls = None
|
||||
|
||||
for plot_id, group in plots_data.group_definitions().items():
|
||||
plot_datapoints: List[Dict] = []
|
||||
plot_datapoints: list[dict] = []
|
||||
props = _squash_plots_properties(group)
|
||||
first_props: Dict = {}
|
||||
first_props: dict = {}
|
||||
|
||||
def_errors: Dict[str, Exception] = {}
|
||||
src_errors: DefaultDict[str, Dict[str, Exception]] = defaultdict(dict)
|
||||
def_errors: dict[str, Exception] = {}
|
||||
src_errors: defaultdict[str, dict[str, Exception]] = defaultdict(dict)
|
||||
|
||||
if out is not None:
|
||||
props["out"] = out
|
||||
|
||||
@@ -1,14 +1,12 @@
|
||||
import os
|
||||
from collections import defaultdict
|
||||
from contextlib import contextmanager
|
||||
from collections.abc import Iterable
|
||||
from contextlib import AbstractContextManager, contextmanager
|
||||
from functools import wraps
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Callable,
|
||||
ContextManager,
|
||||
Iterable,
|
||||
Optional,
|
||||
Tuple,
|
||||
Union,
|
||||
)
|
||||
|
||||
@@ -111,7 +109,7 @@ class Repo:
|
||||
fs: Optional["FileSystem"] = None,
|
||||
uninitialized: bool = False,
|
||||
scm: Optional[Union["Git", "NoSCM"]] = None,
|
||||
) -> Tuple[str, Optional[str]]:
|
||||
) -> tuple[str, Optional[str]]:
|
||||
from dvc.fs import localfs
|
||||
from dvc.scm import SCM, SCMError
|
||||
|
||||
@@ -268,7 +266,7 @@ class Repo:
|
||||
if not isinstance(self.fs, GitFileSystem):
|
||||
return None
|
||||
|
||||
relparts: Tuple[str, ...] = ()
|
||||
relparts: tuple[str, ...] = ()
|
||||
if self.root_dir != "/":
|
||||
# subrepo
|
||||
relparts = self.fs.relparts(self.root_dir, "/")
|
||||
@@ -392,7 +390,7 @@ class Repo:
|
||||
self,
|
||||
path: str,
|
||||
workspace: str = "repo",
|
||||
) -> Tuple["DataIndex", "DataIndexEntry"]:
|
||||
) -> tuple["DataIndex", "DataIndexEntry"]:
|
||||
if self.subrepos:
|
||||
fs_path = self.dvcfs.from_os_path(path)
|
||||
fs = self.dvcfs.fs
|
||||
@@ -476,7 +474,7 @@ class Repo:
|
||||
|
||||
return brancher(self, *args, **kwargs)
|
||||
|
||||
def switch(self, rev: str) -> ContextManager[str]:
|
||||
def switch(self, rev: str) -> AbstractContextManager[str]:
|
||||
from dvc.repo.brancher import switch
|
||||
|
||||
return switch(self, rev)
|
||||
@@ -626,7 +624,7 @@ class Repo:
|
||||
cache_dir = self.config["core"].get("site_cache_dir") or site_cache_dir()
|
||||
|
||||
if isinstance(self.fs, GitFileSystem):
|
||||
relparts: Tuple[str, ...] = ()
|
||||
relparts: tuple[str, ...] = ()
|
||||
if self.root_dir != "/":
|
||||
# subrepo
|
||||
relparts = self.fs.relparts(self.root_dir, "/")
|
||||
|
||||
+7
-10
@@ -1,13 +1,10 @@
|
||||
import os
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Dict,
|
||||
Iterator,
|
||||
List,
|
||||
NamedTuple,
|
||||
Optional,
|
||||
Tuple,
|
||||
Union,
|
||||
)
|
||||
|
||||
@@ -37,7 +34,7 @@ class StageInfo(NamedTuple):
|
||||
|
||||
def find_targets(
|
||||
targets: Union["StrOrBytesPath", Iterator["StrOrBytesPath"]], glob: bool = False
|
||||
) -> List[str]:
|
||||
) -> list[str]:
|
||||
if isinstance(targets, (str, bytes, os.PathLike)):
|
||||
targets_list = [os.fsdecode(targets)]
|
||||
else:
|
||||
@@ -99,7 +96,7 @@ OVERLAPPING_PARENT_FMT = (
|
||||
|
||||
|
||||
@contextmanager
|
||||
def translate_graph_error(stages: List["Stage"]) -> Iterator[None]:
|
||||
def translate_graph_error(stages: list["Stage"]) -> Iterator[None]:
|
||||
try:
|
||||
yield
|
||||
except OverlappingOutputPathsError as exc:
|
||||
@@ -124,7 +121,7 @@ def translate_graph_error(stages: List["Stage"]) -> Iterator[None]:
|
||||
)
|
||||
|
||||
|
||||
def progress_iter(stages: Dict[str, StageInfo]) -> Iterator[Tuple[str, StageInfo]]:
|
||||
def progress_iter(stages: dict[str, StageInfo]) -> Iterator[tuple[str, StageInfo]]:
|
||||
total = len(stages)
|
||||
desc = "Adding..."
|
||||
with ui.progress(
|
||||
@@ -151,8 +148,8 @@ LINK_FAILURE_MESSAGE = (
|
||||
|
||||
|
||||
@contextmanager
|
||||
def warn_link_failures() -> Iterator[List[str]]:
|
||||
link_failures: List[str] = []
|
||||
def warn_link_failures() -> Iterator[list[str]]:
|
||||
link_failures: list[str] = []
|
||||
try:
|
||||
yield link_failures
|
||||
finally:
|
||||
@@ -202,7 +199,7 @@ def add(
|
||||
to_remote: bool = False,
|
||||
remote_jobs: Optional[int] = None,
|
||||
force: bool = False,
|
||||
) -> List["Stage"]:
|
||||
) -> list["Stage"]:
|
||||
add_targets = find_targets(targets, glob=glob)
|
||||
if not add_targets:
|
||||
return []
|
||||
|
||||
+12
-12
@@ -1,7 +1,7 @@
|
||||
import os
|
||||
import posixpath
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
|
||||
from typing import TYPE_CHECKING, Any, Optional, Union
|
||||
|
||||
from dvc.annotations import Artifact
|
||||
from dvc.dvcfile import PROJECT_FILE
|
||||
@@ -95,9 +95,9 @@ class Artifacts:
|
||||
return self.repo.scm
|
||||
return None
|
||||
|
||||
def read(self) -> Dict[str, Dict[str, Artifact]]:
|
||||
def read(self) -> dict[str, dict[str, Artifact]]:
|
||||
"""Read artifacts from dvc.yaml."""
|
||||
artifacts: Dict[str, Dict[str, Artifact]] = {}
|
||||
artifacts: dict[str, dict[str, Artifact]] = {}
|
||||
for (
|
||||
dvcfile,
|
||||
dvcfile_artifacts,
|
||||
@@ -140,14 +140,14 @@ class Artifacts:
|
||||
|
||||
assert not (version and stage)
|
||||
name = _reformat_name(name)
|
||||
tags: List["GitTag"] = find(
|
||||
tags: list["GitTag"] = find(
|
||||
name=name, version=version, stage=stage, scm=self.scm
|
||||
)
|
||||
if not tags:
|
||||
raise ArtifactNotFoundError(name, version=version, stage=stage)
|
||||
if version or stage:
|
||||
return tags[-1].target
|
||||
gto_tags: List["GTOTag"] = sort_versions(parse_tag(tag) for tag in tags)
|
||||
gto_tags: list["GTOTag"] = sort_versions(parse_tag(tag) for tag in tags)
|
||||
return gto_tags[0].tag.target
|
||||
|
||||
def get_path(self, name: str):
|
||||
@@ -177,7 +177,7 @@ class Artifacts:
|
||||
out: Optional[str] = None,
|
||||
force: bool = False,
|
||||
jobs: Optional[int] = None,
|
||||
) -> Tuple[int, str]:
|
||||
) -> tuple[int, str]:
|
||||
"""Download the specified artifact."""
|
||||
from dvc.fs import download as fs_download
|
||||
|
||||
@@ -205,9 +205,9 @@ class Artifacts:
|
||||
out: Optional[str] = None,
|
||||
force: bool = False,
|
||||
jobs: Optional[int] = None,
|
||||
dvc_studio_config: Optional[Dict[str, Any]] = None,
|
||||
dvc_studio_config: Optional[dict[str, Any]] = None,
|
||||
**kwargs,
|
||||
) -> Tuple[int, str]:
|
||||
) -> tuple[int, str]:
|
||||
from dvc_studio_client.model_registry import get_download_uris
|
||||
|
||||
from dvc.fs import HTTPFileSystem, generic, localfs
|
||||
@@ -215,8 +215,8 @@ class Artifacts:
|
||||
|
||||
logger.debug("Trying to download artifact '%s' via studio", name)
|
||||
out = out or os.getcwd()
|
||||
to_infos: List[str] = []
|
||||
from_infos: List[str] = []
|
||||
to_infos: list[str] = []
|
||||
from_infos: list[str] = []
|
||||
if dvc_studio_config is None:
|
||||
dvc_studio_config = {}
|
||||
dvc_studio_config["repo_url"] = repo_url
|
||||
@@ -263,9 +263,9 @@ class Artifacts:
|
||||
name: str,
|
||||
version: Optional[str] = None,
|
||||
stage: Optional[str] = None,
|
||||
config: Optional[Union[str, Dict[str, Any]]] = None,
|
||||
config: Optional[Union[str, dict[str, Any]]] = None,
|
||||
remote: Optional[str] = None,
|
||||
remote_config: Optional[Union[str, Dict[str, Any]]] = None,
|
||||
remote_config: Optional[Union[str, dict[str, Any]]] = None,
|
||||
out: Optional[str] = None,
|
||||
force: bool = False,
|
||||
jobs: Optional[int] = None,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from typing import TYPE_CHECKING, Iterator, Optional, Tuple
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from scmrepo.git import Git
|
||||
|
||||
@@ -59,11 +60,11 @@ def brancher(
|
||||
|
||||
from dvc.fs import LocalFileSystem
|
||||
|
||||
repo_root_parts: Tuple[str, ...] = ()
|
||||
repo_root_parts: tuple[str, ...] = ()
|
||||
if self.fs.isin(self.root_dir, self.scm.root_dir):
|
||||
repo_root_parts = self.fs.relparts(self.root_dir, self.scm.root_dir)
|
||||
|
||||
cwd_parts: Tuple[str, ...] = ()
|
||||
cwd_parts: tuple[str, ...] = ()
|
||||
if self.fs.isin(self.fs.getcwd(), self.scm.root_dir):
|
||||
cwd_parts = self.fs.relparts(self.fs.getcwd(), self.scm.root_dir)
|
||||
|
||||
@@ -112,8 +113,8 @@ def brancher(
|
||||
def _switch_fs(
|
||||
repo: "Repo",
|
||||
rev: str,
|
||||
repo_root_parts: Tuple[str, ...],
|
||||
cwd_parts: Tuple[str, ...],
|
||||
repo_root_parts: tuple[str, ...],
|
||||
cwd_parts: tuple[str, ...],
|
||||
):
|
||||
from dvc.fs import GitFileSystem, LocalFileSystem
|
||||
|
||||
@@ -147,11 +148,11 @@ def switch(repo: "Repo", rev: str) -> Iterator[str]:
|
||||
if rev != "workspace":
|
||||
rev = resolve_rev(repo.scm, rev)
|
||||
|
||||
repo_root_parts: Tuple[str, ...] = ()
|
||||
repo_root_parts: tuple[str, ...] = ()
|
||||
if repo.fs.isin(repo.root_dir, repo.scm.root_dir):
|
||||
repo_root_parts = repo.fs.relparts(repo.root_dir, repo.scm.root_dir)
|
||||
|
||||
cwd_parts: Tuple[str, ...] = ()
|
||||
cwd_parts: tuple[str, ...] = ()
|
||||
if repo.fs.isin(repo.fs.getcwd(), repo.scm.root_dir):
|
||||
cwd_parts = repo.fs.relparts(repo.fs.getcwd(), repo.scm.root_dir)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import os
|
||||
from typing import TYPE_CHECKING, Dict, List
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from dvc.exceptions import (
|
||||
CheckoutError,
|
||||
@@ -66,7 +66,7 @@ def _build_out_changes(index, changes):
|
||||
|
||||
|
||||
def _check_can_delete(
|
||||
entries: List["DataIndexEntry"],
|
||||
entries: list["DataIndexEntry"],
|
||||
index: "BaseDataIndex",
|
||||
path: str,
|
||||
fs: "FileSystem",
|
||||
@@ -107,7 +107,7 @@ def checkout( # noqa: C901
|
||||
from dvc.stage.exceptions import StageFileBadNameError, StageFileDoesNotExistError
|
||||
from dvc_data.index.checkout import ADD, DELETE, MODIFY, apply, compare
|
||||
|
||||
stats: Dict[str, List[str]] = {
|
||||
stats: dict[str, list[str]] = {
|
||||
"added": [],
|
||||
"deleted": [],
|
||||
"modified": [],
|
||||
|
||||
+6
-5
@@ -1,4 +1,5 @@
|
||||
from typing import TYPE_CHECKING, Callable, Iterable, List, Optional, Tuple
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING, Callable, Optional
|
||||
|
||||
from dvc.log import logger
|
||||
|
||||
@@ -10,8 +11,8 @@ logger = logger.getChild(__name__)
|
||||
|
||||
|
||||
FilterFn = Callable[["Output"], bool]
|
||||
Outputs = List["Output"]
|
||||
StrPaths = List[str]
|
||||
Outputs = list["Output"]
|
||||
StrPaths = list[str]
|
||||
|
||||
|
||||
def _collect_outs(
|
||||
@@ -43,7 +44,7 @@ def _collect_paths(
|
||||
|
||||
def _filter_outs(
|
||||
outs: Outputs, fs_paths: StrPaths, duplicates=False
|
||||
) -> Tuple[Outputs, StrPaths]:
|
||||
) -> tuple[Outputs, StrPaths]:
|
||||
res_outs: Outputs = []
|
||||
fs_res_paths = fs_paths
|
||||
|
||||
@@ -65,7 +66,7 @@ def collect(
|
||||
output_filter: Optional[FilterFn] = None,
|
||||
recursive: bool = False,
|
||||
duplicates: bool = False,
|
||||
) -> Tuple[Outputs, StrPaths]:
|
||||
) -> tuple[Outputs, StrPaths]:
|
||||
assert targets or output_filter
|
||||
|
||||
outs: Outputs = _collect_outs(repo, output_filter=output_filter, deps=deps)
|
||||
|
||||
+2
-2
@@ -1,4 +1,4 @@
|
||||
from typing import TYPE_CHECKING, Set
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from dvc import prompt
|
||||
|
||||
@@ -122,7 +122,7 @@ def commit_2_to_3(repo: "Repo", dry: bool = False):
|
||||
stage.dump(update_pipeline=False)
|
||||
|
||||
|
||||
def _migrateable_dvcfiles(view: "IndexView") -> Set[str]:
|
||||
def _migrateable_dvcfiles(view: "IndexView") -> set[str]:
|
||||
from dvc.dvcfile import ProjectFile
|
||||
|
||||
migrated = set()
|
||||
|
||||
+16
-15
@@ -1,6 +1,7 @@
|
||||
import os
|
||||
import posixpath
|
||||
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, TypedDict, Union
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING, Any, TypedDict, Union
|
||||
|
||||
from dvc.fs.callbacks import DEFAULT_CALLBACK
|
||||
from dvc.ui import ui
|
||||
@@ -53,11 +54,11 @@ def _diff(
|
||||
not_in_remote: bool = False,
|
||||
remote_refresh: bool = False,
|
||||
callback: "Callback" = DEFAULT_CALLBACK,
|
||||
) -> Dict[str, List[str]]:
|
||||
) -> dict[str, list[str]]:
|
||||
from dvc_data.index import StorageError
|
||||
from dvc_data.index.diff import UNCHANGED, UNKNOWN, diff
|
||||
|
||||
ret: Dict[str, List[str]] = {}
|
||||
ret: dict[str, list[str]] = {}
|
||||
|
||||
def _add_change(typ, change):
|
||||
typ = _adapt_typ(typ)
|
||||
@@ -115,9 +116,9 @@ def _diff(
|
||||
|
||||
|
||||
class GitInfo(TypedDict, total=False):
|
||||
staged: Dict[str, List[str]]
|
||||
unstaged: Dict[str, List[str]]
|
||||
untracked: List[str]
|
||||
staged: dict[str, list[str]]
|
||||
unstaged: dict[str, list[str]]
|
||||
untracked: list[str]
|
||||
is_empty: bool
|
||||
is_dirty: bool
|
||||
|
||||
@@ -150,7 +151,7 @@ def _git_info(scm: Union["Git", "NoSCM"], untracked_files: str = "all") -> GitIn
|
||||
)
|
||||
|
||||
|
||||
def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> Dict[str, List[str]]:
|
||||
def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> dict[str, list[str]]:
|
||||
from .index import build_data_index
|
||||
|
||||
with ui.progress(
|
||||
@@ -180,7 +181,7 @@ def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> Dict[str, List[str]]:
|
||||
|
||||
def _diff_head_to_index(
|
||||
repo: "Repo", head: str = "HEAD", **kwargs: Any
|
||||
) -> Dict[str, List[str]]:
|
||||
) -> dict[str, list[str]]:
|
||||
index = repo.index.data["repo"]
|
||||
|
||||
with repo.switch(head):
|
||||
@@ -194,16 +195,16 @@ def _diff_head_to_index(
|
||||
|
||||
|
||||
class Status(TypedDict):
|
||||
not_in_cache: List[str]
|
||||
not_in_remote: List[str]
|
||||
committed: Dict[str, List[str]]
|
||||
uncommitted: Dict[str, List[str]]
|
||||
untracked: List[str]
|
||||
unchanged: List[str]
|
||||
not_in_cache: list[str]
|
||||
not_in_remote: list[str]
|
||||
committed: dict[str, list[str]]
|
||||
uncommitted: dict[str, list[str]]
|
||||
untracked: list[str]
|
||||
unchanged: list[str]
|
||||
git: GitInfo
|
||||
|
||||
|
||||
def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> List[str]:
|
||||
def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> list[str]:
|
||||
"""Transform files rel. to Git root to DVC root, and drop outside files."""
|
||||
rel = repo.fs.relpath(repo.root_dir, repo.scm.root_dir).rstrip("/")
|
||||
|
||||
|
||||
+3
-3
@@ -1,7 +1,7 @@
|
||||
import errno
|
||||
import os
|
||||
from collections import defaultdict
|
||||
from typing import Dict, List, Optional
|
||||
from typing import Optional
|
||||
|
||||
from dvc.log import logger
|
||||
from dvc.repo import locked
|
||||
@@ -26,7 +26,7 @@ def _diff(old, new, data_keys, with_missing=False):
|
||||
from dvc_data.index.diff import ADD, DELETE, MODIFY, RENAME
|
||||
from dvc_data.index.diff import diff as idiff
|
||||
|
||||
ret: "Dict[str, List[Dict]]" = {
|
||||
ret: "dict[str, list[dict]]" = {
|
||||
"added": [],
|
||||
"deleted": [],
|
||||
"modified": [],
|
||||
@@ -107,7 +107,7 @@ def diff(
|
||||
self,
|
||||
a_rev: str = "HEAD",
|
||||
b_rev: Optional[str] = None,
|
||||
targets: Optional[List[str]] = None,
|
||||
targets: Optional[list[str]] = None,
|
||||
recursive: bool = False,
|
||||
):
|
||||
"""
|
||||
|
||||
+2
-2
@@ -1,4 +1,4 @@
|
||||
from typing import Any, Dict, Optional, Union
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
|
||||
def du(
|
||||
@@ -6,7 +6,7 @@ def du(
|
||||
path: Optional[str] = None,
|
||||
rev: Optional[str] = None,
|
||||
summarize: bool = False,
|
||||
config: Union[None, Dict[str, Any], str] = None,
|
||||
config: Union[None, dict[str, Any], str] = None,
|
||||
remote: Optional[str] = None,
|
||||
remote_config: Optional[dict] = None,
|
||||
):
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import os
|
||||
import re
|
||||
from typing import TYPE_CHECKING, Dict, Iterable, List, Optional
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from funcy import chain, first
|
||||
|
||||
@@ -104,7 +105,7 @@ class Experiments:
|
||||
return ExpCache(self.repo)
|
||||
|
||||
@property
|
||||
def stash_revs(self) -> Dict[str, "ExpStashEntry"]:
|
||||
def stash_revs(self) -> dict[str, "ExpStashEntry"]:
|
||||
revs = {}
|
||||
for queue in (self.workspace_queue, self.celery_queue):
|
||||
revs.update(queue.stash.stash_revs)
|
||||
@@ -113,7 +114,7 @@ class Experiments:
|
||||
def reproduce_one(
|
||||
self,
|
||||
tmp_dir: bool = False,
|
||||
copy_paths: Optional[List[str]] = None,
|
||||
copy_paths: Optional[list[str]] = None,
|
||||
message: Optional[str] = None,
|
||||
**kwargs,
|
||||
):
|
||||
@@ -143,8 +144,8 @@ class Experiments:
|
||||
|
||||
def reproduce_celery(
|
||||
self, entries: Optional[Iterable["QueueEntry"]] = None, **kwargs
|
||||
) -> Dict[str, str]:
|
||||
results: Dict[str, str] = {}
|
||||
) -> dict[str, str]:
|
||||
results: dict[str, str] = {}
|
||||
if entries is None:
|
||||
entries = list(
|
||||
chain(
|
||||
@@ -250,10 +251,10 @@ class Experiments:
|
||||
def _reproduce_queue(
|
||||
self,
|
||||
queue: "BaseStashQueue",
|
||||
copy_paths: Optional[List[str]] = None,
|
||||
copy_paths: Optional[list[str]] = None,
|
||||
message: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> Dict[str, str]:
|
||||
) -> dict[str, str]:
|
||||
"""Reproduce queued experiments.
|
||||
|
||||
Arguments:
|
||||
@@ -265,7 +266,7 @@ class Experiments:
|
||||
"""
|
||||
exec_results = queue.reproduce(copy_paths=copy_paths, message=message)
|
||||
|
||||
results: Dict[str, str] = {}
|
||||
results: dict[str, str] = {}
|
||||
for _, exp_result in exec_results.items():
|
||||
results.update(exp_result)
|
||||
return results
|
||||
@@ -319,12 +320,12 @@ class Experiments:
|
||||
raise MultipleBranchError(rev, ref_infos)
|
||||
return str(ref_infos[0])
|
||||
|
||||
def get_exact_name(self, revs: Iterable[str]) -> Dict[str, Optional[str]]:
|
||||
def get_exact_name(self, revs: Iterable[str]) -> dict[str, Optional[str]]:
|
||||
"""Returns preferred name for the specified revision.
|
||||
|
||||
Prefers tags, branches (heads), experiments in that order.
|
||||
"""
|
||||
result: Dict[str, Optional[str]] = {}
|
||||
result: dict[str, Optional[str]] = {}
|
||||
exclude = f"{EXEC_NAMESPACE}/*"
|
||||
ref_dict = self.scm.describe(revs, base=EXPS_NAMESPACE, exclude=exclude)
|
||||
for rev in revs:
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from collections.abc import Iterator
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from typing import TYPE_CHECKING, Iterator, Tuple
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from dvc.repo.experiments.exceptions import InvalidExpRevError
|
||||
from dvc.scm import RevError
|
||||
@@ -12,7 +13,7 @@ if TYPE_CHECKING:
|
||||
def switch_repo(
|
||||
repo: "Repo",
|
||||
rev: str,
|
||||
) -> Iterator[Tuple["Repo", str]]:
|
||||
) -> Iterator[tuple["Repo", str]]:
|
||||
"""Return a repo instance (brancher) switched to rev.
|
||||
|
||||
If rev is the name of a running experiment, the returned instance will be
|
||||
|
||||
@@ -1,16 +1,11 @@
|
||||
import itertools
|
||||
import os
|
||||
from collections.abc import Collection, Iterable, Iterator
|
||||
from dataclasses import fields
|
||||
from datetime import datetime
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Collection,
|
||||
Dict,
|
||||
Iterable,
|
||||
Iterator,
|
||||
List,
|
||||
Optional,
|
||||
Tuple,
|
||||
Union,
|
||||
)
|
||||
|
||||
@@ -142,7 +137,7 @@ def collect_queued(
|
||||
repo: "Repo",
|
||||
baseline_revs: Collection[str],
|
||||
**kwargs,
|
||||
) -> Dict[str, List["ExpRange"]]:
|
||||
) -> dict[str, list["ExpRange"]]:
|
||||
"""Collect queued experiments derived from the specified revisions.
|
||||
|
||||
Args:
|
||||
@@ -177,7 +172,7 @@ def collect_active(
|
||||
repo: "Repo",
|
||||
baseline_revs: Collection[str],
|
||||
**kwargs,
|
||||
) -> Dict[str, List["ExpRange"]]:
|
||||
) -> dict[str, list["ExpRange"]]:
|
||||
"""Collect active (running) experiments derived from the specified revisions.
|
||||
|
||||
Args:
|
||||
@@ -189,7 +184,7 @@ def collect_active(
|
||||
"""
|
||||
if not baseline_revs:
|
||||
return {}
|
||||
result: Dict[str, List["ExpRange"]] = {}
|
||||
result: dict[str, list["ExpRange"]] = {}
|
||||
exps = repo.experiments
|
||||
for queue in (exps.workspace_queue, exps.tempdir_queue, exps.celery_queue):
|
||||
for baseline, active_exps in queue.collect_active_data(
|
||||
@@ -206,7 +201,7 @@ def collect_failed(
|
||||
repo: "Repo",
|
||||
baseline_revs: Collection[str],
|
||||
**kwargs,
|
||||
) -> Dict[str, List["ExpRange"]]:
|
||||
) -> dict[str, list["ExpRange"]]:
|
||||
"""Collect failed experiments derived from the specified revisions.
|
||||
|
||||
Args:
|
||||
@@ -225,7 +220,7 @@ def collect_successful(
|
||||
repo: "Repo",
|
||||
baseline_revs: Collection[str],
|
||||
**kwargs,
|
||||
) -> Dict[str, List["ExpRange"]]:
|
||||
) -> dict[str, list["ExpRange"]]:
|
||||
"""Collect successful experiments derived from the specified revisions.
|
||||
|
||||
Args:
|
||||
@@ -235,7 +230,7 @@ def collect_successful(
|
||||
Returns:
|
||||
Dict mapping baseline revision to successful experiments.
|
||||
"""
|
||||
result: Dict[str, List["ExpRange"]] = {}
|
||||
result: dict[str, list["ExpRange"]] = {}
|
||||
for baseline_rev in baseline_revs:
|
||||
result[baseline_rev] = list(_collect_baseline(repo, baseline_rev, **kwargs))
|
||||
return result
|
||||
@@ -282,7 +277,7 @@ def _collect_baseline(
|
||||
|
||||
def collect(
|
||||
repo: "Repo",
|
||||
revs: Union[List[str], str, None] = None,
|
||||
revs: Union[list[str], str, None] = None,
|
||||
all_branches: bool = False,
|
||||
all_tags: bool = False,
|
||||
all_commits: bool = False,
|
||||
@@ -291,7 +286,7 @@ def collect(
|
||||
hide_failed: bool = False,
|
||||
sha_only: bool = False,
|
||||
**kwargs,
|
||||
) -> List["ExpState"]:
|
||||
) -> list["ExpState"]:
|
||||
"""Collect baseline revisions and derived experiments."""
|
||||
assert isinstance(repo.scm, Git)
|
||||
if repo.scm.no_commits:
|
||||
@@ -312,14 +307,14 @@ def collect(
|
||||
)
|
||||
)
|
||||
if sha_only:
|
||||
baseline_names: Dict[str, Optional[str]] = {}
|
||||
baseline_names: dict[str, Optional[str]] = {}
|
||||
else:
|
||||
baseline_names = describe(
|
||||
repo.scm, baseline_revs, refs=cached_refs, logger=logger
|
||||
)
|
||||
|
||||
workspace_data = collect_rev(repo, "workspace", **kwargs)
|
||||
result: List["ExpState"] = [workspace_data]
|
||||
result: list["ExpState"] = [workspace_data]
|
||||
queued = collect_queued(repo, baseline_revs, **kwargs) if not hide_queued else {}
|
||||
active = collect_active(repo, baseline_revs, **kwargs)
|
||||
failed = collect_failed(repo, baseline_revs, **kwargs) if not hide_failed else {}
|
||||
@@ -345,10 +340,10 @@ def collect(
|
||||
return result
|
||||
|
||||
|
||||
def _sorted_ranges(exp_ranges: Iterable["ExpRange"]) -> List["ExpRange"]:
|
||||
def _sorted_ranges(exp_ranges: Iterable["ExpRange"]) -> list["ExpRange"]:
|
||||
"""Return list of ExpRange sorted by (timestamp, rev)."""
|
||||
|
||||
def _head_timestamp(exp_range: "ExpRange") -> Tuple[datetime, str]:
|
||||
def _head_timestamp(exp_range: "ExpRange") -> tuple[datetime, str]:
|
||||
head_exp = first(exp_range.revs)
|
||||
if head_exp and head_exp.data and head_exp.data.timestamp:
|
||||
return head_exp.data.timestamp, head_exp.rev
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
from typing import TYPE_CHECKING, Collection, Iterable, Optional
|
||||
from collections.abc import Collection, Iterable
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from dvc.exceptions import DvcException, InvalidArgumentError
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ import os
|
||||
import pickle
|
||||
import shutil
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Iterable, Iterator
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import asdict, dataclass
|
||||
from enum import IntEnum
|
||||
@@ -11,15 +12,8 @@ from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Callable,
|
||||
Dict,
|
||||
Iterable,
|
||||
Iterator,
|
||||
List,
|
||||
NamedTuple,
|
||||
Optional,
|
||||
Tuple,
|
||||
Type,
|
||||
TypeVar,
|
||||
Union,
|
||||
)
|
||||
|
||||
@@ -41,6 +35,8 @@ from dvc.utils.studio import env_to_config
|
||||
if TYPE_CHECKING:
|
||||
from queue import Queue
|
||||
|
||||
from typing_extensions import Self
|
||||
|
||||
from dvc.repo import Repo
|
||||
from dvc.repo.experiments.stash import ExpStashEntry
|
||||
from dvc.scm import Git
|
||||
@@ -112,9 +108,6 @@ class ExecutorInfo:
|
||||
return cls.from_dict(load_json(filename))
|
||||
|
||||
|
||||
_T = TypeVar("_T", bound="BaseExecutor")
|
||||
|
||||
|
||||
class BaseExecutor(ABC):
|
||||
"""Base class for executing experiments in parallel.
|
||||
|
||||
@@ -185,7 +178,7 @@ class BaseExecutor(ABC):
|
||||
@property
|
||||
def info(self) -> "ExecutorInfo":
|
||||
if self.result is not None:
|
||||
result_dict: Dict[str, Any] = {
|
||||
result_dict: dict[str, Any] = {
|
||||
"result_hash": self.result.exp_hash,
|
||||
"result_ref": (
|
||||
str(self.result.ref_info) if self.result.ref_info else None
|
||||
@@ -207,7 +200,7 @@ class BaseExecutor(ABC):
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_info(cls: Type[_T], info: "ExecutorInfo") -> _T:
|
||||
def from_info(cls, info: "ExecutorInfo") -> "Self":
|
||||
if info.result_hash:
|
||||
result: Optional["ExecutorResult"] = ExecutorResult(
|
||||
info.result_hash,
|
||||
@@ -229,21 +222,21 @@ class BaseExecutor(ABC):
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def from_stash_entry(
|
||||
cls: Type[_T],
|
||||
cls,
|
||||
repo: "Repo",
|
||||
entry: "ExpStashEntry",
|
||||
**kwargs,
|
||||
) -> _T:
|
||||
) -> "Self":
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def _from_stash_entry(
|
||||
cls: Type[_T],
|
||||
cls,
|
||||
repo: "Repo",
|
||||
entry: "ExpStashEntry",
|
||||
root_dir: str,
|
||||
**kwargs,
|
||||
) -> _T:
|
||||
) -> "Self":
|
||||
return cls(
|
||||
root_dir=root_dir,
|
||||
dvc_dir=relpath(repo.dvc_dir, repo.scm.root_dir),
|
||||
@@ -255,7 +248,7 @@ class BaseExecutor(ABC):
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _get_top_level_paths(cls, repo: "Repo") -> List["str"]:
|
||||
def _get_top_level_paths(cls, repo: "Repo") -> list["str"]:
|
||||
return list(
|
||||
chain(
|
||||
_collect_top_level_metrics(repo),
|
||||
@@ -271,7 +264,7 @@ class BaseExecutor(ABC):
|
||||
targets: Optional[Iterable[str]] = None,
|
||||
recursive: bool = False,
|
||||
force: bool = False,
|
||||
include_untracked: Optional[List[str]] = None,
|
||||
include_untracked: Optional[list[str]] = None,
|
||||
message: Optional[str] = None,
|
||||
) -> ExecutorResult:
|
||||
from dvc.dvcfile import LOCK_FILE
|
||||
@@ -382,7 +375,7 @@ class BaseExecutor(ABC):
|
||||
def fetch_exps(
|
||||
self,
|
||||
dest_scm: "Git",
|
||||
refs: List[str],
|
||||
refs: list[str],
|
||||
force: bool = False,
|
||||
on_diverged: Optional[Callable[[str], None]] = None,
|
||||
**kwargs,
|
||||
@@ -457,7 +450,7 @@ class BaseExecutor(ABC):
|
||||
infofile: Optional[str] = None,
|
||||
log_errors: bool = True,
|
||||
log_level: Optional[int] = None,
|
||||
copy_paths: Optional[List[str]] = None,
|
||||
copy_paths: Optional[list[str]] = None,
|
||||
message: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> "ExecutorResult":
|
||||
@@ -565,7 +558,7 @@ class BaseExecutor(ABC):
|
||||
git_remote,
|
||||
repro_force,
|
||||
message: Optional[str] = None,
|
||||
) -> Tuple[Optional[str], Optional["ExpRefInfo"], bool]:
|
||||
) -> tuple[Optional[str], Optional["ExpRefInfo"], bool]:
|
||||
cls.commit(
|
||||
dvc.scm,
|
||||
exp_hash,
|
||||
@@ -598,7 +591,7 @@ class BaseExecutor(ABC):
|
||||
info: "ExecutorInfo",
|
||||
infofile: Optional[str] = None,
|
||||
log_errors: bool = True,
|
||||
copy_paths: Optional[List[str]] = None,
|
||||
copy_paths: Optional[list[str]] = None,
|
||||
message: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> Iterator["Repo"]:
|
||||
@@ -790,7 +783,7 @@ class BaseExecutor(ABC):
|
||||
raise DvcException(f"Unable to copy '{src}' to '{dst}'.") from exc
|
||||
|
||||
@contextmanager
|
||||
def set_temp_refs(self, scm: "Git", temp_dict: Dict[str, str]):
|
||||
def set_temp_refs(self, scm: "Git", temp_dict: dict[str, str]):
|
||||
try:
|
||||
for ref, rev in temp_dict.items():
|
||||
scm.set_ref(ref, rev)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from collections import defaultdict
|
||||
from typing import Dict, List, Optional, Tuple, Union
|
||||
from typing import Optional, Union
|
||||
|
||||
from dvc.log import logger
|
||||
from dvc.repo import locked
|
||||
@@ -15,11 +15,11 @@ logger = logger.getChild(__name__)
|
||||
@scm_context
|
||||
def ls(
|
||||
repo,
|
||||
rev: Optional[Union[List[str], str]] = None,
|
||||
rev: Optional[Union[list[str], str]] = None,
|
||||
all_commits: bool = False,
|
||||
num: int = 1,
|
||||
git_remote: Optional[str] = None,
|
||||
) -> Dict[str, List[Tuple[str, Optional[str]]]]:
|
||||
) -> dict[str, list[tuple[str, Optional[str]]]]:
|
||||
"""List experiments.
|
||||
|
||||
Returns a dict mapping baseline revs to a list of (exp_name, exp_sha) tuples.
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
from typing import Iterable, List, Mapping, Optional, Set, Union
|
||||
from collections.abc import Iterable, Mapping
|
||||
from typing import Optional, Union
|
||||
|
||||
from funcy import group_by
|
||||
from scmrepo.git.backend.base import SyncStatus
|
||||
@@ -23,13 +24,13 @@ def pull( # noqa: C901
|
||||
git_remote: str,
|
||||
exp_names: Optional[Union[Iterable[str], str]] = None,
|
||||
all_commits=False,
|
||||
rev: Optional[Union[List[str], str]] = None,
|
||||
rev: Optional[Union[list[str], str]] = None,
|
||||
num=1,
|
||||
force: bool = False,
|
||||
pull_cache: bool = False,
|
||||
**kwargs,
|
||||
) -> Iterable[str]:
|
||||
exp_ref_set: Set["ExpRefInfo"] = set()
|
||||
exp_ref_set: set["ExpRefInfo"] = set()
|
||||
if all_commits:
|
||||
exp_ref_set.update(exp_refs(repo.scm, git_remote))
|
||||
elif exp_names:
|
||||
@@ -81,7 +82,7 @@ def _pull(
|
||||
git_remote: str,
|
||||
refs: Iterable["ExpRefInfo"],
|
||||
force: bool,
|
||||
) -> Mapping[SyncStatus, List["ExpRefInfo"]]:
|
||||
) -> Mapping[SyncStatus, list["ExpRefInfo"]]:
|
||||
refspec_list = [f"{exp_ref}:{exp_ref}" for exp_ref in refs]
|
||||
logger.debug("git pull experiment '%s' -> '%s'", git_remote, refspec_list)
|
||||
|
||||
@@ -96,7 +97,7 @@ def _pull(
|
||||
def group_result(refspec):
|
||||
return results[str(refspec)]
|
||||
|
||||
pull_result: Mapping[SyncStatus, List["ExpRefInfo"]] = group_by(group_result, refs)
|
||||
pull_result: Mapping[SyncStatus, list["ExpRefInfo"]] = group_by(group_result, refs)
|
||||
|
||||
return pull_result
|
||||
|
||||
|
||||
@@ -1,12 +1,8 @@
|
||||
from collections.abc import Iterable, Mapping
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Dict,
|
||||
Iterable,
|
||||
List,
|
||||
Mapping,
|
||||
Optional,
|
||||
Set,
|
||||
Union,
|
||||
)
|
||||
|
||||
@@ -39,7 +35,7 @@ class UploadError(DvcException):
|
||||
|
||||
|
||||
def notify_refs_to_studio(
|
||||
repo: "Repo", git_remote: str, **refs: List[str]
|
||||
repo: "Repo", git_remote: str, **refs: list[str]
|
||||
) -> Optional[str]:
|
||||
import os
|
||||
|
||||
@@ -67,7 +63,7 @@ def notify_refs_to_studio(
|
||||
return d.get("url")
|
||||
|
||||
|
||||
def exp_refs_from_names(scm: "Git", exp_names: List[str]) -> Set["ExpRefInfo"]:
|
||||
def exp_refs_from_names(scm: "Git", exp_names: list[str]) -> set["ExpRefInfo"]:
|
||||
exp_ref_set = set()
|
||||
exp_ref_dict = resolve_name(scm, exp_names)
|
||||
unresolved_exp_names = []
|
||||
@@ -82,7 +78,7 @@ def exp_refs_from_names(scm: "Git", exp_names: List[str]) -> Set["ExpRefInfo"]:
|
||||
return exp_ref_set
|
||||
|
||||
|
||||
def exp_refs_from_rev(scm: "Git", rev: List[str], num: int = 1) -> Set["ExpRefInfo"]:
|
||||
def exp_refs_from_rev(scm: "Git", rev: list[str], num: int = 1) -> set["ExpRefInfo"]:
|
||||
exp_ref_set = set()
|
||||
rev_dict = iter_revs(scm, rev, num)
|
||||
rev_set = set(rev_dict.keys())
|
||||
@@ -97,15 +93,15 @@ def exp_refs_from_rev(scm: "Git", rev: List[str], num: int = 1) -> Set["ExpRefIn
|
||||
def push(
|
||||
repo: "Repo",
|
||||
git_remote: str,
|
||||
exp_names: Optional[Union[List[str], str]] = None,
|
||||
exp_names: Optional[Union[list[str], str]] = None,
|
||||
all_commits: bool = False,
|
||||
rev: Optional[Union[List[str], str]] = None,
|
||||
rev: Optional[Union[list[str], str]] = None,
|
||||
num: int = 1,
|
||||
force: bool = False,
|
||||
push_cache: bool = False,
|
||||
**kwargs: Any,
|
||||
) -> Dict[str, Any]:
|
||||
exp_ref_set: Set["ExpRefInfo"] = set()
|
||||
) -> dict[str, Any]:
|
||||
exp_ref_set: set["ExpRefInfo"] = set()
|
||||
assert isinstance(repo.scm, Git)
|
||||
if all_commits:
|
||||
exp_ref_set.update(exp_refs(repo.scm))
|
||||
@@ -123,7 +119,7 @@ def push(
|
||||
status.name.lower(): [ref.name for ref in ref_list]
|
||||
for status, ref_list in push_result.items()
|
||||
}
|
||||
result: Dict[str, Any] = {**refs, "uploaded": 0}
|
||||
result: dict[str, Any] = {**refs, "uploaded": 0}
|
||||
|
||||
pushed_refs_info = (
|
||||
push_result[SyncStatus.UP_TO_DATE] + push_result[SyncStatus.SUCCESS]
|
||||
@@ -149,7 +145,7 @@ def _push(
|
||||
git_remote: str,
|
||||
refs: Iterable["ExpRefInfo"],
|
||||
force: bool,
|
||||
) -> Mapping[SyncStatus, List["ExpRefInfo"]]:
|
||||
) -> Mapping[SyncStatus, list["ExpRefInfo"]]:
|
||||
from scmrepo.exceptions import AuthError
|
||||
|
||||
from dvc.scm import GitAuthError
|
||||
@@ -171,7 +167,7 @@ def _push(
|
||||
def group_result(refspec):
|
||||
return results[str(refspec)]
|
||||
|
||||
pull_result: Mapping[SyncStatus, List["ExpRefInfo"]] = group_by(group_result, refs)
|
||||
pull_result: Mapping[SyncStatus, list["ExpRefInfo"]] = group_by(group_result, refs)
|
||||
|
||||
return pull_result
|
||||
|
||||
|
||||
@@ -1,18 +1,12 @@
|
||||
import os
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Collection, Generator, Iterable, Mapping
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Collection,
|
||||
Dict,
|
||||
Generator,
|
||||
Iterable,
|
||||
List,
|
||||
Mapping,
|
||||
NamedTuple,
|
||||
Optional,
|
||||
Type,
|
||||
Union,
|
||||
)
|
||||
|
||||
@@ -69,11 +63,11 @@ class QueueEntry:
|
||||
and self.stash_rev == other.stash_rev
|
||||
)
|
||||
|
||||
def asdict(self) -> Dict[str, Any]:
|
||||
def asdict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d: Dict[str, Any]) -> "QueueEntry":
|
||||
def from_dict(cls, d: dict[str, Any]) -> "QueueEntry":
|
||||
return cls(**d)
|
||||
|
||||
|
||||
@@ -150,7 +144,7 @@ class BaseStashQueue(ABC):
|
||||
all_: bool = False,
|
||||
queued: bool = False,
|
||||
**kwargs,
|
||||
) -> List[str]:
|
||||
) -> list[str]:
|
||||
"""Remove the specified entries from the queue.
|
||||
|
||||
Arguments:
|
||||
@@ -165,8 +159,8 @@ class BaseStashQueue(ABC):
|
||||
if all_ or queued:
|
||||
return self.clear()
|
||||
|
||||
name_to_remove: List[str] = []
|
||||
entry_to_remove: List[ExpStashEntry] = []
|
||||
name_to_remove: list[str] = []
|
||||
entry_to_remove: list[ExpStashEntry] = []
|
||||
queue_entries = self.match_queue_entry_by_name(revs, self.iter_queued())
|
||||
for name, entry in queue_entries.items():
|
||||
if entry:
|
||||
@@ -176,7 +170,7 @@ class BaseStashQueue(ABC):
|
||||
self.stash.remove_revs(entry_to_remove)
|
||||
return name_to_remove
|
||||
|
||||
def clear(self, **kwargs) -> List[str]:
|
||||
def clear(self, **kwargs) -> list[str]:
|
||||
"""Remove all entries from the queue."""
|
||||
stash_revs = self.stash.stash_revs
|
||||
name_to_remove = list(stash_revs)
|
||||
@@ -184,11 +178,11 @@ class BaseStashQueue(ABC):
|
||||
|
||||
return name_to_remove
|
||||
|
||||
def status(self) -> List[Dict[str, Any]]:
|
||||
def status(self) -> list[dict[str, Any]]:
|
||||
"""Show the status of exp tasks in queue"""
|
||||
from datetime import datetime
|
||||
|
||||
result: List[Dict[str, Optional[str]]] = []
|
||||
result: list[dict[str, Optional[str]]] = []
|
||||
|
||||
def _get_timestamp(rev: str) -> datetime:
|
||||
commit = self.scm.resolve_commit(rev)
|
||||
@@ -198,7 +192,7 @@ class BaseStashQueue(ABC):
|
||||
entry: QueueEntry,
|
||||
exp_result: Optional["ExecutorResult"] = None,
|
||||
status: str = "Unknown",
|
||||
) -> Dict[str, Any]:
|
||||
) -> dict[str, Any]:
|
||||
name = entry.name
|
||||
if not name and exp_result and exp_result.ref_info:
|
||||
name = exp_result.ref_info.name
|
||||
@@ -251,7 +245,7 @@ class BaseStashQueue(ABC):
|
||||
|
||||
@abstractmethod
|
||||
def reproduce(
|
||||
self, copy_paths: Optional[List[str]] = None, message: Optional[str] = None
|
||||
self, copy_paths: Optional[list[str]] = None, message: Optional[str] = None
|
||||
) -> Mapping[str, Mapping[str, str]]:
|
||||
"""Reproduce queued experiments sequentially."""
|
||||
|
||||
@@ -300,7 +294,7 @@ class BaseStashQueue(ABC):
|
||||
def _stash_exp(
|
||||
self,
|
||||
*args,
|
||||
params: Optional[Dict[str, List[str]]] = None,
|
||||
params: Optional[dict[str, list[str]]] = None,
|
||||
baseline_rev: Optional[str] = None,
|
||||
branch: Optional[str] = None,
|
||||
name: Optional[str] = None,
|
||||
@@ -460,7 +454,7 @@ class BaseStashQueue(ABC):
|
||||
f"from '{config_path}': {param_list}"
|
||||
)
|
||||
|
||||
def _update_params(self, params: Dict[str, List[str]]):
|
||||
def _update_params(self, params: dict[str, list[str]]):
|
||||
"""Update param files with the provided `Hydra Override`_ patterns.
|
||||
|
||||
Args:
|
||||
@@ -528,7 +522,7 @@ class BaseStashQueue(ABC):
|
||||
cls,
|
||||
exp: "Experiments",
|
||||
queue_entry: QueueEntry,
|
||||
executor_cls: Type[BaseExecutor] = WorkspaceExecutor,
|
||||
executor_cls: type[BaseExecutor] = WorkspaceExecutor,
|
||||
**kwargs,
|
||||
) -> BaseExecutor:
|
||||
stash_entry = cls.get_stash_entry(exp, queue_entry)
|
||||
@@ -563,7 +557,7 @@ class BaseStashQueue(ABC):
|
||||
exp: "Experiments",
|
||||
executor: BaseExecutor,
|
||||
exec_result: "ExecutorResult",
|
||||
) -> Dict[str, str]:
|
||||
) -> dict[str, str]:
|
||||
results = {}
|
||||
|
||||
def on_diverged(ref: str):
|
||||
@@ -593,7 +587,7 @@ class BaseStashQueue(ABC):
|
||||
exp: "Experiments",
|
||||
executor: BaseExecutor,
|
||||
exec_result: "ExecutorResult",
|
||||
) -> Dict[str, str]:
|
||||
) -> dict[str, str]:
|
||||
results = cls.collect_git(exp, executor, exec_result)
|
||||
|
||||
if exec_result.ref_info is not None:
|
||||
@@ -605,11 +599,11 @@ class BaseStashQueue(ABC):
|
||||
self,
|
||||
exp_names: Collection[str],
|
||||
*entries: Iterable[Union[QueueEntry, QueueDoneResult]],
|
||||
) -> Dict[str, Optional[QueueEntry]]:
|
||||
) -> dict[str, Optional[QueueEntry]]:
|
||||
from funcy import concat
|
||||
|
||||
entry_name_dict: Dict[str, QueueEntry] = {}
|
||||
entry_rev_dict: Dict[str, QueueEntry] = {}
|
||||
entry_name_dict: dict[str, QueueEntry] = {}
|
||||
entry_rev_dict: dict[str, QueueEntry] = {}
|
||||
for entry in concat(*entries):
|
||||
if isinstance(entry, QueueDoneResult):
|
||||
queue_entry: QueueEntry = entry.entry
|
||||
@@ -624,7 +618,7 @@ class BaseStashQueue(ABC):
|
||||
entry_name_dict[name] = queue_entry
|
||||
entry_rev_dict[queue_entry.stash_rev] = queue_entry
|
||||
|
||||
result: Dict[str, Optional[QueueEntry]] = {}
|
||||
result: dict[str, Optional[QueueEntry]] = {}
|
||||
for exp_name in exp_names:
|
||||
result[exp_name] = None
|
||||
if exp_name in entry_name_dict:
|
||||
@@ -666,7 +660,7 @@ class BaseStashQueue(ABC):
|
||||
baseline_revs: Optional[Collection[str]],
|
||||
fetch_refs: bool = False,
|
||||
**kwargs,
|
||||
) -> Dict[str, List["ExpRange"]]:
|
||||
) -> dict[str, list["ExpRange"]]:
|
||||
"""Collect data for active (running) experiments.
|
||||
|
||||
Args:
|
||||
@@ -685,7 +679,7 @@ class BaseStashQueue(ABC):
|
||||
self,
|
||||
baseline_revs: Optional[Collection[str]],
|
||||
**kwargs,
|
||||
) -> Dict[str, List["ExpRange"]]:
|
||||
) -> dict[str, list["ExpRange"]]:
|
||||
"""Collect data for queued experiments.
|
||||
|
||||
Args:
|
||||
@@ -702,7 +696,7 @@ class BaseStashQueue(ABC):
|
||||
self,
|
||||
baseline_revs: Optional[Collection[str]],
|
||||
**kwargs,
|
||||
) -> Dict[str, List["ExpRange"]]:
|
||||
) -> dict[str, list["ExpRange"]]:
|
||||
"""Collect data for failed experiments.
|
||||
|
||||
Args:
|
||||
|
||||
@@ -3,16 +3,11 @@ import locale
|
||||
import logging
|
||||
import os
|
||||
from collections import defaultdict
|
||||
from collections.abc import Collection, Generator, Mapping
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Collection,
|
||||
Dict,
|
||||
Generator,
|
||||
List,
|
||||
Mapping,
|
||||
NamedTuple,
|
||||
Optional,
|
||||
Set,
|
||||
Union,
|
||||
)
|
||||
|
||||
@@ -166,7 +161,7 @@ class LocalCeleryQueue(BaseStashQueue):
|
||||
"""
|
||||
|
||||
logger.debug("Spawning %s exp queue workers", count)
|
||||
active_worker: Dict = self.worker_status()
|
||||
active_worker: dict = self.worker_status()
|
||||
|
||||
started = 0
|
||||
for num in range(1, 1 + count):
|
||||
@@ -183,7 +178,7 @@ class LocalCeleryQueue(BaseStashQueue):
|
||||
def put(
|
||||
self,
|
||||
*args,
|
||||
copy_paths: Optional[List[str]] = None,
|
||||
copy_paths: Optional[list[str]] = None,
|
||||
message: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> QueueEntry:
|
||||
@@ -265,7 +260,7 @@ class LocalCeleryQueue(BaseStashQueue):
|
||||
yield QueueDoneResult(queue_entry, exp_result)
|
||||
|
||||
def reproduce(
|
||||
self, copy_paths: Optional[List[str]] = None, message: Optional[str] = None
|
||||
self, copy_paths: Optional[list[str]] = None, message: Optional[str] = None
|
||||
) -> Mapping[str, Mapping[str, str]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -333,8 +328,8 @@ class LocalCeleryQueue(BaseStashQueue):
|
||||
while not self.proc.get(entry.stash_rev):
|
||||
time.sleep(sleep_interval)
|
||||
|
||||
def _get_running_task_ids(self) -> Set[str]:
|
||||
running_task_ids: Set[str] = set()
|
||||
def _get_running_task_ids(self) -> set[str]:
|
||||
running_task_ids: set[str] = set()
|
||||
active_workers = self.worker_status()
|
||||
for _, tasks in active_workers.items():
|
||||
task = first(tasks)
|
||||
@@ -343,9 +338,9 @@ class LocalCeleryQueue(BaseStashQueue):
|
||||
return running_task_ids
|
||||
|
||||
def _try_to_kill_tasks(
|
||||
self, to_kill: Dict[QueueEntry, str], force: bool
|
||||
) -> Dict[QueueEntry, str]:
|
||||
fail_to_kill_entries: Dict[QueueEntry, str] = {}
|
||||
self, to_kill: dict[QueueEntry, str], force: bool
|
||||
) -> dict[QueueEntry, str]:
|
||||
fail_to_kill_entries: dict[QueueEntry, str] = {}
|
||||
for queue_entry, rev in to_kill.items():
|
||||
try:
|
||||
if force:
|
||||
@@ -358,9 +353,9 @@ class LocalCeleryQueue(BaseStashQueue):
|
||||
return fail_to_kill_entries
|
||||
|
||||
def _mark_inactive_tasks_failure(
|
||||
self, remained_entries: Dict[QueueEntry, str]
|
||||
self, remained_entries: dict[QueueEntry, str]
|
||||
) -> None:
|
||||
remained_revs: List[str] = []
|
||||
remained_revs: list[str] = []
|
||||
running_ids = self._get_running_task_ids()
|
||||
logger.debug("Current running tasks ids: %s.", running_ids)
|
||||
for msg, entry in self._iter_processed():
|
||||
@@ -383,12 +378,12 @@ class LocalCeleryQueue(BaseStashQueue):
|
||||
if remained_revs:
|
||||
raise CannotKillTasksError(remained_revs)
|
||||
|
||||
def _kill_entries(self, entries: Dict[QueueEntry, str], force: bool) -> None:
|
||||
def _kill_entries(self, entries: dict[QueueEntry, str], force: bool) -> None:
|
||||
logger.debug(
|
||||
"Found active tasks: '%s' to kill",
|
||||
list(entries.values()),
|
||||
)
|
||||
inactive_entries: Dict[QueueEntry, str] = self._try_to_kill_tasks(
|
||||
inactive_entries: dict[QueueEntry, str] = self._try_to_kill_tasks(
|
||||
entries, force
|
||||
)
|
||||
|
||||
@@ -396,12 +391,12 @@ class LocalCeleryQueue(BaseStashQueue):
|
||||
self._mark_inactive_tasks_failure(inactive_entries)
|
||||
|
||||
def kill(self, revs: Collection[str], force: bool = False) -> None:
|
||||
name_dict: Dict[str, Optional[QueueEntry]] = self.match_queue_entry_by_name(
|
||||
name_dict: dict[str, Optional[QueueEntry]] = self.match_queue_entry_by_name(
|
||||
set(revs), self.iter_active()
|
||||
)
|
||||
|
||||
missing_revs: List[str] = []
|
||||
to_kill: Dict[QueueEntry, str] = {}
|
||||
missing_revs: list[str] = []
|
||||
to_kill: dict[QueueEntry, str] = {}
|
||||
for rev, queue_entry in name_dict.items():
|
||||
if queue_entry is None:
|
||||
missing_revs.append(rev)
|
||||
@@ -417,7 +412,7 @@ class LocalCeleryQueue(BaseStashQueue):
|
||||
def shutdown(self, kill: bool = False):
|
||||
self.celery.control.shutdown()
|
||||
if kill:
|
||||
to_kill: Dict[QueueEntry, str] = {}
|
||||
to_kill: dict[QueueEntry, str] = {}
|
||||
for entry in self.iter_active():
|
||||
to_kill[entry] = entry.name or entry.stash_rev
|
||||
if to_kill:
|
||||
@@ -468,7 +463,7 @@ class LocalCeleryQueue(BaseStashQueue):
|
||||
) as fobj:
|
||||
ui.write(fobj.read())
|
||||
|
||||
def worker_status(self) -> Dict[str, List[Dict]]:
|
||||
def worker_status(self) -> dict[str, list[dict]]:
|
||||
"""Return the current active celery worker"""
|
||||
status = self.celery.control.inspect().active() or {}
|
||||
logger.debug("Worker status: %s", status)
|
||||
@@ -486,21 +481,21 @@ class LocalCeleryQueue(BaseStashQueue):
|
||||
|
||||
def get_ref_and_entry_by_names(
|
||||
self,
|
||||
exp_names: Union[str, List[str]],
|
||||
exp_names: Union[str, list[str]],
|
||||
git_remote: Optional[str] = None,
|
||||
) -> Dict[str, ExpRefAndQueueEntry]:
|
||||
) -> dict[str, ExpRefAndQueueEntry]:
|
||||
"""Find finished ExpRefInfo or queued or failed QueueEntry by name"""
|
||||
from dvc.repo.experiments.utils import resolve_name
|
||||
|
||||
if isinstance(exp_names, str):
|
||||
exp_names = [exp_names]
|
||||
results: Dict[str, ExpRefAndQueueEntry] = {}
|
||||
results: dict[str, ExpRefAndQueueEntry] = {}
|
||||
|
||||
exp_ref_match: Dict[str, Optional["ExpRefInfo"]] = resolve_name(
|
||||
exp_ref_match: dict[str, Optional["ExpRefInfo"]] = resolve_name(
|
||||
self.scm, exp_names, git_remote
|
||||
)
|
||||
if not git_remote:
|
||||
queue_entry_match: Dict[
|
||||
queue_entry_match: dict[
|
||||
str, Optional["QueueEntry"]
|
||||
] = self.match_queue_entry_by_name(
|
||||
exp_names, self.iter_queued(), self.iter_done()
|
||||
@@ -517,7 +512,7 @@ class LocalCeleryQueue(BaseStashQueue):
|
||||
baseline_revs: Optional[Collection[str]],
|
||||
fetch_refs: bool = False,
|
||||
**kwargs,
|
||||
) -> Dict[str, List["ExpRange"]]:
|
||||
) -> dict[str, list["ExpRange"]]:
|
||||
from dvc.repo import Repo
|
||||
from dvc.repo.experiments.collect import collect_exec_branch
|
||||
from dvc.repo.experiments.serialize import (
|
||||
@@ -526,7 +521,7 @@ class LocalCeleryQueue(BaseStashQueue):
|
||||
LocalExpExecutor,
|
||||
)
|
||||
|
||||
result: Dict[str, List[ExpRange]] = defaultdict(list)
|
||||
result: dict[str, list[ExpRange]] = defaultdict(list)
|
||||
for entry in self.iter_active():
|
||||
if baseline_revs and entry.baseline_rev not in baseline_revs:
|
||||
continue
|
||||
@@ -568,7 +563,7 @@ class LocalCeleryQueue(BaseStashQueue):
|
||||
self,
|
||||
baseline_revs: Optional[Collection[str]],
|
||||
**kwargs,
|
||||
) -> Dict[str, List["ExpRange"]]:
|
||||
) -> dict[str, list["ExpRange"]]:
|
||||
from dvc.repo.experiments.collect import collect_rev
|
||||
from dvc.repo.experiments.serialize import (
|
||||
ExpExecutor,
|
||||
@@ -576,7 +571,7 @@ class LocalCeleryQueue(BaseStashQueue):
|
||||
LocalExpExecutor,
|
||||
)
|
||||
|
||||
result: Dict[str, List[ExpRange]] = defaultdict(list)
|
||||
result: dict[str, list[ExpRange]] = defaultdict(list)
|
||||
for entry in self.iter_queued():
|
||||
if baseline_revs and entry.baseline_rev not in baseline_revs:
|
||||
continue
|
||||
@@ -598,7 +593,7 @@ class LocalCeleryQueue(BaseStashQueue):
|
||||
self,
|
||||
baseline_revs: Optional[Collection[str]],
|
||||
**kwargs,
|
||||
) -> Dict[str, List["ExpRange"]]:
|
||||
) -> dict[str, list["ExpRange"]]:
|
||||
from dvc.repo.experiments.collect import collect_rev
|
||||
from dvc.repo.experiments.serialize import (
|
||||
ExpExecutor,
|
||||
@@ -607,7 +602,7 @@ class LocalCeleryQueue(BaseStashQueue):
|
||||
SerializableError,
|
||||
)
|
||||
|
||||
result: Dict[str, List[ExpRange]] = defaultdict(list)
|
||||
result: dict[str, list[ExpRange]] = defaultdict(list)
|
||||
for entry, _ in self.iter_failed():
|
||||
if baseline_revs and entry.baseline_rev not in baseline_revs:
|
||||
continue
|
||||
@@ -637,11 +632,11 @@ class LocalCeleryQueue(BaseStashQueue):
|
||||
self,
|
||||
baseline_revs: Optional[Collection[str]],
|
||||
**kwargs,
|
||||
) -> Dict[str, "ExpExecutor"]:
|
||||
) -> dict[str, "ExpExecutor"]:
|
||||
"""Map exp refs to any available successful executors."""
|
||||
from dvc.repo.experiments.serialize import ExpExecutor, LocalExpExecutor
|
||||
|
||||
result: Dict[str, "ExpExecutor"] = {}
|
||||
result: dict[str, "ExpExecutor"] = {}
|
||||
for entry, exec_result in self.iter_success():
|
||||
if baseline_revs and entry.baseline_rev not in baseline_revs:
|
||||
continue
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from typing import Collection
|
||||
from collections.abc import Collection
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
from typing import TYPE_CHECKING, Collection, Dict, Iterable, List, Set, Union
|
||||
from collections.abc import Collection, Iterable
|
||||
from typing import TYPE_CHECKING, Union
|
||||
|
||||
from dvc.repo.experiments.exceptions import UnresolvedExpNamesError
|
||||
from dvc.repo.experiments.queue.base import QueueDoneResult
|
||||
@@ -20,11 +21,11 @@ def remove_tasks( # noqa: C901, PLR0912
|
||||
"""
|
||||
from celery.result import AsyncResult
|
||||
|
||||
stash_revs: Dict[str, "ExpStashEntry"] = {}
|
||||
failed_stash_revs: List["ExpStashEntry"] = []
|
||||
done_entry_set: Set["QueueEntry"] = set()
|
||||
stash_revs: dict[str, "ExpStashEntry"] = {}
|
||||
failed_stash_revs: list["ExpStashEntry"] = []
|
||||
done_entry_set: set["QueueEntry"] = set()
|
||||
stash_rev_all = celery_queue.stash.stash_revs
|
||||
failed_rev_all: Dict[str, "ExpStashEntry"] = {}
|
||||
failed_rev_all: dict[str, "ExpStashEntry"] = {}
|
||||
if celery_queue.failed_stash:
|
||||
failed_rev_all = celery_queue.failed_stash.stash_revs
|
||||
for entry in queue_entries:
|
||||
@@ -64,7 +65,7 @@ def remove_tasks( # noqa: C901, PLR0912
|
||||
|
||||
|
||||
def _get_names(entries: Iterable[Union["QueueEntry", "QueueDoneResult"]]):
|
||||
names: List[str] = []
|
||||
names: list[str] = []
|
||||
for entry in entries:
|
||||
if isinstance(entry, QueueDoneResult):
|
||||
if entry.result and entry.result.ref_info:
|
||||
@@ -82,7 +83,7 @@ def celery_clear(
|
||||
queued: bool = False,
|
||||
failed: bool = False,
|
||||
success: bool = False,
|
||||
) -> List[str]:
|
||||
) -> list[str]:
|
||||
"""Remove entries from the queue.
|
||||
|
||||
Arguments:
|
||||
@@ -94,18 +95,18 @@ def celery_clear(
|
||||
Revisions which were removed.
|
||||
"""
|
||||
|
||||
removed: List[str] = []
|
||||
entry_list: List["QueueEntry"] = []
|
||||
removed: list[str] = []
|
||||
entry_list: list["QueueEntry"] = []
|
||||
if queued:
|
||||
queue_entries: List["QueueEntry"] = list(self.iter_queued())
|
||||
queue_entries: list["QueueEntry"] = list(self.iter_queued())
|
||||
entry_list.extend(queue_entries)
|
||||
removed.extend(_get_names(queue_entries))
|
||||
if failed:
|
||||
failed_tasks: List["QueueDoneResult"] = list(self.iter_failed())
|
||||
failed_tasks: list["QueueDoneResult"] = list(self.iter_failed())
|
||||
entry_list.extend([result.entry for result in failed_tasks])
|
||||
removed.extend(_get_names(failed_tasks))
|
||||
if success:
|
||||
success_tasks: List["QueueDoneResult"] = list(self.iter_success())
|
||||
success_tasks: list["QueueDoneResult"] = list(self.iter_success())
|
||||
entry_list.extend([result.entry for result in success_tasks])
|
||||
removed.extend(_get_names(success_tasks))
|
||||
|
||||
@@ -117,7 +118,7 @@ def celery_clear(
|
||||
def celery_remove(
|
||||
self: "LocalCeleryQueue",
|
||||
revs: Collection[str],
|
||||
) -> List[str]:
|
||||
) -> list[str]:
|
||||
"""Remove the specified entries from the queue.
|
||||
|
||||
Arguments:
|
||||
@@ -131,9 +132,9 @@ def celery_remove(
|
||||
revs, self.iter_queued(), self.iter_done()
|
||||
)
|
||||
|
||||
remained: List[str] = []
|
||||
removed: List[str] = []
|
||||
entry_to_remove: List["QueueEntry"] = []
|
||||
remained: list[str] = []
|
||||
removed: list[str] = []
|
||||
entry_to_remove: list["QueueEntry"] = []
|
||||
for name, entry in match_results.items():
|
||||
if entry:
|
||||
entry_to_remove.append(entry)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional
|
||||
from typing import TYPE_CHECKING, Any, Optional
|
||||
|
||||
from celery import shared_task
|
||||
from celery.utils.log import get_task_logger
|
||||
@@ -16,7 +16,7 @@ logger = get_task_logger(__name__)
|
||||
|
||||
|
||||
@shared_task
|
||||
def setup_exp(entry_dict: Dict[str, Any]) -> "BaseExecutor":
|
||||
def setup_exp(entry_dict: dict[str, Any]) -> "BaseExecutor":
|
||||
"""Setup an experiment.
|
||||
|
||||
Arguments:
|
||||
@@ -44,8 +44,8 @@ def setup_exp(entry_dict: Dict[str, Any]) -> "BaseExecutor":
|
||||
|
||||
@shared_task
|
||||
def collect_exp(
|
||||
proc_dict: Dict[str, Any], # noqa: ARG001
|
||||
entry_dict: Dict[str, Any],
|
||||
proc_dict: dict[str, Any], # noqa: ARG001
|
||||
entry_dict: dict[str, Any],
|
||||
) -> str:
|
||||
"""Collect results for an experiment.
|
||||
|
||||
@@ -92,8 +92,8 @@ def cleanup_exp(executor: TempDirExecutor, infofile: str) -> None:
|
||||
|
||||
@shared_task
|
||||
def run_exp(
|
||||
entry_dict: Dict[str, Any],
|
||||
copy_paths: Optional[List[str]] = None,
|
||||
entry_dict: dict[str, Any],
|
||||
copy_paths: Optional[list[str]] = None,
|
||||
message: Optional[str] = None,
|
||||
) -> None:
|
||||
"""Run a full experiment.
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import os
|
||||
from collections import defaultdict
|
||||
from typing import TYPE_CHECKING, Collection, Dict, Generator, List, Optional
|
||||
from collections.abc import Collection, Generator
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from funcy import first
|
||||
|
||||
@@ -96,11 +97,11 @@ class TempDirQueue(WorkspaceQueue):
|
||||
self,
|
||||
entry: QueueEntry,
|
||||
executor: "BaseExecutor",
|
||||
copy_paths: Optional[List[str]] = None,
|
||||
copy_paths: Optional[list[str]] = None,
|
||||
message: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> Dict[str, Dict[str, str]]:
|
||||
results: Dict[str, Dict[str, str]] = defaultdict(dict)
|
||||
) -> dict[str, dict[str, str]]:
|
||||
results: dict[str, dict[str, str]] = defaultdict(dict)
|
||||
exec_name = self._EXEC_NAME or entry.stash_rev
|
||||
infofile = self.get_infofile_path(exec_name)
|
||||
try:
|
||||
@@ -135,7 +136,7 @@ class TempDirQueue(WorkspaceQueue):
|
||||
exp: "Experiments",
|
||||
executor: "BaseExecutor",
|
||||
exec_result: "ExecutorResult",
|
||||
) -> Dict[str, str]:
|
||||
) -> dict[str, str]:
|
||||
return BaseStashQueue.collect_executor(exp, executor, exec_result)
|
||||
|
||||
def collect_active_data(
|
||||
@@ -143,7 +144,7 @@ class TempDirQueue(WorkspaceQueue):
|
||||
baseline_revs: Optional[Collection[str]],
|
||||
fetch_refs: bool = False,
|
||||
**kwargs,
|
||||
) -> Dict[str, List["ExpRange"]]:
|
||||
) -> dict[str, list["ExpRange"]]:
|
||||
from dvc.repo import Repo
|
||||
from dvc.repo.experiments.collect import collect_exec_branch
|
||||
from dvc.repo.experiments.serialize import (
|
||||
@@ -152,7 +153,7 @@ class TempDirQueue(WorkspaceQueue):
|
||||
LocalExpExecutor,
|
||||
)
|
||||
|
||||
result: Dict[str, List[ExpRange]] = defaultdict(list)
|
||||
result: dict[str, list[ExpRange]] = defaultdict(list)
|
||||
for entry in self.iter_active():
|
||||
if baseline_revs and entry.baseline_rev not in baseline_revs:
|
||||
continue
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from typing import TYPE_CHECKING, Dict, List
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from scmrepo.exceptions import SCMError
|
||||
|
||||
@@ -16,7 +16,7 @@ if TYPE_CHECKING:
|
||||
from .base import BaseStashQueue
|
||||
|
||||
|
||||
def get_remote_executor_refs(scm: "Git", remote_url: str) -> List[str]:
|
||||
def get_remote_executor_refs(scm: "Git", remote_url: str) -> list[str]:
|
||||
"""Get result list refs from a remote repository
|
||||
|
||||
Args:
|
||||
@@ -35,7 +35,7 @@ def get_remote_executor_refs(scm: "Git", remote_url: str) -> List[str]:
|
||||
|
||||
def fetch_running_exp_from_temp_dir(
|
||||
queue: "BaseStashQueue", rev: str, fetch_refs: bool
|
||||
) -> Dict[str, Dict]:
|
||||
) -> dict[str, dict]:
|
||||
"""Fetch status of running exps out of current working directory
|
||||
|
||||
Args:
|
||||
@@ -50,7 +50,7 @@ def fetch_running_exp_from_temp_dir(
|
||||
from dvc.scm import InvalidRemoteSCMRepo
|
||||
from dvc.utils.serialize import load_json
|
||||
|
||||
result: Dict[str, Dict] = {}
|
||||
result: dict[str, dict] = {}
|
||||
infofile = queue.get_infofile_path(rev)
|
||||
try:
|
||||
info = ExecutorInfo.from_dict(load_json(infofile))
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import json
|
||||
import os
|
||||
from collections import defaultdict
|
||||
from typing import TYPE_CHECKING, Collection, Dict, Generator, List, Optional
|
||||
from collections.abc import Collection, Generator
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
import psutil
|
||||
from funcy import first
|
||||
@@ -82,9 +83,9 @@ class WorkspaceQueue(BaseStashQueue):
|
||||
raise NotImplementedError
|
||||
|
||||
def reproduce(
|
||||
self, copy_paths: Optional[List[str]] = None, message: Optional[str] = None
|
||||
) -> Dict[str, Dict[str, str]]:
|
||||
results: Dict[str, Dict[str, str]] = defaultdict(dict)
|
||||
self, copy_paths: Optional[list[str]] = None, message: Optional[str] = None
|
||||
) -> dict[str, dict[str, str]]:
|
||||
results: dict[str, dict[str, str]] = defaultdict(dict)
|
||||
try:
|
||||
while True:
|
||||
entry, executor = self.get()
|
||||
@@ -99,11 +100,11 @@ class WorkspaceQueue(BaseStashQueue):
|
||||
|
||||
def _reproduce_entry(
|
||||
self, entry: QueueEntry, executor: "BaseExecutor", **kwargs
|
||||
) -> Dict[str, Dict[str, str]]:
|
||||
) -> dict[str, dict[str, str]]:
|
||||
kwargs.pop("copy_paths", None)
|
||||
from dvc_task.proc.process import ProcessInfo
|
||||
|
||||
results: Dict[str, Dict[str, str]] = defaultdict(dict)
|
||||
results: dict[str, dict[str, str]] = defaultdict(dict)
|
||||
exec_name = self._EXEC_NAME or entry.stash_rev
|
||||
proc_info = ProcessInfo(os.getpid(), None, None, None, None)
|
||||
proc_info_path = self._proc_info_path(exec_name)
|
||||
@@ -162,8 +163,8 @@ class WorkspaceQueue(BaseStashQueue):
|
||||
exp: "Experiments",
|
||||
executor: "BaseExecutor", # noqa: ARG004
|
||||
exec_result: "ExecutorResult",
|
||||
) -> Dict[str, str]:
|
||||
results: Dict[str, str] = {}
|
||||
) -> dict[str, str]:
|
||||
results: dict[str, str] = {}
|
||||
exp_rev = exp.scm.get_ref(EXEC_BRANCH)
|
||||
if exp_rev:
|
||||
assert exec_result.exp_hash
|
||||
@@ -207,7 +208,7 @@ class WorkspaceQueue(BaseStashQueue):
|
||||
baseline_revs: Optional[Collection[str]],
|
||||
fetch_refs: bool = False, # noqa: ARG002
|
||||
**kwargs,
|
||||
) -> Dict[str, List["ExpRange"]]:
|
||||
) -> dict[str, list["ExpRange"]]:
|
||||
from dvc.repo.experiments.collect import collect_exec_branch
|
||||
from dvc.repo.experiments.serialize import (
|
||||
ExpExecutor,
|
||||
@@ -215,7 +216,7 @@ class WorkspaceQueue(BaseStashQueue):
|
||||
LocalExpExecutor,
|
||||
)
|
||||
|
||||
result: Dict[str, List[ExpRange]] = defaultdict(list)
|
||||
result: dict[str, list[ExpRange]] = defaultdict(list)
|
||||
pid = self._active_pid
|
||||
if pid is None:
|
||||
return result
|
||||
@@ -248,12 +249,12 @@ class WorkspaceQueue(BaseStashQueue):
|
||||
self,
|
||||
baseline_revs: Optional[Collection[str]],
|
||||
**kwargs,
|
||||
) -> Dict[str, List["ExpRange"]]:
|
||||
) -> dict[str, list["ExpRange"]]:
|
||||
raise NotImplementedError
|
||||
|
||||
def collect_failed_data(
|
||||
self,
|
||||
baseline_revs: Optional[Collection[str]],
|
||||
**kwargs,
|
||||
) -> Dict[str, List["ExpRange"]]:
|
||||
) -> dict[str, list["ExpRange"]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Union
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING, Optional, Union
|
||||
|
||||
from dvc.log import logger
|
||||
from dvc.repo import locked
|
||||
@@ -23,14 +24,14 @@ logger = logger.getChild(__name__)
|
||||
@scm_context
|
||||
def remove( # noqa: C901, PLR0912
|
||||
repo: "Repo",
|
||||
exp_names: Union[None, str, List[str]] = None,
|
||||
rev: Optional[Union[List[str], str]] = None,
|
||||
exp_names: Union[None, str, list[str]] = None,
|
||||
rev: Optional[Union[list[str], str]] = None,
|
||||
all_commits: bool = False,
|
||||
num: int = 1,
|
||||
queue: bool = False,
|
||||
git_remote: Optional[str] = None,
|
||||
) -> List[str]:
|
||||
removed: List[str] = []
|
||||
) -> list[str]:
|
||||
removed: list[str] = []
|
||||
if not any([exp_names, queue, all_commits, rev]):
|
||||
return removed
|
||||
celery_queue: "LocalCeleryQueue" = repo.experiments.celery_queue
|
||||
@@ -40,13 +41,13 @@ def remove( # noqa: C901, PLR0912
|
||||
|
||||
assert isinstance(repo.scm, Git)
|
||||
|
||||
exp_ref_list: List["ExpRefInfo"] = []
|
||||
queue_entry_list: List["QueueEntry"] = []
|
||||
exp_ref_list: list["ExpRefInfo"] = []
|
||||
queue_entry_list: list["QueueEntry"] = []
|
||||
if exp_names:
|
||||
results: Dict[
|
||||
results: dict[
|
||||
str, "ExpRefAndQueueEntry"
|
||||
] = celery_queue.get_ref_and_entry_by_names(exp_names, git_remote)
|
||||
remained: List[str] = []
|
||||
remained: list[str] = []
|
||||
for name, result in results.items():
|
||||
if not result.exp_ref_info and not result.queue_entry:
|
||||
remained.append(name)
|
||||
@@ -87,13 +88,13 @@ def remove( # noqa: C901, PLR0912
|
||||
|
||||
def _resolve_exp_by_baseline(
|
||||
repo: "Repo",
|
||||
rev: List[str],
|
||||
rev: list[str],
|
||||
num: int,
|
||||
git_remote: Optional[str] = None,
|
||||
) -> Dict[str, "ExpRefInfo"]:
|
||||
) -> dict[str, "ExpRefInfo"]:
|
||||
assert isinstance(repo.scm, Git)
|
||||
|
||||
commit_ref_dict: Dict[str, "ExpRefInfo"] = {}
|
||||
commit_ref_dict: dict[str, "ExpRefInfo"] = {}
|
||||
rev_dict = iter_revs(repo.scm, rev, num)
|
||||
rev_set = set(rev_dict.keys())
|
||||
ref_info_dict = exp_refs_by_baseline(repo.scm, rev_set, git_remote)
|
||||
@@ -105,7 +106,7 @@ def _resolve_exp_by_baseline(
|
||||
|
||||
def _remove_commited_exps(
|
||||
scm: "Git", exp_refs_list: Iterable["ExpRefInfo"], remote: Optional[str]
|
||||
) -> List[str]:
|
||||
) -> list[str]:
|
||||
if remote:
|
||||
from dvc.scm import TqdmGit
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from typing import TYPE_CHECKING, Dict, List, Optional, Union
|
||||
from typing import TYPE_CHECKING, Optional, Union
|
||||
|
||||
from dvc.log import logger
|
||||
from dvc.repo.experiments.exceptions import (
|
||||
@@ -22,16 +22,16 @@ def rename(
|
||||
exp_name: Union[str, None] = None,
|
||||
git_remote: Optional[str] = None,
|
||||
force: bool = False,
|
||||
) -> Union[List[str], None]:
|
||||
renamed: List[str] = []
|
||||
remained: List[str] = []
|
||||
) -> Union[list[str], None]:
|
||||
renamed: list[str] = []
|
||||
remained: list[str] = []
|
||||
assert isinstance(repo.scm, Git)
|
||||
|
||||
if exp_name == new_name:
|
||||
return None
|
||||
|
||||
if exp_name:
|
||||
results: Dict[str, Union[ExpRefInfo, None]] = resolve_name(
|
||||
results: dict[str, Union[ExpRefInfo, None]] = resolve_name(
|
||||
scm=repo.scm, exp_names=exp_name, git_remote=git_remote
|
||||
)
|
||||
for name, result in results.items():
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
from typing import Dict, Iterable, Optional
|
||||
from collections.abc import Iterable
|
||||
from typing import Optional
|
||||
|
||||
from dvc.dependency.param import ParamsDependency
|
||||
from dvc.exceptions import InvalidArgumentError
|
||||
@@ -22,7 +23,7 @@ def run( # noqa: C901, PLR0912
|
||||
copy_paths: Optional[Iterable[str]] = None,
|
||||
message: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> Dict[str, str]:
|
||||
) -> dict[str, str]:
|
||||
"""Reproduce the specified targets as an experiment.
|
||||
|
||||
Accepts the same additional kwargs as Repo.reproduce.
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import os
|
||||
from typing import TYPE_CHECKING, Iterable, List, Optional
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from funcy import first
|
||||
|
||||
@@ -18,7 +19,7 @@ def save(
|
||||
name: Optional[str] = None,
|
||||
recursive: bool = False,
|
||||
force: bool = False,
|
||||
include_untracked: Optional[List[str]] = None,
|
||||
include_untracked: Optional[list[str]] = None,
|
||||
message: Optional[str] = None,
|
||||
) -> Optional[str]:
|
||||
"""Save the current workspace status as an experiment.
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import json
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Literal, Optional
|
||||
from typing import TYPE_CHECKING, Any, Literal, Optional
|
||||
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.repo.metrics.show import _gather_metrics
|
||||
@@ -30,11 +31,11 @@ class SerializableExp:
|
||||
|
||||
rev: str
|
||||
timestamp: Optional[datetime] = None
|
||||
params: Dict[str, "FileResult"] = field(default_factory=dict)
|
||||
metrics: Dict[str, "FileResult"] = field(default_factory=dict)
|
||||
deps: Dict[str, "ExpDep"] = field(default_factory=dict)
|
||||
outs: Dict[str, "ExpOut"] = field(default_factory=dict)
|
||||
meta: Dict[str, Any] = field(default_factory=dict)
|
||||
params: dict[str, "FileResult"] = field(default_factory=dict)
|
||||
metrics: dict[str, "FileResult"] = field(default_factory=dict)
|
||||
deps: dict[str, "ExpDep"] = field(default_factory=dict)
|
||||
outs: dict[str, "ExpOut"] = field(default_factory=dict)
|
||||
meta: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
@classmethod
|
||||
def from_repo(
|
||||
@@ -83,7 +84,7 @@ class SerializableExp:
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def dumpd(self) -> Dict[str, Any]:
|
||||
def dumpd(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
def as_bytes(self) -> bytes:
|
||||
@@ -131,7 +132,7 @@ class SerializableError:
|
||||
msg: str
|
||||
type: str = ""
|
||||
|
||||
def dumpd(self) -> Dict[str, Any]:
|
||||
def dumpd(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
def as_bytes(self) -> bytes:
|
||||
@@ -154,15 +155,15 @@ class ExpState:
|
||||
name: Optional[str] = None
|
||||
data: Optional[SerializableExp] = None
|
||||
error: Optional[SerializableError] = None
|
||||
experiments: Optional[List["ExpRange"]] = None
|
||||
experiments: Optional[list["ExpRange"]] = None
|
||||
|
||||
def dumpd(self) -> Dict[str, Any]:
|
||||
def dumpd(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExpRange:
|
||||
revs: List["ExpState"]
|
||||
revs: list["ExpState"]
|
||||
executor: Optional["ExpExecutor"] = None
|
||||
name: Optional[str] = None
|
||||
|
||||
@@ -175,7 +176,7 @@ class ExpRange:
|
||||
def __getitem__(self, index: int) -> "ExpState":
|
||||
return self.revs[index]
|
||||
|
||||
def dumpd(self) -> Dict[str, Any]:
|
||||
def dumpd(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
|
||||
@@ -1,18 +1,12 @@
|
||||
from collections import Counter, defaultdict
|
||||
from collections.abc import Iterable, Iterator, Mapping
|
||||
from datetime import date, datetime
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Dict,
|
||||
Iterable,
|
||||
Iterator,
|
||||
List,
|
||||
Literal,
|
||||
Mapping,
|
||||
NamedTuple,
|
||||
Optional,
|
||||
Set,
|
||||
Tuple,
|
||||
Union,
|
||||
)
|
||||
|
||||
@@ -36,7 +30,7 @@ logger = logger.getChild(__name__)
|
||||
|
||||
def show(
|
||||
repo: "Repo",
|
||||
revs: Union[List[str], str, None] = None,
|
||||
revs: Union[list[str], str, None] = None,
|
||||
all_branches: bool = False,
|
||||
all_tags: bool = False,
|
||||
all_commits: bool = False,
|
||||
@@ -45,7 +39,7 @@ def show(
|
||||
hide_failed: bool = False,
|
||||
sha_only: bool = False,
|
||||
**kwargs,
|
||||
) -> List["ExpState"]:
|
||||
) -> list["ExpState"]:
|
||||
return collect(
|
||||
repo,
|
||||
revs=revs,
|
||||
@@ -65,7 +59,7 @@ def tabulate(
|
||||
fill_value: Optional[str] = "-",
|
||||
error_value: str = "!",
|
||||
**kwargs,
|
||||
) -> Tuple["TabularData", Dict[str, Iterable[str]]]:
|
||||
) -> tuple["TabularData", dict[str, Iterable[str]]]:
|
||||
"""Return table data for experiments.
|
||||
|
||||
Returns:
|
||||
@@ -112,7 +106,7 @@ def tabulate(
|
||||
**kwargs,
|
||||
)
|
||||
)
|
||||
data_headers: Dict[str, Iterable[str]] = {
|
||||
data_headers: dict[str, Iterable[str]] = {
|
||||
"metrics": metrics_headers,
|
||||
"params": params_headers,
|
||||
"deps": deps_names,
|
||||
@@ -128,9 +122,9 @@ def _build_rows(
|
||||
sort_by: Optional[str] = None,
|
||||
sort_order: Optional[Literal["asc", "desc"]] = None,
|
||||
**kwargs,
|
||||
) -> Iterator[Tuple["CellT", ...]]:
|
||||
) -> Iterator[tuple["CellT", ...]]:
|
||||
for baseline in baseline_states:
|
||||
row: Dict[str, "CellT"] = dict.fromkeys(all_headers, fill_value)
|
||||
row: dict[str, "CellT"] = dict.fromkeys(all_headers, fill_value)
|
||||
row["Experiment"] = ""
|
||||
if baseline.name:
|
||||
row["rev"] = baseline.name
|
||||
@@ -177,10 +171,10 @@ def _sort_column( # noqa: C901
|
||||
sort_by: str,
|
||||
metric_names: Mapping[str, Iterable[str]],
|
||||
param_names: Mapping[str, Iterable[str]],
|
||||
) -> Tuple[str, str, str]:
|
||||
) -> tuple[str, str, str]:
|
||||
sep = ":"
|
||||
parts = sort_by.split(sep)
|
||||
matches: Set[Tuple[str, str, str]] = set()
|
||||
matches: set[tuple[str, str, str]] = set()
|
||||
|
||||
for split_num in range(len(parts)):
|
||||
path = sep.join(parts[:split_num])
|
||||
@@ -217,7 +211,7 @@ def _sort_exp(
|
||||
sort_name: str,
|
||||
typ: str,
|
||||
reverse: bool,
|
||||
) -> List["ExpRange"]:
|
||||
) -> list["ExpRange"]:
|
||||
from funcy import first
|
||||
|
||||
def _sort(exp_range: "ExpRange"):
|
||||
@@ -238,14 +232,14 @@ def _exp_range_rows(
|
||||
fill_value: Optional[str],
|
||||
is_base: bool = False,
|
||||
**kwargs,
|
||||
) -> Iterator[Tuple["CellT", ...]]:
|
||||
) -> Iterator[tuple["CellT", ...]]:
|
||||
from funcy import first
|
||||
|
||||
if len(exp_range.revs) > 1:
|
||||
logger.debug("Returning tip commit for legacy checkpoint exp")
|
||||
exp = first(exp_range.revs)
|
||||
if exp:
|
||||
row: Dict[str, "CellT"] = dict.fromkeys(all_headers, fill_value)
|
||||
row: dict[str, "CellT"] = dict.fromkeys(all_headers, fill_value)
|
||||
row["Experiment"] = exp.name or ""
|
||||
row["rev"] = exp.rev[:7] if Git.is_sha(exp.rev) else exp.rev
|
||||
row["typ"] = "branch_base" if is_base else "branch_commit"
|
||||
@@ -274,12 +268,12 @@ def _data_cells(
|
||||
error_value: str = "!",
|
||||
precision: Optional[int] = None,
|
||||
**kwargs,
|
||||
) -> Iterator[Tuple[str, "CellT"]]:
|
||||
) -> Iterator[tuple[str, "CellT"]]:
|
||||
def _d_cells(
|
||||
d: Mapping[str, Any],
|
||||
names: Mapping[str, Iterable[str]],
|
||||
headers: Iterable[str],
|
||||
) -> Iterator[Tuple[str, "CellT"]]:
|
||||
) -> Iterator[tuple[str, "CellT"]]:
|
||||
from dvc.compare import _format_field, with_value
|
||||
|
||||
for fname, data in d.items():
|
||||
@@ -329,9 +323,9 @@ def format_time(
|
||||
class _DataNames(NamedTuple):
|
||||
# NOTE: we use nested dict instead of set for metrics/params names to
|
||||
# preserve key ordering
|
||||
metrics: Dict[str, Dict[str, Any]]
|
||||
params: Dict[str, Dict[str, Any]]
|
||||
deps: Set[str]
|
||||
metrics: dict[str, dict[str, Any]]
|
||||
params: dict[str, dict[str, Any]]
|
||||
deps: set[str]
|
||||
|
||||
@property
|
||||
def sorted_deps(self):
|
||||
@@ -339,7 +333,7 @@ class _DataNames(NamedTuple):
|
||||
|
||||
def update(self, other: "_DataNames"):
|
||||
def _update_d(
|
||||
d: Dict[str, Dict[str, Any]], other_d: Mapping[str, Mapping[str, Any]]
|
||||
d: dict[str, dict[str, Any]], other_d: Mapping[str, Mapping[str, Any]]
|
||||
):
|
||||
for k, v in other_d.items():
|
||||
if k in d:
|
||||
@@ -355,7 +349,7 @@ class _DataNames(NamedTuple):
|
||||
def _collect_names(exp_states: Iterable["ExpState"]) -> _DataNames:
|
||||
result = _DataNames(defaultdict(dict), defaultdict(dict), set())
|
||||
|
||||
def _collect_d(result_d: Dict[str, Dict[str, Any]], data_d: Dict[str, Any]):
|
||||
def _collect_d(result_d: dict[str, dict[str, Any]], data_d: dict[str, Any]):
|
||||
for path, item in data_d.items():
|
||||
item = item.get("data", {})
|
||||
if isinstance(item, dict):
|
||||
@@ -376,7 +370,7 @@ def _collect_names(exp_states: Iterable["ExpState"]) -> _DataNames:
|
||||
|
||||
def _normalize_headers(
|
||||
names: Mapping[str, Mapping[str, Any]], count: Mapping[str, int]
|
||||
) -> List[str]:
|
||||
) -> list[str]:
|
||||
return [
|
||||
name if count[name] == 1 else f"{path}:{name}"
|
||||
for path in names
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import re
|
||||
from collections.abc import Iterable, Iterator
|
||||
from contextlib import contextmanager
|
||||
from typing import Dict, Iterable, Iterator, NamedTuple, Optional
|
||||
from typing import NamedTuple, Optional
|
||||
|
||||
from scmrepo.git import Stash
|
||||
|
||||
@@ -42,7 +43,7 @@ class ExpStash(Stash):
|
||||
)
|
||||
|
||||
@property
|
||||
def stash_revs(self) -> Dict[str, ExpStashEntry]:
|
||||
def stash_revs(self) -> dict[str, ExpStashEntry]:
|
||||
revs = {}
|
||||
for i, entry in enumerate(self):
|
||||
msg = entry.message.decode("utf-8").strip()
|
||||
@@ -110,7 +111,7 @@ class ApplyStash(Stash):
|
||||
)
|
||||
|
||||
@property
|
||||
def stash_revs(self) -> Dict[str, ApplyStashEntry]:
|
||||
def stash_revs(self) -> dict[str, ApplyStashEntry]:
|
||||
revs = {}
|
||||
for i, entry in enumerate(self):
|
||||
msg = entry.message.decode("utf-8").strip()
|
||||
|
||||
@@ -2,18 +2,12 @@ import os
|
||||
import random
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from collections.abc import Generator, Iterable, Mapping
|
||||
from functools import wraps
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Callable,
|
||||
Dict,
|
||||
Generator,
|
||||
Iterable,
|
||||
List,
|
||||
Mapping,
|
||||
Optional,
|
||||
Set,
|
||||
Tuple,
|
||||
Union,
|
||||
)
|
||||
|
||||
@@ -43,8 +37,8 @@ EXEC_PID_DIR = "run"
|
||||
|
||||
def get_exp_rwlock(
|
||||
repo: "Repo",
|
||||
reads: Optional[List[str]] = None,
|
||||
writes: Optional[List[str]] = None,
|
||||
reads: Optional[list[str]] = None,
|
||||
writes: Optional[list[str]] = None,
|
||||
):
|
||||
reads = reads or []
|
||||
writes = writes or []
|
||||
@@ -109,9 +103,9 @@ def exp_refs_by_rev(scm: "Git", rev: str) -> Generator[ExpRefInfo, None, None]:
|
||||
|
||||
def exp_refs_by_baseline(
|
||||
scm: "Git",
|
||||
revs: Optional[Set[str]] = None,
|
||||
revs: Optional[set[str]] = None,
|
||||
url: Optional[str] = None,
|
||||
) -> Mapping[str, List[ExpRefInfo]]:
|
||||
) -> Mapping[str, list[ExpRefInfo]]:
|
||||
"""Iterate over all experiment refs with the specified baseline."""
|
||||
all_exp_refs = exp_refs(scm, url)
|
||||
result = defaultdict(list)
|
||||
@@ -137,7 +131,7 @@ def iter_remote_refs(scm: "Git", url: str, base: Optional[str] = None, **kwargs)
|
||||
def push_refspec(
|
||||
scm: "Git",
|
||||
url: str,
|
||||
push_list=List[Tuple[Optional[str], str]],
|
||||
push_list=list[tuple[Optional[str], str]],
|
||||
force: bool = False,
|
||||
on_diverged: Optional[Callable[[str, str], bool]] = None,
|
||||
**kwargs,
|
||||
@@ -183,8 +177,8 @@ def remote_exp_refs(scm: "Git", url: str) -> Generator[ExpRefInfo, None, None]:
|
||||
|
||||
|
||||
def exp_refs_by_names(
|
||||
scm: "Git", names: Set[str], url: Optional[str] = None
|
||||
) -> Dict[str, List[ExpRefInfo]]:
|
||||
scm: "Git", names: set[str], url: Optional[str] = None
|
||||
) -> dict[str, list[ExpRefInfo]]:
|
||||
"""Iterate over all experiment refs matching the specified names."""
|
||||
resolve_results = defaultdict(list)
|
||||
ref_info_gen = exp_refs(scm, url)
|
||||
@@ -210,7 +204,7 @@ def exp_commits(
|
||||
scm: "Git", ref_infos: Optional[Iterable[ExpRefInfo]] = None
|
||||
) -> Iterable[str]:
|
||||
"""Iterate over all experiment commits."""
|
||||
shas: Set["str"] = set()
|
||||
shas: set["str"] = set()
|
||||
refs = ref_infos if ref_infos else exp_refs(scm)
|
||||
for ref_info in refs:
|
||||
shas.update(scm.branch_revs(str(ref_info), ref_info.baseline_sha))
|
||||
@@ -242,7 +236,7 @@ def resolve_name(
|
||||
scm: "Git",
|
||||
exp_names: Union[Iterable[str], str],
|
||||
git_remote: Optional[str] = None,
|
||||
) -> Dict[str, Optional[ExpRefInfo]]:
|
||||
) -> dict[str, Optional[ExpRefInfo]]:
|
||||
"""find the ref_info of specified names."""
|
||||
if isinstance(exp_names, str):
|
||||
exp_names = [exp_names]
|
||||
@@ -338,7 +332,7 @@ def to_studio_params(dvc_params):
|
||||
"params.yaml": {"foo": 1}
|
||||
}
|
||||
"""
|
||||
result: Dict = {}
|
||||
result: dict = {}
|
||||
if not dvc_params:
|
||||
return result
|
||||
for rev_data in dvc_params.values():
|
||||
@@ -353,7 +347,7 @@ def describe(
|
||||
revs: Iterable[str],
|
||||
logger,
|
||||
refs: Optional[Iterable[str]] = None,
|
||||
) -> Dict[str, Optional[str]]:
|
||||
) -> dict[str, Optional[str]]:
|
||||
"""Describe revisions using a tag, branch.
|
||||
|
||||
The first matching name will be returned for each rev. Names are preferred in this
|
||||
@@ -390,7 +384,7 @@ def describe(
|
||||
if is_branch and rev not in branches:
|
||||
branches[rev] = ref[len("refs/heads/") :]
|
||||
|
||||
names: Dict[str, Optional[str]] = {}
|
||||
names: dict[str, Optional[str]] = {}
|
||||
for rev in revs:
|
||||
if rev == head_rev and head_branch:
|
||||
names[rev] = head_branch
|
||||
|
||||
+4
-4
@@ -1,4 +1,4 @@
|
||||
from typing import TYPE_CHECKING, List, Tuple
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from dvc.exceptions import DownloadError
|
||||
from dvc.log import logger
|
||||
@@ -205,9 +205,9 @@ def fetch( # noqa: PLR0913
|
||||
return transferred_count
|
||||
|
||||
|
||||
def _log_unversioned(data: List["DataIndex"]) -> Tuple[List["DataIndex"], int]:
|
||||
ret: List["DataIndex"] = []
|
||||
unversioned: List[str] = []
|
||||
def _log_unversioned(data: list["DataIndex"]) -> tuple[list["DataIndex"], int]:
|
||||
ret: list["DataIndex"] = []
|
||||
unversioned: list[str] = []
|
||||
for fs_index in data:
|
||||
remote = fs_index.storage_map[()].remote
|
||||
if not isinstance(remote, FileStorage) or not remote.fs.version_aware:
|
||||
|
||||
+2
-2
@@ -1,4 +1,4 @@
|
||||
from typing import TYPE_CHECKING, List, Optional
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from dvc.exceptions import InvalidArgumentError
|
||||
from dvc.log import logger
|
||||
@@ -61,7 +61,7 @@ def gc( # noqa: PLR0913, C901
|
||||
all_experiments: bool = False,
|
||||
force: bool = False,
|
||||
jobs: Optional[int] = None,
|
||||
repos: Optional[List[str]] = None,
|
||||
repos: Optional[list[str]] = None,
|
||||
workspace: bool = False,
|
||||
commit_date: Optional[str] = None,
|
||||
rev: Optional[str] = None,
|
||||
|
||||
+5
-4
@@ -1,4 +1,5 @@
|
||||
from typing import TYPE_CHECKING, Any, Iterator, List, Optional, Set, TypeVar
|
||||
from collections.abc import Iterator
|
||||
from typing import TYPE_CHECKING, Any, Optional, TypeVar
|
||||
|
||||
from dvc.fs import localfs
|
||||
from dvc.utils.fs import path_isin
|
||||
@@ -21,7 +22,7 @@ def check_acyclic(graph: "DiGraph") -> None:
|
||||
except nx.NetworkXNoCycle:
|
||||
return
|
||||
|
||||
stages: Set["Stage"] = set()
|
||||
stages: set["Stage"] = set()
|
||||
for from_node, to_node, _ in edges:
|
||||
stages.add(from_node)
|
||||
stages.add(to_node)
|
||||
@@ -45,7 +46,7 @@ def get_pipelines(graph: "DiGraph"):
|
||||
|
||||
|
||||
def get_subgraph_of_nodes(
|
||||
graph: "DiGraph", sources: Optional[List[Any]] = None, downstream: bool = False
|
||||
graph: "DiGraph", sources: Optional[list[Any]] = None, downstream: bool = False
|
||||
) -> "DiGraph":
|
||||
from networkx import dfs_postorder_nodes, reverse_view
|
||||
|
||||
@@ -69,7 +70,7 @@ def collect_pipeline(stage: "Stage", graph: "DiGraph") -> Iterator["Stage"]:
|
||||
return nx.dfs_postorder_nodes(pipeline, stage)
|
||||
|
||||
|
||||
def collect_inside_path(path: str, graph: "DiGraph") -> List["Stage"]:
|
||||
def collect_inside_path(path: str, graph: "DiGraph") -> list["Stage"]:
|
||||
import networkx as nx
|
||||
|
||||
stages = nx.dfs_postorder_nodes(graph)
|
||||
|
||||
+2
-2
@@ -1,4 +1,4 @@
|
||||
from typing import TYPE_CHECKING, Dict, Optional
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from funcy import compact
|
||||
|
||||
@@ -33,7 +33,7 @@ def imp_db(
|
||||
assert sql or table
|
||||
assert output_format in ("csv", "json")
|
||||
|
||||
db: Dict[str, str] = compact(
|
||||
db: dict[str, str] = compact(
|
||||
{
|
||||
"connection": connection,
|
||||
"file_format": output_format,
|
||||
|
||||
+5
-5
@@ -1,7 +1,7 @@
|
||||
import os
|
||||
from functools import partial
|
||||
from tempfile import TemporaryDirectory
|
||||
from typing import TYPE_CHECKING, List, Set, Tuple, Union
|
||||
from typing import TYPE_CHECKING, Union
|
||||
|
||||
from dvc.log import logger
|
||||
|
||||
@@ -19,7 +19,7 @@ logger = logger.getChild(__name__)
|
||||
|
||||
def unfetched_view(
|
||||
index: "Index", targets: "TargetType", unpartial: bool = False, **kwargs
|
||||
) -> Tuple["IndexView", "IndexView", List["Dependency"]]:
|
||||
) -> tuple["IndexView", "IndexView", list["Dependency"]]:
|
||||
"""Return index view of imports which have not been fetched.
|
||||
|
||||
Returns:
|
||||
@@ -28,7 +28,7 @@ def unfetched_view(
|
||||
"""
|
||||
from dvc.cachemgr import LEGACY_HASH_NAMES
|
||||
|
||||
changed_deps: List["Dependency"] = []
|
||||
changed_deps: list["Dependency"] = []
|
||||
|
||||
def need_fetch(stage: "Stage", legacy: bool = False) -> bool:
|
||||
if not stage.is_import or (stage.is_partial_import and not unpartial):
|
||||
@@ -92,7 +92,7 @@ def unpartial_imports(index: Union["Index", "IndexView"]) -> int:
|
||||
|
||||
def save_imports(
|
||||
repo: "Repo", targets: "TargetType", unpartial: bool = False, **kwargs
|
||||
) -> Set["HashInfo"]:
|
||||
) -> set["HashInfo"]:
|
||||
"""Save (download) imports from their original source location.
|
||||
|
||||
Imports which are already cached will not be downloaded.
|
||||
@@ -105,7 +105,7 @@ def save_imports(
|
||||
from dvc_data.index import md5, save
|
||||
from dvc_data.index.checkout import apply, compare
|
||||
|
||||
downloaded: Set["HashInfo"] = set()
|
||||
downloaded: set["HashInfo"] = set()
|
||||
|
||||
legacy_unfetched, unfetched, changed = unfetched_view(
|
||||
repo.index, targets, unpartial=unpartial, **kwargs
|
||||
|
||||
+35
-40
@@ -1,20 +1,15 @@
|
||||
import logging
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterable, Iterator
|
||||
from functools import partial
|
||||
from itertools import chain
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Callable,
|
||||
Dict,
|
||||
Iterable,
|
||||
Iterator,
|
||||
List,
|
||||
NamedTuple,
|
||||
Optional,
|
||||
Set,
|
||||
Tuple,
|
||||
Union,
|
||||
)
|
||||
|
||||
@@ -44,7 +39,7 @@ if TYPE_CHECKING:
|
||||
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
ObjectContainer = Dict[Optional["HashFileDB"], Set["HashInfo"]]
|
||||
ObjectContainer = dict[Optional["HashFileDB"], set["HashInfo"]]
|
||||
|
||||
|
||||
def log_walk(seq):
|
||||
@@ -74,7 +69,7 @@ def collect_files(
|
||||
scm = repo.scm
|
||||
fs = repo.fs
|
||||
sep = fs.sep
|
||||
outs: Set[str] = set()
|
||||
outs: set[str] = set()
|
||||
|
||||
is_local_fs = isinstance(fs, LocalFileSystem)
|
||||
|
||||
@@ -288,11 +283,11 @@ class Index:
|
||||
def __init__(
|
||||
self,
|
||||
repo: "Repo",
|
||||
stages: Optional[List["Stage"]] = None,
|
||||
metrics: Optional[Dict[str, List[str]]] = None,
|
||||
plots: Optional[Dict[str, List[str]]] = None,
|
||||
params: Optional[Dict[str, Any]] = None,
|
||||
artifacts: Optional[Dict[str, Any]] = None,
|
||||
stages: Optional[list["Stage"]] = None,
|
||||
metrics: Optional[dict[str, list[str]]] = None,
|
||||
plots: Optional[dict[str, list[str]]] = None,
|
||||
params: Optional[dict[str, Any]] = None,
|
||||
artifacts: Optional[dict[str, Any]] = None,
|
||||
) -> None:
|
||||
self.repo = repo
|
||||
self.stages = stages or []
|
||||
@@ -300,7 +295,7 @@ class Index:
|
||||
self._plots = plots or {}
|
||||
self._params = params or {}
|
||||
self._artifacts = artifacts or {}
|
||||
self._collected_targets: Dict[int, List["StageInfo"]] = {}
|
||||
self._collected_targets: dict[int, list["StageInfo"]] = {}
|
||||
|
||||
@cached_property
|
||||
def rev(self) -> Optional[str]:
|
||||
@@ -404,8 +399,8 @@ class Index:
|
||||
yield from stage.outs
|
||||
|
||||
@cached_property
|
||||
def out_data_keys(self) -> Dict[str, Set["DataIndexKey"]]:
|
||||
by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
|
||||
def out_data_keys(self) -> dict[str, set["DataIndexKey"]]:
|
||||
by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set)
|
||||
|
||||
by_workspace["repo"] = set()
|
||||
by_workspace["local"] = set()
|
||||
@@ -443,10 +438,10 @@ class Index:
|
||||
yield from stage.deps
|
||||
|
||||
@cached_property
|
||||
def _plot_sources(self) -> List[str]:
|
||||
def _plot_sources(self) -> list[str]:
|
||||
from dvc.repo.plots import _collect_pipeline_files
|
||||
|
||||
sources: List[str] = []
|
||||
sources: list[str] = []
|
||||
for data in _collect_pipeline_files(self.repo, [], {}).values():
|
||||
for plot_id, props in data.get("data", {}).items():
|
||||
if isinstance(props.get("y"), dict):
|
||||
@@ -458,8 +453,8 @@ class Index:
|
||||
return sources
|
||||
|
||||
@cached_property
|
||||
def data_keys(self) -> Dict[str, Set["DataIndexKey"]]:
|
||||
by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
|
||||
def data_keys(self) -> dict[str, set["DataIndexKey"]]:
|
||||
by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set)
|
||||
|
||||
by_workspace["repo"] = set()
|
||||
by_workspace["local"] = set()
|
||||
@@ -474,10 +469,10 @@ class Index:
|
||||
return dict(by_workspace)
|
||||
|
||||
@cached_property
|
||||
def metric_keys(self) -> Dict[str, Set["DataIndexKey"]]:
|
||||
def metric_keys(self) -> dict[str, set["DataIndexKey"]]:
|
||||
from .metrics.show import _collect_top_level_metrics
|
||||
|
||||
by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
|
||||
by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set)
|
||||
|
||||
by_workspace["repo"] = set()
|
||||
|
||||
@@ -495,10 +490,10 @@ class Index:
|
||||
return dict(by_workspace)
|
||||
|
||||
@cached_property
|
||||
def param_keys(self) -> Dict[str, Set["DataIndexKey"]]:
|
||||
def param_keys(self) -> dict[str, set["DataIndexKey"]]:
|
||||
from .params.show import _collect_top_level_params
|
||||
|
||||
by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
|
||||
by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set)
|
||||
by_workspace["repo"] = set()
|
||||
|
||||
param_paths = _collect_top_level_params(self.repo)
|
||||
@@ -513,8 +508,8 @@ class Index:
|
||||
return dict(by_workspace)
|
||||
|
||||
@cached_property
|
||||
def plot_keys(self) -> Dict[str, Set["DataIndexKey"]]:
|
||||
by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
|
||||
def plot_keys(self) -> dict[str, set["DataIndexKey"]]:
|
||||
by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set)
|
||||
|
||||
by_workspace["repo"] = set()
|
||||
|
||||
@@ -536,7 +531,7 @@ class Index:
|
||||
return _build_tree_from_outs(self.outs)
|
||||
|
||||
@cached_property
|
||||
def data(self) -> "Dict[str, DataIndex]":
|
||||
def data(self) -> "dict[str, DataIndex]":
|
||||
prefix: "DataIndexKey"
|
||||
loaded = False
|
||||
|
||||
@@ -584,7 +579,7 @@ class Index:
|
||||
|
||||
def collect_targets(
|
||||
self, targets: Optional["TargetType"], *, onerror=None, **kwargs: Any
|
||||
) -> List["StageInfo"]:
|
||||
) -> list["StageInfo"]:
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.repo.stage import StageInfo
|
||||
from dvc.utils.collections import ensure_list
|
||||
@@ -658,7 +653,7 @@ class Index:
|
||||
stage_filter: Optional[Callable[["Stage"], bool]] = None,
|
||||
outs_filter: Optional[Callable[["Output"], bool]] = None,
|
||||
max_size: Optional[int] = None,
|
||||
types: Optional[List[str]] = None,
|
||||
types: Optional[list[str]] = None,
|
||||
**kwargs: Any,
|
||||
) -> "IndexView":
|
||||
"""Return read-only view of index for the specified targets.
|
||||
@@ -698,8 +693,8 @@ class Index:
|
||||
|
||||
|
||||
class _DataPrefixes(NamedTuple):
|
||||
explicit: Set["DataIndexKey"]
|
||||
recursive: Set["DataIndexKey"]
|
||||
explicit: set["DataIndexKey"]
|
||||
recursive: set["DataIndexKey"]
|
||||
|
||||
|
||||
class IndexView:
|
||||
@@ -728,7 +723,7 @@ class IndexView:
|
||||
yield from stage.deps
|
||||
|
||||
@property
|
||||
def _filtered_outs(self) -> Iterator[Tuple["Output", Optional[str]]]:
|
||||
def _filtered_outs(self) -> Iterator[tuple["Output", Optional[str]]]:
|
||||
for stage, filter_info in self._stage_infos:
|
||||
for out in stage.filter_outs(filter_info):
|
||||
if not self._outs_filter or self._outs_filter(out):
|
||||
@@ -739,8 +734,8 @@ class IndexView:
|
||||
yield from {out for (out, _) in self._filtered_outs}
|
||||
|
||||
@cached_property
|
||||
def out_data_keys(self) -> Dict[str, Set["DataIndexKey"]]:
|
||||
by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
|
||||
def out_data_keys(self) -> dict[str, set["DataIndexKey"]]:
|
||||
by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set)
|
||||
|
||||
by_workspace["repo"] = set()
|
||||
by_workspace["local"] = set()
|
||||
@@ -755,8 +750,8 @@ class IndexView:
|
||||
return dict(by_workspace)
|
||||
|
||||
@cached_property
|
||||
def _data_prefixes(self) -> Dict[str, "_DataPrefixes"]:
|
||||
prefixes: Dict[str, "_DataPrefixes"] = defaultdict(
|
||||
def _data_prefixes(self) -> dict[str, "_DataPrefixes"]:
|
||||
prefixes: dict[str, "_DataPrefixes"] = defaultdict(
|
||||
lambda: _DataPrefixes(set(), set())
|
||||
)
|
||||
for out, filter_info in self._filtered_outs:
|
||||
@@ -772,8 +767,8 @@ class IndexView:
|
||||
return prefixes
|
||||
|
||||
@cached_property
|
||||
def data_keys(self) -> Dict[str, Set["DataIndexKey"]]:
|
||||
ret: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
|
||||
def data_keys(self) -> dict[str, set["DataIndexKey"]]:
|
||||
ret: dict[str, set["DataIndexKey"]] = defaultdict(set)
|
||||
|
||||
for out, filter_info in self._filtered_outs:
|
||||
if not out.use_cache:
|
||||
@@ -791,7 +786,7 @@ class IndexView:
|
||||
return _build_tree_from_outs(self.outs)
|
||||
|
||||
@cached_property
|
||||
def data(self) -> Dict[str, Union["DataIndex", "DataIndexView"]]:
|
||||
def data(self) -> dict[str, Union["DataIndex", "DataIndexView"]]:
|
||||
from dvc_data.index import DataIndex, view
|
||||
|
||||
def key_filter(workspace: str, key: "DataIndexKey"):
|
||||
@@ -803,7 +798,7 @@ class IndexView:
|
||||
except KeyError:
|
||||
return False
|
||||
|
||||
data: Dict[str, Union["DataIndex", "DataIndexView"]] = {}
|
||||
data: dict[str, Union["DataIndex", "DataIndexView"]] = {}
|
||||
for workspace, data_index in self._index.data.items():
|
||||
if self.stages:
|
||||
data[workspace] = view(data_index, partial(key_filter, workspace))
|
||||
|
||||
+2
-2
@@ -1,5 +1,5 @@
|
||||
import os
|
||||
from typing import TYPE_CHECKING, Any, Dict, Optional, Union
|
||||
from typing import TYPE_CHECKING, Any, Optional, Union
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.fs.dvc import DVCFileSystem
|
||||
@@ -13,7 +13,7 @@ def ls(
|
||||
rev: Optional[str] = None,
|
||||
recursive: Optional[bool] = None,
|
||||
dvc_only: bool = False,
|
||||
config: Union[None, Dict[str, Any], str] = None,
|
||||
config: Union[None, dict[str, Any], str] = None,
|
||||
remote: Optional[str] = None,
|
||||
remote_config: Optional[dict] = None,
|
||||
):
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from typing import TYPE_CHECKING, Dict, TypedDict, Union
|
||||
from typing import TYPE_CHECKING, TypedDict, Union
|
||||
|
||||
from funcy import compact
|
||||
|
||||
@@ -12,12 +12,12 @@ if TYPE_CHECKING:
|
||||
|
||||
|
||||
class DiffResult(TypedDict, total=False):
|
||||
errors: Dict[str, Union[Exception, Dict[str, Exception]]]
|
||||
diff: Dict[str, Dict[str, Dict]]
|
||||
errors: dict[str, Union[Exception, dict[str, Exception]]]
|
||||
diff: dict[str, dict[str, dict]]
|
||||
|
||||
|
||||
def _diff(
|
||||
result: Dict[str, "Result"],
|
||||
result: dict[str, "Result"],
|
||||
old_rev: str,
|
||||
new_rev: str,
|
||||
**kwargs,
|
||||
|
||||
+17
-21
@@ -1,14 +1,10 @@
|
||||
import os
|
||||
from collections.abc import Iterable, Iterator
|
||||
from itertools import chain
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Dict,
|
||||
Iterable,
|
||||
Iterator,
|
||||
List,
|
||||
Optional,
|
||||
Tuple,
|
||||
TypedDict,
|
||||
Union,
|
||||
)
|
||||
@@ -71,7 +67,7 @@ def _read_metric(fs: "FileSystem", path: str, **load_kwargs) -> Any:
|
||||
|
||||
def _read_metrics(
|
||||
fs: "FileSystem", metrics: Iterable[str], **load_kwargs
|
||||
) -> Iterator[Tuple[str, Union[Exception, Any]]]:
|
||||
) -> Iterator[tuple[str, Union[Exception, Any]]]:
|
||||
for metric in metrics:
|
||||
try:
|
||||
yield metric, _read_metric(fs, metric, **load_kwargs)
|
||||
@@ -80,7 +76,7 @@ def _read_metrics(
|
||||
yield metric, exc
|
||||
|
||||
|
||||
def metrics_from_target(repo: "Repo", targets: List[str]) -> Iterator["Output"]:
|
||||
def metrics_from_target(repo: "Repo", targets: list[str]) -> Iterator["Output"]:
|
||||
stages = chain.from_iterable(repo.stage.collect(target) for target in targets)
|
||||
for stage in stages:
|
||||
yield from stage.metrics
|
||||
@@ -88,11 +84,11 @@ def metrics_from_target(repo: "Repo", targets: List[str]) -> Iterator["Output"]:
|
||||
|
||||
def _collect_metrics(
|
||||
repo: "Repo",
|
||||
targets: Optional[List[str]] = None,
|
||||
stages: Optional[List[str]] = None,
|
||||
targets: Optional[list[str]] = None,
|
||||
stages: Optional[list[str]] = None,
|
||||
outs_only: bool = False,
|
||||
) -> List[str]:
|
||||
metrics: List[str] = []
|
||||
) -> list[str]:
|
||||
metrics: list[str] = []
|
||||
|
||||
if targets:
|
||||
# target is a repo-relative path
|
||||
@@ -122,7 +118,7 @@ class FileResult(TypedDict, total=False):
|
||||
|
||||
|
||||
class Result(TypedDict, total=False):
|
||||
data: Dict[str, FileResult]
|
||||
data: dict[str, FileResult]
|
||||
error: Exception
|
||||
|
||||
|
||||
@@ -139,11 +135,11 @@ def to_relpath(fs: "FileSystem", root_dir: str, d: Result) -> Result:
|
||||
|
||||
def _gather_metrics(
|
||||
repo: "Repo",
|
||||
targets: Optional[List[str]] = None,
|
||||
targets: Optional[list[str]] = None,
|
||||
outs_only: bool = False,
|
||||
stages: Optional[List[str]] = None,
|
||||
stages: Optional[list[str]] = None,
|
||||
on_error: str = "return",
|
||||
) -> Dict[str, FileResult]:
|
||||
) -> dict[str, FileResult]:
|
||||
assert on_error in ("raise", "return", "ignore")
|
||||
|
||||
# `files` is a repo-relative posixpath that can be passed to DVCFileSystem
|
||||
@@ -168,8 +164,8 @@ def _gather_metrics(
|
||||
|
||||
|
||||
def _hide_workspace(
|
||||
scm: Union["Git", "NoSCM"], res: Dict[str, Result]
|
||||
) -> Dict[str, Result]:
|
||||
scm: Union["Git", "NoSCM"], res: dict[str, Result]
|
||||
) -> dict[str, Result]:
|
||||
# Hide workspace params if they are the same as in the active branch
|
||||
try:
|
||||
active_branch = scm.active_branch()
|
||||
@@ -186,16 +182,16 @@ def _hide_workspace(
|
||||
|
||||
def show(
|
||||
repo: "Repo",
|
||||
targets: Optional[List[str]] = None,
|
||||
stages: Optional[List[str]] = None,
|
||||
targets: Optional[list[str]] = None,
|
||||
stages: Optional[list[str]] = None,
|
||||
outs_only: bool = False,
|
||||
all_branches: bool = False,
|
||||
all_tags: bool = False,
|
||||
revs: Optional[List[str]] = None,
|
||||
revs: Optional[list[str]] = None,
|
||||
all_commits: bool = False,
|
||||
hide_workspace: bool = True,
|
||||
on_error: str = "return",
|
||||
) -> Dict[str, Result]:
|
||||
) -> dict[str, Result]:
|
||||
assert on_error in ("raise", "return", "ignore")
|
||||
|
||||
targets = [os.path.abspath(target) for target in ensure_list(targets)]
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
from typing import TYPE_CHECKING, Dict, Optional, Tuple
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from funcy import retry, wrap_with
|
||||
|
||||
@@ -78,8 +78,8 @@ def erepo_factory(url, root_dir, cache_config):
|
||||
return make_repo
|
||||
|
||||
|
||||
CLONES: Dict[str, Tuple[str, bool]] = {}
|
||||
CACHE_DIRS: Dict[str, str] = {}
|
||||
CLONES: dict[str, tuple[str, bool]] = {}
|
||||
CACHE_DIRS: dict[str, str] = {}
|
||||
|
||||
|
||||
@wrap_with(threading.Lock())
|
||||
|
||||
+18
-17
@@ -1,7 +1,8 @@
|
||||
import os
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterator
|
||||
from itertools import chain
|
||||
from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Tuple, Union
|
||||
from typing import TYPE_CHECKING, Any, Optional, Union
|
||||
|
||||
from dvc.dependency.param import ParamsDependency, read_param_file
|
||||
from dvc.log import logger
|
||||
@@ -27,7 +28,7 @@ def _collect_top_level_params(repo: "Repo") -> Iterator[str]:
|
||||
|
||||
|
||||
def params_from_target(
|
||||
repo: "Repo", targets: List[str]
|
||||
repo: "Repo", targets: list[str]
|
||||
) -> Iterator["ParamsDependency"]:
|
||||
stages = chain.from_iterable(repo.stage.collect(target) for target in targets)
|
||||
for stage in stages:
|
||||
@@ -36,17 +37,17 @@ def params_from_target(
|
||||
|
||||
def _collect_params(
|
||||
repo: "Repo",
|
||||
targets: Union[List[str], Dict[str, List[str]], None] = None,
|
||||
stages: Optional[List[str]] = None,
|
||||
targets: Union[list[str], dict[str, list[str]], None] = None,
|
||||
stages: Optional[list[str]] = None,
|
||||
deps_only: bool = False,
|
||||
default_file: Optional[str] = None,
|
||||
) -> Dict[str, List[str]]:
|
||||
) -> dict[str, list[str]]:
|
||||
from dvc.dependency import _merge_params
|
||||
|
||||
if isinstance(targets, list):
|
||||
targets = {target: [] for target in targets}
|
||||
|
||||
params: List[Dict[str, List[str]]] = []
|
||||
params: list[dict[str, list[str]]] = []
|
||||
|
||||
if targets:
|
||||
# target is a repo-relative path
|
||||
@@ -80,8 +81,8 @@ def _collect_params(
|
||||
return ret
|
||||
|
||||
|
||||
def _collect_vars(repo, params, stages=None) -> Dict:
|
||||
vars_params: Dict[str, Dict] = defaultdict(dict)
|
||||
def _collect_vars(repo, params, stages=None) -> dict:
|
||||
vars_params: dict[str, dict] = defaultdict(dict)
|
||||
|
||||
for stage in repo.index.stages:
|
||||
if isinstance(stage, PipelineStage) and stage.tracked_vars:
|
||||
@@ -102,8 +103,8 @@ def _collect_vars(repo, params, stages=None) -> Dict:
|
||||
|
||||
|
||||
def _read_params(
|
||||
fs: "FileSystem", params: Dict[str, List[str]], **load_kwargs
|
||||
) -> Iterator[Tuple[str, Union[Exception, Any]]]:
|
||||
fs: "FileSystem", params: dict[str, list[str]], **load_kwargs
|
||||
) -> Iterator[tuple[str, Union[Exception, Any]]]:
|
||||
for file_path, key_paths in params.items():
|
||||
try:
|
||||
yield file_path, read_param_file(fs, file_path, key_paths, **load_kwargs)
|
||||
@@ -114,9 +115,9 @@ def _read_params(
|
||||
|
||||
def _gather_params(
|
||||
repo: "Repo",
|
||||
targets: Union[List[str], Dict[str, List[str]], None] = None,
|
||||
targets: Union[list[str], dict[str, list[str]], None] = None,
|
||||
deps_only: bool = False,
|
||||
stages: Optional[List[str]] = None,
|
||||
stages: Optional[list[str]] = None,
|
||||
on_error: str = "return",
|
||||
):
|
||||
assert on_error in ("raise", "return", "ignore")
|
||||
@@ -132,7 +133,7 @@ def _gather_params(
|
||||
default_file=ParamsDependency.DEFAULT_PARAMS_FILE,
|
||||
)
|
||||
|
||||
data: Dict[str, FileResult] = {}
|
||||
data: dict[str, FileResult] = {}
|
||||
|
||||
fs = repo.dvcfs
|
||||
for fs_path, result in _read_params(fs, files_keypaths, cache=True):
|
||||
@@ -159,16 +160,16 @@ def _gather_params(
|
||||
|
||||
def show(
|
||||
repo: "Repo",
|
||||
targets: Optional[List[str]] = None,
|
||||
stages: Optional[List[str]] = None,
|
||||
targets: Optional[list[str]] = None,
|
||||
stages: Optional[list[str]] = None,
|
||||
deps_only: bool = False,
|
||||
all_branches: bool = False,
|
||||
all_tags: bool = False,
|
||||
revs: Optional[List[str]] = None,
|
||||
revs: Optional[list[str]] = None,
|
||||
all_commits: bool = False,
|
||||
hide_workspace: bool = True,
|
||||
on_error: str = "return",
|
||||
) -> Dict[str, Result]:
|
||||
) -> dict[str, Result]:
|
||||
assert on_error in ("raise", "return", "ignore")
|
||||
res = {}
|
||||
|
||||
|
||||
+30
-33
@@ -2,6 +2,7 @@ import csv
|
||||
import io
|
||||
import os
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterator
|
||||
from copy import deepcopy
|
||||
from functools import partial
|
||||
from multiprocessing import cpu_count
|
||||
@@ -9,11 +10,7 @@ from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Callable,
|
||||
Dict,
|
||||
Iterator,
|
||||
List,
|
||||
Optional,
|
||||
Set,
|
||||
Union,
|
||||
)
|
||||
|
||||
@@ -40,7 +37,7 @@ dpath.options.ALLOW_EMPTY_STRING_KEYS = True
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
|
||||
def onerror_collect(result: Dict, exception: Exception, *args, **kwargs):
|
||||
def onerror_collect(result: dict, exception: Exception, *args, **kwargs):
|
||||
logger.debug("", exc_info=True)
|
||||
result["error"] = exception
|
||||
|
||||
@@ -82,12 +79,12 @@ class Plots:
|
||||
|
||||
def collect(
|
||||
self,
|
||||
targets: Optional[List[str]] = None,
|
||||
revs: Optional[List[str]] = None,
|
||||
targets: Optional[list[str]] = None,
|
||||
revs: Optional[list[str]] = None,
|
||||
recursive: bool = False,
|
||||
onerror: Optional[Callable] = None,
|
||||
props: Optional[Dict] = None,
|
||||
) -> Iterator[Dict]:
|
||||
props: Optional[dict] = None,
|
||||
) -> Iterator[dict]:
|
||||
"""Collects plots definitions and data sources.
|
||||
|
||||
Generator yielding a structure like:
|
||||
@@ -142,7 +139,7 @@ class Plots:
|
||||
revs = ["workspace", *revs]
|
||||
for rev in revs:
|
||||
with switch_repo(self.repo, rev) as (repo, _):
|
||||
res: Dict = {}
|
||||
res: dict = {}
|
||||
definitions = _collect_definitions(
|
||||
repo,
|
||||
targets=targets,
|
||||
@@ -168,9 +165,9 @@ class Plots:
|
||||
def _collect_data_sources(
|
||||
self,
|
||||
repo: "Repo",
|
||||
targets: Optional[List[str]] = None,
|
||||
targets: Optional[list[str]] = None,
|
||||
recursive: bool = False,
|
||||
props: Optional[Dict] = None,
|
||||
props: Optional[dict] = None,
|
||||
onerror: Optional[Callable] = None,
|
||||
):
|
||||
fs = repo.dvcfs
|
||||
@@ -178,7 +175,7 @@ class Plots:
|
||||
props = props or {}
|
||||
|
||||
plots = _collect_plots(repo, targets, recursive)
|
||||
res: Dict[str, Any] = {}
|
||||
res: dict[str, Any] = {}
|
||||
for fs_path, rev_props in plots.items():
|
||||
joined_props = {**rev_props, **props}
|
||||
res[fs_path] = {"props": joined_props}
|
||||
@@ -197,7 +194,7 @@ class Plots:
|
||||
|
||||
def show(
|
||||
self,
|
||||
targets: Optional[List[str]] = None,
|
||||
targets: Optional[list[str]] = None,
|
||||
revs=None,
|
||||
props=None,
|
||||
recursive=False,
|
||||
@@ -206,7 +203,7 @@ class Plots:
|
||||
if onerror is None:
|
||||
onerror = onerror_collect
|
||||
|
||||
result: Dict[str, Dict] = {}
|
||||
result: dict[str, dict] = {}
|
||||
for data in self.collect(
|
||||
targets,
|
||||
revs,
|
||||
@@ -286,7 +283,7 @@ def _is_plot(out: "Output") -> bool:
|
||||
return bool(out.plot)
|
||||
|
||||
|
||||
def _resolve_data_sources(plots_data: Dict, rev: str, cache: bool = False):
|
||||
def _resolve_data_sources(plots_data: dict, rev: str, cache: bool = False):
|
||||
from dvc.progress import Tqdm
|
||||
|
||||
values = list(plots_data.values())
|
||||
@@ -325,9 +322,9 @@ def _resolve_data_sources(plots_data: Dict, rev: str, cache: bool = False):
|
||||
|
||||
def _collect_plots(
|
||||
repo: "Repo",
|
||||
targets: Optional[List[str]] = None,
|
||||
targets: Optional[list[str]] = None,
|
||||
recursive: bool = False,
|
||||
) -> Dict[str, Dict]:
|
||||
) -> dict[str, dict]:
|
||||
from dvc.repo.collect import collect
|
||||
|
||||
plots, fs_paths = collect(
|
||||
@@ -344,8 +341,8 @@ def _collect_plots(
|
||||
return result
|
||||
|
||||
|
||||
def _get_data_targets(definitions: Dict):
|
||||
result: Set = set()
|
||||
def _get_data_targets(definitions: dict):
|
||||
result: set = set()
|
||||
if "data" in definitions:
|
||||
for content in definitions["data"].values():
|
||||
if "data" in content:
|
||||
@@ -404,7 +401,7 @@ def _relpath(fs, path):
|
||||
|
||||
def _collect_output_plots(repo, targets, props, onerror: Optional[Callable] = None):
|
||||
fs = repo.dvcfs
|
||||
result: Dict[str, Dict] = {}
|
||||
result: dict[str, dict] = {}
|
||||
for plot in repo.index.plots:
|
||||
plot_props = _plot_props(plot)
|
||||
dvcfile = plot.stage.dvcfile
|
||||
@@ -445,15 +442,15 @@ def _adjust_sources(fs, plot_props, config_dir):
|
||||
|
||||
def _resolve_definitions(
|
||||
fs: "FileSystem",
|
||||
targets: List[str],
|
||||
props: Dict[str, Any],
|
||||
targets: list[str],
|
||||
props: dict[str, Any],
|
||||
config_path: "StrPath",
|
||||
definitions: "DictStrAny",
|
||||
onerror: Optional[Callable[[Any], Any]] = None,
|
||||
):
|
||||
config_path = os.fspath(config_path)
|
||||
config_dir = fs.dirname(config_path)
|
||||
result: Dict[str, Dict] = {}
|
||||
result: dict[str, dict] = {}
|
||||
for plot_id, plot_props in definitions.items():
|
||||
if plot_props is None:
|
||||
plot_props = {}
|
||||
@@ -477,12 +474,12 @@ def _resolve_definitions(
|
||||
return result
|
||||
|
||||
|
||||
def _collect_pipeline_files(repo, targets: List[str], props, onerror=None):
|
||||
result: Dict[str, Dict] = {}
|
||||
def _collect_pipeline_files(repo, targets: list[str], props, onerror=None):
|
||||
result: dict[str, dict] = {}
|
||||
top_plots = repo.index._plots
|
||||
for dvcfile, plots_def in top_plots.items():
|
||||
dvcfile_path = _relpath(repo.dvcfs, dvcfile)
|
||||
dvcfile_defs_dict: Dict[str, Union[Dict, None]] = {}
|
||||
dvcfile_defs_dict: dict[str, Union[dict, None]] = {}
|
||||
for elem in plots_def:
|
||||
if isinstance(elem, str):
|
||||
dvcfile_defs_dict[elem] = None
|
||||
@@ -510,11 +507,11 @@ def _collect_pipeline_files(repo, targets: List[str], props, onerror=None):
|
||||
def _collect_definitions(
|
||||
repo: "Repo",
|
||||
targets=None,
|
||||
props: Optional[Dict] = None,
|
||||
props: Optional[dict] = None,
|
||||
onerror: Optional[Callable] = None,
|
||||
**kwargs,
|
||||
) -> Dict:
|
||||
result: Dict = defaultdict(dict)
|
||||
) -> dict:
|
||||
result: dict = defaultdict(dict)
|
||||
props = props or {}
|
||||
|
||||
fs = repo.dvcfs
|
||||
@@ -536,8 +533,8 @@ def _collect_definitions(
|
||||
return dict(result)
|
||||
|
||||
|
||||
def unpack_if_dir(fs, path, props: Dict[str, str], onerror: Optional[Callable] = None):
|
||||
result: Dict[str, Dict] = defaultdict(dict)
|
||||
def unpack_if_dir(fs, path, props: dict[str, str], onerror: Optional[Callable] = None):
|
||||
result: dict[str, dict] = defaultdict(dict)
|
||||
if fs.isdir(path):
|
||||
unpacked = _unpack_dir_files(fs, path, onerror=onerror)
|
||||
else:
|
||||
@@ -574,7 +571,7 @@ def parse(fs, path, props=None, **fs_kwargs):
|
||||
return PARSERS[extension](contents, path)
|
||||
|
||||
|
||||
def _plot_props(out: "Output") -> Dict:
|
||||
def _plot_props(out: "Output") -> dict:
|
||||
from dvc.schema import PLOT_PROPS
|
||||
|
||||
if not (out.plot):
|
||||
|
||||
+14
-18
@@ -1,13 +1,9 @@
|
||||
from collections.abc import Iterable
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Callable,
|
||||
Dict,
|
||||
Iterable,
|
||||
List,
|
||||
NoReturn,
|
||||
Optional,
|
||||
Set,
|
||||
Tuple,
|
||||
TypeVar,
|
||||
Union,
|
||||
cast,
|
||||
@@ -40,8 +36,8 @@ def collect_stages(
|
||||
targets: Iterable[str],
|
||||
recursive: bool = False,
|
||||
glob: bool = False,
|
||||
) -> List["Stage"]:
|
||||
stages: List["Stage"] = []
|
||||
) -> list["Stage"]:
|
||||
stages: list["Stage"] = []
|
||||
for target in targets:
|
||||
stages.extend(repo.stage.collect(target, recursive=recursive, glob=glob))
|
||||
return ldistinct(stages)
|
||||
@@ -49,7 +45,7 @@ def collect_stages(
|
||||
|
||||
def get_subgraph(
|
||||
graph: "DiGraph",
|
||||
nodes: Optional[List] = None,
|
||||
nodes: Optional[list] = None,
|
||||
pipeline: bool = False,
|
||||
downstream: bool = False,
|
||||
) -> "DiGraph":
|
||||
@@ -76,10 +72,10 @@ def get_active_graph(graph: "DiGraph") -> "DiGraph":
|
||||
|
||||
def plan_repro(
|
||||
graph: "DiGraph",
|
||||
stages: Optional[List["T"]] = None,
|
||||
stages: Optional[list["T"]] = None,
|
||||
pipeline: bool = False,
|
||||
downstream: bool = False,
|
||||
) -> List["T"]:
|
||||
) -> list["T"]:
|
||||
r"""Derive the evaluation of the given node for the given graph.
|
||||
|
||||
When you _reproduce a stage_, you want to _evaluate the descendants_
|
||||
@@ -134,7 +130,7 @@ def _reproduce_stage(stage: "Stage", **kwargs) -> Optional["Stage"]:
|
||||
|
||||
def _get_upstream_downstream_nodes(
|
||||
graph: Optional["DiGraph"], node: T
|
||||
) -> Tuple[List[T], List[T]]:
|
||||
) -> tuple[list[T], list[T]]:
|
||||
succ = list(graph.successors(node)) if graph else []
|
||||
pre = list(graph.predecessors(node)) if graph else []
|
||||
return succ, pre
|
||||
@@ -146,7 +142,7 @@ def _repr(stages: Iterable["Stage"]) -> str:
|
||||
|
||||
def handle_error(
|
||||
graph: Optional["DiGraph"], on_error: str, exc: Exception, stage: "Stage"
|
||||
) -> Set["Stage"]:
|
||||
) -> set["Stage"]:
|
||||
import networkx as nx
|
||||
|
||||
logger.warning("%s%s", exc, " (ignored)" if on_error == "ignore" else "")
|
||||
@@ -168,19 +164,19 @@ def _raise_error(exc: Optional[Exception], *stages: "Stage") -> NoReturn:
|
||||
|
||||
|
||||
def _reproduce(
|
||||
stages: List["Stage"],
|
||||
stages: list["Stage"],
|
||||
graph: Optional["DiGraph"] = None,
|
||||
force_downstream: bool = False,
|
||||
on_error: str = "fail",
|
||||
force: bool = False,
|
||||
repro_fn: Callable = _reproduce_stage,
|
||||
**kwargs,
|
||||
) -> List["Stage"]:
|
||||
) -> list["Stage"]:
|
||||
assert on_error in ("fail", "keep-going", "ignore")
|
||||
|
||||
result: List["Stage"] = []
|
||||
failed: List["Stage"] = []
|
||||
to_skip: Dict["Stage", "Stage"] = {}
|
||||
result: list["Stage"] = []
|
||||
failed: list["Stage"] = []
|
||||
to_skip: dict["Stage", "Stage"] = {}
|
||||
ret: Optional["Stage"] = None
|
||||
|
||||
force_state = dict.fromkeys(stages, force)
|
||||
@@ -240,7 +236,7 @@ def reproduce(
|
||||
if not kwargs.get("interactive", False):
|
||||
kwargs["interactive"] = self.config["core"].get("interactive", False)
|
||||
|
||||
stages: List["Stage"] = []
|
||||
stages: list["Stage"] = []
|
||||
if not all_pipelines:
|
||||
targets_list = ensure_list(targets or PROJECT_FILE)
|
||||
stages = collect_stages(self, targets_list, recursive=recursive, glob=glob)
|
||||
|
||||
@@ -1,16 +1,12 @@
|
||||
import logging
|
||||
import shlex
|
||||
from collections.abc import Iterable, Iterator
|
||||
from contextlib import contextmanager
|
||||
from functools import wraps
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Dict,
|
||||
Iterable,
|
||||
Iterator,
|
||||
List,
|
||||
Optional,
|
||||
Set,
|
||||
Union,
|
||||
)
|
||||
|
||||
@@ -27,15 +23,15 @@ logger = logger.getChild(__name__)
|
||||
|
||||
|
||||
class SCMContext:
|
||||
def __init__(self, scm: "Base", config: Optional[Dict[str, Any]] = None) -> None:
|
||||
def __init__(self, scm: "Base", config: Optional[dict[str, Any]] = None) -> None:
|
||||
from funcy import get_in
|
||||
|
||||
self.scm: "Base" = scm
|
||||
self.autostage: bool = get_in(
|
||||
config or {}, ["core", "autostage"], default=False
|
||||
)
|
||||
self.ignored_paths: List[str] = []
|
||||
self.files_to_track: Set[str] = set()
|
||||
self.ignored_paths: list[str] = []
|
||||
self.files_to_track: set[str] = set()
|
||||
self.quiet: bool = False
|
||||
|
||||
def track_file(self, paths: Union[str, Iterable[str], None] = None) -> None:
|
||||
|
||||
+6
-5
@@ -1,8 +1,9 @@
|
||||
import fnmatch
|
||||
import typing
|
||||
from collections.abc import Iterable
|
||||
from contextlib import suppress
|
||||
from functools import wraps
|
||||
from typing import Iterable, List, NamedTuple, Optional, Set, Tuple, Union
|
||||
from typing import NamedTuple, Optional, Union
|
||||
|
||||
from dvc.exceptions import (
|
||||
NoOutputOrStageError,
|
||||
@@ -31,9 +32,9 @@ class StageInfo(NamedTuple):
|
||||
filter_info: Optional[str] = None
|
||||
|
||||
|
||||
StageList = List["Stage"]
|
||||
StageList = list["Stage"]
|
||||
StageIter = Iterable["Stage"]
|
||||
StageSet = Set["Stage"]
|
||||
StageSet = set["Stage"]
|
||||
|
||||
|
||||
def _collect_with_deps(stages: StageList, graph: "DiGraph") -> StageSet:
|
||||
@@ -72,7 +73,7 @@ def _collect_specific_target(
|
||||
target: str,
|
||||
with_deps: bool,
|
||||
recursive: bool,
|
||||
) -> Tuple[StageIter, Optional[str], Optional[str]]:
|
||||
) -> tuple[StageIter, Optional[str], Optional[str]]:
|
||||
from dvc.dvcfile import is_valid_filename
|
||||
|
||||
# Optimization: do not collect the graph for a specific target
|
||||
@@ -363,7 +364,7 @@ class StageLoad:
|
||||
with_deps: bool = False,
|
||||
recursive: bool = False,
|
||||
graph: Optional["DiGraph"] = None,
|
||||
) -> List[StageInfo]:
|
||||
) -> list[StageInfo]:
|
||||
"""Collects a list of (stage, filter_info) from the given target.
|
||||
|
||||
Priority is in the order of following in case of ambiguity:
|
||||
|
||||
+2
-2
@@ -1,4 +1,4 @@
|
||||
from typing import TYPE_CHECKING, List
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from dvc.exceptions import InvalidArgumentError
|
||||
|
||||
@@ -34,7 +34,7 @@ def update( # noqa: C901
|
||||
raise InvalidArgumentError("--remote can't be used without --to-remote")
|
||||
|
||||
import_stages = set()
|
||||
other_stage_infos: List["StageInfo"] = []
|
||||
other_stage_infos: list["StageInfo"] = []
|
||||
|
||||
for stage_info in self.index.collect_targets(targets, recursive=recursive):
|
||||
if stage_info.stage.is_import:
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from collections.abc import Iterable
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Tuple, Union
|
||||
from typing import TYPE_CHECKING, Any, Optional, Union
|
||||
|
||||
from funcy import first
|
||||
|
||||
@@ -35,7 +36,7 @@ def worktree_view_by_remotes(
|
||||
targets: Optional["TargetType"] = None,
|
||||
push: bool = False,
|
||||
**kwargs: Any,
|
||||
) -> Iterable[Tuple[Optional[str], "IndexView"]]:
|
||||
) -> Iterable[tuple[Optional[str], "IndexView"]]:
|
||||
from dvc.repo.index import IndexView
|
||||
|
||||
def outs_filter(view: "IndexView", remote: Optional[str]):
|
||||
@@ -176,7 +177,7 @@ def update_worktree_stages(
|
||||
outs_filter=outs_filter,
|
||||
)
|
||||
local_index = view.data["repo"]
|
||||
remote_indexes: Dict[str, Tuple["Remote", "DataIndex"]] = {}
|
||||
remote_indexes: dict[str, tuple["Remote", "DataIndex"]] = {}
|
||||
for stage in view.stages:
|
||||
for out in stage.outs:
|
||||
_update_worktree_out(repo, out, local_index, remote_indexes)
|
||||
@@ -187,7 +188,7 @@ def _update_worktree_out(
|
||||
repo: "Repo",
|
||||
out: "Output",
|
||||
local_index: Union["DataIndex", "DataIndexView"],
|
||||
remote_indexes: Dict[str, Tuple["Remote", "DataIndex"]],
|
||||
remote_indexes: dict[str, tuple["Remote", "DataIndex"]],
|
||||
):
|
||||
from dvc_data.index import build
|
||||
|
||||
@@ -283,7 +284,7 @@ def _get_diff_indexes(
|
||||
out: "Output",
|
||||
local_index: Union["DataIndex", "DataIndexView"],
|
||||
remote_index: Union["DataIndex", "DataIndexView"],
|
||||
) -> Tuple["DataIndex", "DataIndex"]:
|
||||
) -> tuple["DataIndex", "DataIndex"]:
|
||||
from dvc_data.index import DataIndex
|
||||
|
||||
_, key = out.index_key
|
||||
|
||||
+2
-2
@@ -1,5 +1,5 @@
|
||||
from collections.abc import Mapping
|
||||
from typing import Any, Dict
|
||||
from typing import Any
|
||||
|
||||
import voluptuous as vol
|
||||
|
||||
@@ -28,7 +28,7 @@ SINGLE_STAGE_SCHEMA = {
|
||||
StageParams.PARAM_DESC: str,
|
||||
}
|
||||
|
||||
DATA_SCHEMA: Dict[Any, Any] = {
|
||||
DATA_SCHEMA: dict[Any, Any] = {
|
||||
**CHECKSUMS_SCHEMA,
|
||||
**META_SCHEMA,
|
||||
vol.Required("path"): str,
|
||||
|
||||
+6
-8
@@ -1,13 +1,11 @@
|
||||
"""Manages source control systems (e.g. Git)."""
|
||||
import os
|
||||
from collections.abc import Iterator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from functools import partial
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Iterator,
|
||||
List,
|
||||
Literal,
|
||||
Mapping,
|
||||
Optional,
|
||||
Union,
|
||||
overload,
|
||||
@@ -189,7 +187,7 @@ def resolve_rev(scm: Union["Git", "NoSCM"], rev: str) -> str:
|
||||
raise RevError(str(exc)) # noqa: B904
|
||||
|
||||
|
||||
def _get_n_commits(scm: "Git", revs: List[str], num: int) -> List[str]:
|
||||
def _get_n_commits(scm: "Git", revs: list[str], num: int) -> list[str]:
|
||||
results = []
|
||||
for rev in revs:
|
||||
if num == 0:
|
||||
@@ -210,14 +208,14 @@ def _get_n_commits(scm: "Git", revs: List[str], num: int) -> List[str]:
|
||||
|
||||
def iter_revs(
|
||||
scm: "Git",
|
||||
revs: Optional[List[str]] = None,
|
||||
revs: Optional[list[str]] = None,
|
||||
num: int = 1,
|
||||
all_branches: bool = False,
|
||||
all_tags: bool = False,
|
||||
all_commits: bool = False,
|
||||
all_experiments: bool = False,
|
||||
commit_date: Optional[str] = None,
|
||||
) -> Mapping[str, List[str]]:
|
||||
) -> Mapping[str, list[str]]:
|
||||
from scmrepo.exceptions import SCMError as _SCMError
|
||||
|
||||
from dvc.repo.experiments.utils import exp_commits
|
||||
@@ -235,7 +233,7 @@ def iter_revs(
|
||||
return {}
|
||||
|
||||
revs = revs or []
|
||||
results: List[str] = _get_n_commits(scm, revs, num)
|
||||
results: list[str] = _get_n_commits(scm, revs, num)
|
||||
|
||||
if all_commits:
|
||||
results.extend(scm.list_all_commits())
|
||||
@@ -266,7 +264,7 @@ def iter_revs(
|
||||
return group_by(rev_resolver, results)
|
||||
|
||||
|
||||
def lfs_prefetch(fs: "FileSystem", paths: List[str]):
|
||||
def lfs_prefetch(fs: "FileSystem", paths: list[str]):
|
||||
from scmrepo.git.lfs import fetch as _lfs_fetch
|
||||
|
||||
from dvc.fs.dvc import DVCFileSystem
|
||||
|
||||
+21
-26
@@ -1,18 +1,13 @@
|
||||
import os
|
||||
import string
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterable
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Dict,
|
||||
Iterable,
|
||||
List,
|
||||
Optional,
|
||||
Set,
|
||||
Tuple,
|
||||
Type,
|
||||
TypeVar,
|
||||
Union,
|
||||
)
|
||||
@@ -55,14 +50,14 @@ if TYPE_CHECKING:
|
||||
logger = logger.getChild(__name__)
|
||||
# Disallow all punctuation characters except hyphen and underscore
|
||||
INVALID_STAGENAME_CHARS = set(string.punctuation) - {"_", "-"}
|
||||
Env = Dict[str, str]
|
||||
ChangedEntries = Tuple[List[str], List[str], Optional[str]]
|
||||
Env = dict[str, str]
|
||||
ChangedEntries = tuple[list[str], list[str], Optional[str]]
|
||||
|
||||
_T = TypeVar("_T")
|
||||
|
||||
|
||||
def loads_from(
|
||||
cls: Type[_T], repo: "Repo", path: str, wdir: str, data: Dict[str, Any]
|
||||
cls: type[_T], repo: "Repo", path: str, wdir: str, data: dict[str, Any]
|
||||
) -> _T:
|
||||
kw = {
|
||||
"repo": repo,
|
||||
@@ -91,7 +86,7 @@ class RawData:
|
||||
generated_from: Optional[str] = None
|
||||
|
||||
|
||||
def create_stage(cls: Type[_T], repo, path, **kwargs) -> _T:
|
||||
def create_stage(cls: type[_T], repo, path, **kwargs) -> _T:
|
||||
from dvc.dvcfile import check_dvcfile_path
|
||||
|
||||
wdir = os.path.abspath(kwargs.get("wdir", None) or os.curdir)
|
||||
@@ -205,13 +200,13 @@ class Stage(params.StageParams):
|
||||
self._dvcfile = dvcfile
|
||||
|
||||
@property
|
||||
def params(self) -> List["ParamsDependency"]:
|
||||
def params(self) -> list["ParamsDependency"]:
|
||||
from dvc.dependency import ParamsDependency
|
||||
|
||||
return [dep for dep in self.deps if isinstance(dep, ParamsDependency)]
|
||||
|
||||
@property
|
||||
def metrics(self) -> List["Output"]:
|
||||
def metrics(self) -> list["Output"]:
|
||||
return [out for out in self.outs if out.metric]
|
||||
|
||||
def __repr__(self):
|
||||
@@ -310,7 +305,7 @@ class Stage(params.StageParams):
|
||||
return desc
|
||||
|
||||
def changed_deps(
|
||||
self, allow_missing: bool = False, upstream: Optional[List] = None
|
||||
self, allow_missing: bool = False, upstream: Optional[list] = None
|
||||
) -> bool:
|
||||
if self.frozen:
|
||||
return False
|
||||
@@ -322,7 +317,7 @@ class Stage(params.StageParams):
|
||||
|
||||
@rwlocked(read=["deps"])
|
||||
def _changed_deps(
|
||||
self, allow_missing: bool = False, upstream: Optional[List] = None
|
||||
self, allow_missing: bool = False, upstream: Optional[list] = None
|
||||
) -> bool:
|
||||
for dep in self.deps:
|
||||
status = dep.status()
|
||||
@@ -370,7 +365,7 @@ class Stage(params.StageParams):
|
||||
|
||||
@rwlocked(read=["deps", "outs"])
|
||||
def changed(
|
||||
self, allow_missing: bool = False, upstream: Optional[List] = None
|
||||
self, allow_missing: bool = False, upstream: Optional[list] = None
|
||||
) -> bool:
|
||||
is_changed = (
|
||||
# Short-circuit order: stage md5 is fast,
|
||||
@@ -477,7 +472,7 @@ class Stage(params.StageParams):
|
||||
def reload(self) -> "Stage":
|
||||
return self.dvcfile.stage
|
||||
|
||||
def dumpd(self, **kwargs) -> Dict[str, Any]:
|
||||
def dumpd(self, **kwargs) -> dict[str, Any]:
|
||||
return get_dump(self, **kwargs)
|
||||
|
||||
def compute_md5(self) -> Optional[str]:
|
||||
@@ -509,7 +504,7 @@ class Stage(params.StageParams):
|
||||
if not allow_missing:
|
||||
raise
|
||||
|
||||
def get_versioned_outs(self) -> Dict[str, "Output"]:
|
||||
def get_versioned_outs(self) -> dict[str, "Output"]:
|
||||
from .exceptions import StageFileDoesNotExistError, StageNotFound
|
||||
|
||||
try:
|
||||
@@ -543,7 +538,7 @@ class Stage(params.StageParams):
|
||||
out.ignore()
|
||||
|
||||
@staticmethod
|
||||
def _changed_entries(entries) -> List[str]:
|
||||
def _changed_entries(entries) -> list[str]:
|
||||
return [str(entry) for entry in entries if entry.workspace_status()]
|
||||
|
||||
def _changed_stage_entry(self) -> str:
|
||||
@@ -663,8 +658,8 @@ class Stage(params.StageParams):
|
||||
@rwlocked(write=["outs"])
|
||||
def checkout(
|
||||
self, allow_missing: bool = False, **kwargs
|
||||
) -> Dict[str, List["StrPath"]]:
|
||||
stats: Dict[str, List["StrPath"]] = defaultdict(list)
|
||||
) -> dict[str, list["StrPath"]]:
|
||||
stats: dict[str, list["StrPath"]] = defaultdict(list)
|
||||
if self.is_partial_import:
|
||||
return stats
|
||||
|
||||
@@ -679,7 +674,7 @@ class Stage(params.StageParams):
|
||||
return stats
|
||||
|
||||
@staticmethod
|
||||
def _checkout(out, **kwargs) -> Tuple[Optional[str], List[str]]:
|
||||
def _checkout(out, **kwargs) -> tuple[Optional[str], list[str]]:
|
||||
try:
|
||||
result = out.checkout(**kwargs)
|
||||
added, modified = result or (None, None)
|
||||
@@ -692,8 +687,8 @@ class Stage(params.StageParams):
|
||||
@rwlocked(read=["deps", "outs"])
|
||||
def status(
|
||||
self, check_updates: bool = False, filter_info: Optional[bool] = None
|
||||
) -> Dict[str, List[Union[str, Dict[str, str]]]]:
|
||||
ret: List[Union[str, Dict[str, str]]] = []
|
||||
) -> dict[str, list[Union[str, dict[str, str]]]]:
|
||||
ret: list[Union[str, dict[str, str]]] = []
|
||||
show_import = (
|
||||
self.is_repo_import or self.is_versioned_import
|
||||
) and check_updates
|
||||
@@ -706,7 +701,7 @@ class Stage(params.StageParams):
|
||||
return {self.addressing: ret} if ret else {}
|
||||
|
||||
@staticmethod
|
||||
def _status(entries: Iterable["Output"]) -> Dict[str, str]:
|
||||
def _status(entries: Iterable["Output"]) -> dict[str, str]:
|
||||
ret = {}
|
||||
|
||||
for entry in entries:
|
||||
@@ -747,7 +742,7 @@ class Stage(params.StageParams):
|
||||
|
||||
def get_used_objs(
|
||||
self, *args, **kwargs
|
||||
) -> Dict[Optional["HashFileDB"], Set["HashInfo"]]:
|
||||
) -> dict[Optional["HashFileDB"], set["HashInfo"]]:
|
||||
"""Return set of object IDs used by this stage."""
|
||||
if self.is_partial_import and not self.is_repo_import:
|
||||
return {}
|
||||
@@ -802,7 +797,7 @@ class PipelineStage(Stage):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.name = name
|
||||
self.cmd_changed = False
|
||||
self.tracked_vars: Dict[str, Dict[str, Dict[str, str]]] = {}
|
||||
self.tracked_vars: dict[str, dict[str, dict[str, str]]] = {}
|
||||
|
||||
def __eq__(self, other):
|
||||
return super().__eq__(other) and self.name == other.name
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
from typing import TYPE_CHECKING, List, Optional, Tuple
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from funcy import first
|
||||
|
||||
@@ -247,7 +247,7 @@ class StageCache:
|
||||
message = f"run-cache is not supported for http filesystem: {path}"
|
||||
raise RunCacheNotSupported(message)
|
||||
|
||||
ret: List[Tuple[str, str]] = []
|
||||
ret: list[tuple[str, str]] = []
|
||||
if not from_fs.exists(runs):
|
||||
return ret
|
||||
|
||||
|
||||
+7
-5
@@ -1,7 +1,7 @@
|
||||
from collections.abc import Mapping
|
||||
from copy import deepcopy
|
||||
from itertools import chain
|
||||
from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Tuple
|
||||
from typing import TYPE_CHECKING, Any, Optional
|
||||
|
||||
from funcy import get_in, lcat, once, project
|
||||
|
||||
@@ -17,6 +17,8 @@ from .params import StageParams
|
||||
from .utils import fill_stage_dependencies, resolve_paths
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable
|
||||
|
||||
from dvc.dvcfile import ProjectFile, SingleStageFile
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
@@ -39,7 +41,7 @@ class StageLoader(Mapping):
|
||||
self._lockfile_data = lockfile_data.get("stages", {})
|
||||
|
||||
@cached_property
|
||||
def lockfile_data(self) -> Dict[str, Any]:
|
||||
def lockfile_data(self) -> dict[str, Any]:
|
||||
if not self._lockfile_data:
|
||||
logger.debug("Lockfile for '%s' not found", self.dvcfile.relpath)
|
||||
return self._lockfile_data
|
||||
@@ -53,7 +55,7 @@ class StageLoader(Mapping):
|
||||
from dvc.output import Output, merge_file_meta_from_cloud
|
||||
|
||||
assert isinstance(lock_data, dict)
|
||||
items: Iterable[Tuple[str, "Output"]] = chain(
|
||||
items: Iterable[tuple[str, "Output"]] = chain(
|
||||
((StageParams.PARAM_DEPS, dep) for dep in stage.deps),
|
||||
((StageParams.PARAM_OUTS, out) for out in stage.outs),
|
||||
)
|
||||
@@ -174,7 +176,7 @@ class SingleStageLoader(Mapping):
|
||||
def __init__(
|
||||
self,
|
||||
dvcfile: "SingleStageFile",
|
||||
stage_data: Dict[Any, str],
|
||||
stage_data: dict[Any, str],
|
||||
stage_text: Optional[str] = None,
|
||||
):
|
||||
self.dvcfile = dvcfile
|
||||
@@ -196,7 +198,7 @@ class SingleStageLoader(Mapping):
|
||||
def load_stage(
|
||||
cls,
|
||||
dvcfile: "SingleStageFile",
|
||||
d: Dict[str, Any],
|
||||
d: dict[str, Any],
|
||||
stage_text: Optional[str],
|
||||
) -> Stage:
|
||||
path, wdir = resolve_paths(
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Iterable
|
||||
from operator import attrgetter
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Dict,
|
||||
Iterable,
|
||||
List,
|
||||
Optional,
|
||||
Union,
|
||||
no_type_check,
|
||||
@@ -67,7 +65,7 @@ def _serialize_out(out):
|
||||
|
||||
|
||||
@no_type_check
|
||||
def _serialize_outs(outputs: List[Output]):
|
||||
def _serialize_outs(outputs: list[Output]):
|
||||
outs, metrics, plots = [], [], []
|
||||
for out in sorted(outputs, key=attrgetter("def_path")):
|
||||
bucket = outs
|
||||
@@ -88,10 +86,10 @@ def _serialize_params_keys(params: Iterable["ParamsDependency"]):
|
||||
at the first, and then followed by entry of other files in lexicographic
|
||||
order. The keys of those custom files are also sorted in the same order.
|
||||
"""
|
||||
keys: List[Union[str, Dict[str, Optional[List[str]]]]] = []
|
||||
keys: list[Union[str, dict[str, Optional[list[str]]]]] = []
|
||||
for param_dep in sorted(params, key=attrgetter("def_path")):
|
||||
# when on no_exec, params are not filled and are saved as list
|
||||
k: List[str] = sorted(param_dep.params)
|
||||
k: list[str] = sorted(param_dep.params)
|
||||
if k and param_dep.def_path == DEFAULT_PARAMS_FILE:
|
||||
keys = k + keys # type: ignore[operator,assignment]
|
||||
else:
|
||||
@@ -100,7 +98,7 @@ def _serialize_params_keys(params: Iterable["ParamsDependency"]):
|
||||
|
||||
|
||||
@no_type_check
|
||||
def _serialize_params_values(params: List[ParamsDependency]):
|
||||
def _serialize_params_values(params: list[ParamsDependency]):
|
||||
"""Returns output of following format, used for lockfile:
|
||||
{'params.yaml': {'lr': '1', 'train': 2}, {'params2.yaml': {'lr': '1'}}
|
||||
|
||||
@@ -160,7 +158,7 @@ def to_single_stage_lockfile(stage: "Stage", **kwargs) -> dict:
|
||||
assert stage.cmd
|
||||
|
||||
def _dumpd(item: "Output"):
|
||||
ret: Dict[str, Any] = {item.PARAM_PATH: item.def_path}
|
||||
ret: dict[str, Any] = {item.PARAM_PATH: item.def_path}
|
||||
if item.hash_name not in LEGACY_HASH_NAMES:
|
||||
ret[item.PARAM_HASH] = "md5"
|
||||
if item.hash_info.isdir and kwargs.get("with_files"):
|
||||
|
||||
+5
-5
@@ -1,6 +1,6 @@
|
||||
import os
|
||||
import pathlib
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
|
||||
from typing import TYPE_CHECKING, Any, Optional, Union
|
||||
|
||||
from funcy import concat, first, lsplit, rpartial
|
||||
|
||||
@@ -210,7 +210,7 @@ def get_dump(stage: "Stage", **kwargs):
|
||||
|
||||
def split_params_deps(
|
||||
stage: "Stage",
|
||||
) -> Tuple[List["ParamsDependency"], List["Dependency"]]:
|
||||
) -> tuple[list["ParamsDependency"], list["Dependency"]]:
|
||||
from dvc.dependency import ParamsDependency
|
||||
|
||||
return lsplit(rpartial(isinstance, ParamsDependency), stage.deps)
|
||||
@@ -265,7 +265,7 @@ def check_stage_exists(repo: "Repo", stage: Union["Stage", "PipelineStage"], pat
|
||||
|
||||
def validate_kwargs(
|
||||
single_stage: bool = False, fname: Optional[str] = None, **kwargs
|
||||
) -> Dict[str, Any]:
|
||||
) -> dict[str, Any]:
|
||||
"""Prepare, validate and process kwargs passed from cli"""
|
||||
cmd = kwargs.get("cmd")
|
||||
if not cmd and not single_stage:
|
||||
@@ -288,11 +288,11 @@ def validate_kwargs(
|
||||
return kwargs
|
||||
|
||||
|
||||
def _get_stage_files(stage: "Stage") -> List[str]:
|
||||
def _get_stage_files(stage: "Stage") -> list[str]:
|
||||
from dvc.dvcfile import ProjectFile
|
||||
from dvc.utils import relpath
|
||||
|
||||
ret: List[str] = []
|
||||
ret: list[str] = []
|
||||
file = stage.dvcfile
|
||||
ret.append(file.relpath)
|
||||
if isinstance(file, ProjectFile):
|
||||
|
||||
@@ -2,7 +2,7 @@ import os
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from subprocess import check_output
|
||||
from typing import Dict, Optional
|
||||
from typing import Optional
|
||||
|
||||
import pytest
|
||||
from dulwich.porcelain import clone
|
||||
@@ -27,7 +27,7 @@ class VirtualEnv:
|
||||
|
||||
virtualenv.cli_run([os.fspath(self.path)])
|
||||
|
||||
def run(self, cmd: str, *args: str, env: Optional[Dict[str, str]] = None) -> None:
|
||||
def run(self, cmd: str, *args: str, env: Optional[dict[str, str]] = None) -> None:
|
||||
exe = self.which(cmd)
|
||||
check_output([exe, *args], env=env) # noqa: S603
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user