mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
drop Python 3.8 support (#10265)
* drop Python 3.8 Note that this PR only makes typing related changes. Other changes can be done once this gets merged, they are relatively few in numbers. * bump build python version * add typing_extensions to lint group
This commit is contained in:
@@ -25,7 +25,7 @@ jobs:
|
|||||||
|
|
||||||
- uses: actions/setup-python@v5
|
- uses: actions/setup-python@v5
|
||||||
with:
|
with:
|
||||||
python-version: 3.8
|
python-version: 3.12
|
||||||
|
|
||||||
- run: python -m pip install --upgrade pip setuptools_scm build twine
|
- run: python -m pip install --upgrade pip setuptools_scm build twine
|
||||||
|
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ jobs:
|
|||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
os: [ubuntu-20.04, windows-latest, macos-latest]
|
os: [ubuntu-20.04, windows-latest, macos-latest]
|
||||||
pyv: ["3.8", "3.9", "3.10", "3.11", "3.12"]
|
pyv: ["3.9", "3.10", "3.11", "3.12"]
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v4
|
||||||
|
|||||||
+8
-8
@@ -1,5 +1,5 @@
|
|||||||
from dataclasses import asdict, dataclass, field, fields
|
from dataclasses import asdict, dataclass, field, fields
|
||||||
from typing import Any, ClassVar, Dict, List, Optional
|
from typing import Any, ClassVar, Optional
|
||||||
|
|
||||||
from funcy import compact
|
from funcy import compact
|
||||||
from voluptuous import Required
|
from voluptuous import Required
|
||||||
@@ -14,10 +14,10 @@ class Annotation:
|
|||||||
|
|
||||||
desc: Optional[str] = None
|
desc: Optional[str] = None
|
||||||
type: Optional[str] = None
|
type: Optional[str] = None
|
||||||
labels: List[str] = field(default_factory=list)
|
labels: list[str] = field(default_factory=list)
|
||||||
meta: Dict[str, Any] = field(default_factory=dict)
|
meta: dict[str, Any] = field(default_factory=dict)
|
||||||
|
|
||||||
def to_dict(self) -> Dict[str, str]:
|
def to_dict(self) -> dict[str, str]:
|
||||||
return compact(asdict(self))
|
return compact(asdict(self))
|
||||||
|
|
||||||
|
|
||||||
@@ -32,10 +32,10 @@ class Artifact:
|
|||||||
path: str
|
path: str
|
||||||
desc: Optional[str] = None
|
desc: Optional[str] = None
|
||||||
type: Optional[str] = None
|
type: Optional[str] = None
|
||||||
labels: List[str] = field(default_factory=list)
|
labels: list[str] = field(default_factory=list)
|
||||||
meta: Dict[str, Any] = field(default_factory=dict)
|
meta: dict[str, Any] = field(default_factory=dict)
|
||||||
|
|
||||||
def to_dict(self) -> Dict[str, str]:
|
def to_dict(self) -> dict[str, str]:
|
||||||
return compact(asdict(self))
|
return compact(asdict(self))
|
||||||
|
|
||||||
|
|
||||||
@@ -46,7 +46,7 @@ ANNOTATION_SCHEMA = {
|
|||||||
Annotation.PARAM_LABELS: [str],
|
Annotation.PARAM_LABELS: [str],
|
||||||
Annotation.PARAM_META: object,
|
Annotation.PARAM_META: object,
|
||||||
}
|
}
|
||||||
ARTIFACT_SCHEMA: Dict[Any, Any] = {
|
ARTIFACT_SCHEMA: dict[Any, Any] = {
|
||||||
Required(Artifact.PARAM_PATH): str,
|
Required(Artifact.PARAM_PATH): str,
|
||||||
**ANNOTATION_SCHEMA, # type: ignore[arg-type]
|
**ANNOTATION_SCHEMA, # type: ignore[arg-type]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
from typing import Any, Dict, Optional
|
from typing import Any, Optional
|
||||||
|
|
||||||
from dvc.repo import Repo
|
from dvc.repo import Repo
|
||||||
|
|
||||||
@@ -8,7 +8,7 @@ def artifacts_show(
|
|||||||
version: Optional[str] = None,
|
version: Optional[str] = None,
|
||||||
stage: Optional[str] = None,
|
stage: Optional[str] = None,
|
||||||
repo: Optional[str] = None,
|
repo: Optional[str] = None,
|
||||||
) -> Dict[str, str]:
|
) -> dict[str, str]:
|
||||||
"""
|
"""
|
||||||
Return path and Git revision for an artifact in a DVC project.
|
Return path and Git revision for an artifact in a DVC project.
|
||||||
|
|
||||||
@@ -36,7 +36,7 @@ def artifacts_show(
|
|||||||
if version and stage:
|
if version and stage:
|
||||||
raise ValueError("Artifact version and stage are mutually exclusive.")
|
raise ValueError("Artifact version and stage are mutually exclusive.")
|
||||||
|
|
||||||
repo_kwargs: Dict[str, Any] = {
|
repo_kwargs: dict[str, Any] = {
|
||||||
"subrepos": True,
|
"subrepos": True,
|
||||||
"uninitialized": True,
|
"uninitialized": True,
|
||||||
}
|
}
|
||||||
|
|||||||
+5
-5
@@ -1,6 +1,6 @@
|
|||||||
from contextlib import _GeneratorContextManager as GCM
|
from contextlib import _GeneratorContextManager as GCM
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from typing import Any, Dict, Optional
|
from typing import Any, Optional
|
||||||
|
|
||||||
from funcy import reraise
|
from funcy import reraise
|
||||||
|
|
||||||
@@ -39,7 +39,7 @@ def get_url(path, repo=None, rev=None, remote=None):
|
|||||||
from dvc.config import NoRemoteError
|
from dvc.config import NoRemoteError
|
||||||
from dvc_data.index import StorageKeyError
|
from dvc_data.index import StorageKeyError
|
||||||
|
|
||||||
repo_kwargs: Dict[str, Any] = {}
|
repo_kwargs: dict[str, Any] = {}
|
||||||
if remote:
|
if remote:
|
||||||
repo_kwargs["config"] = {"core": {"remote": remote}}
|
repo_kwargs["config"] = {"core": {"remote": remote}}
|
||||||
with Repo.open(
|
with Repo.open(
|
||||||
@@ -74,8 +74,8 @@ def open( # noqa: A001
|
|||||||
remote: Optional[str] = None,
|
remote: Optional[str] = None,
|
||||||
mode: str = "r",
|
mode: str = "r",
|
||||||
encoding: Optional[str] = None,
|
encoding: Optional[str] = None,
|
||||||
config: Optional[Dict[str, Any]] = None,
|
config: Optional[dict[str, Any]] = None,
|
||||||
remote_config: Optional[Dict[str, Any]] = None,
|
remote_config: Optional[dict[str, Any]] = None,
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
Opens a file tracked in a DVC project.
|
Opens a file tracked in a DVC project.
|
||||||
@@ -233,7 +233,7 @@ def _open(
|
|||||||
config=None,
|
config=None,
|
||||||
remote_config=None,
|
remote_config=None,
|
||||||
):
|
):
|
||||||
repo_kwargs: Dict[str, Any] = {
|
repo_kwargs: dict[str, Any] = {
|
||||||
"subrepos": True,
|
"subrepos": True,
|
||||||
"uninitialized": True,
|
"uninitialized": True,
|
||||||
"remote": remote,
|
"remote": remote,
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
from typing import Dict, List, Optional, Union
|
from typing import Optional, Union
|
||||||
|
|
||||||
from rich.text import Text
|
from rich.text import Text
|
||||||
|
|
||||||
@@ -9,7 +9,7 @@ from dvc.repo.experiments.show import tabulate
|
|||||||
def exp_save(
|
def exp_save(
|
||||||
name: Optional[str] = None,
|
name: Optional[str] = None,
|
||||||
force: bool = False,
|
force: bool = False,
|
||||||
include_untracked: Optional[List[str]] = None,
|
include_untracked: Optional[list[str]] = None,
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
Create a new DVC experiment using `exp save`.
|
Create a new DVC experiment using `exp save`.
|
||||||
@@ -61,12 +61,12 @@ def _postprocess(exp_rows):
|
|||||||
|
|
||||||
def exp_show(
|
def exp_show(
|
||||||
repo: Optional[str] = None,
|
repo: Optional[str] = None,
|
||||||
revs: Optional[Union[str, List[str]]] = None,
|
revs: Optional[Union[str, list[str]]] = None,
|
||||||
num: int = 1,
|
num: int = 1,
|
||||||
param_deps: bool = False,
|
param_deps: bool = False,
|
||||||
force: bool = False,
|
force: bool = False,
|
||||||
config: Optional[Dict] = None,
|
config: Optional[dict] = None,
|
||||||
) -> List[Dict]:
|
) -> list[dict]:
|
||||||
"""Get DVC experiments tracked in `repo`.
|
"""Get DVC experiments tracked in `repo`.
|
||||||
|
|
||||||
Without arguments, this function will retrieve all experiments derived from
|
Without arguments, this function will retrieve all experiments derived from
|
||||||
|
|||||||
+4
-4
@@ -1,9 +1,9 @@
|
|||||||
from typing import List, Optional
|
from typing import Optional
|
||||||
|
|
||||||
from dvc.repo import Repo
|
from dvc.repo import Repo
|
||||||
|
|
||||||
|
|
||||||
def all_branches(repo: Optional[str] = None) -> List[str]:
|
def all_branches(repo: Optional[str] = None) -> list[str]:
|
||||||
"""Get all Git branches in a DVC repository.
|
"""Get all Git branches in a DVC repository.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -20,7 +20,7 @@ def all_branches(repo: Optional[str] = None) -> List[str]:
|
|||||||
return _repo.scm.list_branches()
|
return _repo.scm.list_branches()
|
||||||
|
|
||||||
|
|
||||||
def all_commits(repo: Optional[str] = None) -> List[str]:
|
def all_commits(repo: Optional[str] = None) -> list[str]:
|
||||||
"""Get all Git commits in a DVC repository.
|
"""Get all Git commits in a DVC repository.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -37,7 +37,7 @@ def all_commits(repo: Optional[str] = None) -> List[str]:
|
|||||||
return _repo.scm.list_all_commits()
|
return _repo.scm.list_all_commits()
|
||||||
|
|
||||||
|
|
||||||
def all_tags(repo: Optional[str] = None) -> List[str]:
|
def all_tags(repo: Optional[str] = None) -> list[str]:
|
||||||
"""Get all Git tags in a DVC repository.
|
"""Get all Git tags in a DVC repository.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
|
|||||||
+7
-6
@@ -1,6 +1,7 @@
|
|||||||
import typing
|
import typing
|
||||||
from collections import Counter
|
from collections import Counter
|
||||||
from typing import Dict, Iterable, Optional, Union
|
from collections.abc import Iterable
|
||||||
|
from typing import Optional, Union
|
||||||
|
|
||||||
from funcy import first
|
from funcy import first
|
||||||
|
|
||||||
@@ -8,7 +9,7 @@ from dvc.repo import Repo
|
|||||||
|
|
||||||
|
|
||||||
def _postprocess(results):
|
def _postprocess(results):
|
||||||
processed: Dict[str, Dict] = {}
|
processed: dict[str, dict] = {}
|
||||||
for rev, rev_data in results.items():
|
for rev, rev_data in results.items():
|
||||||
if not rev_data:
|
if not rev_data:
|
||||||
continue
|
continue
|
||||||
@@ -36,8 +37,8 @@ def metrics_show(
|
|||||||
*targets: str,
|
*targets: str,
|
||||||
repo: Optional[str] = None,
|
repo: Optional[str] = None,
|
||||||
rev: Optional[str] = None,
|
rev: Optional[str] = None,
|
||||||
config: Optional[Dict] = None,
|
config: Optional[dict] = None,
|
||||||
) -> Dict:
|
) -> dict:
|
||||||
"""Get metrics tracked in `repo`.
|
"""Get metrics tracked in `repo`.
|
||||||
|
|
||||||
Without arguments, this function will retrieve all metrics from all tracked
|
Without arguments, this function will retrieve all metrics from all tracked
|
||||||
@@ -159,8 +160,8 @@ def params_show(
|
|||||||
stages: Optional[Union[str, Iterable[str]]] = None,
|
stages: Optional[Union[str, Iterable[str]]] = None,
|
||||||
rev: Optional[str] = None,
|
rev: Optional[str] = None,
|
||||||
deps: bool = False,
|
deps: bool = False,
|
||||||
config: Optional[Dict] = None,
|
config: Optional[dict] = None,
|
||||||
) -> Dict:
|
) -> dict:
|
||||||
"""Get parameters tracked in `repo`.
|
"""Get parameters tracked in `repo`.
|
||||||
|
|
||||||
Without arguments, this function will retrieve all params from all tracked
|
Without arguments, this function will retrieve all params from all tracked
|
||||||
|
|||||||
+2
-2
@@ -1,5 +1,5 @@
|
|||||||
import os
|
import os
|
||||||
from typing import TYPE_CHECKING, Optional, Tuple
|
from typing import TYPE_CHECKING, Optional
|
||||||
|
|
||||||
from dvc.fs import GitFileSystem, Schemes
|
from dvc.fs import GitFileSystem, Schemes
|
||||||
from dvc_data.hashfile.db import get_odb
|
from dvc_data.hashfile.db import get_odb
|
||||||
@@ -15,7 +15,7 @@ def _get_odb(
|
|||||||
repo,
|
repo,
|
||||||
settings,
|
settings,
|
||||||
fs=None,
|
fs=None,
|
||||||
prefix: Optional[Tuple[str, ...]] = None,
|
prefix: Optional[tuple[str, ...]] = None,
|
||||||
hash_name: Optional[str] = None,
|
hash_name: Optional[str] = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
):
|
):
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
from typing import TYPE_CHECKING, ClassVar, Dict, Tuple
|
from typing import TYPE_CHECKING, ClassVar
|
||||||
|
|
||||||
from funcy import chunks, compact, log_durations
|
from funcy import chunks, compact, log_durations
|
||||||
|
|
||||||
@@ -17,14 +17,14 @@ logger = logger.getChild(__name__)
|
|||||||
|
|
||||||
|
|
||||||
class CmdDataStatus(CmdBase):
|
class CmdDataStatus(CmdBase):
|
||||||
COLORS: ClassVar[Dict[str, str]] = {
|
COLORS: ClassVar[dict[str, str]] = {
|
||||||
"not_in_remote": "red",
|
"not_in_remote": "red",
|
||||||
"not_in_cache": "red",
|
"not_in_cache": "red",
|
||||||
"committed": "green",
|
"committed": "green",
|
||||||
"uncommitted": "yellow",
|
"uncommitted": "yellow",
|
||||||
"untracked": "cyan",
|
"untracked": "cyan",
|
||||||
}
|
}
|
||||||
LABELS: ClassVar[Dict[str, str]] = {
|
LABELS: ClassVar[dict[str, str]] = {
|
||||||
"not_in_remote": "Not in remote",
|
"not_in_remote": "Not in remote",
|
||||||
"not_in_cache": "Not in cache",
|
"not_in_cache": "Not in cache",
|
||||||
"committed": "DVC committed changes",
|
"committed": "DVC committed changes",
|
||||||
@@ -32,7 +32,7 @@ class CmdDataStatus(CmdBase):
|
|||||||
"untracked": "Untracked files",
|
"untracked": "Untracked files",
|
||||||
"unchanged": "DVC unchanged files",
|
"unchanged": "DVC unchanged files",
|
||||||
}
|
}
|
||||||
HINTS: ClassVar[Dict[str, Tuple[str, ...]]] = {
|
HINTS: ClassVar[dict[str, tuple[str, ...]]] = {
|
||||||
"not_in_remote": ('use "dvc push <file>..." to upload files',),
|
"not_in_remote": ('use "dvc push <file>..." to upload files',),
|
||||||
"not_in_cache": ('use "dvc fetch <file>..." to download files',),
|
"not_in_cache": ('use "dvc fetch <file>..." to download files',),
|
||||||
"committed": ("git commit the corresponding dvc files to update the repo",),
|
"committed": ("git commit the corresponding dvc files to update the repo",),
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
from typing import Any, Dict
|
from typing import Any
|
||||||
|
|
||||||
from dvc.cli import completion, formatter
|
from dvc.cli import completion, formatter
|
||||||
from dvc.cli.command import CmdBase
|
from dvc.cli.command import CmdBase
|
||||||
@@ -11,7 +11,7 @@ logger = logger.getChild(__name__)
|
|||||||
|
|
||||||
class CmdExperimentsPush(CmdBase):
|
class CmdExperimentsPush(CmdBase):
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def log_result(result: Dict[str, Any], remote: str):
|
def log_result(result: dict[str, Any], remote: str):
|
||||||
from dvc.utils import humanize
|
from dvc.utils import humanize
|
||||||
|
|
||||||
def join_exps(exps):
|
def join_exps(exps):
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
import argparse
|
import argparse
|
||||||
import re
|
import re
|
||||||
|
from collections.abc import Iterable
|
||||||
from datetime import date, datetime
|
from datetime import date, datetime
|
||||||
from typing import TYPE_CHECKING, Dict, Iterable
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from funcy import lmap
|
from funcy import lmap
|
||||||
|
|
||||||
@@ -57,7 +58,7 @@ def baseline_styler(typ):
|
|||||||
|
|
||||||
def show_experiments(
|
def show_experiments(
|
||||||
td: "TabularData",
|
td: "TabularData",
|
||||||
headers: Dict[str, Iterable[str]],
|
headers: dict[str, Iterable[str]],
|
||||||
keep=None,
|
keep=None,
|
||||||
drop=None,
|
drop=None,
|
||||||
pager=True,
|
pager=True,
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import argparse
|
import argparse
|
||||||
import os
|
import os
|
||||||
from typing import TYPE_CHECKING, Dict, List, Optional
|
from typing import TYPE_CHECKING, Optional
|
||||||
|
|
||||||
from funcy import compact, first, get_in
|
from funcy import compact, first, get_in
|
||||||
|
|
||||||
@@ -20,14 +20,14 @@ logger = logger.getChild(__name__)
|
|||||||
|
|
||||||
|
|
||||||
def _show_json(
|
def _show_json(
|
||||||
renderers_with_errors: List["RendererWithErrors"],
|
renderers_with_errors: list["RendererWithErrors"],
|
||||||
split=False,
|
split=False,
|
||||||
errors: Optional[Dict[str, Exception]] = None,
|
errors: Optional[dict[str, Exception]] = None,
|
||||||
):
|
):
|
||||||
from dvc.render.convert import to_json
|
from dvc.render.convert import to_json
|
||||||
from dvc.utils.serialize import encode_exception
|
from dvc.utils.serialize import encode_exception
|
||||||
|
|
||||||
all_errors: List[Dict] = []
|
all_errors: list[dict] = []
|
||||||
data = {}
|
data = {}
|
||||||
|
|
||||||
for renderer, src_errors, def_errors in renderers_with_errors:
|
for renderer, src_errors, def_errors in renderers_with_errors:
|
||||||
|
|||||||
@@ -1,8 +1,9 @@
|
|||||||
import argparse
|
import argparse
|
||||||
import logging
|
import logging
|
||||||
|
from collections.abc import Iterable
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from itertools import chain, filterfalse
|
from itertools import chain, filterfalse
|
||||||
from typing import TYPE_CHECKING, Dict, Iterable, List
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from dvc.cli import completion, formatter
|
from dvc.cli import completion, formatter
|
||||||
from dvc.cli.command import CmdBase
|
from dvc.cli.command import CmdBase
|
||||||
@@ -33,7 +34,7 @@ def generate_description(stage: "Stage") -> str:
|
|||||||
def is_plot_or_metric(out: "Output"):
|
def is_plot_or_metric(out: "Output"):
|
||||||
return bool(out.plot) or bool(out.metric)
|
return bool(out.plot) or bool(out.metric)
|
||||||
|
|
||||||
desc: List[str] = []
|
desc: list[str] = []
|
||||||
|
|
||||||
outs = list(filterfalse(is_plot_or_metric, stage.outs))
|
outs = list(filterfalse(is_plot_or_metric, stage.outs))
|
||||||
if outs:
|
if outs:
|
||||||
@@ -55,7 +56,7 @@ def prepare_stages_data(
|
|||||||
stages: Iterable["Stage"],
|
stages: Iterable["Stage"],
|
||||||
description: bool = True,
|
description: bool = True,
|
||||||
max_length: int = MAX_TEXT_LENGTH,
|
max_length: int = MAX_TEXT_LENGTH,
|
||||||
) -> Dict[str, str]:
|
) -> dict[str, str]:
|
||||||
return {
|
return {
|
||||||
stage.addressing: (
|
stage.addressing: (
|
||||||
prepare_description(stage, max_length=max_length) if description else ""
|
prepare_description(stage, max_length=max_length) if description else ""
|
||||||
@@ -67,7 +68,7 @@ def prepare_stages_data(
|
|||||||
class CmdStageList(CmdBase):
|
class CmdStageList(CmdBase):
|
||||||
def _get_stages(self) -> Iterable["Stage"]:
|
def _get_stages(self) -> Iterable["Stage"]:
|
||||||
if self.args.all:
|
if self.args.all:
|
||||||
stages: List["Stage"] = self.repo.index.stages
|
stages: list["Stage"] = self.repo.index.stages
|
||||||
logger.trace("%d no. of stages found", len(stages))
|
logger.trace("%d no. of stages found", len(stages))
|
||||||
return stages
|
return stages
|
||||||
|
|
||||||
@@ -96,7 +97,7 @@ class CmdStageList(CmdBase):
|
|||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
def parse_cmd(commands: List[str]) -> str:
|
def parse_cmd(commands: list[str]) -> str:
|
||||||
"""
|
"""
|
||||||
We need to take into account two cases:
|
We need to take into account two cases:
|
||||||
|
|
||||||
|
|||||||
+24
-26
@@ -1,20 +1,18 @@
|
|||||||
from collections import abc
|
from collections import abc
|
||||||
|
from collections.abc import (
|
||||||
|
ItemsView,
|
||||||
|
Iterable,
|
||||||
|
Iterator,
|
||||||
|
Mapping,
|
||||||
|
MutableSequence,
|
||||||
|
Sequence,
|
||||||
|
)
|
||||||
from itertools import chain, repeat, zip_longest
|
from itertools import chain, repeat, zip_longest
|
||||||
from operator import itemgetter
|
from operator import itemgetter
|
||||||
from typing import (
|
from typing import (
|
||||||
TYPE_CHECKING,
|
TYPE_CHECKING,
|
||||||
Any,
|
Any,
|
||||||
Dict,
|
|
||||||
ItemsView,
|
|
||||||
Iterable,
|
|
||||||
Iterator,
|
|
||||||
List,
|
|
||||||
Mapping,
|
|
||||||
MutableSequence,
|
|
||||||
Optional,
|
Optional,
|
||||||
Sequence,
|
|
||||||
Set,
|
|
||||||
Tuple,
|
|
||||||
Union,
|
Union,
|
||||||
overload,
|
overload,
|
||||||
)
|
)
|
||||||
@@ -25,7 +23,7 @@ if TYPE_CHECKING:
|
|||||||
from dvc.ui.table import CellT
|
from dvc.ui.table import CellT
|
||||||
|
|
||||||
|
|
||||||
class Column(List["CellT"]):
|
class Column(list["CellT"]):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
@@ -35,13 +33,13 @@ def with_value(value, default):
|
|||||||
|
|
||||||
class TabularData(MutableSequence[Sequence["CellT"]]):
|
class TabularData(MutableSequence[Sequence["CellT"]]):
|
||||||
def __init__(self, columns: Sequence[str], fill_value: Optional[str] = ""):
|
def __init__(self, columns: Sequence[str], fill_value: Optional[str] = ""):
|
||||||
self._columns: Dict[str, Column] = {name: Column() for name in columns}
|
self._columns: dict[str, Column] = {name: Column() for name in columns}
|
||||||
self._keys: List[str] = list(columns)
|
self._keys: list[str] = list(columns)
|
||||||
self._fill_value = fill_value
|
self._fill_value = fill_value
|
||||||
self._protected: Set[str] = set()
|
self._protected: set[str] = set()
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def columns(self) -> List[Column]:
|
def columns(self) -> list[Column]:
|
||||||
return list(map(self.column, self.keys()))
|
return list(map(self.column, self.keys()))
|
||||||
|
|
||||||
def is_protected(self, col_name) -> bool:
|
def is_protected(self, col_name) -> bool:
|
||||||
@@ -60,10 +58,10 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
|
|||||||
projection = {k: self.column(k) for k in self.keys()}
|
projection = {k: self.column(k) for k in self.keys()}
|
||||||
return projection.items()
|
return projection.items()
|
||||||
|
|
||||||
def keys(self) -> List[str]:
|
def keys(self) -> list[str]:
|
||||||
return self._keys
|
return self._keys
|
||||||
|
|
||||||
def _iter_col_row(self, row: Sequence["CellT"]) -> Iterator[Tuple["CellT", Column]]:
|
def _iter_col_row(self, row: Sequence["CellT"]) -> Iterator[tuple["CellT", Column]]:
|
||||||
for val, col in zip_longest(row, self.columns):
|
for val, col in zip_longest(row, self.columns):
|
||||||
if col is None:
|
if col is None:
|
||||||
break
|
break
|
||||||
@@ -81,7 +79,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
|
|||||||
for val, col in self._iter_col_row(value):
|
for val, col in self._iter_col_row(value):
|
||||||
col.insert(index, val)
|
col.insert(index, val)
|
||||||
|
|
||||||
def __iter__(self) -> Iterator[List["CellT"]]:
|
def __iter__(self) -> Iterator[list["CellT"]]:
|
||||||
return map(list, zip(*self.columns))
|
return map(list, zip(*self.columns))
|
||||||
|
|
||||||
def __getattr__(self, item: str) -> Column:
|
def __getattr__(self, item: str) -> Column:
|
||||||
@@ -127,7 +125,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
|
|||||||
return len(self.columns[0])
|
return len(self.columns[0])
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def shape(self) -> Tuple[int, int]:
|
def shape(self) -> tuple[int, int]:
|
||||||
return len(self.columns), len(self)
|
return len(self.columns), len(self)
|
||||||
|
|
||||||
def drop(self, *col_names: str) -> None:
|
def drop(self, *col_names: str) -> None:
|
||||||
@@ -170,7 +168,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
|
|||||||
if key not in keys:
|
if key not in keys:
|
||||||
self.add_column(key)
|
self.add_column(key)
|
||||||
|
|
||||||
row: List["CellT"] = [
|
row: list["CellT"] = [
|
||||||
with_value(d.get(key), self._fill_value) for key in self.keys()
|
with_value(d.get(key), self._fill_value) for key in self.keys()
|
||||||
]
|
]
|
||||||
self.append(row)
|
self.append(row)
|
||||||
@@ -185,7 +183,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
|
|||||||
|
|
||||||
def as_dict(
|
def as_dict(
|
||||||
self, cols: Optional[Iterable[str]] = None
|
self, cols: Optional[Iterable[str]] = None
|
||||||
) -> Iterable[Dict[str, "CellT"]]:
|
) -> Iterable[dict[str, "CellT"]]:
|
||||||
keys = self.keys() if cols is None else set(cols)
|
keys = self.keys() if cols is None else set(cols)
|
||||||
return [{k: self._columns[k][i] for k in keys} for i in range(len(self))]
|
return [{k: self._columns[k][i] for k in keys} for i in range(len(self))]
|
||||||
|
|
||||||
@@ -202,7 +200,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
|
|||||||
if how not in ["any", "all"]:
|
if how not in ["any", "all"]:
|
||||||
raise ValueError(f"Invalid 'how' value {how}. Choose one of ['any', 'all']")
|
raise ValueError(f"Invalid 'how' value {how}. Choose one of ['any', 'all']")
|
||||||
|
|
||||||
match_line: Set = set()
|
match_line: set = set()
|
||||||
match_any = True
|
match_any = True
|
||||||
if how == "all":
|
if how == "all":
|
||||||
match_any = False
|
match_any = False
|
||||||
@@ -245,7 +243,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
|
|||||||
)
|
)
|
||||||
|
|
||||||
if axis == "cols":
|
if axis == "cols":
|
||||||
cols_to_drop: List[str] = []
|
cols_to_drop: list[str] = []
|
||||||
for n_col, col in enumerate(self.columns):
|
for n_col, col in enumerate(self.columns):
|
||||||
if subset and self.keys()[n_col] not in subset:
|
if subset and self.keys()[n_col] not in subset:
|
||||||
continue
|
continue
|
||||||
@@ -259,7 +257,7 @@ class TabularData(MutableSequence[Sequence["CellT"]]):
|
|||||||
|
|
||||||
elif axis == "rows":
|
elif axis == "rows":
|
||||||
unique_rows = []
|
unique_rows = []
|
||||||
rows_to_drop: List[int] = []
|
rows_to_drop: list[int] = []
|
||||||
for n_row, row in enumerate(self):
|
for n_row, row in enumerate(self):
|
||||||
if subset:
|
if subset:
|
||||||
row = [
|
row = [
|
||||||
@@ -319,7 +317,7 @@ def diff_table(
|
|||||||
) -> TabularData:
|
) -> TabularData:
|
||||||
a_rev = a_rev or "HEAD"
|
a_rev = a_rev or "HEAD"
|
||||||
b_rev = b_rev or "workspace"
|
b_rev = b_rev or "workspace"
|
||||||
headers: List[str] = ["Path", title, a_rev, b_rev, "Change"]
|
headers: list[str] = ["Path", title, a_rev, b_rev, "Change"]
|
||||||
fill_value = "-"
|
fill_value = "-"
|
||||||
td = TabularData(headers, fill_value=fill_value)
|
td = TabularData(headers, fill_value=fill_value)
|
||||||
|
|
||||||
@@ -394,7 +392,7 @@ def metrics_table(
|
|||||||
|
|
||||||
for branch, val in metrics.items():
|
for branch, val in metrics.items():
|
||||||
for fname, metric in val.get("data", {}).items():
|
for fname, metric in val.get("data", {}).items():
|
||||||
row_data: Dict[str, str] = {"Revision": branch, "Path": fname}
|
row_data: dict[str, str] = {"Revision": branch, "Path": fname}
|
||||||
metric = metric.get("data", {})
|
metric = metric.get("data", {})
|
||||||
flattened = (
|
flattened = (
|
||||||
flatten(format_dict(metric))
|
flatten(format_dict(metric))
|
||||||
|
|||||||
+4
-4
@@ -3,7 +3,7 @@ import os
|
|||||||
import re
|
import re
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from functools import partial
|
from functools import partial
|
||||||
from typing import TYPE_CHECKING, Dict, Optional
|
from typing import TYPE_CHECKING, Optional
|
||||||
|
|
||||||
from funcy import compact, memoize, re_find
|
from funcy import compact, memoize, re_find
|
||||||
|
|
||||||
@@ -138,7 +138,7 @@ class Config(dict):
|
|||||||
return system_config_dir()
|
return system_config_dir()
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def files(self) -> Dict[str, str]:
|
def files(self) -> dict[str, str]:
|
||||||
files = {
|
files = {
|
||||||
level: os.path.join(self.get_dir(level), self.CONFIG)
|
level: os.path.join(self.get_dir(level), self.CONFIG)
|
||||||
for level in ("system", "global")
|
for level in ("system", "global")
|
||||||
@@ -344,7 +344,7 @@ class Config(dict):
|
|||||||
return Schema(dirs_schema, extra=ALLOW_EXTRA)(conf)
|
return Schema(dirs_schema, extra=ALLOW_EXTRA)(conf)
|
||||||
|
|
||||||
def load_config_to_level(self, level=None):
|
def load_config_to_level(self, level=None):
|
||||||
merged_conf: Dict = {}
|
merged_conf: dict = {}
|
||||||
for merge_level in self.LEVELS:
|
for merge_level in self.LEVELS:
|
||||||
if merge_level == level:
|
if merge_level == level:
|
||||||
break
|
break
|
||||||
@@ -390,7 +390,7 @@ class Config(dict):
|
|||||||
|
|
||||||
|
|
||||||
def _parse_named(conf):
|
def _parse_named(conf):
|
||||||
result: Dict[str, Dict] = {"remote": {}, "machine": {}, "db": {}}
|
result: dict[str, dict] = {"remote": {}, "machine": {}, "db": {}}
|
||||||
|
|
||||||
for section, val in conf.items():
|
for section, val in conf.items():
|
||||||
match = re_find(r'^\s*(remote|machine|db)\s*"(.*)"\s*$', section)
|
match = re_find(r'^\s*(remote|machine|db)\s*"(.*)"\s*$', section)
|
||||||
|
|||||||
+9
-8
@@ -5,13 +5,14 @@ import logging
|
|||||||
import os
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
|
from collections.abc import Mapping, Sequence
|
||||||
from contextlib import nullcontext
|
from contextlib import nullcontext
|
||||||
from typing import TYPE_CHECKING, Any, List, Mapping, Optional, Sequence, Union
|
from typing import TYPE_CHECKING, Any, Optional, Union
|
||||||
|
|
||||||
from dvc.log import logger
|
from dvc.log import logger
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from typing import ContextManager
|
from contextlib import AbstractContextManager
|
||||||
|
|
||||||
from dvc.env import DVC_DAEMON, DVC_DAEMON_LOGFILE
|
from dvc.env import DVC_DAEMON, DVC_DAEMON_LOGFILE
|
||||||
from dvc.utils import fix_env, is_binary
|
from dvc.utils import fix_env, is_binary
|
||||||
@@ -55,7 +56,7 @@ def _win_detached_subprocess(args: Sequence[str], **kwargs) -> int:
|
|||||||
return popen.pid
|
return popen.pid
|
||||||
|
|
||||||
|
|
||||||
def _get_dvc_args() -> List[str]:
|
def _get_dvc_args() -> list[str]:
|
||||||
args = [sys.executable]
|
args = [sys.executable]
|
||||||
if not is_binary():
|
if not is_binary():
|
||||||
root_dir = os.path.abspath(os.path.dirname(__file__))
|
root_dir = os.path.abspath(os.path.dirname(__file__))
|
||||||
@@ -136,7 +137,7 @@ def _map_log_level_to_flag() -> Optional[str]:
|
|||||||
return flags.get(logger.getEffectiveLevel())
|
return flags.get(logger.getEffectiveLevel())
|
||||||
|
|
||||||
|
|
||||||
def daemon(args: List[str]) -> None:
|
def daemon(args: list[str]) -> None:
|
||||||
"""Launch a `dvc daemon` command in a detached process.
|
"""Launch a `dvc daemon` command in a detached process.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -148,12 +149,12 @@ def daemon(args: List[str]) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def _spawn(
|
def _spawn(
|
||||||
args: List[str],
|
args: list[str],
|
||||||
executable: Optional[Union[str, List[str]]] = None,
|
executable: Optional[Union[str, list[str]]] = None,
|
||||||
env: Optional[Mapping[str, str]] = None,
|
env: Optional[Mapping[str, str]] = None,
|
||||||
output_file: Optional[str] = None,
|
output_file: Optional[str] = None,
|
||||||
) -> int:
|
) -> int:
|
||||||
file: "ContextManager[Any]" = nullcontext()
|
file: "AbstractContextManager[Any]" = nullcontext()
|
||||||
kwargs = {}
|
kwargs = {}
|
||||||
if output_file:
|
if output_file:
|
||||||
file = open(output_file, "ab") # noqa: SIM115
|
file = open(output_file, "ab") # noqa: SIM115
|
||||||
@@ -168,7 +169,7 @@ def _spawn(
|
|||||||
return _detached_subprocess(executable + args, env=env, **kwargs)
|
return _detached_subprocess(executable + args, env=env, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
def daemonize(args: List[str], executable: Union[None, str, List[str]] = None) -> None:
|
def daemonize(args: list[str], executable: Union[None, str, list[str]] = None) -> None:
|
||||||
if os.name not in ("posix", "nt"):
|
if os.name not in ("posix", "nt"):
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|||||||
+3
-2
@@ -1,6 +1,7 @@
|
|||||||
"""Manages dvc remotes that user can use with push/pull/status commands."""
|
"""Manages dvc remotes that user can use with push/pull/status commands."""
|
||||||
|
|
||||||
from typing import TYPE_CHECKING, Iterable, Optional, Set, Tuple
|
from collections.abc import Iterable
|
||||||
|
from typing import TYPE_CHECKING, Optional
|
||||||
|
|
||||||
from dvc.config import NoRemoteError, RemoteConfigError
|
from dvc.config import NoRemoteError, RemoteConfigError
|
||||||
from dvc.log import logger
|
from dvc.log import logger
|
||||||
@@ -50,7 +51,7 @@ class Remote:
|
|||||||
|
|
||||||
def _split_legacy_hash_infos(
|
def _split_legacy_hash_infos(
|
||||||
hash_infos: Iterable["HashInfo"],
|
hash_infos: Iterable["HashInfo"],
|
||||||
) -> Tuple[Set["HashInfo"], Set["HashInfo"]]:
|
) -> tuple[set["HashInfo"], set["HashInfo"]]:
|
||||||
from dvc.cachemgr import LEGACY_HASH_NAMES
|
from dvc.cachemgr import LEGACY_HASH_NAMES
|
||||||
|
|
||||||
legacy = set()
|
legacy = set()
|
||||||
|
|||||||
+4
-3
@@ -1,8 +1,9 @@
|
|||||||
import os
|
import os
|
||||||
|
from collections.abc import Iterator
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from tempfile import NamedTemporaryFile
|
from tempfile import NamedTemporaryFile
|
||||||
from typing import TYPE_CHECKING, Any, Callable, Dict, Iterator, Optional, Union
|
from typing import TYPE_CHECKING, Any, Callable, Optional, Union
|
||||||
|
|
||||||
from sqlalchemy import create_engine
|
from sqlalchemy import create_engine
|
||||||
from sqlalchemy.engine import make_url as _make_url
|
from sqlalchemy.engine import make_url as _make_url
|
||||||
@@ -30,7 +31,7 @@ def make_url(url: Union["URL", str], **kwargs: Any) -> "URL":
|
|||||||
return _make_url(url).set(**kwargs)
|
return _make_url(url).set(**kwargs)
|
||||||
|
|
||||||
|
|
||||||
def url_from_config(config: Union[str, "URL", Dict[str, str]]) -> "URL":
|
def url_from_config(config: Union[str, "URL", dict[str, str]]) -> "URL":
|
||||||
if isinstance(config, dict):
|
if isinstance(config, dict):
|
||||||
return make_url(**config)
|
return make_url(**config)
|
||||||
return make_url(config)
|
return make_url(config)
|
||||||
@@ -118,7 +119,7 @@ def handle_error(url: "URL"):
|
|||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def client(
|
def client(
|
||||||
url_or_config: Union[str, "URL", Dict[str, str]], **engine_kwargs: Any
|
url_or_config: Union[str, "URL", dict[str, str]], **engine_kwargs: Any
|
||||||
) -> Iterator[Client]:
|
) -> Iterator[Client]:
|
||||||
url = url_from_config(url_or_config)
|
url = url_from_config(url_or_config)
|
||||||
echo = env2bool(env.DVC_SQLALCHEMY_ECHO, False)
|
echo = env2bool(env.DVC_SQLALCHEMY_ECHO, False)
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from typing import Any, Dict, List, Mapping, Set
|
from collections.abc import Mapping
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
from dvc.output import ARTIFACT_SCHEMA, DIR_FILES_SCHEMA, Output
|
from dvc.output import ARTIFACT_SCHEMA, DIR_FILES_SCHEMA, Output
|
||||||
|
|
||||||
@@ -66,12 +67,12 @@ def loads_from(stage, s_list, erepo=None, fs_config=None, db=None):
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def _merge_params(s_list) -> Dict[str, List[str]]:
|
def _merge_params(s_list) -> dict[str, list[str]]:
|
||||||
d = defaultdict(list)
|
d = defaultdict(list)
|
||||||
default_file = ParamsDependency.DEFAULT_PARAMS_FILE
|
default_file = ParamsDependency.DEFAULT_PARAMS_FILE
|
||||||
|
|
||||||
# figure out completely tracked params file, and ignore specific keys
|
# figure out completely tracked params file, and ignore specific keys
|
||||||
wholly_tracked: Set[str] = set()
|
wholly_tracked: set[str] = set()
|
||||||
for key in s_list:
|
for key in s_list:
|
||||||
if not isinstance(key, dict):
|
if not isinstance(key, dict):
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -1,5 +1,3 @@
|
|||||||
from typing import Dict, Type
|
|
||||||
|
|
||||||
from dvc.exceptions import DvcException
|
from dvc.exceptions import DvcException
|
||||||
from dvc.fs import download as fs_download
|
from dvc.fs import download as fs_download
|
||||||
from dvc.output import Output
|
from dvc.output import Output
|
||||||
@@ -25,11 +23,11 @@ class DependencyIsStageFileError(DvcException):
|
|||||||
class Dependency(Output):
|
class Dependency(Output):
|
||||||
IS_DEPENDENCY = True
|
IS_DEPENDENCY = True
|
||||||
|
|
||||||
DoesNotExistError: Type[DvcException] = DependencyDoesNotExistError
|
DoesNotExistError: type[DvcException] = DependencyDoesNotExistError
|
||||||
IsNotFileOrDirError: Type[DvcException] = DependencyIsNotFileOrDirError
|
IsNotFileOrDirError: type[DvcException] = DependencyIsNotFileOrDirError
|
||||||
IsStageFileError: Type[DvcException] = DependencyIsStageFileError
|
IsStageFileError: type[DvcException] = DependencyIsStageFileError
|
||||||
|
|
||||||
def workspace_status(self) -> Dict[str, str]:
|
def workspace_status(self) -> dict[str, str]:
|
||||||
if self.fs.version_aware:
|
if self.fs.version_aware:
|
||||||
old_fs_path = self.fs_path
|
old_fs_path = self.fs_path
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
|
from collections.abc import Iterator
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from typing import TYPE_CHECKING, Any, Callable, ClassVar, Dict, Iterator, Optional
|
from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional
|
||||||
|
|
||||||
from funcy import compact, log_durations
|
from funcy import compact, log_durations
|
||||||
|
|
||||||
@@ -31,7 +32,7 @@ def download_progress(to: "Output") -> Iterator[Callable[[int], Any]]:
|
|||||||
class AbstractDependency(Dependency):
|
class AbstractDependency(Dependency):
|
||||||
"""Dependency without workspace/fs/fs_path"""
|
"""Dependency without workspace/fs/fs_path"""
|
||||||
|
|
||||||
def __init__(self, stage: "Stage", info: Dict[str, Any], *args, **kwargs):
|
def __init__(self, stage: "Stage", info: dict[str, Any], *args, **kwargs):
|
||||||
self.repo = stage.repo
|
self.repo = stage.repo
|
||||||
self.stage = stage
|
self.stage = stage
|
||||||
self.fs = None
|
self.fs = None
|
||||||
@@ -46,7 +47,7 @@ class DbDependency(AbstractDependency):
|
|||||||
PARAM_QUERY = "query"
|
PARAM_QUERY = "query"
|
||||||
PARAM_TABLE = "table"
|
PARAM_TABLE = "table"
|
||||||
PARAM_FILE_FORMAT = "file_format"
|
PARAM_FILE_FORMAT = "file_format"
|
||||||
DB_SCHEMA: ClassVar[Dict] = {
|
DB_SCHEMA: ClassVar[dict] = {
|
||||||
PARAM_DB: {
|
PARAM_DB: {
|
||||||
PARAM_QUERY: str,
|
PARAM_QUERY: str,
|
||||||
PARAM_CONNECTION: str,
|
PARAM_CONNECTION: str,
|
||||||
@@ -57,7 +58,7 @@ class DbDependency(AbstractDependency):
|
|||||||
|
|
||||||
def __init__(self, stage: "Stage", info, *args, **kwargs):
|
def __init__(self, stage: "Stage", info, *args, **kwargs):
|
||||||
super().__init__(stage, info, *args, **kwargs)
|
super().__init__(stage, info, *args, **kwargs)
|
||||||
self.db_info: Dict[str, str] = self.info.get(self.PARAM_DB, {})
|
self.db_info: dict[str, str] = self.info.get(self.PARAM_DB, {})
|
||||||
self.connection = self.db_info.get(self.PARAM_CONNECTION)
|
self.connection = self.db_info.get(self.PARAM_CONNECTION)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import os
|
import os
|
||||||
import typing
|
import typing
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional
|
from typing import TYPE_CHECKING, Any, Optional
|
||||||
|
|
||||||
import dpath
|
import dpath
|
||||||
|
|
||||||
@@ -37,7 +37,7 @@ class BadParamFileError(DvcException):
|
|||||||
def read_param_file(
|
def read_param_file(
|
||||||
fs: "FileSystem",
|
fs: "FileSystem",
|
||||||
path: str,
|
path: str,
|
||||||
key_paths: Optional[List[str]] = None,
|
key_paths: Optional[list[str]] = None,
|
||||||
flatten: bool = False,
|
flatten: bool = False,
|
||||||
**load_kwargs,
|
**load_kwargs,
|
||||||
) -> Any:
|
) -> Any:
|
||||||
@@ -110,7 +110,7 @@ class ParamsDependency(Dependency):
|
|||||||
|
|
||||||
def read_params(
|
def read_params(
|
||||||
self, flatten: bool = True, **kwargs: typing.Any
|
self, flatten: bool = True, **kwargs: typing.Any
|
||||||
) -> Dict[str, typing.Any]:
|
) -> dict[str, typing.Any]:
|
||||||
try:
|
try:
|
||||||
self.validate_filepath()
|
self.validate_filepath()
|
||||||
except MissingParamsFile:
|
except MissingParamsFile:
|
||||||
@@ -134,7 +134,7 @@ class ParamsDependency(Dependency):
|
|||||||
|
|
||||||
from funcy import ldistinct
|
from funcy import ldistinct
|
||||||
|
|
||||||
status: Dict[str, Any] = defaultdict(dict)
|
status: dict[str, Any] = defaultdict(dict)
|
||||||
info = self.hash_info.value if self.hash_info else {}
|
info = self.hash_info.value if self.hash_info else {}
|
||||||
assert isinstance(info, dict)
|
assert isinstance(info, dict)
|
||||||
actual = self.read_params()
|
actual = self.read_params()
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
from copy import deepcopy
|
from copy import deepcopy
|
||||||
from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union
|
from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union
|
||||||
|
|
||||||
import voluptuous as vol
|
import voluptuous as vol
|
||||||
|
|
||||||
@@ -20,7 +20,7 @@ class RepoDependency(Dependency):
|
|||||||
PARAM_CONFIG = "config"
|
PARAM_CONFIG = "config"
|
||||||
PARAM_REMOTE = "remote"
|
PARAM_REMOTE = "remote"
|
||||||
|
|
||||||
REPO_SCHEMA: ClassVar[Dict] = {
|
REPO_SCHEMA: ClassVar[dict] = {
|
||||||
PARAM_REPO: {
|
PARAM_REPO: {
|
||||||
vol.Required(PARAM_URL): str,
|
vol.Required(PARAM_URL): str,
|
||||||
PARAM_REV: str,
|
PARAM_REV: str,
|
||||||
@@ -30,7 +30,7 @@ class RepoDependency(Dependency):
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
def __init__(self, def_repo: Dict[str, Any], stage: "Stage", *args, **kwargs):
|
def __init__(self, def_repo: dict[str, Any], stage: "Stage", *args, **kwargs):
|
||||||
self.def_repo = def_repo
|
self.def_repo = def_repo
|
||||||
super().__init__(stage, *args, **kwargs)
|
super().__init__(stage, *args, **kwargs)
|
||||||
|
|
||||||
@@ -65,7 +65,7 @@ class RepoDependency(Dependency):
|
|||||||
self.def_repo[self.PARAM_REV_LOCK] = rev
|
self.def_repo[self.PARAM_REV_LOCK] = rev
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def _dump_def_repo(cls, def_repo) -> Dict[str, str]:
|
def _dump_def_repo(cls, def_repo) -> dict[str, str]:
|
||||||
repo = {cls.PARAM_URL: def_repo[cls.PARAM_URL]}
|
repo = {cls.PARAM_URL: def_repo[cls.PARAM_URL]}
|
||||||
|
|
||||||
rev = def_repo.get(cls.PARAM_REV)
|
rev = def_repo.get(cls.PARAM_REV)
|
||||||
@@ -85,7 +85,7 @@ class RepoDependency(Dependency):
|
|||||||
repo[cls.PARAM_REMOTE] = remote
|
repo[cls.PARAM_REMOTE] = remote
|
||||||
return repo
|
return repo
|
||||||
|
|
||||||
def dumpd(self, **kwargs) -> Dict[str, Union[str, Dict[str, str]]]:
|
def dumpd(self, **kwargs) -> dict[str, Union[str, dict[str, str]]]:
|
||||||
return {
|
return {
|
||||||
self.PARAM_PATH: self.def_path,
|
self.PARAM_PATH: self.def_path,
|
||||||
self.PARAM_REPO: self._dump_def_repo(self.def_repo),
|
self.PARAM_REPO: self._dump_def_repo(self.def_repo),
|
||||||
|
|||||||
+10
-13
@@ -5,10 +5,7 @@ from typing import (
|
|||||||
Any,
|
Any,
|
||||||
Callable,
|
Callable,
|
||||||
ClassVar,
|
ClassVar,
|
||||||
Dict,
|
|
||||||
List,
|
|
||||||
Optional,
|
Optional,
|
||||||
Tuple,
|
|
||||||
TypeVar,
|
TypeVar,
|
||||||
Union,
|
Union,
|
||||||
)
|
)
|
||||||
@@ -134,7 +131,7 @@ class FileMixin:
|
|||||||
d, _ = self._load(**kwargs)
|
d, _ = self._load(**kwargs)
|
||||||
return d
|
return d
|
||||||
|
|
||||||
def _load(self, **kwargs: Any) -> Tuple[Any, str]:
|
def _load(self, **kwargs: Any) -> tuple[Any, str]:
|
||||||
# it raises the proper exceptions by priority:
|
# it raises the proper exceptions by priority:
|
||||||
# 1. when the file doesn't exists
|
# 1. when the file doesn't exists
|
||||||
# 2. filename is not a DVC file
|
# 2. filename is not a DVC file
|
||||||
@@ -157,7 +154,7 @@ class FileMixin:
|
|||||||
|
|
||||||
return validate(d, cls.SCHEMA, path=fname) # type: ignore[arg-type]
|
return validate(d, cls.SCHEMA, path=fname) # type: ignore[arg-type]
|
||||||
|
|
||||||
def _load_yaml(self, **kwargs: Any) -> Tuple[Any, str]:
|
def _load_yaml(self, **kwargs: Any) -> tuple[Any, str]:
|
||||||
from dvc.utils import strictyaml
|
from dvc.utils import strictyaml
|
||||||
|
|
||||||
return strictyaml.load(
|
return strictyaml.load(
|
||||||
@@ -182,10 +179,10 @@ class SingleStageFile(FileMixin):
|
|||||||
from dvc.schema import COMPILED_SINGLE_STAGE_SCHEMA as SCHEMA
|
from dvc.schema import COMPILED_SINGLE_STAGE_SCHEMA as SCHEMA
|
||||||
from dvc.stage.loader import SingleStageLoader as LOADER # noqa: N814
|
from dvc.stage.loader import SingleStageLoader as LOADER # noqa: N814
|
||||||
|
|
||||||
metrics: ClassVar[List[str]] = []
|
metrics: ClassVar[list[str]] = []
|
||||||
plots: ClassVar[Any] = {}
|
plots: ClassVar[Any] = {}
|
||||||
params: ClassVar[List[str]] = []
|
params: ClassVar[list[str]] = []
|
||||||
artifacts: ClassVar[Dict[str, Optional[Dict[str, Any]]]] = {}
|
artifacts: ClassVar[dict[str, Optional[dict[str, Any]]]] = {}
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def stage(self) -> "Stage":
|
def stage(self) -> "Stage":
|
||||||
@@ -284,11 +281,11 @@ class ProjectFile(FileMixin):
|
|||||||
raise DvcException("ProjectFile has multiple stages. Please specify it's name.")
|
raise DvcException("ProjectFile has multiple stages. Please specify it's name.")
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def contents(self) -> Dict[str, Any]:
|
def contents(self) -> dict[str, Any]:
|
||||||
return self._load()[0]
|
return self._load()[0]
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def lockfile_contents(self) -> Dict[str, Any]:
|
def lockfile_contents(self) -> dict[str, Any]:
|
||||||
return self._lockfile.load()
|
return self._lockfile.load()
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
@@ -303,7 +300,7 @@ class ProjectFile(FileMixin):
|
|||||||
return self.LOADER(self, self.contents, self.lockfile_contents)
|
return self.LOADER(self, self.contents, self.lockfile_contents)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def metrics(self) -> List[str]:
|
def metrics(self) -> list[str]:
|
||||||
return self.contents.get("metrics", [])
|
return self.contents.get("metrics", [])
|
||||||
|
|
||||||
@property
|
@property
|
||||||
@@ -311,11 +308,11 @@ class ProjectFile(FileMixin):
|
|||||||
return self.contents.get("plots", {})
|
return self.contents.get("plots", {})
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def params(self) -> List[str]:
|
def params(self) -> list[str]:
|
||||||
return self.contents.get("params", [])
|
return self.contents.get("params", [])
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def artifacts(self) -> Dict[str, Optional[Dict[str, Any]]]:
|
def artifacts(self) -> dict[str, Optional[dict[str, Any]]]:
|
||||||
return self.contents.get("artifacts", {})
|
return self.contents.get("artifacts", {})
|
||||||
|
|
||||||
def remove(self, force=False):
|
def remove(self, force=False):
|
||||||
|
|||||||
+3
-3
@@ -1,6 +1,6 @@
|
|||||||
"""Exceptions raised by the dvc."""
|
"""Exceptions raised by the dvc."""
|
||||||
import errno
|
import errno
|
||||||
from typing import TYPE_CHECKING, Dict, List, Optional, Set
|
from typing import TYPE_CHECKING, Optional
|
||||||
|
|
||||||
from dvc.utils import format_link
|
from dvc.utils import format_link
|
||||||
|
|
||||||
@@ -34,7 +34,7 @@ class OutputDuplicationError(DvcException):
|
|||||||
stages (list): list of paths to stages.
|
stages (list): list of paths to stages.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, output: str, stages: Set["Stage"]):
|
def __init__(self, output: str, stages: set["Stage"]):
|
||||||
from funcy import first
|
from funcy import first
|
||||||
|
|
||||||
assert isinstance(output, str)
|
assert isinstance(output, str)
|
||||||
@@ -241,7 +241,7 @@ class UploadError(FileTransferError):
|
|||||||
|
|
||||||
|
|
||||||
class CheckoutError(DvcException):
|
class CheckoutError(DvcException):
|
||||||
def __init__(self, target_infos: List[str], stats: Dict[str, List[str]]):
|
def __init__(self, target_infos: list[str], stats: dict[str, list[str]]):
|
||||||
from dvc.utils import error_link
|
from dvc.utils import error_link
|
||||||
|
|
||||||
self.target_infos = target_infos
|
self.target_infos = target_infos
|
||||||
|
|||||||
+3
-3
@@ -1,5 +1,5 @@
|
|||||||
from contextlib import ExitStack
|
from contextlib import ExitStack
|
||||||
from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optional, Union
|
from typing import TYPE_CHECKING, Any, BinaryIO, Optional, Union
|
||||||
|
|
||||||
from dvc.progress import Tqdm
|
from dvc.progress import Tqdm
|
||||||
from dvc.utils.objects import cached_property
|
from dvc.utils.objects import cached_property
|
||||||
@@ -55,7 +55,7 @@ class TqdmCallback(Callback):
|
|||||||
self,
|
self,
|
||||||
path_1: "Union[str, BinaryIO]",
|
path_1: "Union[str, BinaryIO]",
|
||||||
path_2: str,
|
path_2: str,
|
||||||
kwargs: Dict[str, Any],
|
kwargs: dict[str, Any],
|
||||||
child: Optional[Callback] = None,
|
child: Optional[Callback] = None,
|
||||||
):
|
):
|
||||||
child = child or TqdmCallback(
|
child = child or TqdmCallback(
|
||||||
@@ -130,7 +130,7 @@ class RichCallback(Callback):
|
|||||||
self,
|
self,
|
||||||
path_1: Union[str, BinaryIO],
|
path_1: Union[str, BinaryIO],
|
||||||
path_2: str,
|
path_2: str,
|
||||||
kwargs: Dict[str, Any],
|
kwargs: dict[str, Any],
|
||||||
child: Optional["Callback"] = None,
|
child: Optional["Callback"] = None,
|
||||||
):
|
):
|
||||||
child = child or RichCallback(
|
child = child or RichCallback(
|
||||||
|
|||||||
+9
-9
@@ -6,7 +6,7 @@ import posixpath
|
|||||||
import threading
|
import threading
|
||||||
from collections import deque
|
from collections import deque
|
||||||
from contextlib import ExitStack, suppress
|
from contextlib import ExitStack, suppress
|
||||||
from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Tuple, Type, Union
|
from typing import TYPE_CHECKING, Any, Callable, Optional, Union
|
||||||
|
|
||||||
from fsspec.spec import AbstractFileSystem
|
from fsspec.spec import AbstractFileSystem
|
||||||
from funcy import wrap_with
|
from funcy import wrap_with
|
||||||
@@ -22,8 +22,8 @@ if TYPE_CHECKING:
|
|||||||
|
|
||||||
logger = logger.getChild(__name__)
|
logger = logger.getChild(__name__)
|
||||||
|
|
||||||
RepoFactory = Union[Callable[..., "Repo"], Type["Repo"]]
|
RepoFactory = Union[Callable[..., "Repo"], type["Repo"]]
|
||||||
Key = Tuple[str, ...]
|
Key = tuple[str, ...]
|
||||||
|
|
||||||
|
|
||||||
def as_posix(path: str) -> str:
|
def as_posix(path: str) -> str:
|
||||||
@@ -85,7 +85,7 @@ class _DVCFileSystem(AbstractFileSystem):
|
|||||||
subrepos: bool = False,
|
subrepos: bool = False,
|
||||||
repo_factory: Optional[RepoFactory] = None,
|
repo_factory: Optional[RepoFactory] = None,
|
||||||
fo: Optional[str] = None,
|
fo: Optional[str] = None,
|
||||||
target_options: Optional[Dict[str, Any]] = None, # noqa: ARG002
|
target_options: Optional[dict[str, Any]] = None, # noqa: ARG002
|
||||||
target_protocol: Optional[str] = None, # noqa: ARG002
|
target_protocol: Optional[str] = None, # noqa: ARG002
|
||||||
config: Optional["DictStrAny"] = None,
|
config: Optional["DictStrAny"] = None,
|
||||||
remote: Optional[str] = None,
|
remote: Optional[str] = None,
|
||||||
@@ -145,7 +145,7 @@ class _DVCFileSystem(AbstractFileSystem):
|
|||||||
}
|
}
|
||||||
|
|
||||||
def getcwd(self):
|
def getcwd(self):
|
||||||
relparts: Tuple[str, ...] = ()
|
relparts: tuple[str, ...] = ()
|
||||||
assert self.repo is not None
|
assert self.repo is not None
|
||||||
if self.repo.fs.isin(self.repo.fs.getcwd(), self.repo.root_dir):
|
if self.repo.fs.isin(self.repo.fs.getcwd(), self.repo.root_dir):
|
||||||
relparts = self.repo.fs.relparts(self.repo.fs.getcwd(), self.repo.root_dir)
|
relparts = self.repo.fs.relparts(self.repo.fs.getcwd(), self.repo.root_dir)
|
||||||
@@ -156,7 +156,7 @@ class _DVCFileSystem(AbstractFileSystem):
|
|||||||
return posixpath.join(*parts)
|
return posixpath.join(*parts)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def parts(cls, path: str) -> Tuple[str, ...]:
|
def parts(cls, path: str) -> tuple[str, ...]:
|
||||||
ret = []
|
ret = []
|
||||||
while True:
|
while True:
|
||||||
path, part = posixpath.split(path)
|
path, part = posixpath.split(path)
|
||||||
@@ -187,7 +187,7 @@ class _DVCFileSystem(AbstractFileSystem):
|
|||||||
start = "."
|
start = "."
|
||||||
return posixpath.relpath(self.abspath(path), start=self.abspath(start))
|
return posixpath.relpath(self.abspath(path), start=self.abspath(start))
|
||||||
|
|
||||||
def relparts(self, path: str, start: Optional[str] = None) -> Tuple[str, ...]:
|
def relparts(self, path: str, start: Optional[str] = None) -> tuple[str, ...]:
|
||||||
return self.parts(self.relpath(path, start=start))
|
return self.parts(self.relpath(path, start=start))
|
||||||
|
|
||||||
@functools.cached_property
|
@functools.cached_property
|
||||||
@@ -321,7 +321,7 @@ class _DVCFileSystem(AbstractFileSystem):
|
|||||||
|
|
||||||
def _get_subrepo_info(
|
def _get_subrepo_info(
|
||||||
self, key: Key
|
self, key: Key
|
||||||
) -> Tuple["Repo", Optional[DataFileSystem], Key]:
|
) -> tuple["Repo", Optional[DataFileSystem], Key]:
|
||||||
"""
|
"""
|
||||||
Returns information about the subrepo the key is part of.
|
Returns information about the subrepo the key is part of.
|
||||||
"""
|
"""
|
||||||
@@ -525,7 +525,7 @@ class DVCFileSystem(FileSystem):
|
|||||||
protocol = "local"
|
protocol = "local"
|
||||||
PARAM_CHECKSUM = "md5"
|
PARAM_CHECKSUM = "md5"
|
||||||
|
|
||||||
def _prepare_credentials(self, **config) -> Dict[str, Any]:
|
def _prepare_credentials(self, **config) -> dict[str, Any]:
|
||||||
return config
|
return config
|
||||||
|
|
||||||
@functools.cached_property
|
@functools.cached_property
|
||||||
|
|||||||
+4
-4
@@ -2,7 +2,7 @@ import os
|
|||||||
import re
|
import re
|
||||||
from collections import namedtuple
|
from collections import namedtuple
|
||||||
from itertools import chain, groupby, takewhile
|
from itertools import chain, groupby, takewhile
|
||||||
from typing import TYPE_CHECKING, List, Optional
|
from typing import TYPE_CHECKING, Optional
|
||||||
|
|
||||||
from pathspec.patterns import GitWildMatchPattern
|
from pathspec.patterns import GitWildMatchPattern
|
||||||
from pathspec.util import normalize_file
|
from pathspec.util import normalize_file
|
||||||
@@ -62,7 +62,7 @@ class DvcIgnorePatterns(DvcIgnore):
|
|||||||
|
|
||||||
return cls(path_spec_lines, dirname, fs.sep)
|
return cls(path_spec_lines, dirname, fs.sep)
|
||||||
|
|
||||||
def __call__(self, root: List[str], dirs: List[str], files: List[str]):
|
def __call__(self, root: list[str], dirs: list[str], files: list[str]):
|
||||||
files = [f for f in files if not self.matches(root, f)]
|
files = [f for f in files if not self.matches(root, f)]
|
||||||
dirs = [d for d in dirs if not self.matches(root, d, True)]
|
dirs = [d for d in dirs if not self.matches(root, d, True)]
|
||||||
|
|
||||||
@@ -226,7 +226,7 @@ class DvcIgnoreFilter:
|
|||||||
self,
|
self,
|
||||||
dirname: str,
|
dirname: str,
|
||||||
ignore_trie: Trie,
|
ignore_trie: Trie,
|
||||||
dnames: Optional["List"],
|
dnames: Optional["list"],
|
||||||
ignore_subrepos: bool,
|
ignore_subrepos: bool,
|
||||||
) -> None:
|
) -> None:
|
||||||
self._update_trie(dirname, ignore_trie)
|
self._update_trie(dirname, ignore_trie)
|
||||||
@@ -331,7 +331,7 @@ class DvcIgnoreFilter:
|
|||||||
yield from fs.find(path)
|
yield from fs.find(path)
|
||||||
|
|
||||||
def _get_trie_pattern(
|
def _get_trie_pattern(
|
||||||
self, dirname, dnames: Optional["List"] = None, ignore_subrepos=True
|
self, dirname, dnames: Optional["list"] = None, ignore_subrepos=True
|
||||||
) -> Optional["DvcIgnorePatterns"]:
|
) -> Optional["DvcIgnorePatterns"]:
|
||||||
if ignore_subrepos:
|
if ignore_subrepos:
|
||||||
ignores_trie = self.ignores_trie_fs
|
ignores_trie = self.ignores_trie_fs
|
||||||
|
|||||||
+2
-2
@@ -5,7 +5,7 @@ import logging.config
|
|||||||
import logging.handlers
|
import logging.handlers
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
from typing import ClassVar, Dict
|
from typing import ClassVar
|
||||||
|
|
||||||
import colorama
|
import colorama
|
||||||
|
|
||||||
@@ -79,7 +79,7 @@ class ColorFormatter(logging.Formatter):
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
reset = colorama.Fore.RESET
|
reset = colorama.Fore.RESET
|
||||||
color_codes: ClassVar[Dict[str, str]] = {
|
color_codes: ClassVar[dict[str, str]] = {
|
||||||
"TRACE": colorama.Fore.GREEN,
|
"TRACE": colorama.Fore.GREEN,
|
||||||
"DEBUG": colorama.Fore.BLUE,
|
"DEBUG": colorama.Fore.BLUE,
|
||||||
"WARNING": colorama.Fore.YELLOW,
|
"WARNING": colorama.Fore.YELLOW,
|
||||||
|
|||||||
+23
-23
@@ -4,7 +4,7 @@ import posixpath
|
|||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from contextlib import suppress
|
from contextlib import suppress
|
||||||
from operator import itemgetter
|
from operator import itemgetter
|
||||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Type, Union
|
from typing import TYPE_CHECKING, Any, Optional, Union
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
import voluptuous as vol
|
import voluptuous as vol
|
||||||
@@ -148,7 +148,7 @@ def _split_dict(d, keys):
|
|||||||
|
|
||||||
|
|
||||||
def _merge_data(s_list):
|
def _merge_data(s_list):
|
||||||
d: Dict[str, Dict] = defaultdict(dict)
|
d: dict[str, dict] = defaultdict(dict)
|
||||||
for key in s_list:
|
for key in s_list:
|
||||||
if isinstance(key, str):
|
if isinstance(key, str):
|
||||||
d[key].update({})
|
d[key].update({})
|
||||||
@@ -207,7 +207,7 @@ def load_from_pipeline(stage, data, typ="outs"):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def split_file_meta_from_cloud(entry: Dict) -> Dict:
|
def split_file_meta_from_cloud(entry: dict) -> dict:
|
||||||
if remote_name := entry.pop(Meta.PARAM_REMOTE, None):
|
if remote_name := entry.pop(Meta.PARAM_REMOTE, None):
|
||||||
remote_meta = {}
|
remote_meta = {}
|
||||||
for key in (
|
for key in (
|
||||||
@@ -223,7 +223,7 @@ def split_file_meta_from_cloud(entry: Dict) -> Dict:
|
|||||||
return entry
|
return entry
|
||||||
|
|
||||||
|
|
||||||
def merge_file_meta_from_cloud(entry: Dict) -> Dict:
|
def merge_file_meta_from_cloud(entry: dict) -> dict:
|
||||||
cloud_meta = entry.pop(Output.PARAM_CLOUD, {})
|
cloud_meta = entry.pop(Output.PARAM_CLOUD, {})
|
||||||
if remote_name := first(cloud_meta):
|
if remote_name := first(cloud_meta):
|
||||||
entry.update(cloud_meta[remote_name])
|
entry.update(cloud_meta[remote_name])
|
||||||
@@ -231,7 +231,7 @@ def merge_file_meta_from_cloud(entry: Dict) -> Dict:
|
|||||||
return entry
|
return entry
|
||||||
|
|
||||||
|
|
||||||
def _serialize_tree_obj_to_files(obj: Tree) -> List[Dict[str, Any]]:
|
def _serialize_tree_obj_to_files(obj: Tree) -> list[dict[str, Any]]:
|
||||||
key = obj.PARAM_RELPATH
|
key = obj.PARAM_RELPATH
|
||||||
return sorted(
|
return sorted(
|
||||||
(
|
(
|
||||||
@@ -246,7 +246,7 @@ def _serialize_tree_obj_to_files(obj: Tree) -> List[Dict[str, Any]]:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _serialize_hi_to_dict(hash_info: Optional[HashInfo]) -> Dict[str, Any]:
|
def _serialize_hi_to_dict(hash_info: Optional[HashInfo]) -> dict[str, Any]:
|
||||||
if hash_info:
|
if hash_info:
|
||||||
if hash_info.name == "md5-dos2unix":
|
if hash_info.name == "md5-dos2unix":
|
||||||
return {"md5": hash_info.value}
|
return {"md5": hash_info.value}
|
||||||
@@ -316,10 +316,10 @@ class Output:
|
|||||||
PARAM_HASH = "hash"
|
PARAM_HASH = "hash"
|
||||||
PARAM_FS_CONFIG = "fs_config"
|
PARAM_FS_CONFIG = "fs_config"
|
||||||
|
|
||||||
DoesNotExistError: Type[DvcException] = OutputDoesNotExistError
|
DoesNotExistError: type[DvcException] = OutputDoesNotExistError
|
||||||
IsNotFileOrDirError: Type[DvcException] = OutputIsNotFileOrDirError
|
IsNotFileOrDirError: type[DvcException] = OutputIsNotFileOrDirError
|
||||||
IsStageFileError: Type[DvcException] = OutputIsStageFileError
|
IsStageFileError: type[DvcException] = OutputIsStageFileError
|
||||||
IsIgnoredError: Type[DvcException] = OutputIsIgnoredError
|
IsIgnoredError: type[DvcException] = OutputIsIgnoredError
|
||||||
|
|
||||||
def __init__( # noqa: PLR0913
|
def __init__( # noqa: PLR0913
|
||||||
self,
|
self,
|
||||||
@@ -337,7 +337,7 @@ class Output:
|
|||||||
remote=None,
|
remote=None,
|
||||||
repo=None,
|
repo=None,
|
||||||
fs_config=None,
|
fs_config=None,
|
||||||
files: Optional[List[Dict[str, Any]]] = None,
|
files: Optional[list[dict[str, Any]]] = None,
|
||||||
push: bool = True,
|
push: bool = True,
|
||||||
hash_name: Optional[str] = DEFAULT_ALGORITHM,
|
hash_name: Optional[str] = DEFAULT_ALGORITHM,
|
||||||
):
|
):
|
||||||
@@ -420,7 +420,7 @@ class Output:
|
|||||||
|
|
||||||
def _compute_hash_info_from_meta(
|
def _compute_hash_info_from_meta(
|
||||||
self, hash_name: Optional[str]
|
self, hash_name: Optional[str]
|
||||||
) -> Tuple[str, HashInfo]:
|
) -> tuple[str, HashInfo]:
|
||||||
if self.is_in_repo:
|
if self.is_in_repo:
|
||||||
if hash_name is None:
|
if hash_name is None:
|
||||||
# Legacy 2.x output, use "md5-dos2unix" but read "md5" from
|
# Legacy 2.x output, use "md5-dos2unix" but read "md5" from
|
||||||
@@ -555,7 +555,7 @@ class Output:
|
|||||||
|
|
||||||
def _build(
|
def _build(
|
||||||
self, *args, no_progress_bar=False, **kwargs
|
self, *args, no_progress_bar=False, **kwargs
|
||||||
) -> Tuple["HashFileDB", "Meta", "HashFile"]:
|
) -> tuple["HashFileDB", "Meta", "HashFile"]:
|
||||||
from dvc.ui import ui
|
from dvc.ui import ui
|
||||||
|
|
||||||
with ui.progress(
|
with ui.progress(
|
||||||
@@ -601,7 +601,7 @@ class Output:
|
|||||||
return self.fs.exists(self.fs_path)
|
return self.fs.exists(self.fs_path)
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def index_key(self) -> Tuple[str, "DataIndexKey"]:
|
def index_key(self) -> tuple[str, "DataIndexKey"]:
|
||||||
if self.is_in_repo:
|
if self.is_in_repo:
|
||||||
workspace = "repo"
|
workspace = "repo"
|
||||||
key = self.repo.fs.relparts(self.fs_path, self.repo.root_dir)
|
key = self.repo.fs.relparts(self.fs_path, self.repo.root_dir)
|
||||||
@@ -633,7 +633,7 @@ class Output:
|
|||||||
return self.meta.version_id != self.get_meta().version_id
|
return self.meta.version_id != self.get_meta().version_id
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def workspace_status(self) -> Dict[str, str]:
|
def workspace_status(self) -> dict[str, str]:
|
||||||
if not self.exists:
|
if not self.exists:
|
||||||
return {str(self): "deleted"}
|
return {str(self): "deleted"}
|
||||||
|
|
||||||
@@ -645,7 +645,7 @@ class Output:
|
|||||||
|
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
def status(self) -> Dict[str, str]:
|
def status(self) -> dict[str, str]:
|
||||||
if self.hash_info and self.use_cache and self.changed_cache():
|
if self.hash_info and self.use_cache and self.changed_cache():
|
||||||
return {str(self): "not in cache"}
|
return {str(self): "not in cache"}
|
||||||
|
|
||||||
@@ -840,7 +840,7 @@ class Output:
|
|||||||
def dumpd(self, **kwargs): # noqa: C901, PLR0912
|
def dumpd(self, **kwargs): # noqa: C901, PLR0912
|
||||||
from dvc.cachemgr import LEGACY_HASH_NAMES
|
from dvc.cachemgr import LEGACY_HASH_NAMES
|
||||||
|
|
||||||
ret: Dict[str, Any] = {}
|
ret: dict[str, Any] = {}
|
||||||
with_files = (
|
with_files = (
|
||||||
(not self.IS_DEPENDENCY or self.stage.is_import)
|
(not self.IS_DEPENDENCY or self.stage.is_import)
|
||||||
and self.hash_info.isdir
|
and self.hash_info.isdir
|
||||||
@@ -962,7 +962,7 @@ class Output:
|
|||||||
filter_info: Optional[str] = None,
|
filter_info: Optional[str] = None,
|
||||||
allow_missing: bool = False,
|
allow_missing: bool = False,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Optional[Tuple[bool, Optional[bool]]]:
|
) -> Optional[tuple[bool, Optional[bool]]]:
|
||||||
# callback passed act as a aggregate callback.
|
# callback passed act as a aggregate callback.
|
||||||
# do not let checkout to call set_size and change progressbar.
|
# do not let checkout to call set_size and change progressbar.
|
||||||
class CallbackProxy(Callback):
|
class CallbackProxy(Callback):
|
||||||
@@ -1157,7 +1157,7 @@ class Output:
|
|||||||
|
|
||||||
def get_used_objs( # noqa: PLR0911
|
def get_used_objs( # noqa: PLR0911
|
||||||
self, **kwargs
|
self, **kwargs
|
||||||
) -> Dict[Optional["HashFileDB"], Set["HashInfo"]]:
|
) -> dict[Optional["HashFileDB"], set["HashInfo"]]:
|
||||||
"""Return filtered set of used object IDs for this out."""
|
"""Return filtered set of used object IDs for this out."""
|
||||||
from dvc.cachemgr import LEGACY_HASH_NAMES
|
from dvc.cachemgr import LEGACY_HASH_NAMES
|
||||||
|
|
||||||
@@ -1291,7 +1291,7 @@ class Output:
|
|||||||
nfiles=len(merged),
|
nfiles=len(merged),
|
||||||
)
|
)
|
||||||
|
|
||||||
def unstage(self, path: str) -> Tuple["Meta", "Tree"]:
|
def unstage(self, path: str) -> tuple["Meta", "Tree"]:
|
||||||
from pygtrie import Trie
|
from pygtrie import Trie
|
||||||
|
|
||||||
rel_key = tuple(self.fs.parts(self.fs.relpath(path, self.fs_path)))
|
rel_key = tuple(self.fs.parts(self.fs.relpath(path, self.fs_path)))
|
||||||
@@ -1324,7 +1324,7 @@ class Output:
|
|||||||
path: str,
|
path: str,
|
||||||
obj: Union["Tree", "HashFile"],
|
obj: Union["Tree", "HashFile"],
|
||||||
meta: "Meta",
|
meta: "Meta",
|
||||||
) -> Tuple["Meta", "Tree"]:
|
) -> tuple["Meta", "Tree"]:
|
||||||
from pygtrie import Trie
|
from pygtrie import Trie
|
||||||
|
|
||||||
append_only = True
|
append_only = True
|
||||||
@@ -1514,7 +1514,7 @@ META_SCHEMA = {
|
|||||||
|
|
||||||
CLOUD_SCHEMA = vol.All({str: {**META_SCHEMA, **CHECKSUMS_SCHEMA}}, vol.Length(max=1))
|
CLOUD_SCHEMA = vol.All({str: {**META_SCHEMA, **CHECKSUMS_SCHEMA}}, vol.Length(max=1))
|
||||||
|
|
||||||
ARTIFACT_SCHEMA: Dict[Any, Any] = {
|
ARTIFACT_SCHEMA: dict[Any, Any] = {
|
||||||
**CHECKSUMS_SCHEMA,
|
**CHECKSUMS_SCHEMA,
|
||||||
**META_SCHEMA,
|
**META_SCHEMA,
|
||||||
Output.PARAM_PATH: str,
|
Output.PARAM_PATH: str,
|
||||||
@@ -1523,7 +1523,7 @@ ARTIFACT_SCHEMA: Dict[Any, Any] = {
|
|||||||
Output.PARAM_HASH: str,
|
Output.PARAM_HASH: str,
|
||||||
}
|
}
|
||||||
|
|
||||||
DIR_FILES_SCHEMA: Dict[Any, Any] = {
|
DIR_FILES_SCHEMA: dict[Any, Any] = {
|
||||||
**CHECKSUMS_SCHEMA,
|
**CHECKSUMS_SCHEMA,
|
||||||
**META_SCHEMA,
|
**META_SCHEMA,
|
||||||
vol.Required(Tree.PARAM_RELPATH): str,
|
vol.Required(Tree.PARAM_RELPATH): str,
|
||||||
|
|||||||
+13
-17
@@ -6,12 +6,8 @@ from itertools import product
|
|||||||
from typing import (
|
from typing import (
|
||||||
TYPE_CHECKING,
|
TYPE_CHECKING,
|
||||||
Any,
|
Any,
|
||||||
Dict,
|
|
||||||
List,
|
|
||||||
NamedTuple,
|
NamedTuple,
|
||||||
Optional,
|
Optional,
|
||||||
Tuple,
|
|
||||||
Type,
|
|
||||||
Union,
|
Union,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -85,7 +81,7 @@ def format_and_raise(exc: Exception, msg: str, path: str) -> "NoReturn":
|
|||||||
|
|
||||||
|
|
||||||
def _reraise_err(
|
def _reraise_err(
|
||||||
exc_cls: Type[Exception], *args, from_exc: Optional[Exception] = None
|
exc_cls: type[Exception], *args, from_exc: Optional[Exception] = None
|
||||||
) -> "NoReturn":
|
) -> "NoReturn":
|
||||||
err = exc_cls(*args)
|
err = exc_cls(*args)
|
||||||
if from_exc and logger.isEnabledFor(logging.DEBUG):
|
if from_exc and logger.isEnabledFor(logging.DEBUG):
|
||||||
@@ -108,7 +104,7 @@ def is_map_or_seq(data: Any) -> bool:
|
|||||||
return not isinstance(data, str) and _is_map_or_seq(data)
|
return not isinstance(data, str) and _is_map_or_seq(data)
|
||||||
|
|
||||||
|
|
||||||
def split_group_name(name: str) -> Tuple[str, Optional[str]]:
|
def split_group_name(name: str) -> tuple[str, Optional[str]]:
|
||||||
group, *keys = name.rsplit(JOIN, maxsplit=1)
|
group, *keys = name.rsplit(JOIN, maxsplit=1)
|
||||||
return group, first(keys)
|
return group, first(keys)
|
||||||
|
|
||||||
@@ -161,14 +157,14 @@ class DataResolver:
|
|||||||
|
|
||||||
# we use `tracked_vars` to keep a dictionary of used variables
|
# we use `tracked_vars` to keep a dictionary of used variables
|
||||||
# by the interpolated entries.
|
# by the interpolated entries.
|
||||||
self.tracked_vars: Dict[str, Mapping] = {}
|
self.tracked_vars: dict[str, Mapping] = {}
|
||||||
|
|
||||||
stages_data = d.get(STAGES_KWD, {})
|
stages_data = d.get(STAGES_KWD, {})
|
||||||
# we wrap the definitions into:
|
# we wrap the definitions into:
|
||||||
# ForeachDefinition, MatrixDefinition, and EntryDefinition
|
# ForeachDefinition, MatrixDefinition, and EntryDefinition
|
||||||
# that helps us to optimize, cache and selectively load each one of
|
# that helps us to optimize, cache and selectively load each one of
|
||||||
# them as we need, and simplify all of this DSL/parsing logic.
|
# them as we need, and simplify all of this DSL/parsing logic.
|
||||||
self.definitions: Dict[str, Definition] = {
|
self.definitions: dict[str, Definition] = {
|
||||||
name: make_definition(self, name, definition)
|
name: make_definition(self, name, definition)
|
||||||
for name, definition in stages_data.items()
|
for name, definition in stages_data.items()
|
||||||
}
|
}
|
||||||
@@ -376,7 +372,7 @@ class ForeachDefinition:
|
|||||||
f" in '{self.relpath}': expected list/dictionary, got " + typ
|
f" in '{self.relpath}': expected list/dictionary, got " + typ
|
||||||
)
|
)
|
||||||
|
|
||||||
def _warn_if_overwriting(self, keys: List[str]):
|
def _warn_if_overwriting(self, keys: list[str]):
|
||||||
warn_for = [k for k in keys if k in self.context]
|
warn_for = [k for k in keys if k in self.context]
|
||||||
if warn_for:
|
if warn_for:
|
||||||
linking_verb = "is" if len(warn_for) == 1 else "are"
|
linking_verb = "is" if len(warn_for) == 1 else "are"
|
||||||
@@ -390,7 +386,7 @@ class ForeachDefinition:
|
|||||||
self.name,
|
self.name,
|
||||||
)
|
)
|
||||||
|
|
||||||
def _inserted_keys(self, iterable) -> List[str]:
|
def _inserted_keys(self, iterable) -> list[str]:
|
||||||
keys = [self.pair.value]
|
keys = [self.pair.value]
|
||||||
if isinstance(iterable, Mapping):
|
if isinstance(iterable, Mapping):
|
||||||
keys.append(self.pair.key)
|
keys.append(self.pair.key)
|
||||||
@@ -490,10 +486,10 @@ class MatrixDefinition:
|
|||||||
return self._template
|
return self._template
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def resolved_iterable(self) -> Dict[str, List]:
|
def resolved_iterable(self) -> dict[str, list]:
|
||||||
return self._resolve_matrix_data()
|
return self._resolve_matrix_data()
|
||||||
|
|
||||||
def _resolve_matrix_data(self) -> Dict[str, List]:
|
def _resolve_matrix_data(self) -> dict[str, list]:
|
||||||
try:
|
try:
|
||||||
iterable = self.context.resolve(self.matrix_data, unwrap=False)
|
iterable = self.context.resolve(self.matrix_data, unwrap=False)
|
||||||
except (ContextError, ParseError) as exc:
|
except (ContextError, ParseError) as exc:
|
||||||
@@ -504,7 +500,7 @@ class MatrixDefinition:
|
|||||||
self._warn_if_overwriting([self.pair.key, self.pair.value])
|
self._warn_if_overwriting([self.pair.key, self.pair.value])
|
||||||
return iterable
|
return iterable
|
||||||
|
|
||||||
def _warn_if_overwriting(self, keys: List[str]):
|
def _warn_if_overwriting(self, keys: list[str]):
|
||||||
warn_for = [k for k in keys if k in self.context]
|
warn_for = [k for k in keys if k in self.context]
|
||||||
if warn_for:
|
if warn_for:
|
||||||
linking_verb = "is" if len(warn_for) == 1 else "are"
|
linking_verb = "is" if len(warn_for) == 1 else "are"
|
||||||
@@ -519,16 +515,16 @@ class MatrixDefinition:
|
|||||||
)
|
)
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def normalized_iterable(self) -> Dict[str, "DictStrAny"]:
|
def normalized_iterable(self) -> dict[str, "DictStrAny"]:
|
||||||
"""Convert sequence to Mapping with keys normalized."""
|
"""Convert sequence to Mapping with keys normalized."""
|
||||||
iterable = self.resolved_iterable
|
iterable = self.resolved_iterable
|
||||||
assert isinstance(iterable, Mapping)
|
assert isinstance(iterable, Mapping)
|
||||||
|
|
||||||
ret: Dict[str, "DictStrAny"] = {}
|
ret: dict[str, "DictStrAny"] = {}
|
||||||
matrix = {key: enumerate(v) for key, v in iterable.items()}
|
matrix = {key: enumerate(v) for key, v in iterable.items()}
|
||||||
for combination in product(*matrix.values()):
|
for combination in product(*matrix.values()):
|
||||||
d: "DictStrAny" = {}
|
d: "DictStrAny" = {}
|
||||||
fragments: List[str] = []
|
fragments: list[str] = []
|
||||||
for k, (i, v) in zip(matrix.keys(), combination):
|
for k, (i, v) in zip(matrix.keys(), combination):
|
||||||
d[k] = v
|
d[k] = v
|
||||||
fragments.append(f"{k}{i}" if is_map_or_seq(v) else to_str(v))
|
fragments.append(f"{k}{i}" if is_map_or_seq(v) else to_str(v))
|
||||||
@@ -540,7 +536,7 @@ class MatrixDefinition:
|
|||||||
def has_member(self, key: str) -> bool:
|
def has_member(self, key: str) -> bool:
|
||||||
return key in self.normalized_iterable
|
return key in self.normalized_iterable
|
||||||
|
|
||||||
def get_generated_names(self) -> List[str]:
|
def get_generated_names(self) -> list[str]:
|
||||||
return list(map(self._generate_name, self.normalized_iterable))
|
return list(map(self._generate_name, self.normalized_iterable))
|
||||||
|
|
||||||
def _generate_name(self, key: str) -> str:
|
def _generate_name(self, key: str) -> str:
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ from collections.abc import Mapping, MutableMapping, MutableSequence, Sequence
|
|||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from copy import deepcopy
|
from copy import deepcopy
|
||||||
from dataclasses import dataclass, field, replace
|
from dataclasses import dataclass, field, replace
|
||||||
from typing import Any, Dict, List, Optional, Union
|
from typing import Any, Optional, Union
|
||||||
|
|
||||||
from funcy import identity, lfilter, nullcontext, select
|
from funcy import identity, lfilter, nullcontext, select
|
||||||
|
|
||||||
@@ -22,7 +22,7 @@ from dvc.parsing.interpolate import (
|
|||||||
|
|
||||||
logger = logger.getChild(__name__)
|
logger = logger.getChild(__name__)
|
||||||
SeqOrMap = Union[Sequence, Mapping]
|
SeqOrMap = Union[Sequence, Mapping]
|
||||||
DictStr = Dict[str, Any]
|
DictStr = dict[str, Any]
|
||||||
|
|
||||||
|
|
||||||
class ContextError(DvcException):
|
class ContextError(DvcException):
|
||||||
@@ -100,7 +100,7 @@ def recurse_not_a_node(data: dict):
|
|||||||
@dataclass
|
@dataclass
|
||||||
class Meta:
|
class Meta:
|
||||||
source: Optional[str] = None
|
source: Optional[str] = None
|
||||||
dpaths: List[str] = field(default_factory=list)
|
dpaths: list[str] = field(default_factory=list)
|
||||||
local: bool = True
|
local: bool = True
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -297,7 +297,7 @@ class Context(CtxDict):
|
|||||||
"""
|
"""
|
||||||
super().__init__(*args, **kwargs)
|
super().__init__(*args, **kwargs)
|
||||||
self._track = False
|
self._track = False
|
||||||
self._tracked_data: Dict[str, Dict] = defaultdict(dict)
|
self._tracked_data: dict[str, dict] = defaultdict(dict)
|
||||||
self.imports = {}
|
self.imports = {}
|
||||||
self._reserved_keys = {}
|
self._reserved_keys = {}
|
||||||
|
|
||||||
@@ -348,7 +348,7 @@ class Context(CtxDict):
|
|||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def load_from(
|
def load_from(
|
||||||
cls, fs, path: str, select_keys: Optional[List[str]] = None
|
cls, fs, path: str, select_keys: Optional[list[str]] = None
|
||||||
) -> "Context":
|
) -> "Context":
|
||||||
from dvc.utils.serialize import load_path
|
from dvc.utils.serialize import load_path
|
||||||
|
|
||||||
@@ -423,7 +423,7 @@ class Context(CtxDict):
|
|||||||
def load_from_vars(
|
def load_from_vars(
|
||||||
self,
|
self,
|
||||||
fs,
|
fs,
|
||||||
vars_: List,
|
vars_: list,
|
||||||
wdir: str,
|
wdir: str,
|
||||||
stage_name: Optional[str] = None,
|
stage_name: Optional[str] = None,
|
||||||
default: Optional[str] = None,
|
default: Optional[str] = None,
|
||||||
|
|||||||
@@ -10,7 +10,8 @@ from dvc.exceptions import DvcException
|
|||||||
from dvc.utils.flatten import flatten
|
from dvc.utils.flatten import flatten
|
||||||
|
|
||||||
if typing.TYPE_CHECKING:
|
if typing.TYPE_CHECKING:
|
||||||
from typing import List, Match, NoReturn
|
from re import Match
|
||||||
|
from typing import NoReturn
|
||||||
|
|
||||||
from pyparsing import ParseException
|
from pyparsing import ParseException
|
||||||
|
|
||||||
@@ -205,7 +206,7 @@ def validate_value(value, key):
|
|||||||
|
|
||||||
def str_interpolate(
|
def str_interpolate(
|
||||||
template: str,
|
template: str,
|
||||||
matches: "List[Match]",
|
matches: "list[Match]",
|
||||||
context: "Context",
|
context: "Context",
|
||||||
skip_checks: bool = False,
|
skip_checks: bool = False,
|
||||||
key=None,
|
key=None,
|
||||||
@@ -225,5 +226,5 @@ def str_interpolate(
|
|||||||
return buf.replace(r"\${", BRACE_OPEN)
|
return buf.replace(r"\${", BRACE_OPEN)
|
||||||
|
|
||||||
|
|
||||||
def is_exact_string(src: str, matches: "List[Match]"):
|
def is_exact_string(src: str, matches: "list[Match]"):
|
||||||
return len(matches) == 1 and src == matches[0].group(0)
|
return len(matches) == 1 and src == matches[0].group(0)
|
||||||
|
|||||||
+2
-2
@@ -2,7 +2,7 @@
|
|||||||
import logging
|
import logging
|
||||||
import sys
|
import sys
|
||||||
from threading import RLock
|
from threading import RLock
|
||||||
from typing import TYPE_CHECKING, Any, ClassVar, Dict
|
from typing import TYPE_CHECKING, Any, ClassVar
|
||||||
|
|
||||||
from tqdm import tqdm
|
from tqdm import tqdm
|
||||||
|
|
||||||
@@ -33,7 +33,7 @@ class Tqdm(tqdm):
|
|||||||
" [{elapsed}<{remaining}, {rate_fmt:>11}]"
|
" [{elapsed}<{remaining}, {rate_fmt:>11}]"
|
||||||
)
|
)
|
||||||
BAR_FMT_NOTOTAL = "{desc}{bar:b}|{postfix[info]}{n_fmt} [{elapsed}, {rate_fmt:>11}]"
|
BAR_FMT_NOTOTAL = "{desc}{bar:b}|{postfix[info]}{n_fmt} [{elapsed}, {rate_fmt:>11}]"
|
||||||
BYTES_DEFAULTS: ClassVar[Dict[str, Any]] = {
|
BYTES_DEFAULTS: ClassVar[dict[str, Any]] = {
|
||||||
"unit": "B",
|
"unit": "B",
|
||||||
"unit_scale": True,
|
"unit_scale": True,
|
||||||
"unit_divisor": 1024,
|
"unit_divisor": 1024,
|
||||||
|
|||||||
+2
-1
@@ -1,7 +1,8 @@
|
|||||||
"""Manages user prompts."""
|
"""Manages user prompts."""
|
||||||
|
|
||||||
|
from collections.abc import Collection
|
||||||
from getpass import getpass
|
from getpass import getpass
|
||||||
from typing import Collection, Optional
|
from typing import Optional
|
||||||
|
|
||||||
from dvc.log import logger
|
from dvc.log import logger
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
from typing import Dict, List, Union
|
from typing import Union
|
||||||
|
|
||||||
from dvc.render import REVISION, REVISIONS, SRC, TYPE_KEY
|
from dvc.render import REVISION, REVISIONS, SRC, TYPE_KEY
|
||||||
from dvc.render.converter.image import ImageConverter
|
from dvc.render.converter.image import ImageConverter
|
||||||
@@ -18,7 +18,7 @@ def _get_converter(
|
|||||||
raise ValueError(f"Invalid renderer class {renderer_class}")
|
raise ValueError(f"Invalid renderer class {renderer_class}")
|
||||||
|
|
||||||
|
|
||||||
def to_json(renderer, split: bool = False) -> List[Dict]:
|
def to_json(renderer, split: bool = False) -> list[dict]:
|
||||||
if renderer.TYPE == "vega":
|
if renderer.TYPE == "vega":
|
||||||
if not renderer.datapoints:
|
if not renderer.datapoints:
|
||||||
return []
|
return []
|
||||||
|
|||||||
@@ -1,19 +1,19 @@
|
|||||||
from typing import Any, Dict, List, Optional, Tuple
|
from typing import Any, Optional
|
||||||
|
|
||||||
|
|
||||||
class Converter:
|
class Converter:
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
plot_id: str,
|
plot_id: str,
|
||||||
data: Optional[Dict[str, Any]] = None,
|
data: Optional[dict[str, Any]] = None,
|
||||||
properties: Optional[Dict] = None,
|
properties: Optional[dict] = None,
|
||||||
):
|
):
|
||||||
self.plot_id = plot_id
|
self.plot_id = plot_id
|
||||||
self.properties = properties or {}
|
self.properties = properties or {}
|
||||||
self.data = data or {}
|
self.data = data or {}
|
||||||
|
|
||||||
def convert(self) -> Tuple[List[Tuple[str, str, Any]], Dict]:
|
def convert(self) -> tuple[list[tuple[str, str, Any]], dict]:
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
def flat_datapoints(self, revision: str) -> Tuple[List[Dict], Dict]:
|
def flat_datapoints(self, revision: str) -> tuple[list[dict], dict]:
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import base64
|
import base64
|
||||||
import os
|
import os
|
||||||
from typing import TYPE_CHECKING, Any, Dict, List, Tuple
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
from dvc.render import FILENAME, REVISION, SRC
|
from dvc.render import FILENAME, REVISION, SRC
|
||||||
|
|
||||||
@@ -34,13 +34,13 @@ class ImageConverter(Converter):
|
|||||||
base64_str = base64.b64encode(image_data).decode()
|
base64_str = base64.b64encode(image_data).decode()
|
||||||
return f"data:image;base64,{base64_str}"
|
return f"data:image;base64,{base64_str}"
|
||||||
|
|
||||||
def convert(self) -> Tuple[List[Tuple[str, str, Any]], Dict]:
|
def convert(self) -> tuple[list[tuple[str, str, Any]], dict]:
|
||||||
datas = []
|
datas = []
|
||||||
for filename, image_data in self.data.items():
|
for filename, image_data in self.data.items():
|
||||||
datas.append((filename, "", image_data))
|
datas.append((filename, "", image_data))
|
||||||
return datas, self.properties
|
return datas, self.properties
|
||||||
|
|
||||||
def flat_datapoints(self, revision: str) -> Tuple[List[Dict], Dict]:
|
def flat_datapoints(self, revision: str) -> tuple[list[dict], dict]:
|
||||||
"""
|
"""
|
||||||
Convert the DVC Plots content to DVC Render datapoints.
|
Convert the DVC Plots content to DVC Render datapoints.
|
||||||
Return both generated datapoints and updated properties.
|
Return both generated datapoints and updated properties.
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import os
|
import os
|
||||||
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
|
from collections.abc import Iterable
|
||||||
|
from typing import Any, Optional, Union
|
||||||
|
|
||||||
from funcy import first, last
|
from funcy import first, last
|
||||||
|
|
||||||
@@ -18,7 +19,7 @@ class FieldNotFoundError(DvcException):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _lists(blob: Union[Dict, List]) -> Iterable[List]:
|
def _lists(blob: Union[dict, list]) -> Iterable[list]:
|
||||||
if isinstance(blob, list):
|
if isinstance(blob, list):
|
||||||
yield blob
|
yield blob
|
||||||
else:
|
else:
|
||||||
@@ -43,7 +44,7 @@ def _file_field(*args):
|
|||||||
def _find(
|
def _find(
|
||||||
filename: str,
|
filename: str,
|
||||||
field: str,
|
field: str,
|
||||||
data_series: List[Tuple[str, str, Any]],
|
data_series: list[tuple[str, str, Any]],
|
||||||
):
|
):
|
||||||
for data_file, data_field, data in data_series:
|
for data_file, data_field, data in data_series:
|
||||||
if data_file == filename and data_field == field:
|
if data_file == filename and data_field == field:
|
||||||
@@ -51,14 +52,14 @@ def _find(
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _verify_field(file2datapoints: Dict[str, List], filename: str, field: str):
|
def _verify_field(file2datapoints: dict[str, list], filename: str, field: str):
|
||||||
if filename in file2datapoints:
|
if filename in file2datapoints:
|
||||||
datapoint = first(file2datapoints[filename])
|
datapoint = first(file2datapoints[filename])
|
||||||
if field not in datapoint:
|
if field not in datapoint:
|
||||||
raise FieldNotFoundError(field, datapoint.keys())
|
raise FieldNotFoundError(field, datapoint.keys())
|
||||||
|
|
||||||
|
|
||||||
def _get_xs(properties: Dict, file2datapoints: Dict[str, List[Dict]]):
|
def _get_xs(properties: dict, file2datapoints: dict[str, list[dict]]):
|
||||||
x = properties.get("x")
|
x = properties.get("x")
|
||||||
if x is not None and isinstance(x, dict):
|
if x is not None and isinstance(x, dict):
|
||||||
for filename, field in _file_field(x):
|
for filename, field in _file_field(x):
|
||||||
@@ -66,7 +67,7 @@ def _get_xs(properties: Dict, file2datapoints: Dict[str, List[Dict]]):
|
|||||||
yield filename, field
|
yield filename, field
|
||||||
|
|
||||||
|
|
||||||
def _get_ys(properties, file2datapoints: Dict[str, List[Dict]]):
|
def _get_ys(properties, file2datapoints: dict[str, list[dict]]):
|
||||||
y = properties.get("y", None)
|
y = properties.get("y", None)
|
||||||
if y is not None:
|
if y is not None:
|
||||||
for filename, field in _file_field(y):
|
for filename, field in _file_field(y):
|
||||||
@@ -74,7 +75,7 @@ def _get_ys(properties, file2datapoints: Dict[str, List[Dict]]):
|
|||||||
yield filename, field
|
yield filename, field
|
||||||
|
|
||||||
|
|
||||||
def _is_datapoints(lst: List[Dict]):
|
def _is_datapoints(lst: list[dict]):
|
||||||
"""
|
"""
|
||||||
check if dict keys match, datapoints with different keys mgiht lead
|
check if dict keys match, datapoints with different keys mgiht lead
|
||||||
to unexpected behavior
|
to unexpected behavior
|
||||||
@@ -85,8 +86,8 @@ def _is_datapoints(lst: List[Dict]):
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def get_datapoints(file_content: Dict):
|
def get_datapoints(file_content: dict):
|
||||||
result: List[Dict[str, Any]] = []
|
result: list[dict[str, Any]] = []
|
||||||
for lst in _lists(file_content):
|
for lst in _lists(file_content):
|
||||||
if _is_datapoints(lst):
|
if _is_datapoints(lst):
|
||||||
for index, datapoint in enumerate(lst):
|
for index, datapoint in enumerate(lst):
|
||||||
@@ -107,8 +108,8 @@ class VegaConverter(Converter):
|
|||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
plot_id: str,
|
plot_id: str,
|
||||||
data: Optional[Dict] = None,
|
data: Optional[dict] = None,
|
||||||
properties: Optional[Dict] = None,
|
properties: Optional[dict] = None,
|
||||||
):
|
):
|
||||||
super().__init__(plot_id, data, properties)
|
super().__init__(plot_id, data, properties)
|
||||||
self.plot_id = plot_id
|
self.plot_id = plot_id
|
||||||
@@ -126,7 +127,7 @@ class VegaConverter(Converter):
|
|||||||
x = self.properties.get("x", None)
|
x = self.properties.get("x", None)
|
||||||
y = self.properties.get("y", None)
|
y = self.properties.get("y", None)
|
||||||
|
|
||||||
inferred_properties: Dict = {}
|
inferred_properties: dict = {}
|
||||||
|
|
||||||
# Infer x.
|
# Infer x.
|
||||||
if isinstance(x, str):
|
if isinstance(x, str):
|
||||||
@@ -195,7 +196,7 @@ class VegaConverter(Converter):
|
|||||||
def flat_datapoints(self, revision): # noqa: C901, PLR0912
|
def flat_datapoints(self, revision): # noqa: C901, PLR0912
|
||||||
file2datapoints, properties = self.convert()
|
file2datapoints, properties = self.convert()
|
||||||
|
|
||||||
props_update: Dict[str, Union[str, List[Dict[str, str]]]] = {}
|
props_update: dict[str, Union[str, list[dict[str, str]]]] = {}
|
||||||
|
|
||||||
xs = list(_get_xs(properties, file2datapoints))
|
xs = list(_get_xs(properties, file2datapoints))
|
||||||
|
|
||||||
@@ -322,9 +323,9 @@ def _get_short_y_file(y_file, common_prefix_len):
|
|||||||
|
|
||||||
|
|
||||||
def _update_from_field(
|
def _update_from_field(
|
||||||
target_datapoints: List[Dict],
|
target_datapoints: list[dict],
|
||||||
field: str,
|
field: str,
|
||||||
source_datapoints: Optional[List[Dict]] = None,
|
source_datapoints: Optional[list[dict]] = None,
|
||||||
source_field: Optional[str] = None,
|
source_field: Optional[str] = None,
|
||||||
):
|
):
|
||||||
if source_datapoints is None:
|
if source_datapoints is None:
|
||||||
@@ -341,11 +342,11 @@ def _update_from_field(
|
|||||||
datapoint[field] = source_datapoint[source_field]
|
datapoint[field] = source_datapoint[source_field]
|
||||||
|
|
||||||
|
|
||||||
def _update_from_index(datapoints: List[Dict], new_field: str):
|
def _update_from_index(datapoints: list[dict], new_field: str):
|
||||||
for index, datapoint in enumerate(datapoints):
|
for index, datapoint in enumerate(datapoints):
|
||||||
datapoint[new_field] = index
|
datapoint[new_field] = index
|
||||||
|
|
||||||
|
|
||||||
def _update_all(datapoints: List[Dict], update_dict: Dict):
|
def _update_all(datapoints: list[dict], update_dict: dict):
|
||||||
for datapoint in datapoints:
|
for datapoint in datapoints:
|
||||||
datapoint.update(update_dict)
|
datapoint.update(update_dict)
|
||||||
|
|||||||
+11
-11
@@ -1,6 +1,6 @@
|
|||||||
import os
|
import os
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from typing import TYPE_CHECKING, DefaultDict, Dict, List, NamedTuple, Optional
|
from typing import TYPE_CHECKING, NamedTuple, Optional
|
||||||
|
|
||||||
import dpath
|
import dpath
|
||||||
import dpath.options
|
import dpath.options
|
||||||
@@ -21,16 +21,16 @@ dpath.options.ALLOW_EMPTY_STRING_KEYS = True
|
|||||||
logger = logger.getChild(__name__)
|
logger = logger.getChild(__name__)
|
||||||
|
|
||||||
|
|
||||||
def _squash_plots_properties(data: List) -> Dict:
|
def _squash_plots_properties(data: list) -> dict:
|
||||||
configs = [last(group) for group in data]
|
configs = [last(group) for group in data]
|
||||||
resolved: Dict = {}
|
resolved: dict = {}
|
||||||
for config in reversed(configs):
|
for config in reversed(configs):
|
||||||
resolved = {**resolved, **config}
|
resolved = {**resolved, **config}
|
||||||
return resolved
|
return resolved
|
||||||
|
|
||||||
|
|
||||||
class PlotsData:
|
class PlotsData:
|
||||||
def __init__(self, data: Dict):
|
def __init__(self, data: dict):
|
||||||
self.data = data
|
self.data = data
|
||||||
|
|
||||||
def group_definitions(self):
|
def group_definitions(self):
|
||||||
@@ -62,15 +62,15 @@ class PlotsData:
|
|||||||
|
|
||||||
class RendererWithErrors(NamedTuple):
|
class RendererWithErrors(NamedTuple):
|
||||||
renderer: "Renderer"
|
renderer: "Renderer"
|
||||||
source_errors: Dict[str, Dict[str, Exception]]
|
source_errors: dict[str, dict[str, Exception]]
|
||||||
definition_errors: Dict[str, Exception]
|
definition_errors: dict[str, Exception]
|
||||||
|
|
||||||
|
|
||||||
def match_defs_renderers( # noqa: C901, PLR0912
|
def match_defs_renderers( # noqa: C901, PLR0912
|
||||||
data,
|
data,
|
||||||
out=None,
|
out=None,
|
||||||
templates_dir: Optional["StrPath"] = None,
|
templates_dir: Optional["StrPath"] = None,
|
||||||
) -> List[RendererWithErrors]:
|
) -> list[RendererWithErrors]:
|
||||||
from dvc_render import ImageRenderer, VegaRenderer
|
from dvc_render import ImageRenderer, VegaRenderer
|
||||||
|
|
||||||
plots_data = PlotsData(data)
|
plots_data = PlotsData(data)
|
||||||
@@ -78,12 +78,12 @@ def match_defs_renderers( # noqa: C901, PLR0912
|
|||||||
renderer_cls = None
|
renderer_cls = None
|
||||||
|
|
||||||
for plot_id, group in plots_data.group_definitions().items():
|
for plot_id, group in plots_data.group_definitions().items():
|
||||||
plot_datapoints: List[Dict] = []
|
plot_datapoints: list[dict] = []
|
||||||
props = _squash_plots_properties(group)
|
props = _squash_plots_properties(group)
|
||||||
first_props: Dict = {}
|
first_props: dict = {}
|
||||||
|
|
||||||
def_errors: Dict[str, Exception] = {}
|
def_errors: dict[str, Exception] = {}
|
||||||
src_errors: DefaultDict[str, Dict[str, Exception]] = defaultdict(dict)
|
src_errors: defaultdict[str, dict[str, Exception]] = defaultdict(dict)
|
||||||
|
|
||||||
if out is not None:
|
if out is not None:
|
||||||
props["out"] = out
|
props["out"] = out
|
||||||
|
|||||||
@@ -1,14 +1,12 @@
|
|||||||
import os
|
import os
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from contextlib import contextmanager
|
from collections.abc import Iterable
|
||||||
|
from contextlib import AbstractContextManager, contextmanager
|
||||||
from functools import wraps
|
from functools import wraps
|
||||||
from typing import (
|
from typing import (
|
||||||
TYPE_CHECKING,
|
TYPE_CHECKING,
|
||||||
Callable,
|
Callable,
|
||||||
ContextManager,
|
|
||||||
Iterable,
|
|
||||||
Optional,
|
Optional,
|
||||||
Tuple,
|
|
||||||
Union,
|
Union,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -111,7 +109,7 @@ class Repo:
|
|||||||
fs: Optional["FileSystem"] = None,
|
fs: Optional["FileSystem"] = None,
|
||||||
uninitialized: bool = False,
|
uninitialized: bool = False,
|
||||||
scm: Optional[Union["Git", "NoSCM"]] = None,
|
scm: Optional[Union["Git", "NoSCM"]] = None,
|
||||||
) -> Tuple[str, Optional[str]]:
|
) -> tuple[str, Optional[str]]:
|
||||||
from dvc.fs import localfs
|
from dvc.fs import localfs
|
||||||
from dvc.scm import SCM, SCMError
|
from dvc.scm import SCM, SCMError
|
||||||
|
|
||||||
@@ -268,7 +266,7 @@ class Repo:
|
|||||||
if not isinstance(self.fs, GitFileSystem):
|
if not isinstance(self.fs, GitFileSystem):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
relparts: Tuple[str, ...] = ()
|
relparts: tuple[str, ...] = ()
|
||||||
if self.root_dir != "/":
|
if self.root_dir != "/":
|
||||||
# subrepo
|
# subrepo
|
||||||
relparts = self.fs.relparts(self.root_dir, "/")
|
relparts = self.fs.relparts(self.root_dir, "/")
|
||||||
@@ -392,7 +390,7 @@ class Repo:
|
|||||||
self,
|
self,
|
||||||
path: str,
|
path: str,
|
||||||
workspace: str = "repo",
|
workspace: str = "repo",
|
||||||
) -> Tuple["DataIndex", "DataIndexEntry"]:
|
) -> tuple["DataIndex", "DataIndexEntry"]:
|
||||||
if self.subrepos:
|
if self.subrepos:
|
||||||
fs_path = self.dvcfs.from_os_path(path)
|
fs_path = self.dvcfs.from_os_path(path)
|
||||||
fs = self.dvcfs.fs
|
fs = self.dvcfs.fs
|
||||||
@@ -476,7 +474,7 @@ class Repo:
|
|||||||
|
|
||||||
return brancher(self, *args, **kwargs)
|
return brancher(self, *args, **kwargs)
|
||||||
|
|
||||||
def switch(self, rev: str) -> ContextManager[str]:
|
def switch(self, rev: str) -> AbstractContextManager[str]:
|
||||||
from dvc.repo.brancher import switch
|
from dvc.repo.brancher import switch
|
||||||
|
|
||||||
return switch(self, rev)
|
return switch(self, rev)
|
||||||
@@ -626,7 +624,7 @@ class Repo:
|
|||||||
cache_dir = self.config["core"].get("site_cache_dir") or site_cache_dir()
|
cache_dir = self.config["core"].get("site_cache_dir") or site_cache_dir()
|
||||||
|
|
||||||
if isinstance(self.fs, GitFileSystem):
|
if isinstance(self.fs, GitFileSystem):
|
||||||
relparts: Tuple[str, ...] = ()
|
relparts: tuple[str, ...] = ()
|
||||||
if self.root_dir != "/":
|
if self.root_dir != "/":
|
||||||
# subrepo
|
# subrepo
|
||||||
relparts = self.fs.relparts(self.root_dir, "/")
|
relparts = self.fs.relparts(self.root_dir, "/")
|
||||||
|
|||||||
+7
-10
@@ -1,13 +1,10 @@
|
|||||||
import os
|
import os
|
||||||
|
from collections.abc import Iterator
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from typing import (
|
from typing import (
|
||||||
TYPE_CHECKING,
|
TYPE_CHECKING,
|
||||||
Dict,
|
|
||||||
Iterator,
|
|
||||||
List,
|
|
||||||
NamedTuple,
|
NamedTuple,
|
||||||
Optional,
|
Optional,
|
||||||
Tuple,
|
|
||||||
Union,
|
Union,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -37,7 +34,7 @@ class StageInfo(NamedTuple):
|
|||||||
|
|
||||||
def find_targets(
|
def find_targets(
|
||||||
targets: Union["StrOrBytesPath", Iterator["StrOrBytesPath"]], glob: bool = False
|
targets: Union["StrOrBytesPath", Iterator["StrOrBytesPath"]], glob: bool = False
|
||||||
) -> List[str]:
|
) -> list[str]:
|
||||||
if isinstance(targets, (str, bytes, os.PathLike)):
|
if isinstance(targets, (str, bytes, os.PathLike)):
|
||||||
targets_list = [os.fsdecode(targets)]
|
targets_list = [os.fsdecode(targets)]
|
||||||
else:
|
else:
|
||||||
@@ -99,7 +96,7 @@ OVERLAPPING_PARENT_FMT = (
|
|||||||
|
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def translate_graph_error(stages: List["Stage"]) -> Iterator[None]:
|
def translate_graph_error(stages: list["Stage"]) -> Iterator[None]:
|
||||||
try:
|
try:
|
||||||
yield
|
yield
|
||||||
except OverlappingOutputPathsError as exc:
|
except OverlappingOutputPathsError as exc:
|
||||||
@@ -124,7 +121,7 @@ def translate_graph_error(stages: List["Stage"]) -> Iterator[None]:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def progress_iter(stages: Dict[str, StageInfo]) -> Iterator[Tuple[str, StageInfo]]:
|
def progress_iter(stages: dict[str, StageInfo]) -> Iterator[tuple[str, StageInfo]]:
|
||||||
total = len(stages)
|
total = len(stages)
|
||||||
desc = "Adding..."
|
desc = "Adding..."
|
||||||
with ui.progress(
|
with ui.progress(
|
||||||
@@ -151,8 +148,8 @@ LINK_FAILURE_MESSAGE = (
|
|||||||
|
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def warn_link_failures() -> Iterator[List[str]]:
|
def warn_link_failures() -> Iterator[list[str]]:
|
||||||
link_failures: List[str] = []
|
link_failures: list[str] = []
|
||||||
try:
|
try:
|
||||||
yield link_failures
|
yield link_failures
|
||||||
finally:
|
finally:
|
||||||
@@ -202,7 +199,7 @@ def add(
|
|||||||
to_remote: bool = False,
|
to_remote: bool = False,
|
||||||
remote_jobs: Optional[int] = None,
|
remote_jobs: Optional[int] = None,
|
||||||
force: bool = False,
|
force: bool = False,
|
||||||
) -> List["Stage"]:
|
) -> list["Stage"]:
|
||||||
add_targets = find_targets(targets, glob=glob)
|
add_targets = find_targets(targets, glob=glob)
|
||||||
if not add_targets:
|
if not add_targets:
|
||||||
return []
|
return []
|
||||||
|
|||||||
+12
-12
@@ -1,7 +1,7 @@
|
|||||||
import os
|
import os
|
||||||
import posixpath
|
import posixpath
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
|
from typing import TYPE_CHECKING, Any, Optional, Union
|
||||||
|
|
||||||
from dvc.annotations import Artifact
|
from dvc.annotations import Artifact
|
||||||
from dvc.dvcfile import PROJECT_FILE
|
from dvc.dvcfile import PROJECT_FILE
|
||||||
@@ -95,9 +95,9 @@ class Artifacts:
|
|||||||
return self.repo.scm
|
return self.repo.scm
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def read(self) -> Dict[str, Dict[str, Artifact]]:
|
def read(self) -> dict[str, dict[str, Artifact]]:
|
||||||
"""Read artifacts from dvc.yaml."""
|
"""Read artifacts from dvc.yaml."""
|
||||||
artifacts: Dict[str, Dict[str, Artifact]] = {}
|
artifacts: dict[str, dict[str, Artifact]] = {}
|
||||||
for (
|
for (
|
||||||
dvcfile,
|
dvcfile,
|
||||||
dvcfile_artifacts,
|
dvcfile_artifacts,
|
||||||
@@ -140,14 +140,14 @@ class Artifacts:
|
|||||||
|
|
||||||
assert not (version and stage)
|
assert not (version and stage)
|
||||||
name = _reformat_name(name)
|
name = _reformat_name(name)
|
||||||
tags: List["GitTag"] = find(
|
tags: list["GitTag"] = find(
|
||||||
name=name, version=version, stage=stage, scm=self.scm
|
name=name, version=version, stage=stage, scm=self.scm
|
||||||
)
|
)
|
||||||
if not tags:
|
if not tags:
|
||||||
raise ArtifactNotFoundError(name, version=version, stage=stage)
|
raise ArtifactNotFoundError(name, version=version, stage=stage)
|
||||||
if version or stage:
|
if version or stage:
|
||||||
return tags[-1].target
|
return tags[-1].target
|
||||||
gto_tags: List["GTOTag"] = sort_versions(parse_tag(tag) for tag in tags)
|
gto_tags: list["GTOTag"] = sort_versions(parse_tag(tag) for tag in tags)
|
||||||
return gto_tags[0].tag.target
|
return gto_tags[0].tag.target
|
||||||
|
|
||||||
def get_path(self, name: str):
|
def get_path(self, name: str):
|
||||||
@@ -177,7 +177,7 @@ class Artifacts:
|
|||||||
out: Optional[str] = None,
|
out: Optional[str] = None,
|
||||||
force: bool = False,
|
force: bool = False,
|
||||||
jobs: Optional[int] = None,
|
jobs: Optional[int] = None,
|
||||||
) -> Tuple[int, str]:
|
) -> tuple[int, str]:
|
||||||
"""Download the specified artifact."""
|
"""Download the specified artifact."""
|
||||||
from dvc.fs import download as fs_download
|
from dvc.fs import download as fs_download
|
||||||
|
|
||||||
@@ -205,9 +205,9 @@ class Artifacts:
|
|||||||
out: Optional[str] = None,
|
out: Optional[str] = None,
|
||||||
force: bool = False,
|
force: bool = False,
|
||||||
jobs: Optional[int] = None,
|
jobs: Optional[int] = None,
|
||||||
dvc_studio_config: Optional[Dict[str, Any]] = None,
|
dvc_studio_config: Optional[dict[str, Any]] = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Tuple[int, str]:
|
) -> tuple[int, str]:
|
||||||
from dvc_studio_client.model_registry import get_download_uris
|
from dvc_studio_client.model_registry import get_download_uris
|
||||||
|
|
||||||
from dvc.fs import HTTPFileSystem, generic, localfs
|
from dvc.fs import HTTPFileSystem, generic, localfs
|
||||||
@@ -215,8 +215,8 @@ class Artifacts:
|
|||||||
|
|
||||||
logger.debug("Trying to download artifact '%s' via studio", name)
|
logger.debug("Trying to download artifact '%s' via studio", name)
|
||||||
out = out or os.getcwd()
|
out = out or os.getcwd()
|
||||||
to_infos: List[str] = []
|
to_infos: list[str] = []
|
||||||
from_infos: List[str] = []
|
from_infos: list[str] = []
|
||||||
if dvc_studio_config is None:
|
if dvc_studio_config is None:
|
||||||
dvc_studio_config = {}
|
dvc_studio_config = {}
|
||||||
dvc_studio_config["repo_url"] = repo_url
|
dvc_studio_config["repo_url"] = repo_url
|
||||||
@@ -263,9 +263,9 @@ class Artifacts:
|
|||||||
name: str,
|
name: str,
|
||||||
version: Optional[str] = None,
|
version: Optional[str] = None,
|
||||||
stage: Optional[str] = None,
|
stage: Optional[str] = None,
|
||||||
config: Optional[Union[str, Dict[str, Any]]] = None,
|
config: Optional[Union[str, dict[str, Any]]] = None,
|
||||||
remote: Optional[str] = None,
|
remote: Optional[str] = None,
|
||||||
remote_config: Optional[Union[str, Dict[str, Any]]] = None,
|
remote_config: Optional[Union[str, dict[str, Any]]] = None,
|
||||||
out: Optional[str] = None,
|
out: Optional[str] = None,
|
||||||
force: bool = False,
|
force: bool = False,
|
||||||
jobs: Optional[int] = None,
|
jobs: Optional[int] = None,
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
|
from collections.abc import Iterator
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from typing import TYPE_CHECKING, Iterator, Optional, Tuple
|
from typing import TYPE_CHECKING, Optional
|
||||||
|
|
||||||
from scmrepo.git import Git
|
from scmrepo.git import Git
|
||||||
|
|
||||||
@@ -59,11 +60,11 @@ def brancher(
|
|||||||
|
|
||||||
from dvc.fs import LocalFileSystem
|
from dvc.fs import LocalFileSystem
|
||||||
|
|
||||||
repo_root_parts: Tuple[str, ...] = ()
|
repo_root_parts: tuple[str, ...] = ()
|
||||||
if self.fs.isin(self.root_dir, self.scm.root_dir):
|
if self.fs.isin(self.root_dir, self.scm.root_dir):
|
||||||
repo_root_parts = self.fs.relparts(self.root_dir, self.scm.root_dir)
|
repo_root_parts = self.fs.relparts(self.root_dir, self.scm.root_dir)
|
||||||
|
|
||||||
cwd_parts: Tuple[str, ...] = ()
|
cwd_parts: tuple[str, ...] = ()
|
||||||
if self.fs.isin(self.fs.getcwd(), self.scm.root_dir):
|
if self.fs.isin(self.fs.getcwd(), self.scm.root_dir):
|
||||||
cwd_parts = self.fs.relparts(self.fs.getcwd(), self.scm.root_dir)
|
cwd_parts = self.fs.relparts(self.fs.getcwd(), self.scm.root_dir)
|
||||||
|
|
||||||
@@ -112,8 +113,8 @@ def brancher(
|
|||||||
def _switch_fs(
|
def _switch_fs(
|
||||||
repo: "Repo",
|
repo: "Repo",
|
||||||
rev: str,
|
rev: str,
|
||||||
repo_root_parts: Tuple[str, ...],
|
repo_root_parts: tuple[str, ...],
|
||||||
cwd_parts: Tuple[str, ...],
|
cwd_parts: tuple[str, ...],
|
||||||
):
|
):
|
||||||
from dvc.fs import GitFileSystem, LocalFileSystem
|
from dvc.fs import GitFileSystem, LocalFileSystem
|
||||||
|
|
||||||
@@ -147,11 +148,11 @@ def switch(repo: "Repo", rev: str) -> Iterator[str]:
|
|||||||
if rev != "workspace":
|
if rev != "workspace":
|
||||||
rev = resolve_rev(repo.scm, rev)
|
rev = resolve_rev(repo.scm, rev)
|
||||||
|
|
||||||
repo_root_parts: Tuple[str, ...] = ()
|
repo_root_parts: tuple[str, ...] = ()
|
||||||
if repo.fs.isin(repo.root_dir, repo.scm.root_dir):
|
if repo.fs.isin(repo.root_dir, repo.scm.root_dir):
|
||||||
repo_root_parts = repo.fs.relparts(repo.root_dir, repo.scm.root_dir)
|
repo_root_parts = repo.fs.relparts(repo.root_dir, repo.scm.root_dir)
|
||||||
|
|
||||||
cwd_parts: Tuple[str, ...] = ()
|
cwd_parts: tuple[str, ...] = ()
|
||||||
if repo.fs.isin(repo.fs.getcwd(), repo.scm.root_dir):
|
if repo.fs.isin(repo.fs.getcwd(), repo.scm.root_dir):
|
||||||
cwd_parts = repo.fs.relparts(repo.fs.getcwd(), repo.scm.root_dir)
|
cwd_parts = repo.fs.relparts(repo.fs.getcwd(), repo.scm.root_dir)
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import os
|
import os
|
||||||
from typing import TYPE_CHECKING, Dict, List
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from dvc.exceptions import (
|
from dvc.exceptions import (
|
||||||
CheckoutError,
|
CheckoutError,
|
||||||
@@ -66,7 +66,7 @@ def _build_out_changes(index, changes):
|
|||||||
|
|
||||||
|
|
||||||
def _check_can_delete(
|
def _check_can_delete(
|
||||||
entries: List["DataIndexEntry"],
|
entries: list["DataIndexEntry"],
|
||||||
index: "BaseDataIndex",
|
index: "BaseDataIndex",
|
||||||
path: str,
|
path: str,
|
||||||
fs: "FileSystem",
|
fs: "FileSystem",
|
||||||
@@ -107,7 +107,7 @@ def checkout( # noqa: C901
|
|||||||
from dvc.stage.exceptions import StageFileBadNameError, StageFileDoesNotExistError
|
from dvc.stage.exceptions import StageFileBadNameError, StageFileDoesNotExistError
|
||||||
from dvc_data.index.checkout import ADD, DELETE, MODIFY, apply, compare
|
from dvc_data.index.checkout import ADD, DELETE, MODIFY, apply, compare
|
||||||
|
|
||||||
stats: Dict[str, List[str]] = {
|
stats: dict[str, list[str]] = {
|
||||||
"added": [],
|
"added": [],
|
||||||
"deleted": [],
|
"deleted": [],
|
||||||
"modified": [],
|
"modified": [],
|
||||||
|
|||||||
+6
-5
@@ -1,4 +1,5 @@
|
|||||||
from typing import TYPE_CHECKING, Callable, Iterable, List, Optional, Tuple
|
from collections.abc import Iterable
|
||||||
|
from typing import TYPE_CHECKING, Callable, Optional
|
||||||
|
|
||||||
from dvc.log import logger
|
from dvc.log import logger
|
||||||
|
|
||||||
@@ -10,8 +11,8 @@ logger = logger.getChild(__name__)
|
|||||||
|
|
||||||
|
|
||||||
FilterFn = Callable[["Output"], bool]
|
FilterFn = Callable[["Output"], bool]
|
||||||
Outputs = List["Output"]
|
Outputs = list["Output"]
|
||||||
StrPaths = List[str]
|
StrPaths = list[str]
|
||||||
|
|
||||||
|
|
||||||
def _collect_outs(
|
def _collect_outs(
|
||||||
@@ -43,7 +44,7 @@ def _collect_paths(
|
|||||||
|
|
||||||
def _filter_outs(
|
def _filter_outs(
|
||||||
outs: Outputs, fs_paths: StrPaths, duplicates=False
|
outs: Outputs, fs_paths: StrPaths, duplicates=False
|
||||||
) -> Tuple[Outputs, StrPaths]:
|
) -> tuple[Outputs, StrPaths]:
|
||||||
res_outs: Outputs = []
|
res_outs: Outputs = []
|
||||||
fs_res_paths = fs_paths
|
fs_res_paths = fs_paths
|
||||||
|
|
||||||
@@ -65,7 +66,7 @@ def collect(
|
|||||||
output_filter: Optional[FilterFn] = None,
|
output_filter: Optional[FilterFn] = None,
|
||||||
recursive: bool = False,
|
recursive: bool = False,
|
||||||
duplicates: bool = False,
|
duplicates: bool = False,
|
||||||
) -> Tuple[Outputs, StrPaths]:
|
) -> tuple[Outputs, StrPaths]:
|
||||||
assert targets or output_filter
|
assert targets or output_filter
|
||||||
|
|
||||||
outs: Outputs = _collect_outs(repo, output_filter=output_filter, deps=deps)
|
outs: Outputs = _collect_outs(repo, output_filter=output_filter, deps=deps)
|
||||||
|
|||||||
+2
-2
@@ -1,4 +1,4 @@
|
|||||||
from typing import TYPE_CHECKING, Set
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from dvc import prompt
|
from dvc import prompt
|
||||||
|
|
||||||
@@ -122,7 +122,7 @@ def commit_2_to_3(repo: "Repo", dry: bool = False):
|
|||||||
stage.dump(update_pipeline=False)
|
stage.dump(update_pipeline=False)
|
||||||
|
|
||||||
|
|
||||||
def _migrateable_dvcfiles(view: "IndexView") -> Set[str]:
|
def _migrateable_dvcfiles(view: "IndexView") -> set[str]:
|
||||||
from dvc.dvcfile import ProjectFile
|
from dvc.dvcfile import ProjectFile
|
||||||
|
|
||||||
migrated = set()
|
migrated = set()
|
||||||
|
|||||||
+16
-15
@@ -1,6 +1,7 @@
|
|||||||
import os
|
import os
|
||||||
import posixpath
|
import posixpath
|
||||||
from typing import TYPE_CHECKING, Any, Dict, Iterable, List, TypedDict, Union
|
from collections.abc import Iterable
|
||||||
|
from typing import TYPE_CHECKING, Any, TypedDict, Union
|
||||||
|
|
||||||
from dvc.fs.callbacks import DEFAULT_CALLBACK
|
from dvc.fs.callbacks import DEFAULT_CALLBACK
|
||||||
from dvc.ui import ui
|
from dvc.ui import ui
|
||||||
@@ -53,11 +54,11 @@ def _diff(
|
|||||||
not_in_remote: bool = False,
|
not_in_remote: bool = False,
|
||||||
remote_refresh: bool = False,
|
remote_refresh: bool = False,
|
||||||
callback: "Callback" = DEFAULT_CALLBACK,
|
callback: "Callback" = DEFAULT_CALLBACK,
|
||||||
) -> Dict[str, List[str]]:
|
) -> dict[str, list[str]]:
|
||||||
from dvc_data.index import StorageError
|
from dvc_data.index import StorageError
|
||||||
from dvc_data.index.diff import UNCHANGED, UNKNOWN, diff
|
from dvc_data.index.diff import UNCHANGED, UNKNOWN, diff
|
||||||
|
|
||||||
ret: Dict[str, List[str]] = {}
|
ret: dict[str, list[str]] = {}
|
||||||
|
|
||||||
def _add_change(typ, change):
|
def _add_change(typ, change):
|
||||||
typ = _adapt_typ(typ)
|
typ = _adapt_typ(typ)
|
||||||
@@ -115,9 +116,9 @@ def _diff(
|
|||||||
|
|
||||||
|
|
||||||
class GitInfo(TypedDict, total=False):
|
class GitInfo(TypedDict, total=False):
|
||||||
staged: Dict[str, List[str]]
|
staged: dict[str, list[str]]
|
||||||
unstaged: Dict[str, List[str]]
|
unstaged: dict[str, list[str]]
|
||||||
untracked: List[str]
|
untracked: list[str]
|
||||||
is_empty: bool
|
is_empty: bool
|
||||||
is_dirty: bool
|
is_dirty: bool
|
||||||
|
|
||||||
@@ -150,7 +151,7 @@ def _git_info(scm: Union["Git", "NoSCM"], untracked_files: str = "all") -> GitIn
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> Dict[str, List[str]]:
|
def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> dict[str, list[str]]:
|
||||||
from .index import build_data_index
|
from .index import build_data_index
|
||||||
|
|
||||||
with ui.progress(
|
with ui.progress(
|
||||||
@@ -180,7 +181,7 @@ def _diff_index_to_wtree(repo: "Repo", **kwargs: Any) -> Dict[str, List[str]]:
|
|||||||
|
|
||||||
def _diff_head_to_index(
|
def _diff_head_to_index(
|
||||||
repo: "Repo", head: str = "HEAD", **kwargs: Any
|
repo: "Repo", head: str = "HEAD", **kwargs: Any
|
||||||
) -> Dict[str, List[str]]:
|
) -> dict[str, list[str]]:
|
||||||
index = repo.index.data["repo"]
|
index = repo.index.data["repo"]
|
||||||
|
|
||||||
with repo.switch(head):
|
with repo.switch(head):
|
||||||
@@ -194,16 +195,16 @@ def _diff_head_to_index(
|
|||||||
|
|
||||||
|
|
||||||
class Status(TypedDict):
|
class Status(TypedDict):
|
||||||
not_in_cache: List[str]
|
not_in_cache: list[str]
|
||||||
not_in_remote: List[str]
|
not_in_remote: list[str]
|
||||||
committed: Dict[str, List[str]]
|
committed: dict[str, list[str]]
|
||||||
uncommitted: Dict[str, List[str]]
|
uncommitted: dict[str, list[str]]
|
||||||
untracked: List[str]
|
untracked: list[str]
|
||||||
unchanged: List[str]
|
unchanged: list[str]
|
||||||
git: GitInfo
|
git: GitInfo
|
||||||
|
|
||||||
|
|
||||||
def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> List[str]:
|
def _transform_git_paths_to_dvc(repo: "Repo", files: Iterable[str]) -> list[str]:
|
||||||
"""Transform files rel. to Git root to DVC root, and drop outside files."""
|
"""Transform files rel. to Git root to DVC root, and drop outside files."""
|
||||||
rel = repo.fs.relpath(repo.root_dir, repo.scm.root_dir).rstrip("/")
|
rel = repo.fs.relpath(repo.root_dir, repo.scm.root_dir).rstrip("/")
|
||||||
|
|
||||||
|
|||||||
+3
-3
@@ -1,7 +1,7 @@
|
|||||||
import errno
|
import errno
|
||||||
import os
|
import os
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from typing import Dict, List, Optional
|
from typing import Optional
|
||||||
|
|
||||||
from dvc.log import logger
|
from dvc.log import logger
|
||||||
from dvc.repo import locked
|
from dvc.repo import locked
|
||||||
@@ -26,7 +26,7 @@ def _diff(old, new, data_keys, with_missing=False):
|
|||||||
from dvc_data.index.diff import ADD, DELETE, MODIFY, RENAME
|
from dvc_data.index.diff import ADD, DELETE, MODIFY, RENAME
|
||||||
from dvc_data.index.diff import diff as idiff
|
from dvc_data.index.diff import diff as idiff
|
||||||
|
|
||||||
ret: "Dict[str, List[Dict]]" = {
|
ret: "dict[str, list[dict]]" = {
|
||||||
"added": [],
|
"added": [],
|
||||||
"deleted": [],
|
"deleted": [],
|
||||||
"modified": [],
|
"modified": [],
|
||||||
@@ -107,7 +107,7 @@ def diff(
|
|||||||
self,
|
self,
|
||||||
a_rev: str = "HEAD",
|
a_rev: str = "HEAD",
|
||||||
b_rev: Optional[str] = None,
|
b_rev: Optional[str] = None,
|
||||||
targets: Optional[List[str]] = None,
|
targets: Optional[list[str]] = None,
|
||||||
recursive: bool = False,
|
recursive: bool = False,
|
||||||
):
|
):
|
||||||
"""
|
"""
|
||||||
|
|||||||
+2
-2
@@ -1,4 +1,4 @@
|
|||||||
from typing import Any, Dict, Optional, Union
|
from typing import Any, Optional, Union
|
||||||
|
|
||||||
|
|
||||||
def du(
|
def du(
|
||||||
@@ -6,7 +6,7 @@ def du(
|
|||||||
path: Optional[str] = None,
|
path: Optional[str] = None,
|
||||||
rev: Optional[str] = None,
|
rev: Optional[str] = None,
|
||||||
summarize: bool = False,
|
summarize: bool = False,
|
||||||
config: Union[None, Dict[str, Any], str] = None,
|
config: Union[None, dict[str, Any], str] = None,
|
||||||
remote: Optional[str] = None,
|
remote: Optional[str] = None,
|
||||||
remote_config: Optional[dict] = None,
|
remote_config: Optional[dict] = None,
|
||||||
):
|
):
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
from typing import TYPE_CHECKING, Dict, Iterable, List, Optional
|
from collections.abc import Iterable
|
||||||
|
from typing import TYPE_CHECKING, Optional
|
||||||
|
|
||||||
from funcy import chain, first
|
from funcy import chain, first
|
||||||
|
|
||||||
@@ -104,7 +105,7 @@ class Experiments:
|
|||||||
return ExpCache(self.repo)
|
return ExpCache(self.repo)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def stash_revs(self) -> Dict[str, "ExpStashEntry"]:
|
def stash_revs(self) -> dict[str, "ExpStashEntry"]:
|
||||||
revs = {}
|
revs = {}
|
||||||
for queue in (self.workspace_queue, self.celery_queue):
|
for queue in (self.workspace_queue, self.celery_queue):
|
||||||
revs.update(queue.stash.stash_revs)
|
revs.update(queue.stash.stash_revs)
|
||||||
@@ -113,7 +114,7 @@ class Experiments:
|
|||||||
def reproduce_one(
|
def reproduce_one(
|
||||||
self,
|
self,
|
||||||
tmp_dir: bool = False,
|
tmp_dir: bool = False,
|
||||||
copy_paths: Optional[List[str]] = None,
|
copy_paths: Optional[list[str]] = None,
|
||||||
message: Optional[str] = None,
|
message: Optional[str] = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
):
|
):
|
||||||
@@ -143,8 +144,8 @@ class Experiments:
|
|||||||
|
|
||||||
def reproduce_celery(
|
def reproduce_celery(
|
||||||
self, entries: Optional[Iterable["QueueEntry"]] = None, **kwargs
|
self, entries: Optional[Iterable["QueueEntry"]] = None, **kwargs
|
||||||
) -> Dict[str, str]:
|
) -> dict[str, str]:
|
||||||
results: Dict[str, str] = {}
|
results: dict[str, str] = {}
|
||||||
if entries is None:
|
if entries is None:
|
||||||
entries = list(
|
entries = list(
|
||||||
chain(
|
chain(
|
||||||
@@ -250,10 +251,10 @@ class Experiments:
|
|||||||
def _reproduce_queue(
|
def _reproduce_queue(
|
||||||
self,
|
self,
|
||||||
queue: "BaseStashQueue",
|
queue: "BaseStashQueue",
|
||||||
copy_paths: Optional[List[str]] = None,
|
copy_paths: Optional[list[str]] = None,
|
||||||
message: Optional[str] = None,
|
message: Optional[str] = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Dict[str, str]:
|
) -> dict[str, str]:
|
||||||
"""Reproduce queued experiments.
|
"""Reproduce queued experiments.
|
||||||
|
|
||||||
Arguments:
|
Arguments:
|
||||||
@@ -265,7 +266,7 @@ class Experiments:
|
|||||||
"""
|
"""
|
||||||
exec_results = queue.reproduce(copy_paths=copy_paths, message=message)
|
exec_results = queue.reproduce(copy_paths=copy_paths, message=message)
|
||||||
|
|
||||||
results: Dict[str, str] = {}
|
results: dict[str, str] = {}
|
||||||
for _, exp_result in exec_results.items():
|
for _, exp_result in exec_results.items():
|
||||||
results.update(exp_result)
|
results.update(exp_result)
|
||||||
return results
|
return results
|
||||||
@@ -319,12 +320,12 @@ class Experiments:
|
|||||||
raise MultipleBranchError(rev, ref_infos)
|
raise MultipleBranchError(rev, ref_infos)
|
||||||
return str(ref_infos[0])
|
return str(ref_infos[0])
|
||||||
|
|
||||||
def get_exact_name(self, revs: Iterable[str]) -> Dict[str, Optional[str]]:
|
def get_exact_name(self, revs: Iterable[str]) -> dict[str, Optional[str]]:
|
||||||
"""Returns preferred name for the specified revision.
|
"""Returns preferred name for the specified revision.
|
||||||
|
|
||||||
Prefers tags, branches (heads), experiments in that order.
|
Prefers tags, branches (heads), experiments in that order.
|
||||||
"""
|
"""
|
||||||
result: Dict[str, Optional[str]] = {}
|
result: dict[str, Optional[str]] = {}
|
||||||
exclude = f"{EXEC_NAMESPACE}/*"
|
exclude = f"{EXEC_NAMESPACE}/*"
|
||||||
ref_dict = self.scm.describe(revs, base=EXPS_NAMESPACE, exclude=exclude)
|
ref_dict = self.scm.describe(revs, base=EXPS_NAMESPACE, exclude=exclude)
|
||||||
for rev in revs:
|
for rev in revs:
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
|
from collections.abc import Iterator
|
||||||
from contextlib import ExitStack, contextmanager
|
from contextlib import ExitStack, contextmanager
|
||||||
from typing import TYPE_CHECKING, Iterator, Tuple
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from dvc.repo.experiments.exceptions import InvalidExpRevError
|
from dvc.repo.experiments.exceptions import InvalidExpRevError
|
||||||
from dvc.scm import RevError
|
from dvc.scm import RevError
|
||||||
@@ -12,7 +13,7 @@ if TYPE_CHECKING:
|
|||||||
def switch_repo(
|
def switch_repo(
|
||||||
repo: "Repo",
|
repo: "Repo",
|
||||||
rev: str,
|
rev: str,
|
||||||
) -> Iterator[Tuple["Repo", str]]:
|
) -> Iterator[tuple["Repo", str]]:
|
||||||
"""Return a repo instance (brancher) switched to rev.
|
"""Return a repo instance (brancher) switched to rev.
|
||||||
|
|
||||||
If rev is the name of a running experiment, the returned instance will be
|
If rev is the name of a running experiment, the returned instance will be
|
||||||
|
|||||||
@@ -1,16 +1,11 @@
|
|||||||
import itertools
|
import itertools
|
||||||
import os
|
import os
|
||||||
|
from collections.abc import Collection, Iterable, Iterator
|
||||||
from dataclasses import fields
|
from dataclasses import fields
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import (
|
from typing import (
|
||||||
TYPE_CHECKING,
|
TYPE_CHECKING,
|
||||||
Collection,
|
|
||||||
Dict,
|
|
||||||
Iterable,
|
|
||||||
Iterator,
|
|
||||||
List,
|
|
||||||
Optional,
|
Optional,
|
||||||
Tuple,
|
|
||||||
Union,
|
Union,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -142,7 +137,7 @@ def collect_queued(
|
|||||||
repo: "Repo",
|
repo: "Repo",
|
||||||
baseline_revs: Collection[str],
|
baseline_revs: Collection[str],
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Dict[str, List["ExpRange"]]:
|
) -> dict[str, list["ExpRange"]]:
|
||||||
"""Collect queued experiments derived from the specified revisions.
|
"""Collect queued experiments derived from the specified revisions.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -177,7 +172,7 @@ def collect_active(
|
|||||||
repo: "Repo",
|
repo: "Repo",
|
||||||
baseline_revs: Collection[str],
|
baseline_revs: Collection[str],
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Dict[str, List["ExpRange"]]:
|
) -> dict[str, list["ExpRange"]]:
|
||||||
"""Collect active (running) experiments derived from the specified revisions.
|
"""Collect active (running) experiments derived from the specified revisions.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -189,7 +184,7 @@ def collect_active(
|
|||||||
"""
|
"""
|
||||||
if not baseline_revs:
|
if not baseline_revs:
|
||||||
return {}
|
return {}
|
||||||
result: Dict[str, List["ExpRange"]] = {}
|
result: dict[str, list["ExpRange"]] = {}
|
||||||
exps = repo.experiments
|
exps = repo.experiments
|
||||||
for queue in (exps.workspace_queue, exps.tempdir_queue, exps.celery_queue):
|
for queue in (exps.workspace_queue, exps.tempdir_queue, exps.celery_queue):
|
||||||
for baseline, active_exps in queue.collect_active_data(
|
for baseline, active_exps in queue.collect_active_data(
|
||||||
@@ -206,7 +201,7 @@ def collect_failed(
|
|||||||
repo: "Repo",
|
repo: "Repo",
|
||||||
baseline_revs: Collection[str],
|
baseline_revs: Collection[str],
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Dict[str, List["ExpRange"]]:
|
) -> dict[str, list["ExpRange"]]:
|
||||||
"""Collect failed experiments derived from the specified revisions.
|
"""Collect failed experiments derived from the specified revisions.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -225,7 +220,7 @@ def collect_successful(
|
|||||||
repo: "Repo",
|
repo: "Repo",
|
||||||
baseline_revs: Collection[str],
|
baseline_revs: Collection[str],
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Dict[str, List["ExpRange"]]:
|
) -> dict[str, list["ExpRange"]]:
|
||||||
"""Collect successful experiments derived from the specified revisions.
|
"""Collect successful experiments derived from the specified revisions.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -235,7 +230,7 @@ def collect_successful(
|
|||||||
Returns:
|
Returns:
|
||||||
Dict mapping baseline revision to successful experiments.
|
Dict mapping baseline revision to successful experiments.
|
||||||
"""
|
"""
|
||||||
result: Dict[str, List["ExpRange"]] = {}
|
result: dict[str, list["ExpRange"]] = {}
|
||||||
for baseline_rev in baseline_revs:
|
for baseline_rev in baseline_revs:
|
||||||
result[baseline_rev] = list(_collect_baseline(repo, baseline_rev, **kwargs))
|
result[baseline_rev] = list(_collect_baseline(repo, baseline_rev, **kwargs))
|
||||||
return result
|
return result
|
||||||
@@ -282,7 +277,7 @@ def _collect_baseline(
|
|||||||
|
|
||||||
def collect(
|
def collect(
|
||||||
repo: "Repo",
|
repo: "Repo",
|
||||||
revs: Union[List[str], str, None] = None,
|
revs: Union[list[str], str, None] = None,
|
||||||
all_branches: bool = False,
|
all_branches: bool = False,
|
||||||
all_tags: bool = False,
|
all_tags: bool = False,
|
||||||
all_commits: bool = False,
|
all_commits: bool = False,
|
||||||
@@ -291,7 +286,7 @@ def collect(
|
|||||||
hide_failed: bool = False,
|
hide_failed: bool = False,
|
||||||
sha_only: bool = False,
|
sha_only: bool = False,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> List["ExpState"]:
|
) -> list["ExpState"]:
|
||||||
"""Collect baseline revisions and derived experiments."""
|
"""Collect baseline revisions and derived experiments."""
|
||||||
assert isinstance(repo.scm, Git)
|
assert isinstance(repo.scm, Git)
|
||||||
if repo.scm.no_commits:
|
if repo.scm.no_commits:
|
||||||
@@ -312,14 +307,14 @@ def collect(
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
if sha_only:
|
if sha_only:
|
||||||
baseline_names: Dict[str, Optional[str]] = {}
|
baseline_names: dict[str, Optional[str]] = {}
|
||||||
else:
|
else:
|
||||||
baseline_names = describe(
|
baseline_names = describe(
|
||||||
repo.scm, baseline_revs, refs=cached_refs, logger=logger
|
repo.scm, baseline_revs, refs=cached_refs, logger=logger
|
||||||
)
|
)
|
||||||
|
|
||||||
workspace_data = collect_rev(repo, "workspace", **kwargs)
|
workspace_data = collect_rev(repo, "workspace", **kwargs)
|
||||||
result: List["ExpState"] = [workspace_data]
|
result: list["ExpState"] = [workspace_data]
|
||||||
queued = collect_queued(repo, baseline_revs, **kwargs) if not hide_queued else {}
|
queued = collect_queued(repo, baseline_revs, **kwargs) if not hide_queued else {}
|
||||||
active = collect_active(repo, baseline_revs, **kwargs)
|
active = collect_active(repo, baseline_revs, **kwargs)
|
||||||
failed = collect_failed(repo, baseline_revs, **kwargs) if not hide_failed else {}
|
failed = collect_failed(repo, baseline_revs, **kwargs) if not hide_failed else {}
|
||||||
@@ -345,10 +340,10 @@ def collect(
|
|||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
def _sorted_ranges(exp_ranges: Iterable["ExpRange"]) -> List["ExpRange"]:
|
def _sorted_ranges(exp_ranges: Iterable["ExpRange"]) -> list["ExpRange"]:
|
||||||
"""Return list of ExpRange sorted by (timestamp, rev)."""
|
"""Return list of ExpRange sorted by (timestamp, rev)."""
|
||||||
|
|
||||||
def _head_timestamp(exp_range: "ExpRange") -> Tuple[datetime, str]:
|
def _head_timestamp(exp_range: "ExpRange") -> tuple[datetime, str]:
|
||||||
head_exp = first(exp_range.revs)
|
head_exp = first(exp_range.revs)
|
||||||
if head_exp and head_exp.data and head_exp.data.timestamp:
|
if head_exp and head_exp.data and head_exp.data.timestamp:
|
||||||
return head_exp.data.timestamp, head_exp.rev
|
return head_exp.data.timestamp, head_exp.rev
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
from typing import TYPE_CHECKING, Collection, Iterable, Optional
|
from collections.abc import Collection, Iterable
|
||||||
|
from typing import TYPE_CHECKING, Optional
|
||||||
|
|
||||||
from dvc.exceptions import DvcException, InvalidArgumentError
|
from dvc.exceptions import DvcException, InvalidArgumentError
|
||||||
|
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import os
|
|||||||
import pickle
|
import pickle
|
||||||
import shutil
|
import shutil
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
|
from collections.abc import Iterable, Iterator
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from dataclasses import asdict, dataclass
|
from dataclasses import asdict, dataclass
|
||||||
from enum import IntEnum
|
from enum import IntEnum
|
||||||
@@ -11,15 +12,8 @@ from typing import (
|
|||||||
TYPE_CHECKING,
|
TYPE_CHECKING,
|
||||||
Any,
|
Any,
|
||||||
Callable,
|
Callable,
|
||||||
Dict,
|
|
||||||
Iterable,
|
|
||||||
Iterator,
|
|
||||||
List,
|
|
||||||
NamedTuple,
|
NamedTuple,
|
||||||
Optional,
|
Optional,
|
||||||
Tuple,
|
|
||||||
Type,
|
|
||||||
TypeVar,
|
|
||||||
Union,
|
Union,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -41,6 +35,8 @@ from dvc.utils.studio import env_to_config
|
|||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from queue import Queue
|
from queue import Queue
|
||||||
|
|
||||||
|
from typing_extensions import Self
|
||||||
|
|
||||||
from dvc.repo import Repo
|
from dvc.repo import Repo
|
||||||
from dvc.repo.experiments.stash import ExpStashEntry
|
from dvc.repo.experiments.stash import ExpStashEntry
|
||||||
from dvc.scm import Git
|
from dvc.scm import Git
|
||||||
@@ -112,9 +108,6 @@ class ExecutorInfo:
|
|||||||
return cls.from_dict(load_json(filename))
|
return cls.from_dict(load_json(filename))
|
||||||
|
|
||||||
|
|
||||||
_T = TypeVar("_T", bound="BaseExecutor")
|
|
||||||
|
|
||||||
|
|
||||||
class BaseExecutor(ABC):
|
class BaseExecutor(ABC):
|
||||||
"""Base class for executing experiments in parallel.
|
"""Base class for executing experiments in parallel.
|
||||||
|
|
||||||
@@ -185,7 +178,7 @@ class BaseExecutor(ABC):
|
|||||||
@property
|
@property
|
||||||
def info(self) -> "ExecutorInfo":
|
def info(self) -> "ExecutorInfo":
|
||||||
if self.result is not None:
|
if self.result is not None:
|
||||||
result_dict: Dict[str, Any] = {
|
result_dict: dict[str, Any] = {
|
||||||
"result_hash": self.result.exp_hash,
|
"result_hash": self.result.exp_hash,
|
||||||
"result_ref": (
|
"result_ref": (
|
||||||
str(self.result.ref_info) if self.result.ref_info else None
|
str(self.result.ref_info) if self.result.ref_info else None
|
||||||
@@ -207,7 +200,7 @@ class BaseExecutor(ABC):
|
|||||||
)
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_info(cls: Type[_T], info: "ExecutorInfo") -> _T:
|
def from_info(cls, info: "ExecutorInfo") -> "Self":
|
||||||
if info.result_hash:
|
if info.result_hash:
|
||||||
result: Optional["ExecutorResult"] = ExecutorResult(
|
result: Optional["ExecutorResult"] = ExecutorResult(
|
||||||
info.result_hash,
|
info.result_hash,
|
||||||
@@ -229,21 +222,21 @@ class BaseExecutor(ABC):
|
|||||||
@classmethod
|
@classmethod
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def from_stash_entry(
|
def from_stash_entry(
|
||||||
cls: Type[_T],
|
cls,
|
||||||
repo: "Repo",
|
repo: "Repo",
|
||||||
entry: "ExpStashEntry",
|
entry: "ExpStashEntry",
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> _T:
|
) -> "Self":
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def _from_stash_entry(
|
def _from_stash_entry(
|
||||||
cls: Type[_T],
|
cls,
|
||||||
repo: "Repo",
|
repo: "Repo",
|
||||||
entry: "ExpStashEntry",
|
entry: "ExpStashEntry",
|
||||||
root_dir: str,
|
root_dir: str,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> _T:
|
) -> "Self":
|
||||||
return cls(
|
return cls(
|
||||||
root_dir=root_dir,
|
root_dir=root_dir,
|
||||||
dvc_dir=relpath(repo.dvc_dir, repo.scm.root_dir),
|
dvc_dir=relpath(repo.dvc_dir, repo.scm.root_dir),
|
||||||
@@ -255,7 +248,7 @@ class BaseExecutor(ABC):
|
|||||||
)
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def _get_top_level_paths(cls, repo: "Repo") -> List["str"]:
|
def _get_top_level_paths(cls, repo: "Repo") -> list["str"]:
|
||||||
return list(
|
return list(
|
||||||
chain(
|
chain(
|
||||||
_collect_top_level_metrics(repo),
|
_collect_top_level_metrics(repo),
|
||||||
@@ -271,7 +264,7 @@ class BaseExecutor(ABC):
|
|||||||
targets: Optional[Iterable[str]] = None,
|
targets: Optional[Iterable[str]] = None,
|
||||||
recursive: bool = False,
|
recursive: bool = False,
|
||||||
force: bool = False,
|
force: bool = False,
|
||||||
include_untracked: Optional[List[str]] = None,
|
include_untracked: Optional[list[str]] = None,
|
||||||
message: Optional[str] = None,
|
message: Optional[str] = None,
|
||||||
) -> ExecutorResult:
|
) -> ExecutorResult:
|
||||||
from dvc.dvcfile import LOCK_FILE
|
from dvc.dvcfile import LOCK_FILE
|
||||||
@@ -382,7 +375,7 @@ class BaseExecutor(ABC):
|
|||||||
def fetch_exps(
|
def fetch_exps(
|
||||||
self,
|
self,
|
||||||
dest_scm: "Git",
|
dest_scm: "Git",
|
||||||
refs: List[str],
|
refs: list[str],
|
||||||
force: bool = False,
|
force: bool = False,
|
||||||
on_diverged: Optional[Callable[[str], None]] = None,
|
on_diverged: Optional[Callable[[str], None]] = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
@@ -457,7 +450,7 @@ class BaseExecutor(ABC):
|
|||||||
infofile: Optional[str] = None,
|
infofile: Optional[str] = None,
|
||||||
log_errors: bool = True,
|
log_errors: bool = True,
|
||||||
log_level: Optional[int] = None,
|
log_level: Optional[int] = None,
|
||||||
copy_paths: Optional[List[str]] = None,
|
copy_paths: Optional[list[str]] = None,
|
||||||
message: Optional[str] = None,
|
message: Optional[str] = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> "ExecutorResult":
|
) -> "ExecutorResult":
|
||||||
@@ -565,7 +558,7 @@ class BaseExecutor(ABC):
|
|||||||
git_remote,
|
git_remote,
|
||||||
repro_force,
|
repro_force,
|
||||||
message: Optional[str] = None,
|
message: Optional[str] = None,
|
||||||
) -> Tuple[Optional[str], Optional["ExpRefInfo"], bool]:
|
) -> tuple[Optional[str], Optional["ExpRefInfo"], bool]:
|
||||||
cls.commit(
|
cls.commit(
|
||||||
dvc.scm,
|
dvc.scm,
|
||||||
exp_hash,
|
exp_hash,
|
||||||
@@ -598,7 +591,7 @@ class BaseExecutor(ABC):
|
|||||||
info: "ExecutorInfo",
|
info: "ExecutorInfo",
|
||||||
infofile: Optional[str] = None,
|
infofile: Optional[str] = None,
|
||||||
log_errors: bool = True,
|
log_errors: bool = True,
|
||||||
copy_paths: Optional[List[str]] = None,
|
copy_paths: Optional[list[str]] = None,
|
||||||
message: Optional[str] = None,
|
message: Optional[str] = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Iterator["Repo"]:
|
) -> Iterator["Repo"]:
|
||||||
@@ -790,7 +783,7 @@ class BaseExecutor(ABC):
|
|||||||
raise DvcException(f"Unable to copy '{src}' to '{dst}'.") from exc
|
raise DvcException(f"Unable to copy '{src}' to '{dst}'.") from exc
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def set_temp_refs(self, scm: "Git", temp_dict: Dict[str, str]):
|
def set_temp_refs(self, scm: "Git", temp_dict: dict[str, str]):
|
||||||
try:
|
try:
|
||||||
for ref, rev in temp_dict.items():
|
for ref, rev in temp_dict.items():
|
||||||
scm.set_ref(ref, rev)
|
scm.set_ref(ref, rev)
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from typing import Dict, List, Optional, Tuple, Union
|
from typing import Optional, Union
|
||||||
|
|
||||||
from dvc.log import logger
|
from dvc.log import logger
|
||||||
from dvc.repo import locked
|
from dvc.repo import locked
|
||||||
@@ -15,11 +15,11 @@ logger = logger.getChild(__name__)
|
|||||||
@scm_context
|
@scm_context
|
||||||
def ls(
|
def ls(
|
||||||
repo,
|
repo,
|
||||||
rev: Optional[Union[List[str], str]] = None,
|
rev: Optional[Union[list[str], str]] = None,
|
||||||
all_commits: bool = False,
|
all_commits: bool = False,
|
||||||
num: int = 1,
|
num: int = 1,
|
||||||
git_remote: Optional[str] = None,
|
git_remote: Optional[str] = None,
|
||||||
) -> Dict[str, List[Tuple[str, Optional[str]]]]:
|
) -> dict[str, list[tuple[str, Optional[str]]]]:
|
||||||
"""List experiments.
|
"""List experiments.
|
||||||
|
|
||||||
Returns a dict mapping baseline revs to a list of (exp_name, exp_sha) tuples.
|
Returns a dict mapping baseline revs to a list of (exp_name, exp_sha) tuples.
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
from typing import Iterable, List, Mapping, Optional, Set, Union
|
from collections.abc import Iterable, Mapping
|
||||||
|
from typing import Optional, Union
|
||||||
|
|
||||||
from funcy import group_by
|
from funcy import group_by
|
||||||
from scmrepo.git.backend.base import SyncStatus
|
from scmrepo.git.backend.base import SyncStatus
|
||||||
@@ -23,13 +24,13 @@ def pull( # noqa: C901
|
|||||||
git_remote: str,
|
git_remote: str,
|
||||||
exp_names: Optional[Union[Iterable[str], str]] = None,
|
exp_names: Optional[Union[Iterable[str], str]] = None,
|
||||||
all_commits=False,
|
all_commits=False,
|
||||||
rev: Optional[Union[List[str], str]] = None,
|
rev: Optional[Union[list[str], str]] = None,
|
||||||
num=1,
|
num=1,
|
||||||
force: bool = False,
|
force: bool = False,
|
||||||
pull_cache: bool = False,
|
pull_cache: bool = False,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Iterable[str]:
|
) -> Iterable[str]:
|
||||||
exp_ref_set: Set["ExpRefInfo"] = set()
|
exp_ref_set: set["ExpRefInfo"] = set()
|
||||||
if all_commits:
|
if all_commits:
|
||||||
exp_ref_set.update(exp_refs(repo.scm, git_remote))
|
exp_ref_set.update(exp_refs(repo.scm, git_remote))
|
||||||
elif exp_names:
|
elif exp_names:
|
||||||
@@ -81,7 +82,7 @@ def _pull(
|
|||||||
git_remote: str,
|
git_remote: str,
|
||||||
refs: Iterable["ExpRefInfo"],
|
refs: Iterable["ExpRefInfo"],
|
||||||
force: bool,
|
force: bool,
|
||||||
) -> Mapping[SyncStatus, List["ExpRefInfo"]]:
|
) -> Mapping[SyncStatus, list["ExpRefInfo"]]:
|
||||||
refspec_list = [f"{exp_ref}:{exp_ref}" for exp_ref in refs]
|
refspec_list = [f"{exp_ref}:{exp_ref}" for exp_ref in refs]
|
||||||
logger.debug("git pull experiment '%s' -> '%s'", git_remote, refspec_list)
|
logger.debug("git pull experiment '%s' -> '%s'", git_remote, refspec_list)
|
||||||
|
|
||||||
@@ -96,7 +97,7 @@ def _pull(
|
|||||||
def group_result(refspec):
|
def group_result(refspec):
|
||||||
return results[str(refspec)]
|
return results[str(refspec)]
|
||||||
|
|
||||||
pull_result: Mapping[SyncStatus, List["ExpRefInfo"]] = group_by(group_result, refs)
|
pull_result: Mapping[SyncStatus, list["ExpRefInfo"]] = group_by(group_result, refs)
|
||||||
|
|
||||||
return pull_result
|
return pull_result
|
||||||
|
|
||||||
|
|||||||
@@ -1,12 +1,8 @@
|
|||||||
|
from collections.abc import Iterable, Mapping
|
||||||
from typing import (
|
from typing import (
|
||||||
TYPE_CHECKING,
|
TYPE_CHECKING,
|
||||||
Any,
|
Any,
|
||||||
Dict,
|
|
||||||
Iterable,
|
|
||||||
List,
|
|
||||||
Mapping,
|
|
||||||
Optional,
|
Optional,
|
||||||
Set,
|
|
||||||
Union,
|
Union,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -39,7 +35,7 @@ class UploadError(DvcException):
|
|||||||
|
|
||||||
|
|
||||||
def notify_refs_to_studio(
|
def notify_refs_to_studio(
|
||||||
repo: "Repo", git_remote: str, **refs: List[str]
|
repo: "Repo", git_remote: str, **refs: list[str]
|
||||||
) -> Optional[str]:
|
) -> Optional[str]:
|
||||||
import os
|
import os
|
||||||
|
|
||||||
@@ -67,7 +63,7 @@ def notify_refs_to_studio(
|
|||||||
return d.get("url")
|
return d.get("url")
|
||||||
|
|
||||||
|
|
||||||
def exp_refs_from_names(scm: "Git", exp_names: List[str]) -> Set["ExpRefInfo"]:
|
def exp_refs_from_names(scm: "Git", exp_names: list[str]) -> set["ExpRefInfo"]:
|
||||||
exp_ref_set = set()
|
exp_ref_set = set()
|
||||||
exp_ref_dict = resolve_name(scm, exp_names)
|
exp_ref_dict = resolve_name(scm, exp_names)
|
||||||
unresolved_exp_names = []
|
unresolved_exp_names = []
|
||||||
@@ -82,7 +78,7 @@ def exp_refs_from_names(scm: "Git", exp_names: List[str]) -> Set["ExpRefInfo"]:
|
|||||||
return exp_ref_set
|
return exp_ref_set
|
||||||
|
|
||||||
|
|
||||||
def exp_refs_from_rev(scm: "Git", rev: List[str], num: int = 1) -> Set["ExpRefInfo"]:
|
def exp_refs_from_rev(scm: "Git", rev: list[str], num: int = 1) -> set["ExpRefInfo"]:
|
||||||
exp_ref_set = set()
|
exp_ref_set = set()
|
||||||
rev_dict = iter_revs(scm, rev, num)
|
rev_dict = iter_revs(scm, rev, num)
|
||||||
rev_set = set(rev_dict.keys())
|
rev_set = set(rev_dict.keys())
|
||||||
@@ -97,15 +93,15 @@ def exp_refs_from_rev(scm: "Git", rev: List[str], num: int = 1) -> Set["ExpRefIn
|
|||||||
def push(
|
def push(
|
||||||
repo: "Repo",
|
repo: "Repo",
|
||||||
git_remote: str,
|
git_remote: str,
|
||||||
exp_names: Optional[Union[List[str], str]] = None,
|
exp_names: Optional[Union[list[str], str]] = None,
|
||||||
all_commits: bool = False,
|
all_commits: bool = False,
|
||||||
rev: Optional[Union[List[str], str]] = None,
|
rev: Optional[Union[list[str], str]] = None,
|
||||||
num: int = 1,
|
num: int = 1,
|
||||||
force: bool = False,
|
force: bool = False,
|
||||||
push_cache: bool = False,
|
push_cache: bool = False,
|
||||||
**kwargs: Any,
|
**kwargs: Any,
|
||||||
) -> Dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
exp_ref_set: Set["ExpRefInfo"] = set()
|
exp_ref_set: set["ExpRefInfo"] = set()
|
||||||
assert isinstance(repo.scm, Git)
|
assert isinstance(repo.scm, Git)
|
||||||
if all_commits:
|
if all_commits:
|
||||||
exp_ref_set.update(exp_refs(repo.scm))
|
exp_ref_set.update(exp_refs(repo.scm))
|
||||||
@@ -123,7 +119,7 @@ def push(
|
|||||||
status.name.lower(): [ref.name for ref in ref_list]
|
status.name.lower(): [ref.name for ref in ref_list]
|
||||||
for status, ref_list in push_result.items()
|
for status, ref_list in push_result.items()
|
||||||
}
|
}
|
||||||
result: Dict[str, Any] = {**refs, "uploaded": 0}
|
result: dict[str, Any] = {**refs, "uploaded": 0}
|
||||||
|
|
||||||
pushed_refs_info = (
|
pushed_refs_info = (
|
||||||
push_result[SyncStatus.UP_TO_DATE] + push_result[SyncStatus.SUCCESS]
|
push_result[SyncStatus.UP_TO_DATE] + push_result[SyncStatus.SUCCESS]
|
||||||
@@ -149,7 +145,7 @@ def _push(
|
|||||||
git_remote: str,
|
git_remote: str,
|
||||||
refs: Iterable["ExpRefInfo"],
|
refs: Iterable["ExpRefInfo"],
|
||||||
force: bool,
|
force: bool,
|
||||||
) -> Mapping[SyncStatus, List["ExpRefInfo"]]:
|
) -> Mapping[SyncStatus, list["ExpRefInfo"]]:
|
||||||
from scmrepo.exceptions import AuthError
|
from scmrepo.exceptions import AuthError
|
||||||
|
|
||||||
from dvc.scm import GitAuthError
|
from dvc.scm import GitAuthError
|
||||||
@@ -171,7 +167,7 @@ def _push(
|
|||||||
def group_result(refspec):
|
def group_result(refspec):
|
||||||
return results[str(refspec)]
|
return results[str(refspec)]
|
||||||
|
|
||||||
pull_result: Mapping[SyncStatus, List["ExpRefInfo"]] = group_by(group_result, refs)
|
pull_result: Mapping[SyncStatus, list["ExpRefInfo"]] = group_by(group_result, refs)
|
||||||
|
|
||||||
return pull_result
|
return pull_result
|
||||||
|
|
||||||
|
|||||||
@@ -1,18 +1,12 @@
|
|||||||
import os
|
import os
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
|
from collections.abc import Collection, Generator, Iterable, Mapping
|
||||||
from dataclasses import asdict, dataclass
|
from dataclasses import asdict, dataclass
|
||||||
from typing import (
|
from typing import (
|
||||||
TYPE_CHECKING,
|
TYPE_CHECKING,
|
||||||
Any,
|
Any,
|
||||||
Collection,
|
|
||||||
Dict,
|
|
||||||
Generator,
|
|
||||||
Iterable,
|
|
||||||
List,
|
|
||||||
Mapping,
|
|
||||||
NamedTuple,
|
NamedTuple,
|
||||||
Optional,
|
Optional,
|
||||||
Type,
|
|
||||||
Union,
|
Union,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -69,11 +63,11 @@ class QueueEntry:
|
|||||||
and self.stash_rev == other.stash_rev
|
and self.stash_rev == other.stash_rev
|
||||||
)
|
)
|
||||||
|
|
||||||
def asdict(self) -> Dict[str, Any]:
|
def asdict(self) -> dict[str, Any]:
|
||||||
return asdict(self)
|
return asdict(self)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_dict(cls, d: Dict[str, Any]) -> "QueueEntry":
|
def from_dict(cls, d: dict[str, Any]) -> "QueueEntry":
|
||||||
return cls(**d)
|
return cls(**d)
|
||||||
|
|
||||||
|
|
||||||
@@ -150,7 +144,7 @@ class BaseStashQueue(ABC):
|
|||||||
all_: bool = False,
|
all_: bool = False,
|
||||||
queued: bool = False,
|
queued: bool = False,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> List[str]:
|
) -> list[str]:
|
||||||
"""Remove the specified entries from the queue.
|
"""Remove the specified entries from the queue.
|
||||||
|
|
||||||
Arguments:
|
Arguments:
|
||||||
@@ -165,8 +159,8 @@ class BaseStashQueue(ABC):
|
|||||||
if all_ or queued:
|
if all_ or queued:
|
||||||
return self.clear()
|
return self.clear()
|
||||||
|
|
||||||
name_to_remove: List[str] = []
|
name_to_remove: list[str] = []
|
||||||
entry_to_remove: List[ExpStashEntry] = []
|
entry_to_remove: list[ExpStashEntry] = []
|
||||||
queue_entries = self.match_queue_entry_by_name(revs, self.iter_queued())
|
queue_entries = self.match_queue_entry_by_name(revs, self.iter_queued())
|
||||||
for name, entry in queue_entries.items():
|
for name, entry in queue_entries.items():
|
||||||
if entry:
|
if entry:
|
||||||
@@ -176,7 +170,7 @@ class BaseStashQueue(ABC):
|
|||||||
self.stash.remove_revs(entry_to_remove)
|
self.stash.remove_revs(entry_to_remove)
|
||||||
return name_to_remove
|
return name_to_remove
|
||||||
|
|
||||||
def clear(self, **kwargs) -> List[str]:
|
def clear(self, **kwargs) -> list[str]:
|
||||||
"""Remove all entries from the queue."""
|
"""Remove all entries from the queue."""
|
||||||
stash_revs = self.stash.stash_revs
|
stash_revs = self.stash.stash_revs
|
||||||
name_to_remove = list(stash_revs)
|
name_to_remove = list(stash_revs)
|
||||||
@@ -184,11 +178,11 @@ class BaseStashQueue(ABC):
|
|||||||
|
|
||||||
return name_to_remove
|
return name_to_remove
|
||||||
|
|
||||||
def status(self) -> List[Dict[str, Any]]:
|
def status(self) -> list[dict[str, Any]]:
|
||||||
"""Show the status of exp tasks in queue"""
|
"""Show the status of exp tasks in queue"""
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
result: List[Dict[str, Optional[str]]] = []
|
result: list[dict[str, Optional[str]]] = []
|
||||||
|
|
||||||
def _get_timestamp(rev: str) -> datetime:
|
def _get_timestamp(rev: str) -> datetime:
|
||||||
commit = self.scm.resolve_commit(rev)
|
commit = self.scm.resolve_commit(rev)
|
||||||
@@ -198,7 +192,7 @@ class BaseStashQueue(ABC):
|
|||||||
entry: QueueEntry,
|
entry: QueueEntry,
|
||||||
exp_result: Optional["ExecutorResult"] = None,
|
exp_result: Optional["ExecutorResult"] = None,
|
||||||
status: str = "Unknown",
|
status: str = "Unknown",
|
||||||
) -> Dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
name = entry.name
|
name = entry.name
|
||||||
if not name and exp_result and exp_result.ref_info:
|
if not name and exp_result and exp_result.ref_info:
|
||||||
name = exp_result.ref_info.name
|
name = exp_result.ref_info.name
|
||||||
@@ -251,7 +245,7 @@ class BaseStashQueue(ABC):
|
|||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def reproduce(
|
def reproduce(
|
||||||
self, copy_paths: Optional[List[str]] = None, message: Optional[str] = None
|
self, copy_paths: Optional[list[str]] = None, message: Optional[str] = None
|
||||||
) -> Mapping[str, Mapping[str, str]]:
|
) -> Mapping[str, Mapping[str, str]]:
|
||||||
"""Reproduce queued experiments sequentially."""
|
"""Reproduce queued experiments sequentially."""
|
||||||
|
|
||||||
@@ -300,7 +294,7 @@ class BaseStashQueue(ABC):
|
|||||||
def _stash_exp(
|
def _stash_exp(
|
||||||
self,
|
self,
|
||||||
*args,
|
*args,
|
||||||
params: Optional[Dict[str, List[str]]] = None,
|
params: Optional[dict[str, list[str]]] = None,
|
||||||
baseline_rev: Optional[str] = None,
|
baseline_rev: Optional[str] = None,
|
||||||
branch: Optional[str] = None,
|
branch: Optional[str] = None,
|
||||||
name: Optional[str] = None,
|
name: Optional[str] = None,
|
||||||
@@ -460,7 +454,7 @@ class BaseStashQueue(ABC):
|
|||||||
f"from '{config_path}': {param_list}"
|
f"from '{config_path}': {param_list}"
|
||||||
)
|
)
|
||||||
|
|
||||||
def _update_params(self, params: Dict[str, List[str]]):
|
def _update_params(self, params: dict[str, list[str]]):
|
||||||
"""Update param files with the provided `Hydra Override`_ patterns.
|
"""Update param files with the provided `Hydra Override`_ patterns.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -528,7 +522,7 @@ class BaseStashQueue(ABC):
|
|||||||
cls,
|
cls,
|
||||||
exp: "Experiments",
|
exp: "Experiments",
|
||||||
queue_entry: QueueEntry,
|
queue_entry: QueueEntry,
|
||||||
executor_cls: Type[BaseExecutor] = WorkspaceExecutor,
|
executor_cls: type[BaseExecutor] = WorkspaceExecutor,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> BaseExecutor:
|
) -> BaseExecutor:
|
||||||
stash_entry = cls.get_stash_entry(exp, queue_entry)
|
stash_entry = cls.get_stash_entry(exp, queue_entry)
|
||||||
@@ -563,7 +557,7 @@ class BaseStashQueue(ABC):
|
|||||||
exp: "Experiments",
|
exp: "Experiments",
|
||||||
executor: BaseExecutor,
|
executor: BaseExecutor,
|
||||||
exec_result: "ExecutorResult",
|
exec_result: "ExecutorResult",
|
||||||
) -> Dict[str, str]:
|
) -> dict[str, str]:
|
||||||
results = {}
|
results = {}
|
||||||
|
|
||||||
def on_diverged(ref: str):
|
def on_diverged(ref: str):
|
||||||
@@ -593,7 +587,7 @@ class BaseStashQueue(ABC):
|
|||||||
exp: "Experiments",
|
exp: "Experiments",
|
||||||
executor: BaseExecutor,
|
executor: BaseExecutor,
|
||||||
exec_result: "ExecutorResult",
|
exec_result: "ExecutorResult",
|
||||||
) -> Dict[str, str]:
|
) -> dict[str, str]:
|
||||||
results = cls.collect_git(exp, executor, exec_result)
|
results = cls.collect_git(exp, executor, exec_result)
|
||||||
|
|
||||||
if exec_result.ref_info is not None:
|
if exec_result.ref_info is not None:
|
||||||
@@ -605,11 +599,11 @@ class BaseStashQueue(ABC):
|
|||||||
self,
|
self,
|
||||||
exp_names: Collection[str],
|
exp_names: Collection[str],
|
||||||
*entries: Iterable[Union[QueueEntry, QueueDoneResult]],
|
*entries: Iterable[Union[QueueEntry, QueueDoneResult]],
|
||||||
) -> Dict[str, Optional[QueueEntry]]:
|
) -> dict[str, Optional[QueueEntry]]:
|
||||||
from funcy import concat
|
from funcy import concat
|
||||||
|
|
||||||
entry_name_dict: Dict[str, QueueEntry] = {}
|
entry_name_dict: dict[str, QueueEntry] = {}
|
||||||
entry_rev_dict: Dict[str, QueueEntry] = {}
|
entry_rev_dict: dict[str, QueueEntry] = {}
|
||||||
for entry in concat(*entries):
|
for entry in concat(*entries):
|
||||||
if isinstance(entry, QueueDoneResult):
|
if isinstance(entry, QueueDoneResult):
|
||||||
queue_entry: QueueEntry = entry.entry
|
queue_entry: QueueEntry = entry.entry
|
||||||
@@ -624,7 +618,7 @@ class BaseStashQueue(ABC):
|
|||||||
entry_name_dict[name] = queue_entry
|
entry_name_dict[name] = queue_entry
|
||||||
entry_rev_dict[queue_entry.stash_rev] = queue_entry
|
entry_rev_dict[queue_entry.stash_rev] = queue_entry
|
||||||
|
|
||||||
result: Dict[str, Optional[QueueEntry]] = {}
|
result: dict[str, Optional[QueueEntry]] = {}
|
||||||
for exp_name in exp_names:
|
for exp_name in exp_names:
|
||||||
result[exp_name] = None
|
result[exp_name] = None
|
||||||
if exp_name in entry_name_dict:
|
if exp_name in entry_name_dict:
|
||||||
@@ -666,7 +660,7 @@ class BaseStashQueue(ABC):
|
|||||||
baseline_revs: Optional[Collection[str]],
|
baseline_revs: Optional[Collection[str]],
|
||||||
fetch_refs: bool = False,
|
fetch_refs: bool = False,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Dict[str, List["ExpRange"]]:
|
) -> dict[str, list["ExpRange"]]:
|
||||||
"""Collect data for active (running) experiments.
|
"""Collect data for active (running) experiments.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -685,7 +679,7 @@ class BaseStashQueue(ABC):
|
|||||||
self,
|
self,
|
||||||
baseline_revs: Optional[Collection[str]],
|
baseline_revs: Optional[Collection[str]],
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Dict[str, List["ExpRange"]]:
|
) -> dict[str, list["ExpRange"]]:
|
||||||
"""Collect data for queued experiments.
|
"""Collect data for queued experiments.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -702,7 +696,7 @@ class BaseStashQueue(ABC):
|
|||||||
self,
|
self,
|
||||||
baseline_revs: Optional[Collection[str]],
|
baseline_revs: Optional[Collection[str]],
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Dict[str, List["ExpRange"]]:
|
) -> dict[str, list["ExpRange"]]:
|
||||||
"""Collect data for failed experiments.
|
"""Collect data for failed experiments.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
|
|||||||
@@ -3,16 +3,11 @@ import locale
|
|||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
|
from collections.abc import Collection, Generator, Mapping
|
||||||
from typing import (
|
from typing import (
|
||||||
TYPE_CHECKING,
|
TYPE_CHECKING,
|
||||||
Collection,
|
|
||||||
Dict,
|
|
||||||
Generator,
|
|
||||||
List,
|
|
||||||
Mapping,
|
|
||||||
NamedTuple,
|
NamedTuple,
|
||||||
Optional,
|
Optional,
|
||||||
Set,
|
|
||||||
Union,
|
Union,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -166,7 +161,7 @@ class LocalCeleryQueue(BaseStashQueue):
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
logger.debug("Spawning %s exp queue workers", count)
|
logger.debug("Spawning %s exp queue workers", count)
|
||||||
active_worker: Dict = self.worker_status()
|
active_worker: dict = self.worker_status()
|
||||||
|
|
||||||
started = 0
|
started = 0
|
||||||
for num in range(1, 1 + count):
|
for num in range(1, 1 + count):
|
||||||
@@ -183,7 +178,7 @@ class LocalCeleryQueue(BaseStashQueue):
|
|||||||
def put(
|
def put(
|
||||||
self,
|
self,
|
||||||
*args,
|
*args,
|
||||||
copy_paths: Optional[List[str]] = None,
|
copy_paths: Optional[list[str]] = None,
|
||||||
message: Optional[str] = None,
|
message: Optional[str] = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> QueueEntry:
|
) -> QueueEntry:
|
||||||
@@ -265,7 +260,7 @@ class LocalCeleryQueue(BaseStashQueue):
|
|||||||
yield QueueDoneResult(queue_entry, exp_result)
|
yield QueueDoneResult(queue_entry, exp_result)
|
||||||
|
|
||||||
def reproduce(
|
def reproduce(
|
||||||
self, copy_paths: Optional[List[str]] = None, message: Optional[str] = None
|
self, copy_paths: Optional[list[str]] = None, message: Optional[str] = None
|
||||||
) -> Mapping[str, Mapping[str, str]]:
|
) -> Mapping[str, Mapping[str, str]]:
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
@@ -333,8 +328,8 @@ class LocalCeleryQueue(BaseStashQueue):
|
|||||||
while not self.proc.get(entry.stash_rev):
|
while not self.proc.get(entry.stash_rev):
|
||||||
time.sleep(sleep_interval)
|
time.sleep(sleep_interval)
|
||||||
|
|
||||||
def _get_running_task_ids(self) -> Set[str]:
|
def _get_running_task_ids(self) -> set[str]:
|
||||||
running_task_ids: Set[str] = set()
|
running_task_ids: set[str] = set()
|
||||||
active_workers = self.worker_status()
|
active_workers = self.worker_status()
|
||||||
for _, tasks in active_workers.items():
|
for _, tasks in active_workers.items():
|
||||||
task = first(tasks)
|
task = first(tasks)
|
||||||
@@ -343,9 +338,9 @@ class LocalCeleryQueue(BaseStashQueue):
|
|||||||
return running_task_ids
|
return running_task_ids
|
||||||
|
|
||||||
def _try_to_kill_tasks(
|
def _try_to_kill_tasks(
|
||||||
self, to_kill: Dict[QueueEntry, str], force: bool
|
self, to_kill: dict[QueueEntry, str], force: bool
|
||||||
) -> Dict[QueueEntry, str]:
|
) -> dict[QueueEntry, str]:
|
||||||
fail_to_kill_entries: Dict[QueueEntry, str] = {}
|
fail_to_kill_entries: dict[QueueEntry, str] = {}
|
||||||
for queue_entry, rev in to_kill.items():
|
for queue_entry, rev in to_kill.items():
|
||||||
try:
|
try:
|
||||||
if force:
|
if force:
|
||||||
@@ -358,9 +353,9 @@ class LocalCeleryQueue(BaseStashQueue):
|
|||||||
return fail_to_kill_entries
|
return fail_to_kill_entries
|
||||||
|
|
||||||
def _mark_inactive_tasks_failure(
|
def _mark_inactive_tasks_failure(
|
||||||
self, remained_entries: Dict[QueueEntry, str]
|
self, remained_entries: dict[QueueEntry, str]
|
||||||
) -> None:
|
) -> None:
|
||||||
remained_revs: List[str] = []
|
remained_revs: list[str] = []
|
||||||
running_ids = self._get_running_task_ids()
|
running_ids = self._get_running_task_ids()
|
||||||
logger.debug("Current running tasks ids: %s.", running_ids)
|
logger.debug("Current running tasks ids: %s.", running_ids)
|
||||||
for msg, entry in self._iter_processed():
|
for msg, entry in self._iter_processed():
|
||||||
@@ -383,12 +378,12 @@ class LocalCeleryQueue(BaseStashQueue):
|
|||||||
if remained_revs:
|
if remained_revs:
|
||||||
raise CannotKillTasksError(remained_revs)
|
raise CannotKillTasksError(remained_revs)
|
||||||
|
|
||||||
def _kill_entries(self, entries: Dict[QueueEntry, str], force: bool) -> None:
|
def _kill_entries(self, entries: dict[QueueEntry, str], force: bool) -> None:
|
||||||
logger.debug(
|
logger.debug(
|
||||||
"Found active tasks: '%s' to kill",
|
"Found active tasks: '%s' to kill",
|
||||||
list(entries.values()),
|
list(entries.values()),
|
||||||
)
|
)
|
||||||
inactive_entries: Dict[QueueEntry, str] = self._try_to_kill_tasks(
|
inactive_entries: dict[QueueEntry, str] = self._try_to_kill_tasks(
|
||||||
entries, force
|
entries, force
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -396,12 +391,12 @@ class LocalCeleryQueue(BaseStashQueue):
|
|||||||
self._mark_inactive_tasks_failure(inactive_entries)
|
self._mark_inactive_tasks_failure(inactive_entries)
|
||||||
|
|
||||||
def kill(self, revs: Collection[str], force: bool = False) -> None:
|
def kill(self, revs: Collection[str], force: bool = False) -> None:
|
||||||
name_dict: Dict[str, Optional[QueueEntry]] = self.match_queue_entry_by_name(
|
name_dict: dict[str, Optional[QueueEntry]] = self.match_queue_entry_by_name(
|
||||||
set(revs), self.iter_active()
|
set(revs), self.iter_active()
|
||||||
)
|
)
|
||||||
|
|
||||||
missing_revs: List[str] = []
|
missing_revs: list[str] = []
|
||||||
to_kill: Dict[QueueEntry, str] = {}
|
to_kill: dict[QueueEntry, str] = {}
|
||||||
for rev, queue_entry in name_dict.items():
|
for rev, queue_entry in name_dict.items():
|
||||||
if queue_entry is None:
|
if queue_entry is None:
|
||||||
missing_revs.append(rev)
|
missing_revs.append(rev)
|
||||||
@@ -417,7 +412,7 @@ class LocalCeleryQueue(BaseStashQueue):
|
|||||||
def shutdown(self, kill: bool = False):
|
def shutdown(self, kill: bool = False):
|
||||||
self.celery.control.shutdown()
|
self.celery.control.shutdown()
|
||||||
if kill:
|
if kill:
|
||||||
to_kill: Dict[QueueEntry, str] = {}
|
to_kill: dict[QueueEntry, str] = {}
|
||||||
for entry in self.iter_active():
|
for entry in self.iter_active():
|
||||||
to_kill[entry] = entry.name or entry.stash_rev
|
to_kill[entry] = entry.name or entry.stash_rev
|
||||||
if to_kill:
|
if to_kill:
|
||||||
@@ -468,7 +463,7 @@ class LocalCeleryQueue(BaseStashQueue):
|
|||||||
) as fobj:
|
) as fobj:
|
||||||
ui.write(fobj.read())
|
ui.write(fobj.read())
|
||||||
|
|
||||||
def worker_status(self) -> Dict[str, List[Dict]]:
|
def worker_status(self) -> dict[str, list[dict]]:
|
||||||
"""Return the current active celery worker"""
|
"""Return the current active celery worker"""
|
||||||
status = self.celery.control.inspect().active() or {}
|
status = self.celery.control.inspect().active() or {}
|
||||||
logger.debug("Worker status: %s", status)
|
logger.debug("Worker status: %s", status)
|
||||||
@@ -486,21 +481,21 @@ class LocalCeleryQueue(BaseStashQueue):
|
|||||||
|
|
||||||
def get_ref_and_entry_by_names(
|
def get_ref_and_entry_by_names(
|
||||||
self,
|
self,
|
||||||
exp_names: Union[str, List[str]],
|
exp_names: Union[str, list[str]],
|
||||||
git_remote: Optional[str] = None,
|
git_remote: Optional[str] = None,
|
||||||
) -> Dict[str, ExpRefAndQueueEntry]:
|
) -> dict[str, ExpRefAndQueueEntry]:
|
||||||
"""Find finished ExpRefInfo or queued or failed QueueEntry by name"""
|
"""Find finished ExpRefInfo or queued or failed QueueEntry by name"""
|
||||||
from dvc.repo.experiments.utils import resolve_name
|
from dvc.repo.experiments.utils import resolve_name
|
||||||
|
|
||||||
if isinstance(exp_names, str):
|
if isinstance(exp_names, str):
|
||||||
exp_names = [exp_names]
|
exp_names = [exp_names]
|
||||||
results: Dict[str, ExpRefAndQueueEntry] = {}
|
results: dict[str, ExpRefAndQueueEntry] = {}
|
||||||
|
|
||||||
exp_ref_match: Dict[str, Optional["ExpRefInfo"]] = resolve_name(
|
exp_ref_match: dict[str, Optional["ExpRefInfo"]] = resolve_name(
|
||||||
self.scm, exp_names, git_remote
|
self.scm, exp_names, git_remote
|
||||||
)
|
)
|
||||||
if not git_remote:
|
if not git_remote:
|
||||||
queue_entry_match: Dict[
|
queue_entry_match: dict[
|
||||||
str, Optional["QueueEntry"]
|
str, Optional["QueueEntry"]
|
||||||
] = self.match_queue_entry_by_name(
|
] = self.match_queue_entry_by_name(
|
||||||
exp_names, self.iter_queued(), self.iter_done()
|
exp_names, self.iter_queued(), self.iter_done()
|
||||||
@@ -517,7 +512,7 @@ class LocalCeleryQueue(BaseStashQueue):
|
|||||||
baseline_revs: Optional[Collection[str]],
|
baseline_revs: Optional[Collection[str]],
|
||||||
fetch_refs: bool = False,
|
fetch_refs: bool = False,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Dict[str, List["ExpRange"]]:
|
) -> dict[str, list["ExpRange"]]:
|
||||||
from dvc.repo import Repo
|
from dvc.repo import Repo
|
||||||
from dvc.repo.experiments.collect import collect_exec_branch
|
from dvc.repo.experiments.collect import collect_exec_branch
|
||||||
from dvc.repo.experiments.serialize import (
|
from dvc.repo.experiments.serialize import (
|
||||||
@@ -526,7 +521,7 @@ class LocalCeleryQueue(BaseStashQueue):
|
|||||||
LocalExpExecutor,
|
LocalExpExecutor,
|
||||||
)
|
)
|
||||||
|
|
||||||
result: Dict[str, List[ExpRange]] = defaultdict(list)
|
result: dict[str, list[ExpRange]] = defaultdict(list)
|
||||||
for entry in self.iter_active():
|
for entry in self.iter_active():
|
||||||
if baseline_revs and entry.baseline_rev not in baseline_revs:
|
if baseline_revs and entry.baseline_rev not in baseline_revs:
|
||||||
continue
|
continue
|
||||||
@@ -568,7 +563,7 @@ class LocalCeleryQueue(BaseStashQueue):
|
|||||||
self,
|
self,
|
||||||
baseline_revs: Optional[Collection[str]],
|
baseline_revs: Optional[Collection[str]],
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Dict[str, List["ExpRange"]]:
|
) -> dict[str, list["ExpRange"]]:
|
||||||
from dvc.repo.experiments.collect import collect_rev
|
from dvc.repo.experiments.collect import collect_rev
|
||||||
from dvc.repo.experiments.serialize import (
|
from dvc.repo.experiments.serialize import (
|
||||||
ExpExecutor,
|
ExpExecutor,
|
||||||
@@ -576,7 +571,7 @@ class LocalCeleryQueue(BaseStashQueue):
|
|||||||
LocalExpExecutor,
|
LocalExpExecutor,
|
||||||
)
|
)
|
||||||
|
|
||||||
result: Dict[str, List[ExpRange]] = defaultdict(list)
|
result: dict[str, list[ExpRange]] = defaultdict(list)
|
||||||
for entry in self.iter_queued():
|
for entry in self.iter_queued():
|
||||||
if baseline_revs and entry.baseline_rev not in baseline_revs:
|
if baseline_revs and entry.baseline_rev not in baseline_revs:
|
||||||
continue
|
continue
|
||||||
@@ -598,7 +593,7 @@ class LocalCeleryQueue(BaseStashQueue):
|
|||||||
self,
|
self,
|
||||||
baseline_revs: Optional[Collection[str]],
|
baseline_revs: Optional[Collection[str]],
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Dict[str, List["ExpRange"]]:
|
) -> dict[str, list["ExpRange"]]:
|
||||||
from dvc.repo.experiments.collect import collect_rev
|
from dvc.repo.experiments.collect import collect_rev
|
||||||
from dvc.repo.experiments.serialize import (
|
from dvc.repo.experiments.serialize import (
|
||||||
ExpExecutor,
|
ExpExecutor,
|
||||||
@@ -607,7 +602,7 @@ class LocalCeleryQueue(BaseStashQueue):
|
|||||||
SerializableError,
|
SerializableError,
|
||||||
)
|
)
|
||||||
|
|
||||||
result: Dict[str, List[ExpRange]] = defaultdict(list)
|
result: dict[str, list[ExpRange]] = defaultdict(list)
|
||||||
for entry, _ in self.iter_failed():
|
for entry, _ in self.iter_failed():
|
||||||
if baseline_revs and entry.baseline_rev not in baseline_revs:
|
if baseline_revs and entry.baseline_rev not in baseline_revs:
|
||||||
continue
|
continue
|
||||||
@@ -637,11 +632,11 @@ class LocalCeleryQueue(BaseStashQueue):
|
|||||||
self,
|
self,
|
||||||
baseline_revs: Optional[Collection[str]],
|
baseline_revs: Optional[Collection[str]],
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Dict[str, "ExpExecutor"]:
|
) -> dict[str, "ExpExecutor"]:
|
||||||
"""Map exp refs to any available successful executors."""
|
"""Map exp refs to any available successful executors."""
|
||||||
from dvc.repo.experiments.serialize import ExpExecutor, LocalExpExecutor
|
from dvc.repo.experiments.serialize import ExpExecutor, LocalExpExecutor
|
||||||
|
|
||||||
result: Dict[str, "ExpExecutor"] = {}
|
result: dict[str, "ExpExecutor"] = {}
|
||||||
for entry, exec_result in self.iter_success():
|
for entry, exec_result in self.iter_success():
|
||||||
if baseline_revs and entry.baseline_rev not in baseline_revs:
|
if baseline_revs and entry.baseline_rev not in baseline_revs:
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
from typing import Collection
|
from collections.abc import Collection
|
||||||
|
|
||||||
from dvc.exceptions import DvcException
|
from dvc.exceptions import DvcException
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
from typing import TYPE_CHECKING, Collection, Dict, Iterable, List, Set, Union
|
from collections.abc import Collection, Iterable
|
||||||
|
from typing import TYPE_CHECKING, Union
|
||||||
|
|
||||||
from dvc.repo.experiments.exceptions import UnresolvedExpNamesError
|
from dvc.repo.experiments.exceptions import UnresolvedExpNamesError
|
||||||
from dvc.repo.experiments.queue.base import QueueDoneResult
|
from dvc.repo.experiments.queue.base import QueueDoneResult
|
||||||
@@ -20,11 +21,11 @@ def remove_tasks( # noqa: C901, PLR0912
|
|||||||
"""
|
"""
|
||||||
from celery.result import AsyncResult
|
from celery.result import AsyncResult
|
||||||
|
|
||||||
stash_revs: Dict[str, "ExpStashEntry"] = {}
|
stash_revs: dict[str, "ExpStashEntry"] = {}
|
||||||
failed_stash_revs: List["ExpStashEntry"] = []
|
failed_stash_revs: list["ExpStashEntry"] = []
|
||||||
done_entry_set: Set["QueueEntry"] = set()
|
done_entry_set: set["QueueEntry"] = set()
|
||||||
stash_rev_all = celery_queue.stash.stash_revs
|
stash_rev_all = celery_queue.stash.stash_revs
|
||||||
failed_rev_all: Dict[str, "ExpStashEntry"] = {}
|
failed_rev_all: dict[str, "ExpStashEntry"] = {}
|
||||||
if celery_queue.failed_stash:
|
if celery_queue.failed_stash:
|
||||||
failed_rev_all = celery_queue.failed_stash.stash_revs
|
failed_rev_all = celery_queue.failed_stash.stash_revs
|
||||||
for entry in queue_entries:
|
for entry in queue_entries:
|
||||||
@@ -64,7 +65,7 @@ def remove_tasks( # noqa: C901, PLR0912
|
|||||||
|
|
||||||
|
|
||||||
def _get_names(entries: Iterable[Union["QueueEntry", "QueueDoneResult"]]):
|
def _get_names(entries: Iterable[Union["QueueEntry", "QueueDoneResult"]]):
|
||||||
names: List[str] = []
|
names: list[str] = []
|
||||||
for entry in entries:
|
for entry in entries:
|
||||||
if isinstance(entry, QueueDoneResult):
|
if isinstance(entry, QueueDoneResult):
|
||||||
if entry.result and entry.result.ref_info:
|
if entry.result and entry.result.ref_info:
|
||||||
@@ -82,7 +83,7 @@ def celery_clear(
|
|||||||
queued: bool = False,
|
queued: bool = False,
|
||||||
failed: bool = False,
|
failed: bool = False,
|
||||||
success: bool = False,
|
success: bool = False,
|
||||||
) -> List[str]:
|
) -> list[str]:
|
||||||
"""Remove entries from the queue.
|
"""Remove entries from the queue.
|
||||||
|
|
||||||
Arguments:
|
Arguments:
|
||||||
@@ -94,18 +95,18 @@ def celery_clear(
|
|||||||
Revisions which were removed.
|
Revisions which were removed.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
removed: List[str] = []
|
removed: list[str] = []
|
||||||
entry_list: List["QueueEntry"] = []
|
entry_list: list["QueueEntry"] = []
|
||||||
if queued:
|
if queued:
|
||||||
queue_entries: List["QueueEntry"] = list(self.iter_queued())
|
queue_entries: list["QueueEntry"] = list(self.iter_queued())
|
||||||
entry_list.extend(queue_entries)
|
entry_list.extend(queue_entries)
|
||||||
removed.extend(_get_names(queue_entries))
|
removed.extend(_get_names(queue_entries))
|
||||||
if failed:
|
if failed:
|
||||||
failed_tasks: List["QueueDoneResult"] = list(self.iter_failed())
|
failed_tasks: list["QueueDoneResult"] = list(self.iter_failed())
|
||||||
entry_list.extend([result.entry for result in failed_tasks])
|
entry_list.extend([result.entry for result in failed_tasks])
|
||||||
removed.extend(_get_names(failed_tasks))
|
removed.extend(_get_names(failed_tasks))
|
||||||
if success:
|
if success:
|
||||||
success_tasks: List["QueueDoneResult"] = list(self.iter_success())
|
success_tasks: list["QueueDoneResult"] = list(self.iter_success())
|
||||||
entry_list.extend([result.entry for result in success_tasks])
|
entry_list.extend([result.entry for result in success_tasks])
|
||||||
removed.extend(_get_names(success_tasks))
|
removed.extend(_get_names(success_tasks))
|
||||||
|
|
||||||
@@ -117,7 +118,7 @@ def celery_clear(
|
|||||||
def celery_remove(
|
def celery_remove(
|
||||||
self: "LocalCeleryQueue",
|
self: "LocalCeleryQueue",
|
||||||
revs: Collection[str],
|
revs: Collection[str],
|
||||||
) -> List[str]:
|
) -> list[str]:
|
||||||
"""Remove the specified entries from the queue.
|
"""Remove the specified entries from the queue.
|
||||||
|
|
||||||
Arguments:
|
Arguments:
|
||||||
@@ -131,9 +132,9 @@ def celery_remove(
|
|||||||
revs, self.iter_queued(), self.iter_done()
|
revs, self.iter_queued(), self.iter_done()
|
||||||
)
|
)
|
||||||
|
|
||||||
remained: List[str] = []
|
remained: list[str] = []
|
||||||
removed: List[str] = []
|
removed: list[str] = []
|
||||||
entry_to_remove: List["QueueEntry"] = []
|
entry_to_remove: list["QueueEntry"] = []
|
||||||
for name, entry in match_results.items():
|
for name, entry in match_results.items():
|
||||||
if entry:
|
if entry:
|
||||||
entry_to_remove.append(entry)
|
entry_to_remove.append(entry)
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional
|
from typing import TYPE_CHECKING, Any, Optional
|
||||||
|
|
||||||
from celery import shared_task
|
from celery import shared_task
|
||||||
from celery.utils.log import get_task_logger
|
from celery.utils.log import get_task_logger
|
||||||
@@ -16,7 +16,7 @@ logger = get_task_logger(__name__)
|
|||||||
|
|
||||||
|
|
||||||
@shared_task
|
@shared_task
|
||||||
def setup_exp(entry_dict: Dict[str, Any]) -> "BaseExecutor":
|
def setup_exp(entry_dict: dict[str, Any]) -> "BaseExecutor":
|
||||||
"""Setup an experiment.
|
"""Setup an experiment.
|
||||||
|
|
||||||
Arguments:
|
Arguments:
|
||||||
@@ -44,8 +44,8 @@ def setup_exp(entry_dict: Dict[str, Any]) -> "BaseExecutor":
|
|||||||
|
|
||||||
@shared_task
|
@shared_task
|
||||||
def collect_exp(
|
def collect_exp(
|
||||||
proc_dict: Dict[str, Any], # noqa: ARG001
|
proc_dict: dict[str, Any], # noqa: ARG001
|
||||||
entry_dict: Dict[str, Any],
|
entry_dict: dict[str, Any],
|
||||||
) -> str:
|
) -> str:
|
||||||
"""Collect results for an experiment.
|
"""Collect results for an experiment.
|
||||||
|
|
||||||
@@ -92,8 +92,8 @@ def cleanup_exp(executor: TempDirExecutor, infofile: str) -> None:
|
|||||||
|
|
||||||
@shared_task
|
@shared_task
|
||||||
def run_exp(
|
def run_exp(
|
||||||
entry_dict: Dict[str, Any],
|
entry_dict: dict[str, Any],
|
||||||
copy_paths: Optional[List[str]] = None,
|
copy_paths: Optional[list[str]] = None,
|
||||||
message: Optional[str] = None,
|
message: Optional[str] = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Run a full experiment.
|
"""Run a full experiment.
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import os
|
import os
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from typing import TYPE_CHECKING, Collection, Dict, Generator, List, Optional
|
from collections.abc import Collection, Generator
|
||||||
|
from typing import TYPE_CHECKING, Optional
|
||||||
|
|
||||||
from funcy import first
|
from funcy import first
|
||||||
|
|
||||||
@@ -96,11 +97,11 @@ class TempDirQueue(WorkspaceQueue):
|
|||||||
self,
|
self,
|
||||||
entry: QueueEntry,
|
entry: QueueEntry,
|
||||||
executor: "BaseExecutor",
|
executor: "BaseExecutor",
|
||||||
copy_paths: Optional[List[str]] = None,
|
copy_paths: Optional[list[str]] = None,
|
||||||
message: Optional[str] = None,
|
message: Optional[str] = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Dict[str, Dict[str, str]]:
|
) -> dict[str, dict[str, str]]:
|
||||||
results: Dict[str, Dict[str, str]] = defaultdict(dict)
|
results: dict[str, dict[str, str]] = defaultdict(dict)
|
||||||
exec_name = self._EXEC_NAME or entry.stash_rev
|
exec_name = self._EXEC_NAME or entry.stash_rev
|
||||||
infofile = self.get_infofile_path(exec_name)
|
infofile = self.get_infofile_path(exec_name)
|
||||||
try:
|
try:
|
||||||
@@ -135,7 +136,7 @@ class TempDirQueue(WorkspaceQueue):
|
|||||||
exp: "Experiments",
|
exp: "Experiments",
|
||||||
executor: "BaseExecutor",
|
executor: "BaseExecutor",
|
||||||
exec_result: "ExecutorResult",
|
exec_result: "ExecutorResult",
|
||||||
) -> Dict[str, str]:
|
) -> dict[str, str]:
|
||||||
return BaseStashQueue.collect_executor(exp, executor, exec_result)
|
return BaseStashQueue.collect_executor(exp, executor, exec_result)
|
||||||
|
|
||||||
def collect_active_data(
|
def collect_active_data(
|
||||||
@@ -143,7 +144,7 @@ class TempDirQueue(WorkspaceQueue):
|
|||||||
baseline_revs: Optional[Collection[str]],
|
baseline_revs: Optional[Collection[str]],
|
||||||
fetch_refs: bool = False,
|
fetch_refs: bool = False,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Dict[str, List["ExpRange"]]:
|
) -> dict[str, list["ExpRange"]]:
|
||||||
from dvc.repo import Repo
|
from dvc.repo import Repo
|
||||||
from dvc.repo.experiments.collect import collect_exec_branch
|
from dvc.repo.experiments.collect import collect_exec_branch
|
||||||
from dvc.repo.experiments.serialize import (
|
from dvc.repo.experiments.serialize import (
|
||||||
@@ -152,7 +153,7 @@ class TempDirQueue(WorkspaceQueue):
|
|||||||
LocalExpExecutor,
|
LocalExpExecutor,
|
||||||
)
|
)
|
||||||
|
|
||||||
result: Dict[str, List[ExpRange]] = defaultdict(list)
|
result: dict[str, list[ExpRange]] = defaultdict(list)
|
||||||
for entry in self.iter_active():
|
for entry in self.iter_active():
|
||||||
if baseline_revs and entry.baseline_rev not in baseline_revs:
|
if baseline_revs and entry.baseline_rev not in baseline_revs:
|
||||||
continue
|
continue
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
from typing import TYPE_CHECKING, Dict, List
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from scmrepo.exceptions import SCMError
|
from scmrepo.exceptions import SCMError
|
||||||
|
|
||||||
@@ -16,7 +16,7 @@ if TYPE_CHECKING:
|
|||||||
from .base import BaseStashQueue
|
from .base import BaseStashQueue
|
||||||
|
|
||||||
|
|
||||||
def get_remote_executor_refs(scm: "Git", remote_url: str) -> List[str]:
|
def get_remote_executor_refs(scm: "Git", remote_url: str) -> list[str]:
|
||||||
"""Get result list refs from a remote repository
|
"""Get result list refs from a remote repository
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -35,7 +35,7 @@ def get_remote_executor_refs(scm: "Git", remote_url: str) -> List[str]:
|
|||||||
|
|
||||||
def fetch_running_exp_from_temp_dir(
|
def fetch_running_exp_from_temp_dir(
|
||||||
queue: "BaseStashQueue", rev: str, fetch_refs: bool
|
queue: "BaseStashQueue", rev: str, fetch_refs: bool
|
||||||
) -> Dict[str, Dict]:
|
) -> dict[str, dict]:
|
||||||
"""Fetch status of running exps out of current working directory
|
"""Fetch status of running exps out of current working directory
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -50,7 +50,7 @@ def fetch_running_exp_from_temp_dir(
|
|||||||
from dvc.scm import InvalidRemoteSCMRepo
|
from dvc.scm import InvalidRemoteSCMRepo
|
||||||
from dvc.utils.serialize import load_json
|
from dvc.utils.serialize import load_json
|
||||||
|
|
||||||
result: Dict[str, Dict] = {}
|
result: dict[str, dict] = {}
|
||||||
infofile = queue.get_infofile_path(rev)
|
infofile = queue.get_infofile_path(rev)
|
||||||
try:
|
try:
|
||||||
info = ExecutorInfo.from_dict(load_json(infofile))
|
info = ExecutorInfo.from_dict(load_json(infofile))
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from typing import TYPE_CHECKING, Collection, Dict, Generator, List, Optional
|
from collections.abc import Collection, Generator
|
||||||
|
from typing import TYPE_CHECKING, Optional
|
||||||
|
|
||||||
import psutil
|
import psutil
|
||||||
from funcy import first
|
from funcy import first
|
||||||
@@ -82,9 +83,9 @@ class WorkspaceQueue(BaseStashQueue):
|
|||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
def reproduce(
|
def reproduce(
|
||||||
self, copy_paths: Optional[List[str]] = None, message: Optional[str] = None
|
self, copy_paths: Optional[list[str]] = None, message: Optional[str] = None
|
||||||
) -> Dict[str, Dict[str, str]]:
|
) -> dict[str, dict[str, str]]:
|
||||||
results: Dict[str, Dict[str, str]] = defaultdict(dict)
|
results: dict[str, dict[str, str]] = defaultdict(dict)
|
||||||
try:
|
try:
|
||||||
while True:
|
while True:
|
||||||
entry, executor = self.get()
|
entry, executor = self.get()
|
||||||
@@ -99,11 +100,11 @@ class WorkspaceQueue(BaseStashQueue):
|
|||||||
|
|
||||||
def _reproduce_entry(
|
def _reproduce_entry(
|
||||||
self, entry: QueueEntry, executor: "BaseExecutor", **kwargs
|
self, entry: QueueEntry, executor: "BaseExecutor", **kwargs
|
||||||
) -> Dict[str, Dict[str, str]]:
|
) -> dict[str, dict[str, str]]:
|
||||||
kwargs.pop("copy_paths", None)
|
kwargs.pop("copy_paths", None)
|
||||||
from dvc_task.proc.process import ProcessInfo
|
from dvc_task.proc.process import ProcessInfo
|
||||||
|
|
||||||
results: Dict[str, Dict[str, str]] = defaultdict(dict)
|
results: dict[str, dict[str, str]] = defaultdict(dict)
|
||||||
exec_name = self._EXEC_NAME or entry.stash_rev
|
exec_name = self._EXEC_NAME or entry.stash_rev
|
||||||
proc_info = ProcessInfo(os.getpid(), None, None, None, None)
|
proc_info = ProcessInfo(os.getpid(), None, None, None, None)
|
||||||
proc_info_path = self._proc_info_path(exec_name)
|
proc_info_path = self._proc_info_path(exec_name)
|
||||||
@@ -162,8 +163,8 @@ class WorkspaceQueue(BaseStashQueue):
|
|||||||
exp: "Experiments",
|
exp: "Experiments",
|
||||||
executor: "BaseExecutor", # noqa: ARG004
|
executor: "BaseExecutor", # noqa: ARG004
|
||||||
exec_result: "ExecutorResult",
|
exec_result: "ExecutorResult",
|
||||||
) -> Dict[str, str]:
|
) -> dict[str, str]:
|
||||||
results: Dict[str, str] = {}
|
results: dict[str, str] = {}
|
||||||
exp_rev = exp.scm.get_ref(EXEC_BRANCH)
|
exp_rev = exp.scm.get_ref(EXEC_BRANCH)
|
||||||
if exp_rev:
|
if exp_rev:
|
||||||
assert exec_result.exp_hash
|
assert exec_result.exp_hash
|
||||||
@@ -207,7 +208,7 @@ class WorkspaceQueue(BaseStashQueue):
|
|||||||
baseline_revs: Optional[Collection[str]],
|
baseline_revs: Optional[Collection[str]],
|
||||||
fetch_refs: bool = False, # noqa: ARG002
|
fetch_refs: bool = False, # noqa: ARG002
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Dict[str, List["ExpRange"]]:
|
) -> dict[str, list["ExpRange"]]:
|
||||||
from dvc.repo.experiments.collect import collect_exec_branch
|
from dvc.repo.experiments.collect import collect_exec_branch
|
||||||
from dvc.repo.experiments.serialize import (
|
from dvc.repo.experiments.serialize import (
|
||||||
ExpExecutor,
|
ExpExecutor,
|
||||||
@@ -215,7 +216,7 @@ class WorkspaceQueue(BaseStashQueue):
|
|||||||
LocalExpExecutor,
|
LocalExpExecutor,
|
||||||
)
|
)
|
||||||
|
|
||||||
result: Dict[str, List[ExpRange]] = defaultdict(list)
|
result: dict[str, list[ExpRange]] = defaultdict(list)
|
||||||
pid = self._active_pid
|
pid = self._active_pid
|
||||||
if pid is None:
|
if pid is None:
|
||||||
return result
|
return result
|
||||||
@@ -248,12 +249,12 @@ class WorkspaceQueue(BaseStashQueue):
|
|||||||
self,
|
self,
|
||||||
baseline_revs: Optional[Collection[str]],
|
baseline_revs: Optional[Collection[str]],
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Dict[str, List["ExpRange"]]:
|
) -> dict[str, list["ExpRange"]]:
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
def collect_failed_data(
|
def collect_failed_data(
|
||||||
self,
|
self,
|
||||||
baseline_revs: Optional[Collection[str]],
|
baseline_revs: Optional[Collection[str]],
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Dict[str, List["ExpRange"]]:
|
) -> dict[str, list["ExpRange"]]:
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Union
|
from collections.abc import Iterable
|
||||||
|
from typing import TYPE_CHECKING, Optional, Union
|
||||||
|
|
||||||
from dvc.log import logger
|
from dvc.log import logger
|
||||||
from dvc.repo import locked
|
from dvc.repo import locked
|
||||||
@@ -23,14 +24,14 @@ logger = logger.getChild(__name__)
|
|||||||
@scm_context
|
@scm_context
|
||||||
def remove( # noqa: C901, PLR0912
|
def remove( # noqa: C901, PLR0912
|
||||||
repo: "Repo",
|
repo: "Repo",
|
||||||
exp_names: Union[None, str, List[str]] = None,
|
exp_names: Union[None, str, list[str]] = None,
|
||||||
rev: Optional[Union[List[str], str]] = None,
|
rev: Optional[Union[list[str], str]] = None,
|
||||||
all_commits: bool = False,
|
all_commits: bool = False,
|
||||||
num: int = 1,
|
num: int = 1,
|
||||||
queue: bool = False,
|
queue: bool = False,
|
||||||
git_remote: Optional[str] = None,
|
git_remote: Optional[str] = None,
|
||||||
) -> List[str]:
|
) -> list[str]:
|
||||||
removed: List[str] = []
|
removed: list[str] = []
|
||||||
if not any([exp_names, queue, all_commits, rev]):
|
if not any([exp_names, queue, all_commits, rev]):
|
||||||
return removed
|
return removed
|
||||||
celery_queue: "LocalCeleryQueue" = repo.experiments.celery_queue
|
celery_queue: "LocalCeleryQueue" = repo.experiments.celery_queue
|
||||||
@@ -40,13 +41,13 @@ def remove( # noqa: C901, PLR0912
|
|||||||
|
|
||||||
assert isinstance(repo.scm, Git)
|
assert isinstance(repo.scm, Git)
|
||||||
|
|
||||||
exp_ref_list: List["ExpRefInfo"] = []
|
exp_ref_list: list["ExpRefInfo"] = []
|
||||||
queue_entry_list: List["QueueEntry"] = []
|
queue_entry_list: list["QueueEntry"] = []
|
||||||
if exp_names:
|
if exp_names:
|
||||||
results: Dict[
|
results: dict[
|
||||||
str, "ExpRefAndQueueEntry"
|
str, "ExpRefAndQueueEntry"
|
||||||
] = celery_queue.get_ref_and_entry_by_names(exp_names, git_remote)
|
] = celery_queue.get_ref_and_entry_by_names(exp_names, git_remote)
|
||||||
remained: List[str] = []
|
remained: list[str] = []
|
||||||
for name, result in results.items():
|
for name, result in results.items():
|
||||||
if not result.exp_ref_info and not result.queue_entry:
|
if not result.exp_ref_info and not result.queue_entry:
|
||||||
remained.append(name)
|
remained.append(name)
|
||||||
@@ -87,13 +88,13 @@ def remove( # noqa: C901, PLR0912
|
|||||||
|
|
||||||
def _resolve_exp_by_baseline(
|
def _resolve_exp_by_baseline(
|
||||||
repo: "Repo",
|
repo: "Repo",
|
||||||
rev: List[str],
|
rev: list[str],
|
||||||
num: int,
|
num: int,
|
||||||
git_remote: Optional[str] = None,
|
git_remote: Optional[str] = None,
|
||||||
) -> Dict[str, "ExpRefInfo"]:
|
) -> dict[str, "ExpRefInfo"]:
|
||||||
assert isinstance(repo.scm, Git)
|
assert isinstance(repo.scm, Git)
|
||||||
|
|
||||||
commit_ref_dict: Dict[str, "ExpRefInfo"] = {}
|
commit_ref_dict: dict[str, "ExpRefInfo"] = {}
|
||||||
rev_dict = iter_revs(repo.scm, rev, num)
|
rev_dict = iter_revs(repo.scm, rev, num)
|
||||||
rev_set = set(rev_dict.keys())
|
rev_set = set(rev_dict.keys())
|
||||||
ref_info_dict = exp_refs_by_baseline(repo.scm, rev_set, git_remote)
|
ref_info_dict = exp_refs_by_baseline(repo.scm, rev_set, git_remote)
|
||||||
@@ -105,7 +106,7 @@ def _resolve_exp_by_baseline(
|
|||||||
|
|
||||||
def _remove_commited_exps(
|
def _remove_commited_exps(
|
||||||
scm: "Git", exp_refs_list: Iterable["ExpRefInfo"], remote: Optional[str]
|
scm: "Git", exp_refs_list: Iterable["ExpRefInfo"], remote: Optional[str]
|
||||||
) -> List[str]:
|
) -> list[str]:
|
||||||
if remote:
|
if remote:
|
||||||
from dvc.scm import TqdmGit
|
from dvc.scm import TqdmGit
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
from typing import TYPE_CHECKING, Dict, List, Optional, Union
|
from typing import TYPE_CHECKING, Optional, Union
|
||||||
|
|
||||||
from dvc.log import logger
|
from dvc.log import logger
|
||||||
from dvc.repo.experiments.exceptions import (
|
from dvc.repo.experiments.exceptions import (
|
||||||
@@ -22,16 +22,16 @@ def rename(
|
|||||||
exp_name: Union[str, None] = None,
|
exp_name: Union[str, None] = None,
|
||||||
git_remote: Optional[str] = None,
|
git_remote: Optional[str] = None,
|
||||||
force: bool = False,
|
force: bool = False,
|
||||||
) -> Union[List[str], None]:
|
) -> Union[list[str], None]:
|
||||||
renamed: List[str] = []
|
renamed: list[str] = []
|
||||||
remained: List[str] = []
|
remained: list[str] = []
|
||||||
assert isinstance(repo.scm, Git)
|
assert isinstance(repo.scm, Git)
|
||||||
|
|
||||||
if exp_name == new_name:
|
if exp_name == new_name:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
if exp_name:
|
if exp_name:
|
||||||
results: Dict[str, Union[ExpRefInfo, None]] = resolve_name(
|
results: dict[str, Union[ExpRefInfo, None]] = resolve_name(
|
||||||
scm=repo.scm, exp_names=exp_name, git_remote=git_remote
|
scm=repo.scm, exp_names=exp_name, git_remote=git_remote
|
||||||
)
|
)
|
||||||
for name, result in results.items():
|
for name, result in results.items():
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
from typing import Dict, Iterable, Optional
|
from collections.abc import Iterable
|
||||||
|
from typing import Optional
|
||||||
|
|
||||||
from dvc.dependency.param import ParamsDependency
|
from dvc.dependency.param import ParamsDependency
|
||||||
from dvc.exceptions import InvalidArgumentError
|
from dvc.exceptions import InvalidArgumentError
|
||||||
@@ -22,7 +23,7 @@ def run( # noqa: C901, PLR0912
|
|||||||
copy_paths: Optional[Iterable[str]] = None,
|
copy_paths: Optional[Iterable[str]] = None,
|
||||||
message: Optional[str] = None,
|
message: Optional[str] = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Dict[str, str]:
|
) -> dict[str, str]:
|
||||||
"""Reproduce the specified targets as an experiment.
|
"""Reproduce the specified targets as an experiment.
|
||||||
|
|
||||||
Accepts the same additional kwargs as Repo.reproduce.
|
Accepts the same additional kwargs as Repo.reproduce.
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import os
|
import os
|
||||||
from typing import TYPE_CHECKING, Iterable, List, Optional
|
from collections.abc import Iterable
|
||||||
|
from typing import TYPE_CHECKING, Optional
|
||||||
|
|
||||||
from funcy import first
|
from funcy import first
|
||||||
|
|
||||||
@@ -18,7 +19,7 @@ def save(
|
|||||||
name: Optional[str] = None,
|
name: Optional[str] = None,
|
||||||
recursive: bool = False,
|
recursive: bool = False,
|
||||||
force: bool = False,
|
force: bool = False,
|
||||||
include_untracked: Optional[List[str]] = None,
|
include_untracked: Optional[list[str]] = None,
|
||||||
message: Optional[str] = None,
|
message: Optional[str] = None,
|
||||||
) -> Optional[str]:
|
) -> Optional[str]:
|
||||||
"""Save the current workspace status as an experiment.
|
"""Save the current workspace status as an experiment.
|
||||||
|
|||||||
@@ -1,7 +1,8 @@
|
|||||||
import json
|
import json
|
||||||
|
from collections.abc import Iterator
|
||||||
from dataclasses import asdict, dataclass, field
|
from dataclasses import asdict, dataclass, field
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Literal, Optional
|
from typing import TYPE_CHECKING, Any, Literal, Optional
|
||||||
|
|
||||||
from dvc.exceptions import DvcException
|
from dvc.exceptions import DvcException
|
||||||
from dvc.repo.metrics.show import _gather_metrics
|
from dvc.repo.metrics.show import _gather_metrics
|
||||||
@@ -30,11 +31,11 @@ class SerializableExp:
|
|||||||
|
|
||||||
rev: str
|
rev: str
|
||||||
timestamp: Optional[datetime] = None
|
timestamp: Optional[datetime] = None
|
||||||
params: Dict[str, "FileResult"] = field(default_factory=dict)
|
params: dict[str, "FileResult"] = field(default_factory=dict)
|
||||||
metrics: Dict[str, "FileResult"] = field(default_factory=dict)
|
metrics: dict[str, "FileResult"] = field(default_factory=dict)
|
||||||
deps: Dict[str, "ExpDep"] = field(default_factory=dict)
|
deps: dict[str, "ExpDep"] = field(default_factory=dict)
|
||||||
outs: Dict[str, "ExpOut"] = field(default_factory=dict)
|
outs: dict[str, "ExpOut"] = field(default_factory=dict)
|
||||||
meta: Dict[str, Any] = field(default_factory=dict)
|
meta: dict[str, Any] = field(default_factory=dict)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_repo(
|
def from_repo(
|
||||||
@@ -83,7 +84,7 @@ class SerializableExp:
|
|||||||
**kwargs,
|
**kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
def dumpd(self) -> Dict[str, Any]:
|
def dumpd(self) -> dict[str, Any]:
|
||||||
return asdict(self)
|
return asdict(self)
|
||||||
|
|
||||||
def as_bytes(self) -> bytes:
|
def as_bytes(self) -> bytes:
|
||||||
@@ -131,7 +132,7 @@ class SerializableError:
|
|||||||
msg: str
|
msg: str
|
||||||
type: str = ""
|
type: str = ""
|
||||||
|
|
||||||
def dumpd(self) -> Dict[str, Any]:
|
def dumpd(self) -> dict[str, Any]:
|
||||||
return asdict(self)
|
return asdict(self)
|
||||||
|
|
||||||
def as_bytes(self) -> bytes:
|
def as_bytes(self) -> bytes:
|
||||||
@@ -154,15 +155,15 @@ class ExpState:
|
|||||||
name: Optional[str] = None
|
name: Optional[str] = None
|
||||||
data: Optional[SerializableExp] = None
|
data: Optional[SerializableExp] = None
|
||||||
error: Optional[SerializableError] = None
|
error: Optional[SerializableError] = None
|
||||||
experiments: Optional[List["ExpRange"]] = None
|
experiments: Optional[list["ExpRange"]] = None
|
||||||
|
|
||||||
def dumpd(self) -> Dict[str, Any]:
|
def dumpd(self) -> dict[str, Any]:
|
||||||
return asdict(self)
|
return asdict(self)
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class ExpRange:
|
class ExpRange:
|
||||||
revs: List["ExpState"]
|
revs: list["ExpState"]
|
||||||
executor: Optional["ExpExecutor"] = None
|
executor: Optional["ExpExecutor"] = None
|
||||||
name: Optional[str] = None
|
name: Optional[str] = None
|
||||||
|
|
||||||
@@ -175,7 +176,7 @@ class ExpRange:
|
|||||||
def __getitem__(self, index: int) -> "ExpState":
|
def __getitem__(self, index: int) -> "ExpState":
|
||||||
return self.revs[index]
|
return self.revs[index]
|
||||||
|
|
||||||
def dumpd(self) -> Dict[str, Any]:
|
def dumpd(self) -> dict[str, Any]:
|
||||||
return asdict(self)
|
return asdict(self)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,18 +1,12 @@
|
|||||||
from collections import Counter, defaultdict
|
from collections import Counter, defaultdict
|
||||||
|
from collections.abc import Iterable, Iterator, Mapping
|
||||||
from datetime import date, datetime
|
from datetime import date, datetime
|
||||||
from typing import (
|
from typing import (
|
||||||
TYPE_CHECKING,
|
TYPE_CHECKING,
|
||||||
Any,
|
Any,
|
||||||
Dict,
|
|
||||||
Iterable,
|
|
||||||
Iterator,
|
|
||||||
List,
|
|
||||||
Literal,
|
Literal,
|
||||||
Mapping,
|
|
||||||
NamedTuple,
|
NamedTuple,
|
||||||
Optional,
|
Optional,
|
||||||
Set,
|
|
||||||
Tuple,
|
|
||||||
Union,
|
Union,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -36,7 +30,7 @@ logger = logger.getChild(__name__)
|
|||||||
|
|
||||||
def show(
|
def show(
|
||||||
repo: "Repo",
|
repo: "Repo",
|
||||||
revs: Union[List[str], str, None] = None,
|
revs: Union[list[str], str, None] = None,
|
||||||
all_branches: bool = False,
|
all_branches: bool = False,
|
||||||
all_tags: bool = False,
|
all_tags: bool = False,
|
||||||
all_commits: bool = False,
|
all_commits: bool = False,
|
||||||
@@ -45,7 +39,7 @@ def show(
|
|||||||
hide_failed: bool = False,
|
hide_failed: bool = False,
|
||||||
sha_only: bool = False,
|
sha_only: bool = False,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> List["ExpState"]:
|
) -> list["ExpState"]:
|
||||||
return collect(
|
return collect(
|
||||||
repo,
|
repo,
|
||||||
revs=revs,
|
revs=revs,
|
||||||
@@ -65,7 +59,7 @@ def tabulate(
|
|||||||
fill_value: Optional[str] = "-",
|
fill_value: Optional[str] = "-",
|
||||||
error_value: str = "!",
|
error_value: str = "!",
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Tuple["TabularData", Dict[str, Iterable[str]]]:
|
) -> tuple["TabularData", dict[str, Iterable[str]]]:
|
||||||
"""Return table data for experiments.
|
"""Return table data for experiments.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
@@ -112,7 +106,7 @@ def tabulate(
|
|||||||
**kwargs,
|
**kwargs,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
data_headers: Dict[str, Iterable[str]] = {
|
data_headers: dict[str, Iterable[str]] = {
|
||||||
"metrics": metrics_headers,
|
"metrics": metrics_headers,
|
||||||
"params": params_headers,
|
"params": params_headers,
|
||||||
"deps": deps_names,
|
"deps": deps_names,
|
||||||
@@ -128,9 +122,9 @@ def _build_rows(
|
|||||||
sort_by: Optional[str] = None,
|
sort_by: Optional[str] = None,
|
||||||
sort_order: Optional[Literal["asc", "desc"]] = None,
|
sort_order: Optional[Literal["asc", "desc"]] = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Iterator[Tuple["CellT", ...]]:
|
) -> Iterator[tuple["CellT", ...]]:
|
||||||
for baseline in baseline_states:
|
for baseline in baseline_states:
|
||||||
row: Dict[str, "CellT"] = dict.fromkeys(all_headers, fill_value)
|
row: dict[str, "CellT"] = dict.fromkeys(all_headers, fill_value)
|
||||||
row["Experiment"] = ""
|
row["Experiment"] = ""
|
||||||
if baseline.name:
|
if baseline.name:
|
||||||
row["rev"] = baseline.name
|
row["rev"] = baseline.name
|
||||||
@@ -177,10 +171,10 @@ def _sort_column( # noqa: C901
|
|||||||
sort_by: str,
|
sort_by: str,
|
||||||
metric_names: Mapping[str, Iterable[str]],
|
metric_names: Mapping[str, Iterable[str]],
|
||||||
param_names: Mapping[str, Iterable[str]],
|
param_names: Mapping[str, Iterable[str]],
|
||||||
) -> Tuple[str, str, str]:
|
) -> tuple[str, str, str]:
|
||||||
sep = ":"
|
sep = ":"
|
||||||
parts = sort_by.split(sep)
|
parts = sort_by.split(sep)
|
||||||
matches: Set[Tuple[str, str, str]] = set()
|
matches: set[tuple[str, str, str]] = set()
|
||||||
|
|
||||||
for split_num in range(len(parts)):
|
for split_num in range(len(parts)):
|
||||||
path = sep.join(parts[:split_num])
|
path = sep.join(parts[:split_num])
|
||||||
@@ -217,7 +211,7 @@ def _sort_exp(
|
|||||||
sort_name: str,
|
sort_name: str,
|
||||||
typ: str,
|
typ: str,
|
||||||
reverse: bool,
|
reverse: bool,
|
||||||
) -> List["ExpRange"]:
|
) -> list["ExpRange"]:
|
||||||
from funcy import first
|
from funcy import first
|
||||||
|
|
||||||
def _sort(exp_range: "ExpRange"):
|
def _sort(exp_range: "ExpRange"):
|
||||||
@@ -238,14 +232,14 @@ def _exp_range_rows(
|
|||||||
fill_value: Optional[str],
|
fill_value: Optional[str],
|
||||||
is_base: bool = False,
|
is_base: bool = False,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Iterator[Tuple["CellT", ...]]:
|
) -> Iterator[tuple["CellT", ...]]:
|
||||||
from funcy import first
|
from funcy import first
|
||||||
|
|
||||||
if len(exp_range.revs) > 1:
|
if len(exp_range.revs) > 1:
|
||||||
logger.debug("Returning tip commit for legacy checkpoint exp")
|
logger.debug("Returning tip commit for legacy checkpoint exp")
|
||||||
exp = first(exp_range.revs)
|
exp = first(exp_range.revs)
|
||||||
if exp:
|
if exp:
|
||||||
row: Dict[str, "CellT"] = dict.fromkeys(all_headers, fill_value)
|
row: dict[str, "CellT"] = dict.fromkeys(all_headers, fill_value)
|
||||||
row["Experiment"] = exp.name or ""
|
row["Experiment"] = exp.name or ""
|
||||||
row["rev"] = exp.rev[:7] if Git.is_sha(exp.rev) else exp.rev
|
row["rev"] = exp.rev[:7] if Git.is_sha(exp.rev) else exp.rev
|
||||||
row["typ"] = "branch_base" if is_base else "branch_commit"
|
row["typ"] = "branch_base" if is_base else "branch_commit"
|
||||||
@@ -274,12 +268,12 @@ def _data_cells(
|
|||||||
error_value: str = "!",
|
error_value: str = "!",
|
||||||
precision: Optional[int] = None,
|
precision: Optional[int] = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Iterator[Tuple[str, "CellT"]]:
|
) -> Iterator[tuple[str, "CellT"]]:
|
||||||
def _d_cells(
|
def _d_cells(
|
||||||
d: Mapping[str, Any],
|
d: Mapping[str, Any],
|
||||||
names: Mapping[str, Iterable[str]],
|
names: Mapping[str, Iterable[str]],
|
||||||
headers: Iterable[str],
|
headers: Iterable[str],
|
||||||
) -> Iterator[Tuple[str, "CellT"]]:
|
) -> Iterator[tuple[str, "CellT"]]:
|
||||||
from dvc.compare import _format_field, with_value
|
from dvc.compare import _format_field, with_value
|
||||||
|
|
||||||
for fname, data in d.items():
|
for fname, data in d.items():
|
||||||
@@ -329,9 +323,9 @@ def format_time(
|
|||||||
class _DataNames(NamedTuple):
|
class _DataNames(NamedTuple):
|
||||||
# NOTE: we use nested dict instead of set for metrics/params names to
|
# NOTE: we use nested dict instead of set for metrics/params names to
|
||||||
# preserve key ordering
|
# preserve key ordering
|
||||||
metrics: Dict[str, Dict[str, Any]]
|
metrics: dict[str, dict[str, Any]]
|
||||||
params: Dict[str, Dict[str, Any]]
|
params: dict[str, dict[str, Any]]
|
||||||
deps: Set[str]
|
deps: set[str]
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def sorted_deps(self):
|
def sorted_deps(self):
|
||||||
@@ -339,7 +333,7 @@ class _DataNames(NamedTuple):
|
|||||||
|
|
||||||
def update(self, other: "_DataNames"):
|
def update(self, other: "_DataNames"):
|
||||||
def _update_d(
|
def _update_d(
|
||||||
d: Dict[str, Dict[str, Any]], other_d: Mapping[str, Mapping[str, Any]]
|
d: dict[str, dict[str, Any]], other_d: Mapping[str, Mapping[str, Any]]
|
||||||
):
|
):
|
||||||
for k, v in other_d.items():
|
for k, v in other_d.items():
|
||||||
if k in d:
|
if k in d:
|
||||||
@@ -355,7 +349,7 @@ class _DataNames(NamedTuple):
|
|||||||
def _collect_names(exp_states: Iterable["ExpState"]) -> _DataNames:
|
def _collect_names(exp_states: Iterable["ExpState"]) -> _DataNames:
|
||||||
result = _DataNames(defaultdict(dict), defaultdict(dict), set())
|
result = _DataNames(defaultdict(dict), defaultdict(dict), set())
|
||||||
|
|
||||||
def _collect_d(result_d: Dict[str, Dict[str, Any]], data_d: Dict[str, Any]):
|
def _collect_d(result_d: dict[str, dict[str, Any]], data_d: dict[str, Any]):
|
||||||
for path, item in data_d.items():
|
for path, item in data_d.items():
|
||||||
item = item.get("data", {})
|
item = item.get("data", {})
|
||||||
if isinstance(item, dict):
|
if isinstance(item, dict):
|
||||||
@@ -376,7 +370,7 @@ def _collect_names(exp_states: Iterable["ExpState"]) -> _DataNames:
|
|||||||
|
|
||||||
def _normalize_headers(
|
def _normalize_headers(
|
||||||
names: Mapping[str, Mapping[str, Any]], count: Mapping[str, int]
|
names: Mapping[str, Mapping[str, Any]], count: Mapping[str, int]
|
||||||
) -> List[str]:
|
) -> list[str]:
|
||||||
return [
|
return [
|
||||||
name if count[name] == 1 else f"{path}:{name}"
|
name if count[name] == 1 else f"{path}:{name}"
|
||||||
for path in names
|
for path in names
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import re
|
import re
|
||||||
|
from collections.abc import Iterable, Iterator
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from typing import Dict, Iterable, Iterator, NamedTuple, Optional
|
from typing import NamedTuple, Optional
|
||||||
|
|
||||||
from scmrepo.git import Stash
|
from scmrepo.git import Stash
|
||||||
|
|
||||||
@@ -42,7 +43,7 @@ class ExpStash(Stash):
|
|||||||
)
|
)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def stash_revs(self) -> Dict[str, ExpStashEntry]:
|
def stash_revs(self) -> dict[str, ExpStashEntry]:
|
||||||
revs = {}
|
revs = {}
|
||||||
for i, entry in enumerate(self):
|
for i, entry in enumerate(self):
|
||||||
msg = entry.message.decode("utf-8").strip()
|
msg = entry.message.decode("utf-8").strip()
|
||||||
@@ -110,7 +111,7 @@ class ApplyStash(Stash):
|
|||||||
)
|
)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def stash_revs(self) -> Dict[str, ApplyStashEntry]:
|
def stash_revs(self) -> dict[str, ApplyStashEntry]:
|
||||||
revs = {}
|
revs = {}
|
||||||
for i, entry in enumerate(self):
|
for i, entry in enumerate(self):
|
||||||
msg = entry.message.decode("utf-8").strip()
|
msg = entry.message.decode("utf-8").strip()
|
||||||
|
|||||||
@@ -2,18 +2,12 @@ import os
|
|||||||
import random
|
import random
|
||||||
import sys
|
import sys
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
|
from collections.abc import Generator, Iterable, Mapping
|
||||||
from functools import wraps
|
from functools import wraps
|
||||||
from typing import (
|
from typing import (
|
||||||
TYPE_CHECKING,
|
TYPE_CHECKING,
|
||||||
Callable,
|
Callable,
|
||||||
Dict,
|
|
||||||
Generator,
|
|
||||||
Iterable,
|
|
||||||
List,
|
|
||||||
Mapping,
|
|
||||||
Optional,
|
Optional,
|
||||||
Set,
|
|
||||||
Tuple,
|
|
||||||
Union,
|
Union,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -43,8 +37,8 @@ EXEC_PID_DIR = "run"
|
|||||||
|
|
||||||
def get_exp_rwlock(
|
def get_exp_rwlock(
|
||||||
repo: "Repo",
|
repo: "Repo",
|
||||||
reads: Optional[List[str]] = None,
|
reads: Optional[list[str]] = None,
|
||||||
writes: Optional[List[str]] = None,
|
writes: Optional[list[str]] = None,
|
||||||
):
|
):
|
||||||
reads = reads or []
|
reads = reads or []
|
||||||
writes = writes or []
|
writes = writes or []
|
||||||
@@ -109,9 +103,9 @@ def exp_refs_by_rev(scm: "Git", rev: str) -> Generator[ExpRefInfo, None, None]:
|
|||||||
|
|
||||||
def exp_refs_by_baseline(
|
def exp_refs_by_baseline(
|
||||||
scm: "Git",
|
scm: "Git",
|
||||||
revs: Optional[Set[str]] = None,
|
revs: Optional[set[str]] = None,
|
||||||
url: Optional[str] = None,
|
url: Optional[str] = None,
|
||||||
) -> Mapping[str, List[ExpRefInfo]]:
|
) -> Mapping[str, list[ExpRefInfo]]:
|
||||||
"""Iterate over all experiment refs with the specified baseline."""
|
"""Iterate over all experiment refs with the specified baseline."""
|
||||||
all_exp_refs = exp_refs(scm, url)
|
all_exp_refs = exp_refs(scm, url)
|
||||||
result = defaultdict(list)
|
result = defaultdict(list)
|
||||||
@@ -137,7 +131,7 @@ def iter_remote_refs(scm: "Git", url: str, base: Optional[str] = None, **kwargs)
|
|||||||
def push_refspec(
|
def push_refspec(
|
||||||
scm: "Git",
|
scm: "Git",
|
||||||
url: str,
|
url: str,
|
||||||
push_list=List[Tuple[Optional[str], str]],
|
push_list=list[tuple[Optional[str], str]],
|
||||||
force: bool = False,
|
force: bool = False,
|
||||||
on_diverged: Optional[Callable[[str, str], bool]] = None,
|
on_diverged: Optional[Callable[[str, str], bool]] = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
@@ -183,8 +177,8 @@ def remote_exp_refs(scm: "Git", url: str) -> Generator[ExpRefInfo, None, None]:
|
|||||||
|
|
||||||
|
|
||||||
def exp_refs_by_names(
|
def exp_refs_by_names(
|
||||||
scm: "Git", names: Set[str], url: Optional[str] = None
|
scm: "Git", names: set[str], url: Optional[str] = None
|
||||||
) -> Dict[str, List[ExpRefInfo]]:
|
) -> dict[str, list[ExpRefInfo]]:
|
||||||
"""Iterate over all experiment refs matching the specified names."""
|
"""Iterate over all experiment refs matching the specified names."""
|
||||||
resolve_results = defaultdict(list)
|
resolve_results = defaultdict(list)
|
||||||
ref_info_gen = exp_refs(scm, url)
|
ref_info_gen = exp_refs(scm, url)
|
||||||
@@ -210,7 +204,7 @@ def exp_commits(
|
|||||||
scm: "Git", ref_infos: Optional[Iterable[ExpRefInfo]] = None
|
scm: "Git", ref_infos: Optional[Iterable[ExpRefInfo]] = None
|
||||||
) -> Iterable[str]:
|
) -> Iterable[str]:
|
||||||
"""Iterate over all experiment commits."""
|
"""Iterate over all experiment commits."""
|
||||||
shas: Set["str"] = set()
|
shas: set["str"] = set()
|
||||||
refs = ref_infos if ref_infos else exp_refs(scm)
|
refs = ref_infos if ref_infos else exp_refs(scm)
|
||||||
for ref_info in refs:
|
for ref_info in refs:
|
||||||
shas.update(scm.branch_revs(str(ref_info), ref_info.baseline_sha))
|
shas.update(scm.branch_revs(str(ref_info), ref_info.baseline_sha))
|
||||||
@@ -242,7 +236,7 @@ def resolve_name(
|
|||||||
scm: "Git",
|
scm: "Git",
|
||||||
exp_names: Union[Iterable[str], str],
|
exp_names: Union[Iterable[str], str],
|
||||||
git_remote: Optional[str] = None,
|
git_remote: Optional[str] = None,
|
||||||
) -> Dict[str, Optional[ExpRefInfo]]:
|
) -> dict[str, Optional[ExpRefInfo]]:
|
||||||
"""find the ref_info of specified names."""
|
"""find the ref_info of specified names."""
|
||||||
if isinstance(exp_names, str):
|
if isinstance(exp_names, str):
|
||||||
exp_names = [exp_names]
|
exp_names = [exp_names]
|
||||||
@@ -338,7 +332,7 @@ def to_studio_params(dvc_params):
|
|||||||
"params.yaml": {"foo": 1}
|
"params.yaml": {"foo": 1}
|
||||||
}
|
}
|
||||||
"""
|
"""
|
||||||
result: Dict = {}
|
result: dict = {}
|
||||||
if not dvc_params:
|
if not dvc_params:
|
||||||
return result
|
return result
|
||||||
for rev_data in dvc_params.values():
|
for rev_data in dvc_params.values():
|
||||||
@@ -353,7 +347,7 @@ def describe(
|
|||||||
revs: Iterable[str],
|
revs: Iterable[str],
|
||||||
logger,
|
logger,
|
||||||
refs: Optional[Iterable[str]] = None,
|
refs: Optional[Iterable[str]] = None,
|
||||||
) -> Dict[str, Optional[str]]:
|
) -> dict[str, Optional[str]]:
|
||||||
"""Describe revisions using a tag, branch.
|
"""Describe revisions using a tag, branch.
|
||||||
|
|
||||||
The first matching name will be returned for each rev. Names are preferred in this
|
The first matching name will be returned for each rev. Names are preferred in this
|
||||||
@@ -390,7 +384,7 @@ def describe(
|
|||||||
if is_branch and rev not in branches:
|
if is_branch and rev not in branches:
|
||||||
branches[rev] = ref[len("refs/heads/") :]
|
branches[rev] = ref[len("refs/heads/") :]
|
||||||
|
|
||||||
names: Dict[str, Optional[str]] = {}
|
names: dict[str, Optional[str]] = {}
|
||||||
for rev in revs:
|
for rev in revs:
|
||||||
if rev == head_rev and head_branch:
|
if rev == head_rev and head_branch:
|
||||||
names[rev] = head_branch
|
names[rev] = head_branch
|
||||||
|
|||||||
+4
-4
@@ -1,4 +1,4 @@
|
|||||||
from typing import TYPE_CHECKING, List, Tuple
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from dvc.exceptions import DownloadError
|
from dvc.exceptions import DownloadError
|
||||||
from dvc.log import logger
|
from dvc.log import logger
|
||||||
@@ -205,9 +205,9 @@ def fetch( # noqa: PLR0913
|
|||||||
return transferred_count
|
return transferred_count
|
||||||
|
|
||||||
|
|
||||||
def _log_unversioned(data: List["DataIndex"]) -> Tuple[List["DataIndex"], int]:
|
def _log_unversioned(data: list["DataIndex"]) -> tuple[list["DataIndex"], int]:
|
||||||
ret: List["DataIndex"] = []
|
ret: list["DataIndex"] = []
|
||||||
unversioned: List[str] = []
|
unversioned: list[str] = []
|
||||||
for fs_index in data:
|
for fs_index in data:
|
||||||
remote = fs_index.storage_map[()].remote
|
remote = fs_index.storage_map[()].remote
|
||||||
if not isinstance(remote, FileStorage) or not remote.fs.version_aware:
|
if not isinstance(remote, FileStorage) or not remote.fs.version_aware:
|
||||||
|
|||||||
+2
-2
@@ -1,4 +1,4 @@
|
|||||||
from typing import TYPE_CHECKING, List, Optional
|
from typing import TYPE_CHECKING, Optional
|
||||||
|
|
||||||
from dvc.exceptions import InvalidArgumentError
|
from dvc.exceptions import InvalidArgumentError
|
||||||
from dvc.log import logger
|
from dvc.log import logger
|
||||||
@@ -61,7 +61,7 @@ def gc( # noqa: PLR0913, C901
|
|||||||
all_experiments: bool = False,
|
all_experiments: bool = False,
|
||||||
force: bool = False,
|
force: bool = False,
|
||||||
jobs: Optional[int] = None,
|
jobs: Optional[int] = None,
|
||||||
repos: Optional[List[str]] = None,
|
repos: Optional[list[str]] = None,
|
||||||
workspace: bool = False,
|
workspace: bool = False,
|
||||||
commit_date: Optional[str] = None,
|
commit_date: Optional[str] = None,
|
||||||
rev: Optional[str] = None,
|
rev: Optional[str] = None,
|
||||||
|
|||||||
+5
-4
@@ -1,4 +1,5 @@
|
|||||||
from typing import TYPE_CHECKING, Any, Iterator, List, Optional, Set, TypeVar
|
from collections.abc import Iterator
|
||||||
|
from typing import TYPE_CHECKING, Any, Optional, TypeVar
|
||||||
|
|
||||||
from dvc.fs import localfs
|
from dvc.fs import localfs
|
||||||
from dvc.utils.fs import path_isin
|
from dvc.utils.fs import path_isin
|
||||||
@@ -21,7 +22,7 @@ def check_acyclic(graph: "DiGraph") -> None:
|
|||||||
except nx.NetworkXNoCycle:
|
except nx.NetworkXNoCycle:
|
||||||
return
|
return
|
||||||
|
|
||||||
stages: Set["Stage"] = set()
|
stages: set["Stage"] = set()
|
||||||
for from_node, to_node, _ in edges:
|
for from_node, to_node, _ in edges:
|
||||||
stages.add(from_node)
|
stages.add(from_node)
|
||||||
stages.add(to_node)
|
stages.add(to_node)
|
||||||
@@ -45,7 +46,7 @@ def get_pipelines(graph: "DiGraph"):
|
|||||||
|
|
||||||
|
|
||||||
def get_subgraph_of_nodes(
|
def get_subgraph_of_nodes(
|
||||||
graph: "DiGraph", sources: Optional[List[Any]] = None, downstream: bool = False
|
graph: "DiGraph", sources: Optional[list[Any]] = None, downstream: bool = False
|
||||||
) -> "DiGraph":
|
) -> "DiGraph":
|
||||||
from networkx import dfs_postorder_nodes, reverse_view
|
from networkx import dfs_postorder_nodes, reverse_view
|
||||||
|
|
||||||
@@ -69,7 +70,7 @@ def collect_pipeline(stage: "Stage", graph: "DiGraph") -> Iterator["Stage"]:
|
|||||||
return nx.dfs_postorder_nodes(pipeline, stage)
|
return nx.dfs_postorder_nodes(pipeline, stage)
|
||||||
|
|
||||||
|
|
||||||
def collect_inside_path(path: str, graph: "DiGraph") -> List["Stage"]:
|
def collect_inside_path(path: str, graph: "DiGraph") -> list["Stage"]:
|
||||||
import networkx as nx
|
import networkx as nx
|
||||||
|
|
||||||
stages = nx.dfs_postorder_nodes(graph)
|
stages = nx.dfs_postorder_nodes(graph)
|
||||||
|
|||||||
+2
-2
@@ -1,4 +1,4 @@
|
|||||||
from typing import TYPE_CHECKING, Dict, Optional
|
from typing import TYPE_CHECKING, Optional
|
||||||
|
|
||||||
from funcy import compact
|
from funcy import compact
|
||||||
|
|
||||||
@@ -33,7 +33,7 @@ def imp_db(
|
|||||||
assert sql or table
|
assert sql or table
|
||||||
assert output_format in ("csv", "json")
|
assert output_format in ("csv", "json")
|
||||||
|
|
||||||
db: Dict[str, str] = compact(
|
db: dict[str, str] = compact(
|
||||||
{
|
{
|
||||||
"connection": connection,
|
"connection": connection,
|
||||||
"file_format": output_format,
|
"file_format": output_format,
|
||||||
|
|||||||
+5
-5
@@ -1,7 +1,7 @@
|
|||||||
import os
|
import os
|
||||||
from functools import partial
|
from functools import partial
|
||||||
from tempfile import TemporaryDirectory
|
from tempfile import TemporaryDirectory
|
||||||
from typing import TYPE_CHECKING, List, Set, Tuple, Union
|
from typing import TYPE_CHECKING, Union
|
||||||
|
|
||||||
from dvc.log import logger
|
from dvc.log import logger
|
||||||
|
|
||||||
@@ -19,7 +19,7 @@ logger = logger.getChild(__name__)
|
|||||||
|
|
||||||
def unfetched_view(
|
def unfetched_view(
|
||||||
index: "Index", targets: "TargetType", unpartial: bool = False, **kwargs
|
index: "Index", targets: "TargetType", unpartial: bool = False, **kwargs
|
||||||
) -> Tuple["IndexView", "IndexView", List["Dependency"]]:
|
) -> tuple["IndexView", "IndexView", list["Dependency"]]:
|
||||||
"""Return index view of imports which have not been fetched.
|
"""Return index view of imports which have not been fetched.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
@@ -28,7 +28,7 @@ def unfetched_view(
|
|||||||
"""
|
"""
|
||||||
from dvc.cachemgr import LEGACY_HASH_NAMES
|
from dvc.cachemgr import LEGACY_HASH_NAMES
|
||||||
|
|
||||||
changed_deps: List["Dependency"] = []
|
changed_deps: list["Dependency"] = []
|
||||||
|
|
||||||
def need_fetch(stage: "Stage", legacy: bool = False) -> bool:
|
def need_fetch(stage: "Stage", legacy: bool = False) -> bool:
|
||||||
if not stage.is_import or (stage.is_partial_import and not unpartial):
|
if not stage.is_import or (stage.is_partial_import and not unpartial):
|
||||||
@@ -92,7 +92,7 @@ def unpartial_imports(index: Union["Index", "IndexView"]) -> int:
|
|||||||
|
|
||||||
def save_imports(
|
def save_imports(
|
||||||
repo: "Repo", targets: "TargetType", unpartial: bool = False, **kwargs
|
repo: "Repo", targets: "TargetType", unpartial: bool = False, **kwargs
|
||||||
) -> Set["HashInfo"]:
|
) -> set["HashInfo"]:
|
||||||
"""Save (download) imports from their original source location.
|
"""Save (download) imports from their original source location.
|
||||||
|
|
||||||
Imports which are already cached will not be downloaded.
|
Imports which are already cached will not be downloaded.
|
||||||
@@ -105,7 +105,7 @@ def save_imports(
|
|||||||
from dvc_data.index import md5, save
|
from dvc_data.index import md5, save
|
||||||
from dvc_data.index.checkout import apply, compare
|
from dvc_data.index.checkout import apply, compare
|
||||||
|
|
||||||
downloaded: Set["HashInfo"] = set()
|
downloaded: set["HashInfo"] = set()
|
||||||
|
|
||||||
legacy_unfetched, unfetched, changed = unfetched_view(
|
legacy_unfetched, unfetched, changed = unfetched_view(
|
||||||
repo.index, targets, unpartial=unpartial, **kwargs
|
repo.index, targets, unpartial=unpartial, **kwargs
|
||||||
|
|||||||
+35
-40
@@ -1,20 +1,15 @@
|
|||||||
import logging
|
import logging
|
||||||
import time
|
import time
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
|
from collections.abc import Iterable, Iterator
|
||||||
from functools import partial
|
from functools import partial
|
||||||
from itertools import chain
|
from itertools import chain
|
||||||
from typing import (
|
from typing import (
|
||||||
TYPE_CHECKING,
|
TYPE_CHECKING,
|
||||||
Any,
|
Any,
|
||||||
Callable,
|
Callable,
|
||||||
Dict,
|
|
||||||
Iterable,
|
|
||||||
Iterator,
|
|
||||||
List,
|
|
||||||
NamedTuple,
|
NamedTuple,
|
||||||
Optional,
|
Optional,
|
||||||
Set,
|
|
||||||
Tuple,
|
|
||||||
Union,
|
Union,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -44,7 +39,7 @@ if TYPE_CHECKING:
|
|||||||
|
|
||||||
|
|
||||||
logger = logger.getChild(__name__)
|
logger = logger.getChild(__name__)
|
||||||
ObjectContainer = Dict[Optional["HashFileDB"], Set["HashInfo"]]
|
ObjectContainer = dict[Optional["HashFileDB"], set["HashInfo"]]
|
||||||
|
|
||||||
|
|
||||||
def log_walk(seq):
|
def log_walk(seq):
|
||||||
@@ -74,7 +69,7 @@ def collect_files(
|
|||||||
scm = repo.scm
|
scm = repo.scm
|
||||||
fs = repo.fs
|
fs = repo.fs
|
||||||
sep = fs.sep
|
sep = fs.sep
|
||||||
outs: Set[str] = set()
|
outs: set[str] = set()
|
||||||
|
|
||||||
is_local_fs = isinstance(fs, LocalFileSystem)
|
is_local_fs = isinstance(fs, LocalFileSystem)
|
||||||
|
|
||||||
@@ -288,11 +283,11 @@ class Index:
|
|||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
repo: "Repo",
|
repo: "Repo",
|
||||||
stages: Optional[List["Stage"]] = None,
|
stages: Optional[list["Stage"]] = None,
|
||||||
metrics: Optional[Dict[str, List[str]]] = None,
|
metrics: Optional[dict[str, list[str]]] = None,
|
||||||
plots: Optional[Dict[str, List[str]]] = None,
|
plots: Optional[dict[str, list[str]]] = None,
|
||||||
params: Optional[Dict[str, Any]] = None,
|
params: Optional[dict[str, Any]] = None,
|
||||||
artifacts: Optional[Dict[str, Any]] = None,
|
artifacts: Optional[dict[str, Any]] = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.repo = repo
|
self.repo = repo
|
||||||
self.stages = stages or []
|
self.stages = stages or []
|
||||||
@@ -300,7 +295,7 @@ class Index:
|
|||||||
self._plots = plots or {}
|
self._plots = plots or {}
|
||||||
self._params = params or {}
|
self._params = params or {}
|
||||||
self._artifacts = artifacts or {}
|
self._artifacts = artifacts or {}
|
||||||
self._collected_targets: Dict[int, List["StageInfo"]] = {}
|
self._collected_targets: dict[int, list["StageInfo"]] = {}
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def rev(self) -> Optional[str]:
|
def rev(self) -> Optional[str]:
|
||||||
@@ -404,8 +399,8 @@ class Index:
|
|||||||
yield from stage.outs
|
yield from stage.outs
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def out_data_keys(self) -> Dict[str, Set["DataIndexKey"]]:
|
def out_data_keys(self) -> dict[str, set["DataIndexKey"]]:
|
||||||
by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
|
by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set)
|
||||||
|
|
||||||
by_workspace["repo"] = set()
|
by_workspace["repo"] = set()
|
||||||
by_workspace["local"] = set()
|
by_workspace["local"] = set()
|
||||||
@@ -443,10 +438,10 @@ class Index:
|
|||||||
yield from stage.deps
|
yield from stage.deps
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def _plot_sources(self) -> List[str]:
|
def _plot_sources(self) -> list[str]:
|
||||||
from dvc.repo.plots import _collect_pipeline_files
|
from dvc.repo.plots import _collect_pipeline_files
|
||||||
|
|
||||||
sources: List[str] = []
|
sources: list[str] = []
|
||||||
for data in _collect_pipeline_files(self.repo, [], {}).values():
|
for data in _collect_pipeline_files(self.repo, [], {}).values():
|
||||||
for plot_id, props in data.get("data", {}).items():
|
for plot_id, props in data.get("data", {}).items():
|
||||||
if isinstance(props.get("y"), dict):
|
if isinstance(props.get("y"), dict):
|
||||||
@@ -458,8 +453,8 @@ class Index:
|
|||||||
return sources
|
return sources
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def data_keys(self) -> Dict[str, Set["DataIndexKey"]]:
|
def data_keys(self) -> dict[str, set["DataIndexKey"]]:
|
||||||
by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
|
by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set)
|
||||||
|
|
||||||
by_workspace["repo"] = set()
|
by_workspace["repo"] = set()
|
||||||
by_workspace["local"] = set()
|
by_workspace["local"] = set()
|
||||||
@@ -474,10 +469,10 @@ class Index:
|
|||||||
return dict(by_workspace)
|
return dict(by_workspace)
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def metric_keys(self) -> Dict[str, Set["DataIndexKey"]]:
|
def metric_keys(self) -> dict[str, set["DataIndexKey"]]:
|
||||||
from .metrics.show import _collect_top_level_metrics
|
from .metrics.show import _collect_top_level_metrics
|
||||||
|
|
||||||
by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
|
by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set)
|
||||||
|
|
||||||
by_workspace["repo"] = set()
|
by_workspace["repo"] = set()
|
||||||
|
|
||||||
@@ -495,10 +490,10 @@ class Index:
|
|||||||
return dict(by_workspace)
|
return dict(by_workspace)
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def param_keys(self) -> Dict[str, Set["DataIndexKey"]]:
|
def param_keys(self) -> dict[str, set["DataIndexKey"]]:
|
||||||
from .params.show import _collect_top_level_params
|
from .params.show import _collect_top_level_params
|
||||||
|
|
||||||
by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
|
by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set)
|
||||||
by_workspace["repo"] = set()
|
by_workspace["repo"] = set()
|
||||||
|
|
||||||
param_paths = _collect_top_level_params(self.repo)
|
param_paths = _collect_top_level_params(self.repo)
|
||||||
@@ -513,8 +508,8 @@ class Index:
|
|||||||
return dict(by_workspace)
|
return dict(by_workspace)
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def plot_keys(self) -> Dict[str, Set["DataIndexKey"]]:
|
def plot_keys(self) -> dict[str, set["DataIndexKey"]]:
|
||||||
by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
|
by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set)
|
||||||
|
|
||||||
by_workspace["repo"] = set()
|
by_workspace["repo"] = set()
|
||||||
|
|
||||||
@@ -536,7 +531,7 @@ class Index:
|
|||||||
return _build_tree_from_outs(self.outs)
|
return _build_tree_from_outs(self.outs)
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def data(self) -> "Dict[str, DataIndex]":
|
def data(self) -> "dict[str, DataIndex]":
|
||||||
prefix: "DataIndexKey"
|
prefix: "DataIndexKey"
|
||||||
loaded = False
|
loaded = False
|
||||||
|
|
||||||
@@ -584,7 +579,7 @@ class Index:
|
|||||||
|
|
||||||
def collect_targets(
|
def collect_targets(
|
||||||
self, targets: Optional["TargetType"], *, onerror=None, **kwargs: Any
|
self, targets: Optional["TargetType"], *, onerror=None, **kwargs: Any
|
||||||
) -> List["StageInfo"]:
|
) -> list["StageInfo"]:
|
||||||
from dvc.exceptions import DvcException
|
from dvc.exceptions import DvcException
|
||||||
from dvc.repo.stage import StageInfo
|
from dvc.repo.stage import StageInfo
|
||||||
from dvc.utils.collections import ensure_list
|
from dvc.utils.collections import ensure_list
|
||||||
@@ -658,7 +653,7 @@ class Index:
|
|||||||
stage_filter: Optional[Callable[["Stage"], bool]] = None,
|
stage_filter: Optional[Callable[["Stage"], bool]] = None,
|
||||||
outs_filter: Optional[Callable[["Output"], bool]] = None,
|
outs_filter: Optional[Callable[["Output"], bool]] = None,
|
||||||
max_size: Optional[int] = None,
|
max_size: Optional[int] = None,
|
||||||
types: Optional[List[str]] = None,
|
types: Optional[list[str]] = None,
|
||||||
**kwargs: Any,
|
**kwargs: Any,
|
||||||
) -> "IndexView":
|
) -> "IndexView":
|
||||||
"""Return read-only view of index for the specified targets.
|
"""Return read-only view of index for the specified targets.
|
||||||
@@ -698,8 +693,8 @@ class Index:
|
|||||||
|
|
||||||
|
|
||||||
class _DataPrefixes(NamedTuple):
|
class _DataPrefixes(NamedTuple):
|
||||||
explicit: Set["DataIndexKey"]
|
explicit: set["DataIndexKey"]
|
||||||
recursive: Set["DataIndexKey"]
|
recursive: set["DataIndexKey"]
|
||||||
|
|
||||||
|
|
||||||
class IndexView:
|
class IndexView:
|
||||||
@@ -728,7 +723,7 @@ class IndexView:
|
|||||||
yield from stage.deps
|
yield from stage.deps
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def _filtered_outs(self) -> Iterator[Tuple["Output", Optional[str]]]:
|
def _filtered_outs(self) -> Iterator[tuple["Output", Optional[str]]]:
|
||||||
for stage, filter_info in self._stage_infos:
|
for stage, filter_info in self._stage_infos:
|
||||||
for out in stage.filter_outs(filter_info):
|
for out in stage.filter_outs(filter_info):
|
||||||
if not self._outs_filter or self._outs_filter(out):
|
if not self._outs_filter or self._outs_filter(out):
|
||||||
@@ -739,8 +734,8 @@ class IndexView:
|
|||||||
yield from {out for (out, _) in self._filtered_outs}
|
yield from {out for (out, _) in self._filtered_outs}
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def out_data_keys(self) -> Dict[str, Set["DataIndexKey"]]:
|
def out_data_keys(self) -> dict[str, set["DataIndexKey"]]:
|
||||||
by_workspace: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
|
by_workspace: dict[str, set["DataIndexKey"]] = defaultdict(set)
|
||||||
|
|
||||||
by_workspace["repo"] = set()
|
by_workspace["repo"] = set()
|
||||||
by_workspace["local"] = set()
|
by_workspace["local"] = set()
|
||||||
@@ -755,8 +750,8 @@ class IndexView:
|
|||||||
return dict(by_workspace)
|
return dict(by_workspace)
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def _data_prefixes(self) -> Dict[str, "_DataPrefixes"]:
|
def _data_prefixes(self) -> dict[str, "_DataPrefixes"]:
|
||||||
prefixes: Dict[str, "_DataPrefixes"] = defaultdict(
|
prefixes: dict[str, "_DataPrefixes"] = defaultdict(
|
||||||
lambda: _DataPrefixes(set(), set())
|
lambda: _DataPrefixes(set(), set())
|
||||||
)
|
)
|
||||||
for out, filter_info in self._filtered_outs:
|
for out, filter_info in self._filtered_outs:
|
||||||
@@ -772,8 +767,8 @@ class IndexView:
|
|||||||
return prefixes
|
return prefixes
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def data_keys(self) -> Dict[str, Set["DataIndexKey"]]:
|
def data_keys(self) -> dict[str, set["DataIndexKey"]]:
|
||||||
ret: Dict[str, Set["DataIndexKey"]] = defaultdict(set)
|
ret: dict[str, set["DataIndexKey"]] = defaultdict(set)
|
||||||
|
|
||||||
for out, filter_info in self._filtered_outs:
|
for out, filter_info in self._filtered_outs:
|
||||||
if not out.use_cache:
|
if not out.use_cache:
|
||||||
@@ -791,7 +786,7 @@ class IndexView:
|
|||||||
return _build_tree_from_outs(self.outs)
|
return _build_tree_from_outs(self.outs)
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def data(self) -> Dict[str, Union["DataIndex", "DataIndexView"]]:
|
def data(self) -> dict[str, Union["DataIndex", "DataIndexView"]]:
|
||||||
from dvc_data.index import DataIndex, view
|
from dvc_data.index import DataIndex, view
|
||||||
|
|
||||||
def key_filter(workspace: str, key: "DataIndexKey"):
|
def key_filter(workspace: str, key: "DataIndexKey"):
|
||||||
@@ -803,7 +798,7 @@ class IndexView:
|
|||||||
except KeyError:
|
except KeyError:
|
||||||
return False
|
return False
|
||||||
|
|
||||||
data: Dict[str, Union["DataIndex", "DataIndexView"]] = {}
|
data: dict[str, Union["DataIndex", "DataIndexView"]] = {}
|
||||||
for workspace, data_index in self._index.data.items():
|
for workspace, data_index in self._index.data.items():
|
||||||
if self.stages:
|
if self.stages:
|
||||||
data[workspace] = view(data_index, partial(key_filter, workspace))
|
data[workspace] = view(data_index, partial(key_filter, workspace))
|
||||||
|
|||||||
+2
-2
@@ -1,5 +1,5 @@
|
|||||||
import os
|
import os
|
||||||
from typing import TYPE_CHECKING, Any, Dict, Optional, Union
|
from typing import TYPE_CHECKING, Any, Optional, Union
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from dvc.fs.dvc import DVCFileSystem
|
from dvc.fs.dvc import DVCFileSystem
|
||||||
@@ -13,7 +13,7 @@ def ls(
|
|||||||
rev: Optional[str] = None,
|
rev: Optional[str] = None,
|
||||||
recursive: Optional[bool] = None,
|
recursive: Optional[bool] = None,
|
||||||
dvc_only: bool = False,
|
dvc_only: bool = False,
|
||||||
config: Union[None, Dict[str, Any], str] = None,
|
config: Union[None, dict[str, Any], str] = None,
|
||||||
remote: Optional[str] = None,
|
remote: Optional[str] = None,
|
||||||
remote_config: Optional[dict] = None,
|
remote_config: Optional[dict] = None,
|
||||||
):
|
):
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
from typing import TYPE_CHECKING, Dict, TypedDict, Union
|
from typing import TYPE_CHECKING, TypedDict, Union
|
||||||
|
|
||||||
from funcy import compact
|
from funcy import compact
|
||||||
|
|
||||||
@@ -12,12 +12,12 @@ if TYPE_CHECKING:
|
|||||||
|
|
||||||
|
|
||||||
class DiffResult(TypedDict, total=False):
|
class DiffResult(TypedDict, total=False):
|
||||||
errors: Dict[str, Union[Exception, Dict[str, Exception]]]
|
errors: dict[str, Union[Exception, dict[str, Exception]]]
|
||||||
diff: Dict[str, Dict[str, Dict]]
|
diff: dict[str, dict[str, dict]]
|
||||||
|
|
||||||
|
|
||||||
def _diff(
|
def _diff(
|
||||||
result: Dict[str, "Result"],
|
result: dict[str, "Result"],
|
||||||
old_rev: str,
|
old_rev: str,
|
||||||
new_rev: str,
|
new_rev: str,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
|
|||||||
+17
-21
@@ -1,14 +1,10 @@
|
|||||||
import os
|
import os
|
||||||
|
from collections.abc import Iterable, Iterator
|
||||||
from itertools import chain
|
from itertools import chain
|
||||||
from typing import (
|
from typing import (
|
||||||
TYPE_CHECKING,
|
TYPE_CHECKING,
|
||||||
Any,
|
Any,
|
||||||
Dict,
|
|
||||||
Iterable,
|
|
||||||
Iterator,
|
|
||||||
List,
|
|
||||||
Optional,
|
Optional,
|
||||||
Tuple,
|
|
||||||
TypedDict,
|
TypedDict,
|
||||||
Union,
|
Union,
|
||||||
)
|
)
|
||||||
@@ -71,7 +67,7 @@ def _read_metric(fs: "FileSystem", path: str, **load_kwargs) -> Any:
|
|||||||
|
|
||||||
def _read_metrics(
|
def _read_metrics(
|
||||||
fs: "FileSystem", metrics: Iterable[str], **load_kwargs
|
fs: "FileSystem", metrics: Iterable[str], **load_kwargs
|
||||||
) -> Iterator[Tuple[str, Union[Exception, Any]]]:
|
) -> Iterator[tuple[str, Union[Exception, Any]]]:
|
||||||
for metric in metrics:
|
for metric in metrics:
|
||||||
try:
|
try:
|
||||||
yield metric, _read_metric(fs, metric, **load_kwargs)
|
yield metric, _read_metric(fs, metric, **load_kwargs)
|
||||||
@@ -80,7 +76,7 @@ def _read_metrics(
|
|||||||
yield metric, exc
|
yield metric, exc
|
||||||
|
|
||||||
|
|
||||||
def metrics_from_target(repo: "Repo", targets: List[str]) -> Iterator["Output"]:
|
def metrics_from_target(repo: "Repo", targets: list[str]) -> Iterator["Output"]:
|
||||||
stages = chain.from_iterable(repo.stage.collect(target) for target in targets)
|
stages = chain.from_iterable(repo.stage.collect(target) for target in targets)
|
||||||
for stage in stages:
|
for stage in stages:
|
||||||
yield from stage.metrics
|
yield from stage.metrics
|
||||||
@@ -88,11 +84,11 @@ def metrics_from_target(repo: "Repo", targets: List[str]) -> Iterator["Output"]:
|
|||||||
|
|
||||||
def _collect_metrics(
|
def _collect_metrics(
|
||||||
repo: "Repo",
|
repo: "Repo",
|
||||||
targets: Optional[List[str]] = None,
|
targets: Optional[list[str]] = None,
|
||||||
stages: Optional[List[str]] = None,
|
stages: Optional[list[str]] = None,
|
||||||
outs_only: bool = False,
|
outs_only: bool = False,
|
||||||
) -> List[str]:
|
) -> list[str]:
|
||||||
metrics: List[str] = []
|
metrics: list[str] = []
|
||||||
|
|
||||||
if targets:
|
if targets:
|
||||||
# target is a repo-relative path
|
# target is a repo-relative path
|
||||||
@@ -122,7 +118,7 @@ class FileResult(TypedDict, total=False):
|
|||||||
|
|
||||||
|
|
||||||
class Result(TypedDict, total=False):
|
class Result(TypedDict, total=False):
|
||||||
data: Dict[str, FileResult]
|
data: dict[str, FileResult]
|
||||||
error: Exception
|
error: Exception
|
||||||
|
|
||||||
|
|
||||||
@@ -139,11 +135,11 @@ def to_relpath(fs: "FileSystem", root_dir: str, d: Result) -> Result:
|
|||||||
|
|
||||||
def _gather_metrics(
|
def _gather_metrics(
|
||||||
repo: "Repo",
|
repo: "Repo",
|
||||||
targets: Optional[List[str]] = None,
|
targets: Optional[list[str]] = None,
|
||||||
outs_only: bool = False,
|
outs_only: bool = False,
|
||||||
stages: Optional[List[str]] = None,
|
stages: Optional[list[str]] = None,
|
||||||
on_error: str = "return",
|
on_error: str = "return",
|
||||||
) -> Dict[str, FileResult]:
|
) -> dict[str, FileResult]:
|
||||||
assert on_error in ("raise", "return", "ignore")
|
assert on_error in ("raise", "return", "ignore")
|
||||||
|
|
||||||
# `files` is a repo-relative posixpath that can be passed to DVCFileSystem
|
# `files` is a repo-relative posixpath that can be passed to DVCFileSystem
|
||||||
@@ -168,8 +164,8 @@ def _gather_metrics(
|
|||||||
|
|
||||||
|
|
||||||
def _hide_workspace(
|
def _hide_workspace(
|
||||||
scm: Union["Git", "NoSCM"], res: Dict[str, Result]
|
scm: Union["Git", "NoSCM"], res: dict[str, Result]
|
||||||
) -> Dict[str, Result]:
|
) -> dict[str, Result]:
|
||||||
# Hide workspace params if they are the same as in the active branch
|
# Hide workspace params if they are the same as in the active branch
|
||||||
try:
|
try:
|
||||||
active_branch = scm.active_branch()
|
active_branch = scm.active_branch()
|
||||||
@@ -186,16 +182,16 @@ def _hide_workspace(
|
|||||||
|
|
||||||
def show(
|
def show(
|
||||||
repo: "Repo",
|
repo: "Repo",
|
||||||
targets: Optional[List[str]] = None,
|
targets: Optional[list[str]] = None,
|
||||||
stages: Optional[List[str]] = None,
|
stages: Optional[list[str]] = None,
|
||||||
outs_only: bool = False,
|
outs_only: bool = False,
|
||||||
all_branches: bool = False,
|
all_branches: bool = False,
|
||||||
all_tags: bool = False,
|
all_tags: bool = False,
|
||||||
revs: Optional[List[str]] = None,
|
revs: Optional[list[str]] = None,
|
||||||
all_commits: bool = False,
|
all_commits: bool = False,
|
||||||
hide_workspace: bool = True,
|
hide_workspace: bool = True,
|
||||||
on_error: str = "return",
|
on_error: str = "return",
|
||||||
) -> Dict[str, Result]:
|
) -> dict[str, Result]:
|
||||||
assert on_error in ("raise", "return", "ignore")
|
assert on_error in ("raise", "return", "ignore")
|
||||||
|
|
||||||
targets = [os.path.abspath(target) for target in ensure_list(targets)]
|
targets = [os.path.abspath(target) for target in ensure_list(targets)]
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import os
|
import os
|
||||||
import tempfile
|
import tempfile
|
||||||
import threading
|
import threading
|
||||||
from typing import TYPE_CHECKING, Dict, Optional, Tuple
|
from typing import TYPE_CHECKING, Optional
|
||||||
|
|
||||||
from funcy import retry, wrap_with
|
from funcy import retry, wrap_with
|
||||||
|
|
||||||
@@ -78,8 +78,8 @@ def erepo_factory(url, root_dir, cache_config):
|
|||||||
return make_repo
|
return make_repo
|
||||||
|
|
||||||
|
|
||||||
CLONES: Dict[str, Tuple[str, bool]] = {}
|
CLONES: dict[str, tuple[str, bool]] = {}
|
||||||
CACHE_DIRS: Dict[str, str] = {}
|
CACHE_DIRS: dict[str, str] = {}
|
||||||
|
|
||||||
|
|
||||||
@wrap_with(threading.Lock())
|
@wrap_with(threading.Lock())
|
||||||
|
|||||||
+18
-17
@@ -1,7 +1,8 @@
|
|||||||
import os
|
import os
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
|
from collections.abc import Iterator
|
||||||
from itertools import chain
|
from itertools import chain
|
||||||
from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Tuple, Union
|
from typing import TYPE_CHECKING, Any, Optional, Union
|
||||||
|
|
||||||
from dvc.dependency.param import ParamsDependency, read_param_file
|
from dvc.dependency.param import ParamsDependency, read_param_file
|
||||||
from dvc.log import logger
|
from dvc.log import logger
|
||||||
@@ -27,7 +28,7 @@ def _collect_top_level_params(repo: "Repo") -> Iterator[str]:
|
|||||||
|
|
||||||
|
|
||||||
def params_from_target(
|
def params_from_target(
|
||||||
repo: "Repo", targets: List[str]
|
repo: "Repo", targets: list[str]
|
||||||
) -> Iterator["ParamsDependency"]:
|
) -> Iterator["ParamsDependency"]:
|
||||||
stages = chain.from_iterable(repo.stage.collect(target) for target in targets)
|
stages = chain.from_iterable(repo.stage.collect(target) for target in targets)
|
||||||
for stage in stages:
|
for stage in stages:
|
||||||
@@ -36,17 +37,17 @@ def params_from_target(
|
|||||||
|
|
||||||
def _collect_params(
|
def _collect_params(
|
||||||
repo: "Repo",
|
repo: "Repo",
|
||||||
targets: Union[List[str], Dict[str, List[str]], None] = None,
|
targets: Union[list[str], dict[str, list[str]], None] = None,
|
||||||
stages: Optional[List[str]] = None,
|
stages: Optional[list[str]] = None,
|
||||||
deps_only: bool = False,
|
deps_only: bool = False,
|
||||||
default_file: Optional[str] = None,
|
default_file: Optional[str] = None,
|
||||||
) -> Dict[str, List[str]]:
|
) -> dict[str, list[str]]:
|
||||||
from dvc.dependency import _merge_params
|
from dvc.dependency import _merge_params
|
||||||
|
|
||||||
if isinstance(targets, list):
|
if isinstance(targets, list):
|
||||||
targets = {target: [] for target in targets}
|
targets = {target: [] for target in targets}
|
||||||
|
|
||||||
params: List[Dict[str, List[str]]] = []
|
params: list[dict[str, list[str]]] = []
|
||||||
|
|
||||||
if targets:
|
if targets:
|
||||||
# target is a repo-relative path
|
# target is a repo-relative path
|
||||||
@@ -80,8 +81,8 @@ def _collect_params(
|
|||||||
return ret
|
return ret
|
||||||
|
|
||||||
|
|
||||||
def _collect_vars(repo, params, stages=None) -> Dict:
|
def _collect_vars(repo, params, stages=None) -> dict:
|
||||||
vars_params: Dict[str, Dict] = defaultdict(dict)
|
vars_params: dict[str, dict] = defaultdict(dict)
|
||||||
|
|
||||||
for stage in repo.index.stages:
|
for stage in repo.index.stages:
|
||||||
if isinstance(stage, PipelineStage) and stage.tracked_vars:
|
if isinstance(stage, PipelineStage) and stage.tracked_vars:
|
||||||
@@ -102,8 +103,8 @@ def _collect_vars(repo, params, stages=None) -> Dict:
|
|||||||
|
|
||||||
|
|
||||||
def _read_params(
|
def _read_params(
|
||||||
fs: "FileSystem", params: Dict[str, List[str]], **load_kwargs
|
fs: "FileSystem", params: dict[str, list[str]], **load_kwargs
|
||||||
) -> Iterator[Tuple[str, Union[Exception, Any]]]:
|
) -> Iterator[tuple[str, Union[Exception, Any]]]:
|
||||||
for file_path, key_paths in params.items():
|
for file_path, key_paths in params.items():
|
||||||
try:
|
try:
|
||||||
yield file_path, read_param_file(fs, file_path, key_paths, **load_kwargs)
|
yield file_path, read_param_file(fs, file_path, key_paths, **load_kwargs)
|
||||||
@@ -114,9 +115,9 @@ def _read_params(
|
|||||||
|
|
||||||
def _gather_params(
|
def _gather_params(
|
||||||
repo: "Repo",
|
repo: "Repo",
|
||||||
targets: Union[List[str], Dict[str, List[str]], None] = None,
|
targets: Union[list[str], dict[str, list[str]], None] = None,
|
||||||
deps_only: bool = False,
|
deps_only: bool = False,
|
||||||
stages: Optional[List[str]] = None,
|
stages: Optional[list[str]] = None,
|
||||||
on_error: str = "return",
|
on_error: str = "return",
|
||||||
):
|
):
|
||||||
assert on_error in ("raise", "return", "ignore")
|
assert on_error in ("raise", "return", "ignore")
|
||||||
@@ -132,7 +133,7 @@ def _gather_params(
|
|||||||
default_file=ParamsDependency.DEFAULT_PARAMS_FILE,
|
default_file=ParamsDependency.DEFAULT_PARAMS_FILE,
|
||||||
)
|
)
|
||||||
|
|
||||||
data: Dict[str, FileResult] = {}
|
data: dict[str, FileResult] = {}
|
||||||
|
|
||||||
fs = repo.dvcfs
|
fs = repo.dvcfs
|
||||||
for fs_path, result in _read_params(fs, files_keypaths, cache=True):
|
for fs_path, result in _read_params(fs, files_keypaths, cache=True):
|
||||||
@@ -159,16 +160,16 @@ def _gather_params(
|
|||||||
|
|
||||||
def show(
|
def show(
|
||||||
repo: "Repo",
|
repo: "Repo",
|
||||||
targets: Optional[List[str]] = None,
|
targets: Optional[list[str]] = None,
|
||||||
stages: Optional[List[str]] = None,
|
stages: Optional[list[str]] = None,
|
||||||
deps_only: bool = False,
|
deps_only: bool = False,
|
||||||
all_branches: bool = False,
|
all_branches: bool = False,
|
||||||
all_tags: bool = False,
|
all_tags: bool = False,
|
||||||
revs: Optional[List[str]] = None,
|
revs: Optional[list[str]] = None,
|
||||||
all_commits: bool = False,
|
all_commits: bool = False,
|
||||||
hide_workspace: bool = True,
|
hide_workspace: bool = True,
|
||||||
on_error: str = "return",
|
on_error: str = "return",
|
||||||
) -> Dict[str, Result]:
|
) -> dict[str, Result]:
|
||||||
assert on_error in ("raise", "return", "ignore")
|
assert on_error in ("raise", "return", "ignore")
|
||||||
res = {}
|
res = {}
|
||||||
|
|
||||||
|
|||||||
+30
-33
@@ -2,6 +2,7 @@ import csv
|
|||||||
import io
|
import io
|
||||||
import os
|
import os
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
|
from collections.abc import Iterator
|
||||||
from copy import deepcopy
|
from copy import deepcopy
|
||||||
from functools import partial
|
from functools import partial
|
||||||
from multiprocessing import cpu_count
|
from multiprocessing import cpu_count
|
||||||
@@ -9,11 +10,7 @@ from typing import (
|
|||||||
TYPE_CHECKING,
|
TYPE_CHECKING,
|
||||||
Any,
|
Any,
|
||||||
Callable,
|
Callable,
|
||||||
Dict,
|
|
||||||
Iterator,
|
|
||||||
List,
|
|
||||||
Optional,
|
Optional,
|
||||||
Set,
|
|
||||||
Union,
|
Union,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -40,7 +37,7 @@ dpath.options.ALLOW_EMPTY_STRING_KEYS = True
|
|||||||
logger = logger.getChild(__name__)
|
logger = logger.getChild(__name__)
|
||||||
|
|
||||||
|
|
||||||
def onerror_collect(result: Dict, exception: Exception, *args, **kwargs):
|
def onerror_collect(result: dict, exception: Exception, *args, **kwargs):
|
||||||
logger.debug("", exc_info=True)
|
logger.debug("", exc_info=True)
|
||||||
result["error"] = exception
|
result["error"] = exception
|
||||||
|
|
||||||
@@ -82,12 +79,12 @@ class Plots:
|
|||||||
|
|
||||||
def collect(
|
def collect(
|
||||||
self,
|
self,
|
||||||
targets: Optional[List[str]] = None,
|
targets: Optional[list[str]] = None,
|
||||||
revs: Optional[List[str]] = None,
|
revs: Optional[list[str]] = None,
|
||||||
recursive: bool = False,
|
recursive: bool = False,
|
||||||
onerror: Optional[Callable] = None,
|
onerror: Optional[Callable] = None,
|
||||||
props: Optional[Dict] = None,
|
props: Optional[dict] = None,
|
||||||
) -> Iterator[Dict]:
|
) -> Iterator[dict]:
|
||||||
"""Collects plots definitions and data sources.
|
"""Collects plots definitions and data sources.
|
||||||
|
|
||||||
Generator yielding a structure like:
|
Generator yielding a structure like:
|
||||||
@@ -142,7 +139,7 @@ class Plots:
|
|||||||
revs = ["workspace", *revs]
|
revs = ["workspace", *revs]
|
||||||
for rev in revs:
|
for rev in revs:
|
||||||
with switch_repo(self.repo, rev) as (repo, _):
|
with switch_repo(self.repo, rev) as (repo, _):
|
||||||
res: Dict = {}
|
res: dict = {}
|
||||||
definitions = _collect_definitions(
|
definitions = _collect_definitions(
|
||||||
repo,
|
repo,
|
||||||
targets=targets,
|
targets=targets,
|
||||||
@@ -168,9 +165,9 @@ class Plots:
|
|||||||
def _collect_data_sources(
|
def _collect_data_sources(
|
||||||
self,
|
self,
|
||||||
repo: "Repo",
|
repo: "Repo",
|
||||||
targets: Optional[List[str]] = None,
|
targets: Optional[list[str]] = None,
|
||||||
recursive: bool = False,
|
recursive: bool = False,
|
||||||
props: Optional[Dict] = None,
|
props: Optional[dict] = None,
|
||||||
onerror: Optional[Callable] = None,
|
onerror: Optional[Callable] = None,
|
||||||
):
|
):
|
||||||
fs = repo.dvcfs
|
fs = repo.dvcfs
|
||||||
@@ -178,7 +175,7 @@ class Plots:
|
|||||||
props = props or {}
|
props = props or {}
|
||||||
|
|
||||||
plots = _collect_plots(repo, targets, recursive)
|
plots = _collect_plots(repo, targets, recursive)
|
||||||
res: Dict[str, Any] = {}
|
res: dict[str, Any] = {}
|
||||||
for fs_path, rev_props in plots.items():
|
for fs_path, rev_props in plots.items():
|
||||||
joined_props = {**rev_props, **props}
|
joined_props = {**rev_props, **props}
|
||||||
res[fs_path] = {"props": joined_props}
|
res[fs_path] = {"props": joined_props}
|
||||||
@@ -197,7 +194,7 @@ class Plots:
|
|||||||
|
|
||||||
def show(
|
def show(
|
||||||
self,
|
self,
|
||||||
targets: Optional[List[str]] = None,
|
targets: Optional[list[str]] = None,
|
||||||
revs=None,
|
revs=None,
|
||||||
props=None,
|
props=None,
|
||||||
recursive=False,
|
recursive=False,
|
||||||
@@ -206,7 +203,7 @@ class Plots:
|
|||||||
if onerror is None:
|
if onerror is None:
|
||||||
onerror = onerror_collect
|
onerror = onerror_collect
|
||||||
|
|
||||||
result: Dict[str, Dict] = {}
|
result: dict[str, dict] = {}
|
||||||
for data in self.collect(
|
for data in self.collect(
|
||||||
targets,
|
targets,
|
||||||
revs,
|
revs,
|
||||||
@@ -286,7 +283,7 @@ def _is_plot(out: "Output") -> bool:
|
|||||||
return bool(out.plot)
|
return bool(out.plot)
|
||||||
|
|
||||||
|
|
||||||
def _resolve_data_sources(plots_data: Dict, rev: str, cache: bool = False):
|
def _resolve_data_sources(plots_data: dict, rev: str, cache: bool = False):
|
||||||
from dvc.progress import Tqdm
|
from dvc.progress import Tqdm
|
||||||
|
|
||||||
values = list(plots_data.values())
|
values = list(plots_data.values())
|
||||||
@@ -325,9 +322,9 @@ def _resolve_data_sources(plots_data: Dict, rev: str, cache: bool = False):
|
|||||||
|
|
||||||
def _collect_plots(
|
def _collect_plots(
|
||||||
repo: "Repo",
|
repo: "Repo",
|
||||||
targets: Optional[List[str]] = None,
|
targets: Optional[list[str]] = None,
|
||||||
recursive: bool = False,
|
recursive: bool = False,
|
||||||
) -> Dict[str, Dict]:
|
) -> dict[str, dict]:
|
||||||
from dvc.repo.collect import collect
|
from dvc.repo.collect import collect
|
||||||
|
|
||||||
plots, fs_paths = collect(
|
plots, fs_paths = collect(
|
||||||
@@ -344,8 +341,8 @@ def _collect_plots(
|
|||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
def _get_data_targets(definitions: Dict):
|
def _get_data_targets(definitions: dict):
|
||||||
result: Set = set()
|
result: set = set()
|
||||||
if "data" in definitions:
|
if "data" in definitions:
|
||||||
for content in definitions["data"].values():
|
for content in definitions["data"].values():
|
||||||
if "data" in content:
|
if "data" in content:
|
||||||
@@ -404,7 +401,7 @@ def _relpath(fs, path):
|
|||||||
|
|
||||||
def _collect_output_plots(repo, targets, props, onerror: Optional[Callable] = None):
|
def _collect_output_plots(repo, targets, props, onerror: Optional[Callable] = None):
|
||||||
fs = repo.dvcfs
|
fs = repo.dvcfs
|
||||||
result: Dict[str, Dict] = {}
|
result: dict[str, dict] = {}
|
||||||
for plot in repo.index.plots:
|
for plot in repo.index.plots:
|
||||||
plot_props = _plot_props(plot)
|
plot_props = _plot_props(plot)
|
||||||
dvcfile = plot.stage.dvcfile
|
dvcfile = plot.stage.dvcfile
|
||||||
@@ -445,15 +442,15 @@ def _adjust_sources(fs, plot_props, config_dir):
|
|||||||
|
|
||||||
def _resolve_definitions(
|
def _resolve_definitions(
|
||||||
fs: "FileSystem",
|
fs: "FileSystem",
|
||||||
targets: List[str],
|
targets: list[str],
|
||||||
props: Dict[str, Any],
|
props: dict[str, Any],
|
||||||
config_path: "StrPath",
|
config_path: "StrPath",
|
||||||
definitions: "DictStrAny",
|
definitions: "DictStrAny",
|
||||||
onerror: Optional[Callable[[Any], Any]] = None,
|
onerror: Optional[Callable[[Any], Any]] = None,
|
||||||
):
|
):
|
||||||
config_path = os.fspath(config_path)
|
config_path = os.fspath(config_path)
|
||||||
config_dir = fs.dirname(config_path)
|
config_dir = fs.dirname(config_path)
|
||||||
result: Dict[str, Dict] = {}
|
result: dict[str, dict] = {}
|
||||||
for plot_id, plot_props in definitions.items():
|
for plot_id, plot_props in definitions.items():
|
||||||
if plot_props is None:
|
if plot_props is None:
|
||||||
plot_props = {}
|
plot_props = {}
|
||||||
@@ -477,12 +474,12 @@ def _resolve_definitions(
|
|||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
def _collect_pipeline_files(repo, targets: List[str], props, onerror=None):
|
def _collect_pipeline_files(repo, targets: list[str], props, onerror=None):
|
||||||
result: Dict[str, Dict] = {}
|
result: dict[str, dict] = {}
|
||||||
top_plots = repo.index._plots
|
top_plots = repo.index._plots
|
||||||
for dvcfile, plots_def in top_plots.items():
|
for dvcfile, plots_def in top_plots.items():
|
||||||
dvcfile_path = _relpath(repo.dvcfs, dvcfile)
|
dvcfile_path = _relpath(repo.dvcfs, dvcfile)
|
||||||
dvcfile_defs_dict: Dict[str, Union[Dict, None]] = {}
|
dvcfile_defs_dict: dict[str, Union[dict, None]] = {}
|
||||||
for elem in plots_def:
|
for elem in plots_def:
|
||||||
if isinstance(elem, str):
|
if isinstance(elem, str):
|
||||||
dvcfile_defs_dict[elem] = None
|
dvcfile_defs_dict[elem] = None
|
||||||
@@ -510,11 +507,11 @@ def _collect_pipeline_files(repo, targets: List[str], props, onerror=None):
|
|||||||
def _collect_definitions(
|
def _collect_definitions(
|
||||||
repo: "Repo",
|
repo: "Repo",
|
||||||
targets=None,
|
targets=None,
|
||||||
props: Optional[Dict] = None,
|
props: Optional[dict] = None,
|
||||||
onerror: Optional[Callable] = None,
|
onerror: Optional[Callable] = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> Dict:
|
) -> dict:
|
||||||
result: Dict = defaultdict(dict)
|
result: dict = defaultdict(dict)
|
||||||
props = props or {}
|
props = props or {}
|
||||||
|
|
||||||
fs = repo.dvcfs
|
fs = repo.dvcfs
|
||||||
@@ -536,8 +533,8 @@ def _collect_definitions(
|
|||||||
return dict(result)
|
return dict(result)
|
||||||
|
|
||||||
|
|
||||||
def unpack_if_dir(fs, path, props: Dict[str, str], onerror: Optional[Callable] = None):
|
def unpack_if_dir(fs, path, props: dict[str, str], onerror: Optional[Callable] = None):
|
||||||
result: Dict[str, Dict] = defaultdict(dict)
|
result: dict[str, dict] = defaultdict(dict)
|
||||||
if fs.isdir(path):
|
if fs.isdir(path):
|
||||||
unpacked = _unpack_dir_files(fs, path, onerror=onerror)
|
unpacked = _unpack_dir_files(fs, path, onerror=onerror)
|
||||||
else:
|
else:
|
||||||
@@ -574,7 +571,7 @@ def parse(fs, path, props=None, **fs_kwargs):
|
|||||||
return PARSERS[extension](contents, path)
|
return PARSERS[extension](contents, path)
|
||||||
|
|
||||||
|
|
||||||
def _plot_props(out: "Output") -> Dict:
|
def _plot_props(out: "Output") -> dict:
|
||||||
from dvc.schema import PLOT_PROPS
|
from dvc.schema import PLOT_PROPS
|
||||||
|
|
||||||
if not (out.plot):
|
if not (out.plot):
|
||||||
|
|||||||
+14
-18
@@ -1,13 +1,9 @@
|
|||||||
|
from collections.abc import Iterable
|
||||||
from typing import (
|
from typing import (
|
||||||
TYPE_CHECKING,
|
TYPE_CHECKING,
|
||||||
Callable,
|
Callable,
|
||||||
Dict,
|
|
||||||
Iterable,
|
|
||||||
List,
|
|
||||||
NoReturn,
|
NoReturn,
|
||||||
Optional,
|
Optional,
|
||||||
Set,
|
|
||||||
Tuple,
|
|
||||||
TypeVar,
|
TypeVar,
|
||||||
Union,
|
Union,
|
||||||
cast,
|
cast,
|
||||||
@@ -40,8 +36,8 @@ def collect_stages(
|
|||||||
targets: Iterable[str],
|
targets: Iterable[str],
|
||||||
recursive: bool = False,
|
recursive: bool = False,
|
||||||
glob: bool = False,
|
glob: bool = False,
|
||||||
) -> List["Stage"]:
|
) -> list["Stage"]:
|
||||||
stages: List["Stage"] = []
|
stages: list["Stage"] = []
|
||||||
for target in targets:
|
for target in targets:
|
||||||
stages.extend(repo.stage.collect(target, recursive=recursive, glob=glob))
|
stages.extend(repo.stage.collect(target, recursive=recursive, glob=glob))
|
||||||
return ldistinct(stages)
|
return ldistinct(stages)
|
||||||
@@ -49,7 +45,7 @@ def collect_stages(
|
|||||||
|
|
||||||
def get_subgraph(
|
def get_subgraph(
|
||||||
graph: "DiGraph",
|
graph: "DiGraph",
|
||||||
nodes: Optional[List] = None,
|
nodes: Optional[list] = None,
|
||||||
pipeline: bool = False,
|
pipeline: bool = False,
|
||||||
downstream: bool = False,
|
downstream: bool = False,
|
||||||
) -> "DiGraph":
|
) -> "DiGraph":
|
||||||
@@ -76,10 +72,10 @@ def get_active_graph(graph: "DiGraph") -> "DiGraph":
|
|||||||
|
|
||||||
def plan_repro(
|
def plan_repro(
|
||||||
graph: "DiGraph",
|
graph: "DiGraph",
|
||||||
stages: Optional[List["T"]] = None,
|
stages: Optional[list["T"]] = None,
|
||||||
pipeline: bool = False,
|
pipeline: bool = False,
|
||||||
downstream: bool = False,
|
downstream: bool = False,
|
||||||
) -> List["T"]:
|
) -> list["T"]:
|
||||||
r"""Derive the evaluation of the given node for the given graph.
|
r"""Derive the evaluation of the given node for the given graph.
|
||||||
|
|
||||||
When you _reproduce a stage_, you want to _evaluate the descendants_
|
When you _reproduce a stage_, you want to _evaluate the descendants_
|
||||||
@@ -134,7 +130,7 @@ def _reproduce_stage(stage: "Stage", **kwargs) -> Optional["Stage"]:
|
|||||||
|
|
||||||
def _get_upstream_downstream_nodes(
|
def _get_upstream_downstream_nodes(
|
||||||
graph: Optional["DiGraph"], node: T
|
graph: Optional["DiGraph"], node: T
|
||||||
) -> Tuple[List[T], List[T]]:
|
) -> tuple[list[T], list[T]]:
|
||||||
succ = list(graph.successors(node)) if graph else []
|
succ = list(graph.successors(node)) if graph else []
|
||||||
pre = list(graph.predecessors(node)) if graph else []
|
pre = list(graph.predecessors(node)) if graph else []
|
||||||
return succ, pre
|
return succ, pre
|
||||||
@@ -146,7 +142,7 @@ def _repr(stages: Iterable["Stage"]) -> str:
|
|||||||
|
|
||||||
def handle_error(
|
def handle_error(
|
||||||
graph: Optional["DiGraph"], on_error: str, exc: Exception, stage: "Stage"
|
graph: Optional["DiGraph"], on_error: str, exc: Exception, stage: "Stage"
|
||||||
) -> Set["Stage"]:
|
) -> set["Stage"]:
|
||||||
import networkx as nx
|
import networkx as nx
|
||||||
|
|
||||||
logger.warning("%s%s", exc, " (ignored)" if on_error == "ignore" else "")
|
logger.warning("%s%s", exc, " (ignored)" if on_error == "ignore" else "")
|
||||||
@@ -168,19 +164,19 @@ def _raise_error(exc: Optional[Exception], *stages: "Stage") -> NoReturn:
|
|||||||
|
|
||||||
|
|
||||||
def _reproduce(
|
def _reproduce(
|
||||||
stages: List["Stage"],
|
stages: list["Stage"],
|
||||||
graph: Optional["DiGraph"] = None,
|
graph: Optional["DiGraph"] = None,
|
||||||
force_downstream: bool = False,
|
force_downstream: bool = False,
|
||||||
on_error: str = "fail",
|
on_error: str = "fail",
|
||||||
force: bool = False,
|
force: bool = False,
|
||||||
repro_fn: Callable = _reproduce_stage,
|
repro_fn: Callable = _reproduce_stage,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
) -> List["Stage"]:
|
) -> list["Stage"]:
|
||||||
assert on_error in ("fail", "keep-going", "ignore")
|
assert on_error in ("fail", "keep-going", "ignore")
|
||||||
|
|
||||||
result: List["Stage"] = []
|
result: list["Stage"] = []
|
||||||
failed: List["Stage"] = []
|
failed: list["Stage"] = []
|
||||||
to_skip: Dict["Stage", "Stage"] = {}
|
to_skip: dict["Stage", "Stage"] = {}
|
||||||
ret: Optional["Stage"] = None
|
ret: Optional["Stage"] = None
|
||||||
|
|
||||||
force_state = dict.fromkeys(stages, force)
|
force_state = dict.fromkeys(stages, force)
|
||||||
@@ -240,7 +236,7 @@ def reproduce(
|
|||||||
if not kwargs.get("interactive", False):
|
if not kwargs.get("interactive", False):
|
||||||
kwargs["interactive"] = self.config["core"].get("interactive", False)
|
kwargs["interactive"] = self.config["core"].get("interactive", False)
|
||||||
|
|
||||||
stages: List["Stage"] = []
|
stages: list["Stage"] = []
|
||||||
if not all_pipelines:
|
if not all_pipelines:
|
||||||
targets_list = ensure_list(targets or PROJECT_FILE)
|
targets_list = ensure_list(targets or PROJECT_FILE)
|
||||||
stages = collect_stages(self, targets_list, recursive=recursive, glob=glob)
|
stages = collect_stages(self, targets_list, recursive=recursive, glob=glob)
|
||||||
|
|||||||
@@ -1,16 +1,12 @@
|
|||||||
import logging
|
import logging
|
||||||
import shlex
|
import shlex
|
||||||
|
from collections.abc import Iterable, Iterator
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from functools import wraps
|
from functools import wraps
|
||||||
from typing import (
|
from typing import (
|
||||||
TYPE_CHECKING,
|
TYPE_CHECKING,
|
||||||
Any,
|
Any,
|
||||||
Dict,
|
|
||||||
Iterable,
|
|
||||||
Iterator,
|
|
||||||
List,
|
|
||||||
Optional,
|
Optional,
|
||||||
Set,
|
|
||||||
Union,
|
Union,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -27,15 +23,15 @@ logger = logger.getChild(__name__)
|
|||||||
|
|
||||||
|
|
||||||
class SCMContext:
|
class SCMContext:
|
||||||
def __init__(self, scm: "Base", config: Optional[Dict[str, Any]] = None) -> None:
|
def __init__(self, scm: "Base", config: Optional[dict[str, Any]] = None) -> None:
|
||||||
from funcy import get_in
|
from funcy import get_in
|
||||||
|
|
||||||
self.scm: "Base" = scm
|
self.scm: "Base" = scm
|
||||||
self.autostage: bool = get_in(
|
self.autostage: bool = get_in(
|
||||||
config or {}, ["core", "autostage"], default=False
|
config or {}, ["core", "autostage"], default=False
|
||||||
)
|
)
|
||||||
self.ignored_paths: List[str] = []
|
self.ignored_paths: list[str] = []
|
||||||
self.files_to_track: Set[str] = set()
|
self.files_to_track: set[str] = set()
|
||||||
self.quiet: bool = False
|
self.quiet: bool = False
|
||||||
|
|
||||||
def track_file(self, paths: Union[str, Iterable[str], None] = None) -> None:
|
def track_file(self, paths: Union[str, Iterable[str], None] = None) -> None:
|
||||||
|
|||||||
+6
-5
@@ -1,8 +1,9 @@
|
|||||||
import fnmatch
|
import fnmatch
|
||||||
import typing
|
import typing
|
||||||
|
from collections.abc import Iterable
|
||||||
from contextlib import suppress
|
from contextlib import suppress
|
||||||
from functools import wraps
|
from functools import wraps
|
||||||
from typing import Iterable, List, NamedTuple, Optional, Set, Tuple, Union
|
from typing import NamedTuple, Optional, Union
|
||||||
|
|
||||||
from dvc.exceptions import (
|
from dvc.exceptions import (
|
||||||
NoOutputOrStageError,
|
NoOutputOrStageError,
|
||||||
@@ -31,9 +32,9 @@ class StageInfo(NamedTuple):
|
|||||||
filter_info: Optional[str] = None
|
filter_info: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
StageList = List["Stage"]
|
StageList = list["Stage"]
|
||||||
StageIter = Iterable["Stage"]
|
StageIter = Iterable["Stage"]
|
||||||
StageSet = Set["Stage"]
|
StageSet = set["Stage"]
|
||||||
|
|
||||||
|
|
||||||
def _collect_with_deps(stages: StageList, graph: "DiGraph") -> StageSet:
|
def _collect_with_deps(stages: StageList, graph: "DiGraph") -> StageSet:
|
||||||
@@ -72,7 +73,7 @@ def _collect_specific_target(
|
|||||||
target: str,
|
target: str,
|
||||||
with_deps: bool,
|
with_deps: bool,
|
||||||
recursive: bool,
|
recursive: bool,
|
||||||
) -> Tuple[StageIter, Optional[str], Optional[str]]:
|
) -> tuple[StageIter, Optional[str], Optional[str]]:
|
||||||
from dvc.dvcfile import is_valid_filename
|
from dvc.dvcfile import is_valid_filename
|
||||||
|
|
||||||
# Optimization: do not collect the graph for a specific target
|
# Optimization: do not collect the graph for a specific target
|
||||||
@@ -363,7 +364,7 @@ class StageLoad:
|
|||||||
with_deps: bool = False,
|
with_deps: bool = False,
|
||||||
recursive: bool = False,
|
recursive: bool = False,
|
||||||
graph: Optional["DiGraph"] = None,
|
graph: Optional["DiGraph"] = None,
|
||||||
) -> List[StageInfo]:
|
) -> list[StageInfo]:
|
||||||
"""Collects a list of (stage, filter_info) from the given target.
|
"""Collects a list of (stage, filter_info) from the given target.
|
||||||
|
|
||||||
Priority is in the order of following in case of ambiguity:
|
Priority is in the order of following in case of ambiguity:
|
||||||
|
|||||||
+2
-2
@@ -1,4 +1,4 @@
|
|||||||
from typing import TYPE_CHECKING, List
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from dvc.exceptions import InvalidArgumentError
|
from dvc.exceptions import InvalidArgumentError
|
||||||
|
|
||||||
@@ -34,7 +34,7 @@ def update( # noqa: C901
|
|||||||
raise InvalidArgumentError("--remote can't be used without --to-remote")
|
raise InvalidArgumentError("--remote can't be used without --to-remote")
|
||||||
|
|
||||||
import_stages = set()
|
import_stages = set()
|
||||||
other_stage_infos: List["StageInfo"] = []
|
other_stage_infos: list["StageInfo"] = []
|
||||||
|
|
||||||
for stage_info in self.index.collect_targets(targets, recursive=recursive):
|
for stage_info in self.index.collect_targets(targets, recursive=recursive):
|
||||||
if stage_info.stage.is_import:
|
if stage_info.stage.is_import:
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
|
from collections.abc import Iterable
|
||||||
from functools import partial
|
from functools import partial
|
||||||
from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Tuple, Union
|
from typing import TYPE_CHECKING, Any, Optional, Union
|
||||||
|
|
||||||
from funcy import first
|
from funcy import first
|
||||||
|
|
||||||
@@ -35,7 +36,7 @@ def worktree_view_by_remotes(
|
|||||||
targets: Optional["TargetType"] = None,
|
targets: Optional["TargetType"] = None,
|
||||||
push: bool = False,
|
push: bool = False,
|
||||||
**kwargs: Any,
|
**kwargs: Any,
|
||||||
) -> Iterable[Tuple[Optional[str], "IndexView"]]:
|
) -> Iterable[tuple[Optional[str], "IndexView"]]:
|
||||||
from dvc.repo.index import IndexView
|
from dvc.repo.index import IndexView
|
||||||
|
|
||||||
def outs_filter(view: "IndexView", remote: Optional[str]):
|
def outs_filter(view: "IndexView", remote: Optional[str]):
|
||||||
@@ -176,7 +177,7 @@ def update_worktree_stages(
|
|||||||
outs_filter=outs_filter,
|
outs_filter=outs_filter,
|
||||||
)
|
)
|
||||||
local_index = view.data["repo"]
|
local_index = view.data["repo"]
|
||||||
remote_indexes: Dict[str, Tuple["Remote", "DataIndex"]] = {}
|
remote_indexes: dict[str, tuple["Remote", "DataIndex"]] = {}
|
||||||
for stage in view.stages:
|
for stage in view.stages:
|
||||||
for out in stage.outs:
|
for out in stage.outs:
|
||||||
_update_worktree_out(repo, out, local_index, remote_indexes)
|
_update_worktree_out(repo, out, local_index, remote_indexes)
|
||||||
@@ -187,7 +188,7 @@ def _update_worktree_out(
|
|||||||
repo: "Repo",
|
repo: "Repo",
|
||||||
out: "Output",
|
out: "Output",
|
||||||
local_index: Union["DataIndex", "DataIndexView"],
|
local_index: Union["DataIndex", "DataIndexView"],
|
||||||
remote_indexes: Dict[str, Tuple["Remote", "DataIndex"]],
|
remote_indexes: dict[str, tuple["Remote", "DataIndex"]],
|
||||||
):
|
):
|
||||||
from dvc_data.index import build
|
from dvc_data.index import build
|
||||||
|
|
||||||
@@ -283,7 +284,7 @@ def _get_diff_indexes(
|
|||||||
out: "Output",
|
out: "Output",
|
||||||
local_index: Union["DataIndex", "DataIndexView"],
|
local_index: Union["DataIndex", "DataIndexView"],
|
||||||
remote_index: Union["DataIndex", "DataIndexView"],
|
remote_index: Union["DataIndex", "DataIndexView"],
|
||||||
) -> Tuple["DataIndex", "DataIndex"]:
|
) -> tuple["DataIndex", "DataIndex"]:
|
||||||
from dvc_data.index import DataIndex
|
from dvc_data.index import DataIndex
|
||||||
|
|
||||||
_, key = out.index_key
|
_, key = out.index_key
|
||||||
|
|||||||
+2
-2
@@ -1,5 +1,5 @@
|
|||||||
from collections.abc import Mapping
|
from collections.abc import Mapping
|
||||||
from typing import Any, Dict
|
from typing import Any
|
||||||
|
|
||||||
import voluptuous as vol
|
import voluptuous as vol
|
||||||
|
|
||||||
@@ -28,7 +28,7 @@ SINGLE_STAGE_SCHEMA = {
|
|||||||
StageParams.PARAM_DESC: str,
|
StageParams.PARAM_DESC: str,
|
||||||
}
|
}
|
||||||
|
|
||||||
DATA_SCHEMA: Dict[Any, Any] = {
|
DATA_SCHEMA: dict[Any, Any] = {
|
||||||
**CHECKSUMS_SCHEMA,
|
**CHECKSUMS_SCHEMA,
|
||||||
**META_SCHEMA,
|
**META_SCHEMA,
|
||||||
vol.Required("path"): str,
|
vol.Required("path"): str,
|
||||||
|
|||||||
+6
-8
@@ -1,13 +1,11 @@
|
|||||||
"""Manages source control systems (e.g. Git)."""
|
"""Manages source control systems (e.g. Git)."""
|
||||||
import os
|
import os
|
||||||
|
from collections.abc import Iterator, Mapping
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from functools import partial
|
from functools import partial
|
||||||
from typing import (
|
from typing import (
|
||||||
TYPE_CHECKING,
|
TYPE_CHECKING,
|
||||||
Iterator,
|
|
||||||
List,
|
|
||||||
Literal,
|
Literal,
|
||||||
Mapping,
|
|
||||||
Optional,
|
Optional,
|
||||||
Union,
|
Union,
|
||||||
overload,
|
overload,
|
||||||
@@ -189,7 +187,7 @@ def resolve_rev(scm: Union["Git", "NoSCM"], rev: str) -> str:
|
|||||||
raise RevError(str(exc)) # noqa: B904
|
raise RevError(str(exc)) # noqa: B904
|
||||||
|
|
||||||
|
|
||||||
def _get_n_commits(scm: "Git", revs: List[str], num: int) -> List[str]:
|
def _get_n_commits(scm: "Git", revs: list[str], num: int) -> list[str]:
|
||||||
results = []
|
results = []
|
||||||
for rev in revs:
|
for rev in revs:
|
||||||
if num == 0:
|
if num == 0:
|
||||||
@@ -210,14 +208,14 @@ def _get_n_commits(scm: "Git", revs: List[str], num: int) -> List[str]:
|
|||||||
|
|
||||||
def iter_revs(
|
def iter_revs(
|
||||||
scm: "Git",
|
scm: "Git",
|
||||||
revs: Optional[List[str]] = None,
|
revs: Optional[list[str]] = None,
|
||||||
num: int = 1,
|
num: int = 1,
|
||||||
all_branches: bool = False,
|
all_branches: bool = False,
|
||||||
all_tags: bool = False,
|
all_tags: bool = False,
|
||||||
all_commits: bool = False,
|
all_commits: bool = False,
|
||||||
all_experiments: bool = False,
|
all_experiments: bool = False,
|
||||||
commit_date: Optional[str] = None,
|
commit_date: Optional[str] = None,
|
||||||
) -> Mapping[str, List[str]]:
|
) -> Mapping[str, list[str]]:
|
||||||
from scmrepo.exceptions import SCMError as _SCMError
|
from scmrepo.exceptions import SCMError as _SCMError
|
||||||
|
|
||||||
from dvc.repo.experiments.utils import exp_commits
|
from dvc.repo.experiments.utils import exp_commits
|
||||||
@@ -235,7 +233,7 @@ def iter_revs(
|
|||||||
return {}
|
return {}
|
||||||
|
|
||||||
revs = revs or []
|
revs = revs or []
|
||||||
results: List[str] = _get_n_commits(scm, revs, num)
|
results: list[str] = _get_n_commits(scm, revs, num)
|
||||||
|
|
||||||
if all_commits:
|
if all_commits:
|
||||||
results.extend(scm.list_all_commits())
|
results.extend(scm.list_all_commits())
|
||||||
@@ -266,7 +264,7 @@ def iter_revs(
|
|||||||
return group_by(rev_resolver, results)
|
return group_by(rev_resolver, results)
|
||||||
|
|
||||||
|
|
||||||
def lfs_prefetch(fs: "FileSystem", paths: List[str]):
|
def lfs_prefetch(fs: "FileSystem", paths: list[str]):
|
||||||
from scmrepo.git.lfs import fetch as _lfs_fetch
|
from scmrepo.git.lfs import fetch as _lfs_fetch
|
||||||
|
|
||||||
from dvc.fs.dvc import DVCFileSystem
|
from dvc.fs.dvc import DVCFileSystem
|
||||||
|
|||||||
+21
-26
@@ -1,18 +1,13 @@
|
|||||||
import os
|
import os
|
||||||
import string
|
import string
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
|
from collections.abc import Iterable
|
||||||
from contextlib import suppress
|
from contextlib import suppress
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import (
|
from typing import (
|
||||||
TYPE_CHECKING,
|
TYPE_CHECKING,
|
||||||
Any,
|
Any,
|
||||||
Dict,
|
|
||||||
Iterable,
|
|
||||||
List,
|
|
||||||
Optional,
|
Optional,
|
||||||
Set,
|
|
||||||
Tuple,
|
|
||||||
Type,
|
|
||||||
TypeVar,
|
TypeVar,
|
||||||
Union,
|
Union,
|
||||||
)
|
)
|
||||||
@@ -55,14 +50,14 @@ if TYPE_CHECKING:
|
|||||||
logger = logger.getChild(__name__)
|
logger = logger.getChild(__name__)
|
||||||
# Disallow all punctuation characters except hyphen and underscore
|
# Disallow all punctuation characters except hyphen and underscore
|
||||||
INVALID_STAGENAME_CHARS = set(string.punctuation) - {"_", "-"}
|
INVALID_STAGENAME_CHARS = set(string.punctuation) - {"_", "-"}
|
||||||
Env = Dict[str, str]
|
Env = dict[str, str]
|
||||||
ChangedEntries = Tuple[List[str], List[str], Optional[str]]
|
ChangedEntries = tuple[list[str], list[str], Optional[str]]
|
||||||
|
|
||||||
_T = TypeVar("_T")
|
_T = TypeVar("_T")
|
||||||
|
|
||||||
|
|
||||||
def loads_from(
|
def loads_from(
|
||||||
cls: Type[_T], repo: "Repo", path: str, wdir: str, data: Dict[str, Any]
|
cls: type[_T], repo: "Repo", path: str, wdir: str, data: dict[str, Any]
|
||||||
) -> _T:
|
) -> _T:
|
||||||
kw = {
|
kw = {
|
||||||
"repo": repo,
|
"repo": repo,
|
||||||
@@ -91,7 +86,7 @@ class RawData:
|
|||||||
generated_from: Optional[str] = None
|
generated_from: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
def create_stage(cls: Type[_T], repo, path, **kwargs) -> _T:
|
def create_stage(cls: type[_T], repo, path, **kwargs) -> _T:
|
||||||
from dvc.dvcfile import check_dvcfile_path
|
from dvc.dvcfile import check_dvcfile_path
|
||||||
|
|
||||||
wdir = os.path.abspath(kwargs.get("wdir", None) or os.curdir)
|
wdir = os.path.abspath(kwargs.get("wdir", None) or os.curdir)
|
||||||
@@ -205,13 +200,13 @@ class Stage(params.StageParams):
|
|||||||
self._dvcfile = dvcfile
|
self._dvcfile = dvcfile
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def params(self) -> List["ParamsDependency"]:
|
def params(self) -> list["ParamsDependency"]:
|
||||||
from dvc.dependency import ParamsDependency
|
from dvc.dependency import ParamsDependency
|
||||||
|
|
||||||
return [dep for dep in self.deps if isinstance(dep, ParamsDependency)]
|
return [dep for dep in self.deps if isinstance(dep, ParamsDependency)]
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def metrics(self) -> List["Output"]:
|
def metrics(self) -> list["Output"]:
|
||||||
return [out for out in self.outs if out.metric]
|
return [out for out in self.outs if out.metric]
|
||||||
|
|
||||||
def __repr__(self):
|
def __repr__(self):
|
||||||
@@ -310,7 +305,7 @@ class Stage(params.StageParams):
|
|||||||
return desc
|
return desc
|
||||||
|
|
||||||
def changed_deps(
|
def changed_deps(
|
||||||
self, allow_missing: bool = False, upstream: Optional[List] = None
|
self, allow_missing: bool = False, upstream: Optional[list] = None
|
||||||
) -> bool:
|
) -> bool:
|
||||||
if self.frozen:
|
if self.frozen:
|
||||||
return False
|
return False
|
||||||
@@ -322,7 +317,7 @@ class Stage(params.StageParams):
|
|||||||
|
|
||||||
@rwlocked(read=["deps"])
|
@rwlocked(read=["deps"])
|
||||||
def _changed_deps(
|
def _changed_deps(
|
||||||
self, allow_missing: bool = False, upstream: Optional[List] = None
|
self, allow_missing: bool = False, upstream: Optional[list] = None
|
||||||
) -> bool:
|
) -> bool:
|
||||||
for dep in self.deps:
|
for dep in self.deps:
|
||||||
status = dep.status()
|
status = dep.status()
|
||||||
@@ -370,7 +365,7 @@ class Stage(params.StageParams):
|
|||||||
|
|
||||||
@rwlocked(read=["deps", "outs"])
|
@rwlocked(read=["deps", "outs"])
|
||||||
def changed(
|
def changed(
|
||||||
self, allow_missing: bool = False, upstream: Optional[List] = None
|
self, allow_missing: bool = False, upstream: Optional[list] = None
|
||||||
) -> bool:
|
) -> bool:
|
||||||
is_changed = (
|
is_changed = (
|
||||||
# Short-circuit order: stage md5 is fast,
|
# Short-circuit order: stage md5 is fast,
|
||||||
@@ -477,7 +472,7 @@ class Stage(params.StageParams):
|
|||||||
def reload(self) -> "Stage":
|
def reload(self) -> "Stage":
|
||||||
return self.dvcfile.stage
|
return self.dvcfile.stage
|
||||||
|
|
||||||
def dumpd(self, **kwargs) -> Dict[str, Any]:
|
def dumpd(self, **kwargs) -> dict[str, Any]:
|
||||||
return get_dump(self, **kwargs)
|
return get_dump(self, **kwargs)
|
||||||
|
|
||||||
def compute_md5(self) -> Optional[str]:
|
def compute_md5(self) -> Optional[str]:
|
||||||
@@ -509,7 +504,7 @@ class Stage(params.StageParams):
|
|||||||
if not allow_missing:
|
if not allow_missing:
|
||||||
raise
|
raise
|
||||||
|
|
||||||
def get_versioned_outs(self) -> Dict[str, "Output"]:
|
def get_versioned_outs(self) -> dict[str, "Output"]:
|
||||||
from .exceptions import StageFileDoesNotExistError, StageNotFound
|
from .exceptions import StageFileDoesNotExistError, StageNotFound
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -543,7 +538,7 @@ class Stage(params.StageParams):
|
|||||||
out.ignore()
|
out.ignore()
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _changed_entries(entries) -> List[str]:
|
def _changed_entries(entries) -> list[str]:
|
||||||
return [str(entry) for entry in entries if entry.workspace_status()]
|
return [str(entry) for entry in entries if entry.workspace_status()]
|
||||||
|
|
||||||
def _changed_stage_entry(self) -> str:
|
def _changed_stage_entry(self) -> str:
|
||||||
@@ -663,8 +658,8 @@ class Stage(params.StageParams):
|
|||||||
@rwlocked(write=["outs"])
|
@rwlocked(write=["outs"])
|
||||||
def checkout(
|
def checkout(
|
||||||
self, allow_missing: bool = False, **kwargs
|
self, allow_missing: bool = False, **kwargs
|
||||||
) -> Dict[str, List["StrPath"]]:
|
) -> dict[str, list["StrPath"]]:
|
||||||
stats: Dict[str, List["StrPath"]] = defaultdict(list)
|
stats: dict[str, list["StrPath"]] = defaultdict(list)
|
||||||
if self.is_partial_import:
|
if self.is_partial_import:
|
||||||
return stats
|
return stats
|
||||||
|
|
||||||
@@ -679,7 +674,7 @@ class Stage(params.StageParams):
|
|||||||
return stats
|
return stats
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _checkout(out, **kwargs) -> Tuple[Optional[str], List[str]]:
|
def _checkout(out, **kwargs) -> tuple[Optional[str], list[str]]:
|
||||||
try:
|
try:
|
||||||
result = out.checkout(**kwargs)
|
result = out.checkout(**kwargs)
|
||||||
added, modified = result or (None, None)
|
added, modified = result or (None, None)
|
||||||
@@ -692,8 +687,8 @@ class Stage(params.StageParams):
|
|||||||
@rwlocked(read=["deps", "outs"])
|
@rwlocked(read=["deps", "outs"])
|
||||||
def status(
|
def status(
|
||||||
self, check_updates: bool = False, filter_info: Optional[bool] = None
|
self, check_updates: bool = False, filter_info: Optional[bool] = None
|
||||||
) -> Dict[str, List[Union[str, Dict[str, str]]]]:
|
) -> dict[str, list[Union[str, dict[str, str]]]]:
|
||||||
ret: List[Union[str, Dict[str, str]]] = []
|
ret: list[Union[str, dict[str, str]]] = []
|
||||||
show_import = (
|
show_import = (
|
||||||
self.is_repo_import or self.is_versioned_import
|
self.is_repo_import or self.is_versioned_import
|
||||||
) and check_updates
|
) and check_updates
|
||||||
@@ -706,7 +701,7 @@ class Stage(params.StageParams):
|
|||||||
return {self.addressing: ret} if ret else {}
|
return {self.addressing: ret} if ret else {}
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _status(entries: Iterable["Output"]) -> Dict[str, str]:
|
def _status(entries: Iterable["Output"]) -> dict[str, str]:
|
||||||
ret = {}
|
ret = {}
|
||||||
|
|
||||||
for entry in entries:
|
for entry in entries:
|
||||||
@@ -747,7 +742,7 @@ class Stage(params.StageParams):
|
|||||||
|
|
||||||
def get_used_objs(
|
def get_used_objs(
|
||||||
self, *args, **kwargs
|
self, *args, **kwargs
|
||||||
) -> Dict[Optional["HashFileDB"], Set["HashInfo"]]:
|
) -> dict[Optional["HashFileDB"], set["HashInfo"]]:
|
||||||
"""Return set of object IDs used by this stage."""
|
"""Return set of object IDs used by this stage."""
|
||||||
if self.is_partial_import and not self.is_repo_import:
|
if self.is_partial_import and not self.is_repo_import:
|
||||||
return {}
|
return {}
|
||||||
@@ -802,7 +797,7 @@ class PipelineStage(Stage):
|
|||||||
super().__init__(*args, **kwargs)
|
super().__init__(*args, **kwargs)
|
||||||
self.name = name
|
self.name = name
|
||||||
self.cmd_changed = False
|
self.cmd_changed = False
|
||||||
self.tracked_vars: Dict[str, Dict[str, Dict[str, str]]] = {}
|
self.tracked_vars: dict[str, dict[str, dict[str, str]]] = {}
|
||||||
|
|
||||||
def __eq__(self, other):
|
def __eq__(self, other):
|
||||||
return super().__eq__(other) and self.name == other.name
|
return super().__eq__(other) and self.name == other.name
|
||||||
|
|||||||
+2
-2
@@ -1,6 +1,6 @@
|
|||||||
import os
|
import os
|
||||||
from contextlib import contextmanager
|
from contextlib import contextmanager
|
||||||
from typing import TYPE_CHECKING, List, Optional, Tuple
|
from typing import TYPE_CHECKING, Optional
|
||||||
|
|
||||||
from funcy import first
|
from funcy import first
|
||||||
|
|
||||||
@@ -247,7 +247,7 @@ class StageCache:
|
|||||||
message = f"run-cache is not supported for http filesystem: {path}"
|
message = f"run-cache is not supported for http filesystem: {path}"
|
||||||
raise RunCacheNotSupported(message)
|
raise RunCacheNotSupported(message)
|
||||||
|
|
||||||
ret: List[Tuple[str, str]] = []
|
ret: list[tuple[str, str]] = []
|
||||||
if not from_fs.exists(runs):
|
if not from_fs.exists(runs):
|
||||||
return ret
|
return ret
|
||||||
|
|
||||||
|
|||||||
+7
-5
@@ -1,7 +1,7 @@
|
|||||||
from collections.abc import Mapping
|
from collections.abc import Mapping
|
||||||
from copy import deepcopy
|
from copy import deepcopy
|
||||||
from itertools import chain
|
from itertools import chain
|
||||||
from typing import TYPE_CHECKING, Any, Dict, Iterable, Optional, Tuple
|
from typing import TYPE_CHECKING, Any, Optional
|
||||||
|
|
||||||
from funcy import get_in, lcat, once, project
|
from funcy import get_in, lcat, once, project
|
||||||
|
|
||||||
@@ -17,6 +17,8 @@ from .params import StageParams
|
|||||||
from .utils import fill_stage_dependencies, resolve_paths
|
from .utils import fill_stage_dependencies, resolve_paths
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
|
from collections.abc import Iterable
|
||||||
|
|
||||||
from dvc.dvcfile import ProjectFile, SingleStageFile
|
from dvc.dvcfile import ProjectFile, SingleStageFile
|
||||||
|
|
||||||
logger = logger.getChild(__name__)
|
logger = logger.getChild(__name__)
|
||||||
@@ -39,7 +41,7 @@ class StageLoader(Mapping):
|
|||||||
self._lockfile_data = lockfile_data.get("stages", {})
|
self._lockfile_data = lockfile_data.get("stages", {})
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def lockfile_data(self) -> Dict[str, Any]:
|
def lockfile_data(self) -> dict[str, Any]:
|
||||||
if not self._lockfile_data:
|
if not self._lockfile_data:
|
||||||
logger.debug("Lockfile for '%s' not found", self.dvcfile.relpath)
|
logger.debug("Lockfile for '%s' not found", self.dvcfile.relpath)
|
||||||
return self._lockfile_data
|
return self._lockfile_data
|
||||||
@@ -53,7 +55,7 @@ class StageLoader(Mapping):
|
|||||||
from dvc.output import Output, merge_file_meta_from_cloud
|
from dvc.output import Output, merge_file_meta_from_cloud
|
||||||
|
|
||||||
assert isinstance(lock_data, dict)
|
assert isinstance(lock_data, dict)
|
||||||
items: Iterable[Tuple[str, "Output"]] = chain(
|
items: Iterable[tuple[str, "Output"]] = chain(
|
||||||
((StageParams.PARAM_DEPS, dep) for dep in stage.deps),
|
((StageParams.PARAM_DEPS, dep) for dep in stage.deps),
|
||||||
((StageParams.PARAM_OUTS, out) for out in stage.outs),
|
((StageParams.PARAM_OUTS, out) for out in stage.outs),
|
||||||
)
|
)
|
||||||
@@ -174,7 +176,7 @@ class SingleStageLoader(Mapping):
|
|||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
dvcfile: "SingleStageFile",
|
dvcfile: "SingleStageFile",
|
||||||
stage_data: Dict[Any, str],
|
stage_data: dict[Any, str],
|
||||||
stage_text: Optional[str] = None,
|
stage_text: Optional[str] = None,
|
||||||
):
|
):
|
||||||
self.dvcfile = dvcfile
|
self.dvcfile = dvcfile
|
||||||
@@ -196,7 +198,7 @@ class SingleStageLoader(Mapping):
|
|||||||
def load_stage(
|
def load_stage(
|
||||||
cls,
|
cls,
|
||||||
dvcfile: "SingleStageFile",
|
dvcfile: "SingleStageFile",
|
||||||
d: Dict[str, Any],
|
d: dict[str, Any],
|
||||||
stage_text: Optional[str],
|
stage_text: Optional[str],
|
||||||
) -> Stage:
|
) -> Stage:
|
||||||
path, wdir = resolve_paths(
|
path, wdir = resolve_paths(
|
||||||
|
|||||||
@@ -1,11 +1,9 @@
|
|||||||
from collections import OrderedDict
|
from collections import OrderedDict
|
||||||
|
from collections.abc import Iterable
|
||||||
from operator import attrgetter
|
from operator import attrgetter
|
||||||
from typing import (
|
from typing import (
|
||||||
TYPE_CHECKING,
|
TYPE_CHECKING,
|
||||||
Any,
|
Any,
|
||||||
Dict,
|
|
||||||
Iterable,
|
|
||||||
List,
|
|
||||||
Optional,
|
Optional,
|
||||||
Union,
|
Union,
|
||||||
no_type_check,
|
no_type_check,
|
||||||
@@ -67,7 +65,7 @@ def _serialize_out(out):
|
|||||||
|
|
||||||
|
|
||||||
@no_type_check
|
@no_type_check
|
||||||
def _serialize_outs(outputs: List[Output]):
|
def _serialize_outs(outputs: list[Output]):
|
||||||
outs, metrics, plots = [], [], []
|
outs, metrics, plots = [], [], []
|
||||||
for out in sorted(outputs, key=attrgetter("def_path")):
|
for out in sorted(outputs, key=attrgetter("def_path")):
|
||||||
bucket = outs
|
bucket = outs
|
||||||
@@ -88,10 +86,10 @@ def _serialize_params_keys(params: Iterable["ParamsDependency"]):
|
|||||||
at the first, and then followed by entry of other files in lexicographic
|
at the first, and then followed by entry of other files in lexicographic
|
||||||
order. The keys of those custom files are also sorted in the same order.
|
order. The keys of those custom files are also sorted in the same order.
|
||||||
"""
|
"""
|
||||||
keys: List[Union[str, Dict[str, Optional[List[str]]]]] = []
|
keys: list[Union[str, dict[str, Optional[list[str]]]]] = []
|
||||||
for param_dep in sorted(params, key=attrgetter("def_path")):
|
for param_dep in sorted(params, key=attrgetter("def_path")):
|
||||||
# when on no_exec, params are not filled and are saved as list
|
# when on no_exec, params are not filled and are saved as list
|
||||||
k: List[str] = sorted(param_dep.params)
|
k: list[str] = sorted(param_dep.params)
|
||||||
if k and param_dep.def_path == DEFAULT_PARAMS_FILE:
|
if k and param_dep.def_path == DEFAULT_PARAMS_FILE:
|
||||||
keys = k + keys # type: ignore[operator,assignment]
|
keys = k + keys # type: ignore[operator,assignment]
|
||||||
else:
|
else:
|
||||||
@@ -100,7 +98,7 @@ def _serialize_params_keys(params: Iterable["ParamsDependency"]):
|
|||||||
|
|
||||||
|
|
||||||
@no_type_check
|
@no_type_check
|
||||||
def _serialize_params_values(params: List[ParamsDependency]):
|
def _serialize_params_values(params: list[ParamsDependency]):
|
||||||
"""Returns output of following format, used for lockfile:
|
"""Returns output of following format, used for lockfile:
|
||||||
{'params.yaml': {'lr': '1', 'train': 2}, {'params2.yaml': {'lr': '1'}}
|
{'params.yaml': {'lr': '1', 'train': 2}, {'params2.yaml': {'lr': '1'}}
|
||||||
|
|
||||||
@@ -160,7 +158,7 @@ def to_single_stage_lockfile(stage: "Stage", **kwargs) -> dict:
|
|||||||
assert stage.cmd
|
assert stage.cmd
|
||||||
|
|
||||||
def _dumpd(item: "Output"):
|
def _dumpd(item: "Output"):
|
||||||
ret: Dict[str, Any] = {item.PARAM_PATH: item.def_path}
|
ret: dict[str, Any] = {item.PARAM_PATH: item.def_path}
|
||||||
if item.hash_name not in LEGACY_HASH_NAMES:
|
if item.hash_name not in LEGACY_HASH_NAMES:
|
||||||
ret[item.PARAM_HASH] = "md5"
|
ret[item.PARAM_HASH] = "md5"
|
||||||
if item.hash_info.isdir and kwargs.get("with_files"):
|
if item.hash_info.isdir and kwargs.get("with_files"):
|
||||||
|
|||||||
+5
-5
@@ -1,6 +1,6 @@
|
|||||||
import os
|
import os
|
||||||
import pathlib
|
import pathlib
|
||||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
|
from typing import TYPE_CHECKING, Any, Optional, Union
|
||||||
|
|
||||||
from funcy import concat, first, lsplit, rpartial
|
from funcy import concat, first, lsplit, rpartial
|
||||||
|
|
||||||
@@ -210,7 +210,7 @@ def get_dump(stage: "Stage", **kwargs):
|
|||||||
|
|
||||||
def split_params_deps(
|
def split_params_deps(
|
||||||
stage: "Stage",
|
stage: "Stage",
|
||||||
) -> Tuple[List["ParamsDependency"], List["Dependency"]]:
|
) -> tuple[list["ParamsDependency"], list["Dependency"]]:
|
||||||
from dvc.dependency import ParamsDependency
|
from dvc.dependency import ParamsDependency
|
||||||
|
|
||||||
return lsplit(rpartial(isinstance, ParamsDependency), stage.deps)
|
return lsplit(rpartial(isinstance, ParamsDependency), stage.deps)
|
||||||
@@ -265,7 +265,7 @@ def check_stage_exists(repo: "Repo", stage: Union["Stage", "PipelineStage"], pat
|
|||||||
|
|
||||||
def validate_kwargs(
|
def validate_kwargs(
|
||||||
single_stage: bool = False, fname: Optional[str] = None, **kwargs
|
single_stage: bool = False, fname: Optional[str] = None, **kwargs
|
||||||
) -> Dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
"""Prepare, validate and process kwargs passed from cli"""
|
"""Prepare, validate and process kwargs passed from cli"""
|
||||||
cmd = kwargs.get("cmd")
|
cmd = kwargs.get("cmd")
|
||||||
if not cmd and not single_stage:
|
if not cmd and not single_stage:
|
||||||
@@ -288,11 +288,11 @@ def validate_kwargs(
|
|||||||
return kwargs
|
return kwargs
|
||||||
|
|
||||||
|
|
||||||
def _get_stage_files(stage: "Stage") -> List[str]:
|
def _get_stage_files(stage: "Stage") -> list[str]:
|
||||||
from dvc.dvcfile import ProjectFile
|
from dvc.dvcfile import ProjectFile
|
||||||
from dvc.utils import relpath
|
from dvc.utils import relpath
|
||||||
|
|
||||||
ret: List[str] = []
|
ret: list[str] = []
|
||||||
file = stage.dvcfile
|
file = stage.dvcfile
|
||||||
ret.append(file.relpath)
|
ret.append(file.relpath)
|
||||||
if isinstance(file, ProjectFile):
|
if isinstance(file, ProjectFile):
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import os
|
|||||||
import shutil
|
import shutil
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from subprocess import check_output
|
from subprocess import check_output
|
||||||
from typing import Dict, Optional
|
from typing import Optional
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from dulwich.porcelain import clone
|
from dulwich.porcelain import clone
|
||||||
@@ -27,7 +27,7 @@ class VirtualEnv:
|
|||||||
|
|
||||||
virtualenv.cli_run([os.fspath(self.path)])
|
virtualenv.cli_run([os.fspath(self.path)])
|
||||||
|
|
||||||
def run(self, cmd: str, *args: str, env: Optional[Dict[str, str]] = None) -> None:
|
def run(self, cmd: str, *args: str, env: Optional[dict[str, str]] = None) -> None:
|
||||||
exe = self.which(cmd)
|
exe = self.which(cmd)
|
||||||
check_output([exe, *args], env=env) # noqa: S603
|
check_output([exe, *args], env=env) # noqa: S603
|
||||||
|
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user