mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-11 23:10:10 +01:00
exp: refactor base.py into refs.py and exceptions.py
This commit is contained in:
committed by
Peter Rowlands (변기호)
parent
6143146974
commit
7113568c60
@@ -11,16 +11,9 @@ from dvc.env import DVCLIVE_RESUME
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.utils import relpath
|
||||
|
||||
from .base import (
|
||||
EXEC_APPLY,
|
||||
EXEC_BRANCH,
|
||||
EXEC_CHECKPOINT,
|
||||
EXEC_NAMESPACE,
|
||||
EXPS_NAMESPACE,
|
||||
EXPS_STASH,
|
||||
from .exceptions import (
|
||||
BaselineMismatchError,
|
||||
ExperimentExistsError,
|
||||
ExpRefInfo,
|
||||
InvalidExpRefError,
|
||||
MultipleBranchError,
|
||||
)
|
||||
@@ -36,6 +29,15 @@ from .executor.manager.local import (
|
||||
WorkspaceExecutorManager,
|
||||
)
|
||||
from .executor.manager.ssh import SSHExecutorManager
|
||||
from .refs import (
|
||||
EXEC_APPLY,
|
||||
EXEC_BRANCH,
|
||||
EXEC_CHECKPOINT,
|
||||
EXEC_NAMESPACE,
|
||||
EXPS_NAMESPACE,
|
||||
EXPS_STASH,
|
||||
ExpRefInfo,
|
||||
)
|
||||
from .stash import ExpStash, ExpStashEntry
|
||||
from .utils import exp_refs_by_rev
|
||||
|
||||
|
||||
@@ -7,13 +7,13 @@ from dvc.repo import locked
|
||||
from dvc.repo.scm_context import scm_context
|
||||
from dvc.utils.fs import remove
|
||||
|
||||
from .base import (
|
||||
EXEC_APPLY,
|
||||
from .exceptions import (
|
||||
ApplyConflictError,
|
||||
BaselineMismatchError,
|
||||
InvalidExpRevError,
|
||||
)
|
||||
from .executor.base import BaseExecutor
|
||||
from .refs import EXEC_APPLY
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scmrepo import Git
|
||||
|
||||
@@ -1,144 +0,0 @@
|
||||
from typing import Optional
|
||||
|
||||
from dvc.exceptions import DvcException, InvalidArgumentError
|
||||
|
||||
# Experiment refs are stored according baseline git SHA:
|
||||
# refs/exps/01/234abcd.../<exp_name>
|
||||
EXPS_NAMESPACE = "refs/exps"
|
||||
EXPS_STASH = f"{EXPS_NAMESPACE}/stash"
|
||||
EXEC_NAMESPACE = f"{EXPS_NAMESPACE}/exec"
|
||||
EXEC_APPLY = f"{EXEC_NAMESPACE}/EXEC_APPLY"
|
||||
EXEC_CHECKPOINT = f"{EXEC_NAMESPACE}/EXEC_CHECKPOINT"
|
||||
EXEC_BRANCH = f"{EXEC_NAMESPACE}/EXEC_BRANCH"
|
||||
EXEC_BASELINE = f"{EXEC_NAMESPACE}/EXEC_BASELINE"
|
||||
EXEC_HEAD = f"{EXEC_NAMESPACE}/EXEC_HEAD"
|
||||
EXEC_MERGE = f"{EXEC_NAMESPACE}/EXEC_MERGE"
|
||||
|
||||
|
||||
class UnchangedExperimentError(DvcException):
|
||||
def __init__(self, rev):
|
||||
super().__init__(f"Experiment unchanged from '{rev[:7]}'.")
|
||||
self.rev = rev
|
||||
|
||||
|
||||
class BaselineMismatchError(DvcException):
|
||||
def __init__(self, rev, expected):
|
||||
if hasattr(rev, "hexsha"):
|
||||
rev = rev.hexsha
|
||||
rev_str = f"{rev[:7]}" if rev is not None else "invalid commit"
|
||||
super().__init__(
|
||||
f"Experiment derived from '{rev_str}', expected '{expected[:7]}'."
|
||||
)
|
||||
self.rev = rev
|
||||
self.expected_rev = expected
|
||||
|
||||
|
||||
class ExperimentExistsError(DvcException):
|
||||
def __init__(self, name: str):
|
||||
msg = (
|
||||
"Experiment conflicts with existing experiment "
|
||||
f"'{name}'. To overwrite the existing experiment run:\n\n"
|
||||
"\tdvc exp run -f ...\n\n"
|
||||
"To run this experiment with a different name run:\n\n"
|
||||
f"\tdvc exp run -n <new_name> ...\n"
|
||||
)
|
||||
super().__init__(msg)
|
||||
self.name = name
|
||||
|
||||
|
||||
class CheckpointExistsError(DvcException):
|
||||
def __init__(self, name: str):
|
||||
msg = (
|
||||
"Reproduced checkpoint experiment conflicts with existing "
|
||||
f"experiment '{name}'. To restart (and overwrite) the existing "
|
||||
"experiment run:\n\n"
|
||||
"\tdvc exp run -f ...\n\n"
|
||||
"To resume the existing experiment, run:\n\n"
|
||||
f"\tdvc exp apply {name}\n"
|
||||
f"\tdvc exp run\n"
|
||||
)
|
||||
super().__init__(msg)
|
||||
self.name = name
|
||||
|
||||
|
||||
class InvalidExpRefError(DvcException):
|
||||
def __init__(self, ref):
|
||||
super().__init__(f"'{ref}' is not a valid experiment refname.")
|
||||
self.ref = ref
|
||||
|
||||
|
||||
class InvalidExpRevError(InvalidArgumentError):
|
||||
def __init__(self, rev):
|
||||
super().__init__(
|
||||
f"'{rev}' does not appear to be an experiment commit."
|
||||
)
|
||||
|
||||
|
||||
class MultipleBranchError(DvcException):
|
||||
def __init__(self, rev, ref_infos):
|
||||
super().__init__(
|
||||
f"Ambiguous commit '{rev[:7]}' belongs to multiple experiment "
|
||||
"branches."
|
||||
)
|
||||
self.rev = rev
|
||||
self.ref_infos = ref_infos
|
||||
|
||||
|
||||
class ApplyConflictError(InvalidArgumentError):
|
||||
def __init__(self, name):
|
||||
super().__init__(
|
||||
f"Experiment '{name}' cannot be applied to because your current "
|
||||
"workspace contains changes which would be overwritten. Either "
|
||||
"'git stash' your current changes before applying this "
|
||||
"experiment, or re-run this command with '--force' to overwrite "
|
||||
"your current changes."
|
||||
)
|
||||
self.name = name
|
||||
|
||||
|
||||
class ExpRefInfo:
|
||||
|
||||
namespace = EXPS_NAMESPACE
|
||||
|
||||
def __init__(
|
||||
self, baseline_sha: Optional[str] = None, name: Optional[str] = None
|
||||
):
|
||||
self.baseline_sha = baseline_sha
|
||||
self.name: str = name if name else ""
|
||||
|
||||
def __str__(self):
|
||||
return "/".join(self.parts)
|
||||
|
||||
def __repr__(self):
|
||||
baseline = f"'{self.baseline_sha}'" if self.baseline_sha else "None"
|
||||
name = f"'{self.name}'" if self.name else "None"
|
||||
return f"ExpRefInfo(baseline_sha={baseline}, name={name})"
|
||||
|
||||
@property
|
||||
def parts(self):
|
||||
return (
|
||||
(self.namespace,)
|
||||
+ (
|
||||
(self.baseline_sha[:2], self.baseline_sha[2:])
|
||||
if self.baseline_sha
|
||||
else ()
|
||||
)
|
||||
+ ((self.name,) if self.name else ())
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_ref(cls, ref: str):
|
||||
try:
|
||||
parts = ref.split("/")
|
||||
if (
|
||||
len(parts) < 2
|
||||
or len(parts) == 3
|
||||
or len(parts) > 5
|
||||
or "/".join(parts[:2]) != EXPS_NAMESPACE
|
||||
):
|
||||
raise InvalidExpRefError(ref)
|
||||
except ValueError:
|
||||
raise InvalidExpRefError(ref)
|
||||
baseline_sha = parts[2] + parts[3] if len(parts) >= 4 else None
|
||||
name = parts[4] if len(parts) == 5 else None
|
||||
return cls(baseline_sha, name)
|
||||
@@ -5,7 +5,7 @@ from dvc.repo import locked
|
||||
from dvc.repo.scm_context import scm_context
|
||||
from dvc.scm import RevError
|
||||
|
||||
from .base import InvalidExpRevError
|
||||
from .exceptions import InvalidExpRevError
|
||||
from .utils import exp_refs_by_rev
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -1,15 +1,97 @@
|
||||
from typing import Collection, Iterable
|
||||
from typing import TYPE_CHECKING, Collection, Iterable
|
||||
|
||||
from dvc.exceptions import InvalidArgumentError
|
||||
from dvc.exceptions import DvcException, InvalidArgumentError
|
||||
|
||||
from .base import ExpRefInfo
|
||||
if TYPE_CHECKING:
|
||||
from .refs import ExpRefInfo
|
||||
|
||||
|
||||
class UnchangedExperimentError(DvcException):
|
||||
def __init__(self, rev):
|
||||
super().__init__(f"Experiment unchanged from '{rev[:7]}'.")
|
||||
self.rev = rev
|
||||
|
||||
|
||||
class BaselineMismatchError(DvcException):
|
||||
def __init__(self, rev, expected):
|
||||
if hasattr(rev, "hexsha"):
|
||||
rev = rev.hexsha
|
||||
rev_str = f"{rev[:7]}" if rev is not None else "invalid commit"
|
||||
super().__init__(
|
||||
f"Experiment derived from '{rev_str}', expected '{expected[:7]}'."
|
||||
)
|
||||
self.rev = rev
|
||||
self.expected_rev = expected
|
||||
|
||||
|
||||
class ExperimentExistsError(DvcException):
|
||||
def __init__(self, name: str):
|
||||
msg = (
|
||||
"Experiment conflicts with existing experiment "
|
||||
f"'{name}'. To overwrite the existing experiment run:\n\n"
|
||||
"\tdvc exp run -f ...\n\n"
|
||||
"To run this experiment with a different name run:\n\n"
|
||||
f"\tdvc exp run -n <new_name> ...\n"
|
||||
)
|
||||
super().__init__(msg)
|
||||
self.name = name
|
||||
|
||||
|
||||
class CheckpointExistsError(DvcException):
|
||||
def __init__(self, name: str):
|
||||
msg = (
|
||||
"Reproduced checkpoint experiment conflicts with existing "
|
||||
f"experiment '{name}'. To restart (and overwrite) the existing "
|
||||
"experiment run:\n\n"
|
||||
"\tdvc exp run -f ...\n\n"
|
||||
"To resume the existing experiment, run:\n\n"
|
||||
f"\tdvc exp apply {name}\n"
|
||||
f"\tdvc exp run\n"
|
||||
)
|
||||
super().__init__(msg)
|
||||
self.name = name
|
||||
|
||||
|
||||
class InvalidExpRefError(DvcException):
|
||||
def __init__(self, ref):
|
||||
super().__init__(f"'{ref}' is not a valid experiment refname.")
|
||||
self.ref = ref
|
||||
|
||||
|
||||
class InvalidExpRevError(InvalidArgumentError):
|
||||
def __init__(self, rev):
|
||||
super().__init__(
|
||||
f"'{rev}' does not appear to be an experiment commit."
|
||||
)
|
||||
|
||||
|
||||
class MultipleBranchError(DvcException):
|
||||
def __init__(self, rev, ref_infos):
|
||||
super().__init__(
|
||||
f"Ambiguous commit '{rev[:7]}' belongs to multiple experiment "
|
||||
"branches."
|
||||
)
|
||||
self.rev = rev
|
||||
self.ref_infos = ref_infos
|
||||
|
||||
|
||||
class ApplyConflictError(InvalidArgumentError):
|
||||
def __init__(self, name):
|
||||
super().__init__(
|
||||
f"Experiment '{name}' cannot be applied to because your current "
|
||||
"workspace contains changes which would be overwritten. Either "
|
||||
"'git stash' your current changes before applying this "
|
||||
"experiment, or re-run this command with '--force' to overwrite "
|
||||
"your current changes."
|
||||
)
|
||||
self.name = name
|
||||
|
||||
|
||||
class AmbiguousExpRefInfo(InvalidArgumentError):
|
||||
def __init__(
|
||||
self,
|
||||
exp_name: str,
|
||||
exp_ref_list: Iterable[ExpRefInfo],
|
||||
exp_ref_list: Iterable["ExpRefInfo"],
|
||||
):
|
||||
msg = [
|
||||
(
|
||||
|
||||
@@ -25,17 +25,19 @@ from dvc.stage.serialize import to_lockfile
|
||||
from dvc.utils import dict_sha256, env2bool, relpath
|
||||
from dvc.utils.fs import remove
|
||||
|
||||
from ..base import (
|
||||
from ..exceptions import (
|
||||
CheckpointExistsError,
|
||||
ExperimentExistsError,
|
||||
UnchangedExperimentError,
|
||||
)
|
||||
from ..refs import (
|
||||
EXEC_BASELINE,
|
||||
EXEC_BRANCH,
|
||||
EXEC_CHECKPOINT,
|
||||
EXEC_NAMESPACE,
|
||||
EXPS_NAMESPACE,
|
||||
EXPS_STASH,
|
||||
CheckpointExistsError,
|
||||
ExperimentExistsError,
|
||||
ExpRefInfo,
|
||||
UnchangedExperimentError,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
||||
@@ -10,7 +10,7 @@ from scmrepo.exceptions import SCMError as _SCMError
|
||||
from dvc.scm import SCM, GitMergeError
|
||||
from dvc.utils.fs import remove
|
||||
|
||||
from ..base import (
|
||||
from ..refs import (
|
||||
EXEC_APPLY,
|
||||
EXEC_BASELINE,
|
||||
EXEC_BRANCH,
|
||||
@@ -26,7 +26,8 @@ if TYPE_CHECKING:
|
||||
|
||||
from dvc.repo import Repo
|
||||
|
||||
from ..base import ExpRefInfo, ExpStashEntry
|
||||
from ..refs import ExpRefInfo
|
||||
from ..stash import ExpStashEntry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -7,14 +7,8 @@ from typing import TYPE_CHECKING, Deque, Dict, Generator, Optional, Tuple, Type
|
||||
|
||||
from dvc.proc.manager import ProcessManager
|
||||
|
||||
from ...base import (
|
||||
EXEC_BASELINE,
|
||||
EXEC_HEAD,
|
||||
EXEC_MERGE,
|
||||
CheckpointExistsError,
|
||||
ExperimentExistsError,
|
||||
ExpRefInfo,
|
||||
)
|
||||
from ...exceptions import CheckpointExistsError, ExperimentExistsError
|
||||
from ...refs import EXEC_BASELINE, EXEC_HEAD, EXEC_MERGE, ExpRefInfo
|
||||
from ...stash import ExpStashEntry
|
||||
from ..base import EXEC_PID_DIR, BaseExecutor
|
||||
from ..local import TempDirExecutor, WorkspaceExecutor
|
||||
|
||||
@@ -2,7 +2,7 @@ import logging
|
||||
from collections import defaultdict
|
||||
from typing import TYPE_CHECKING, Dict, Optional
|
||||
|
||||
from ...base import EXEC_BASELINE, EXEC_BRANCH, EXEC_HEAD, EXEC_MERGE
|
||||
from ...refs import EXEC_BASELINE, EXEC_BRANCH, EXEC_HEAD, EXEC_MERGE
|
||||
from ...stash import ExpStashEntry
|
||||
from ..local import TempDirExecutor, WorkspaceExecutor
|
||||
from .base import BaseExecutorManager
|
||||
|
||||
@@ -8,14 +8,14 @@ from typing import TYPE_CHECKING, Callable, Iterable, Optional
|
||||
from funcy import first
|
||||
|
||||
from dvc.fs import SSHFileSystem
|
||||
from dvc.repo.experiments.base import (
|
||||
|
||||
from ..refs import (
|
||||
EXEC_BRANCH,
|
||||
EXEC_CHECKPOINT,
|
||||
EXEC_HEAD,
|
||||
EXEC_MERGE,
|
||||
EXEC_NAMESPACE,
|
||||
)
|
||||
|
||||
from .base import BaseExecutor, ExecutorInfo, ExecutorResult
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
||||
@@ -9,8 +9,8 @@ from dvc.repo.scm_context import scm_context
|
||||
from dvc.scm import TqdmGit, iter_revs
|
||||
from dvc.ui import ui
|
||||
|
||||
from .base import ExpRefInfo
|
||||
from .exceptions import UnresolvedExpNamesError
|
||||
from .refs import ExpRefInfo
|
||||
from .utils import exp_commits, exp_refs, exp_refs_by_baseline, resolve_name
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -9,8 +9,8 @@ from dvc.repo.scm_context import scm_context
|
||||
from dvc.scm import TqdmGit, iter_revs
|
||||
from dvc.ui import ui
|
||||
|
||||
from .base import ExpRefInfo
|
||||
from .exceptions import UnresolvedExpNamesError
|
||||
from .refs import ExpRefInfo
|
||||
from .utils import exp_commits, exp_refs, exp_refs_by_baseline, resolve_name
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
from typing import Optional
|
||||
|
||||
from .exceptions import InvalidExpRefError
|
||||
|
||||
# Experiment refs are stored according baseline git SHA:
|
||||
# refs/exps/01/234abcd.../<exp_name>
|
||||
EXPS_NAMESPACE = "refs/exps"
|
||||
EXPS_STASH = f"{EXPS_NAMESPACE}/stash"
|
||||
EXEC_NAMESPACE = f"{EXPS_NAMESPACE}/exec"
|
||||
EXEC_APPLY = f"{EXEC_NAMESPACE}/EXEC_APPLY"
|
||||
EXEC_CHECKPOINT = f"{EXEC_NAMESPACE}/EXEC_CHECKPOINT"
|
||||
EXEC_BRANCH = f"{EXEC_NAMESPACE}/EXEC_BRANCH"
|
||||
EXEC_BASELINE = f"{EXEC_NAMESPACE}/EXEC_BASELINE"
|
||||
EXEC_HEAD = f"{EXEC_NAMESPACE}/EXEC_HEAD"
|
||||
EXEC_MERGE = f"{EXEC_NAMESPACE}/EXEC_MERGE"
|
||||
|
||||
|
||||
class ExpRefInfo:
|
||||
|
||||
namespace = EXPS_NAMESPACE
|
||||
|
||||
def __init__(
|
||||
self, baseline_sha: Optional[str] = None, name: Optional[str] = None
|
||||
):
|
||||
self.baseline_sha = baseline_sha
|
||||
self.name: str = name if name else ""
|
||||
|
||||
def __str__(self):
|
||||
return "/".join(self.parts)
|
||||
|
||||
def __repr__(self):
|
||||
baseline = f"'{self.baseline_sha}'" if self.baseline_sha else "None"
|
||||
name = f"'{self.name}'" if self.name else "None"
|
||||
return f"ExpRefInfo(baseline_sha={baseline}, name={name})"
|
||||
|
||||
@property
|
||||
def parts(self):
|
||||
return (
|
||||
(self.namespace,)
|
||||
+ (
|
||||
(self.baseline_sha[:2], self.baseline_sha[2:])
|
||||
if self.baseline_sha
|
||||
else ()
|
||||
)
|
||||
+ ((self.name,) if self.name else ())
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_ref(cls, ref: str):
|
||||
try:
|
||||
parts = ref.split("/")
|
||||
if (
|
||||
len(parts) < 2
|
||||
or len(parts) == 3
|
||||
or len(parts) > 5
|
||||
or "/".join(parts[:2]) != EXPS_NAMESPACE
|
||||
):
|
||||
raise InvalidExpRefError(ref)
|
||||
except ValueError:
|
||||
raise InvalidExpRefError(ref)
|
||||
baseline_sha = parts[2] + parts[3] if len(parts) >= 4 else None
|
||||
name = parts[4] if len(parts) == 5 else None
|
||||
return cls(baseline_sha, name)
|
||||
@@ -6,6 +6,7 @@ from dvc.repo.scm_context import scm_context
|
||||
from dvc.scm import iter_revs
|
||||
|
||||
from .exceptions import UnresolvedExpNamesError
|
||||
from .refs import ExpRefInfo
|
||||
from .utils import (
|
||||
exp_refs,
|
||||
exp_refs_by_baseline,
|
||||
@@ -18,8 +19,6 @@ if TYPE_CHECKING:
|
||||
from dvc.repo import Repo
|
||||
from dvc.scm import Git
|
||||
|
||||
from .base import ExpRefInfo
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -45,7 +44,7 @@ def remove(
|
||||
removed.extend(_clear_all_commits(repo, git_remote))
|
||||
return removed
|
||||
|
||||
commit_ref_dict: Dict["ExpRefInfo", str] = {}
|
||||
commit_ref_dict: Dict[ExpRefInfo, str] = {}
|
||||
queued_ref_dict: Dict[int, str] = {}
|
||||
if exp_names:
|
||||
_resolve_exp_by_name(
|
||||
@@ -69,7 +68,7 @@ def remove(
|
||||
def _resolve_exp_by_name(
|
||||
repo: "Repo",
|
||||
exp_names: Union[str, List[str]],
|
||||
commit_ref_dict: Dict["ExpRefInfo", str],
|
||||
commit_ref_dict: Dict[ExpRefInfo, str],
|
||||
queued_ref_dict: Dict[int, str],
|
||||
git_remote: Optional[str],
|
||||
):
|
||||
@@ -99,7 +98,7 @@ def _resolve_exp_by_baseline(
|
||||
repo,
|
||||
rev: str,
|
||||
num: int,
|
||||
commit_ref_dict: Dict["ExpRefInfo", str],
|
||||
commit_ref_dict: Dict[ExpRefInfo, str],
|
||||
git_remote: Optional[str] = None,
|
||||
):
|
||||
rev_dict = iter_revs(repo.scm, [rev], num)
|
||||
@@ -156,7 +155,7 @@ def _get_queued_index_by_names(
|
||||
|
||||
|
||||
def _remove_commited_exps(
|
||||
scm: "Git", exp_ref_dict: Mapping["ExpRefInfo", str], remote: Optional[str]
|
||||
scm: "Git", exp_ref_dict: Mapping[ExpRefInfo, str], remote: Optional[str]
|
||||
) -> List[str]:
|
||||
if remote:
|
||||
from dvc.scm import TqdmGit
|
||||
|
||||
@@ -3,12 +3,13 @@ from collections import OrderedDict, defaultdict
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union
|
||||
|
||||
from dvc.repo.experiments.base import ExpRefInfo
|
||||
from dvc.repo.metrics.show import _gather_metrics
|
||||
from dvc.repo.params.show import _gather_params
|
||||
from dvc.scm import iter_revs
|
||||
from dvc.utils import error_handler, onerror_collect, relpath
|
||||
|
||||
from .refs import ExpRefInfo
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.repo import Repo
|
||||
|
||||
|
||||
@@ -16,8 +16,11 @@ from scmrepo.git import Git
|
||||
from dvc.exceptions import InvalidArgumentError
|
||||
from dvc.repo.experiments.exceptions import AmbiguousExpRefInfo
|
||||
|
||||
from .base import (
|
||||
from .refs import (
|
||||
EXEC_APPLY,
|
||||
EXEC_BASELINE,
|
||||
EXEC_BRANCH,
|
||||
EXEC_CHECKPOINT,
|
||||
EXEC_NAMESPACE,
|
||||
EXPS_NAMESPACE,
|
||||
EXPS_STASH,
|
||||
@@ -40,9 +43,7 @@ def exp_refs(
|
||||
yield ExpRefInfo.from_ref(ref)
|
||||
|
||||
|
||||
def exp_refs_by_rev(
|
||||
scm: "Git", rev: str
|
||||
) -> Generator["ExpRefInfo", None, None]:
|
||||
def exp_refs_by_rev(scm: "Git", rev: str) -> Generator[ExpRefInfo, None, None]:
|
||||
"""Iterate over all experiment refs pointing to the specified revision."""
|
||||
for ref in scm.get_refs_containing(rev, EXPS_NAMESPACE):
|
||||
if not (ref.startswith(EXEC_NAMESPACE) or ref == EXPS_STASH):
|
||||
@@ -121,6 +122,14 @@ def push_refspec(
|
||||
raise GitAuthError(str(exc))
|
||||
|
||||
|
||||
def remote_exp_refs(scm: "Git", url: str) -> Generator[ExpRefInfo, None, None]:
|
||||
"""Iterate over all remote experiment refs."""
|
||||
for ref in iter_remote_refs(scm, url, base=EXPS_NAMESPACE):
|
||||
if ref.startswith(EXEC_NAMESPACE) or ref == EXPS_STASH:
|
||||
continue
|
||||
yield ExpRefInfo.from_ref(ref)
|
||||
|
||||
|
||||
def exp_refs_by_names(
|
||||
scm: "Git", names: Set[str], url: Optional[str] = None
|
||||
) -> Dict[str, List[ExpRefInfo]]:
|
||||
@@ -134,8 +143,19 @@ def exp_refs_by_names(
|
||||
return resolve_results
|
||||
|
||||
|
||||
def remote_exp_refs_by_baseline(
|
||||
scm: "Git", url: str, rev: str
|
||||
) -> Generator[ExpRefInfo, None, None]:
|
||||
"""Iterate over all remote experiment refs with the specified baseline."""
|
||||
ref_info = ExpRefInfo(baseline_sha=rev)
|
||||
for ref in iter_remote_refs(scm, url, base=str(ref_info)):
|
||||
if ref.startswith(EXEC_NAMESPACE) or ref == EXPS_STASH:
|
||||
continue
|
||||
yield ExpRefInfo.from_ref(ref)
|
||||
|
||||
|
||||
def exp_commits(
|
||||
scm: "Git", ref_infos: Iterable["ExpRefInfo"] = None
|
||||
scm: "Git", ref_infos: Iterable[ExpRefInfo] = None
|
||||
) -> Iterable[str]:
|
||||
"""Iterate over all experiment commits."""
|
||||
shas: Set["str"] = set()
|
||||
@@ -147,9 +167,7 @@ def exp_commits(
|
||||
yield from shas
|
||||
|
||||
|
||||
def remove_exp_refs(scm: "Git", ref_infos: Iterable["ExpRefInfo"]):
|
||||
from .base import EXEC_APPLY, EXEC_BRANCH, EXEC_CHECKPOINT
|
||||
|
||||
def remove_exp_refs(scm: "Git", ref_infos: Iterable[ExpRefInfo]):
|
||||
exec_branch = scm.get_ref(EXEC_BRANCH, follow=False)
|
||||
exec_apply = scm.get_ref(EXEC_APPLY)
|
||||
exec_checkpoint = scm.get_ref(EXEC_CHECKPOINT)
|
||||
|
||||
@@ -7,9 +7,9 @@ import pytest
|
||||
from dvc_ssh.tests.cloud import TEST_SSH_KEY_PATH, TEST_SSH_USER
|
||||
|
||||
from dvc.fs import SSHFileSystem
|
||||
from dvc.repo.experiments.base import EXEC_HEAD, EXEC_MERGE
|
||||
from dvc.repo.experiments.executor.base import ExecutorInfo, ExecutorResult
|
||||
from dvc.repo.experiments.executor.ssh import SSHExecutor
|
||||
from dvc.repo.experiments.refs import EXEC_HEAD, EXEC_MERGE
|
||||
from tests.func.machine.conftest import * # noqa, pylint: disable=wildcard-import
|
||||
|
||||
|
||||
|
||||
@@ -6,9 +6,9 @@ from funcy import first
|
||||
from dvc.config import NoRemoteError
|
||||
from dvc.env import DVC_EXP_AUTO_PUSH, DVC_EXP_GIT_REMOTE
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.repo.experiments import MultipleBranchError
|
||||
from dvc.repo.experiments.base import EXEC_APPLY, EXEC_CHECKPOINT
|
||||
from dvc.repo.experiments.exceptions import MultipleBranchError
|
||||
from dvc.repo.experiments.executor.base import BaseExecutor
|
||||
from dvc.repo.experiments.refs import EXEC_APPLY, EXEC_CHECKPOINT
|
||||
from dvc.repo.experiments.utils import exp_refs_by_rev
|
||||
from dvc.scm import InvalidRemoteSCMRepo
|
||||
|
||||
|
||||
@@ -47,7 +47,7 @@ def test_new_simple(tmp_dir, scm, dvc, exp_stage, mocker, name, workspace):
|
||||
|
||||
@pytest.mark.parametrize("workspace", [True, False])
|
||||
def test_experiment_exists(tmp_dir, scm, dvc, exp_stage, mocker, workspace):
|
||||
from dvc.repo.experiments.base import ExperimentExistsError
|
||||
from dvc.repo.experiments.exceptions import ExperimentExistsError
|
||||
|
||||
dvc.experiments.run(
|
||||
exp_stage.addressing,
|
||||
@@ -175,7 +175,7 @@ def test_add_params(tmp_dir, scm, dvc, changes):
|
||||
@pytest.mark.parametrize("queue", [True, False])
|
||||
def test_apply(tmp_dir, scm, dvc, exp_stage, queue):
|
||||
from dvc.exceptions import InvalidArgumentError
|
||||
from dvc.repo.experiments.base import ApplyConflictError
|
||||
from dvc.repo.experiments.exceptions import ApplyConflictError
|
||||
|
||||
metrics_original = (tmp_dir / "metrics.yaml").read_text().strip()
|
||||
results = dvc.experiments.run(
|
||||
@@ -219,7 +219,7 @@ def test_apply(tmp_dir, scm, dvc, exp_stage, queue):
|
||||
|
||||
|
||||
def test_apply_untracked(tmp_dir, scm, dvc, exp_stage):
|
||||
from dvc.repo.experiments.base import ApplyConflictError
|
||||
from dvc.repo.experiments.exceptions import ApplyConflictError
|
||||
|
||||
results = dvc.experiments.run(exp_stage.addressing, params=["foo=2"])
|
||||
exp = first(results)
|
||||
@@ -238,7 +238,7 @@ def test_apply_untracked(tmp_dir, scm, dvc, exp_stage):
|
||||
|
||||
|
||||
def test_get_baseline(tmp_dir, scm, dvc, exp_stage):
|
||||
from dvc.repo.experiments.base import EXPS_STASH
|
||||
from dvc.repo.experiments.refs import EXPS_STASH
|
||||
|
||||
init_rev = scm.get_rev()
|
||||
assert dvc.experiments.get_baseline(init_rev) is None
|
||||
@@ -642,7 +642,7 @@ def test_checkout_targets_deps(tmp_dir, scm, dvc, exp_stage):
|
||||
|
||||
@pytest.mark.parametrize("tail", ["", "~1", "^"])
|
||||
def test_fix_exp_head(tmp_dir, scm, tail):
|
||||
from dvc.repo.experiments.base import EXEC_BASELINE
|
||||
from dvc.repo.experiments.refs import EXEC_BASELINE
|
||||
from dvc.repo.experiments.utils import fix_exp_head
|
||||
|
||||
head = "HEAD" + tail
|
||||
|
||||
@@ -7,13 +7,13 @@ import pytest
|
||||
from funcy import first, get_in
|
||||
|
||||
from dvc.cli import main
|
||||
from dvc.repo.experiments.base import EXPS_STASH, ExpRefInfo
|
||||
from dvc.repo.experiments.executor.base import (
|
||||
EXEC_PID_DIR,
|
||||
EXEC_TMP_DIR,
|
||||
BaseExecutor,
|
||||
ExecutorInfo,
|
||||
)
|
||||
from dvc.repo.experiments.refs import EXPS_STASH, ExpRefInfo
|
||||
from dvc.repo.experiments.utils import exp_refs_by_rev
|
||||
from dvc.utils.fs import makedirs
|
||||
from dvc.utils.serialize import YAMLFileCorruptedError
|
||||
@@ -392,8 +392,8 @@ def test_show_running_executor(tmp_dir, scm, dvc, exp_stage):
|
||||
def test_show_running_checkpoint(
|
||||
tmp_dir, scm, dvc, checkpoint_stage, workspace, mocker
|
||||
):
|
||||
from dvc.repo.experiments.base import EXEC_BASELINE, EXEC_BRANCH
|
||||
from dvc.repo.experiments.executor.local import TempDirExecutor
|
||||
from dvc.repo.experiments.refs import EXEC_BASELINE, EXEC_BRANCH
|
||||
|
||||
baseline_rev = scm.get_rev()
|
||||
dvc.experiments.run(
|
||||
|
||||
@@ -364,7 +364,7 @@ def test_gc_external_output(tmp_dir, dvc, workspace):
|
||||
|
||||
|
||||
def test_gc_all_experiments(tmp_dir, scm, dvc):
|
||||
from dvc.repo.experiments.base import ExpRefInfo
|
||||
from dvc.repo.experiments.refs import ExpRefInfo
|
||||
|
||||
(foo,) = tmp_dir.dvc_gen("foo", "foo", commit="foo")
|
||||
foo_hash = foo.outs[0].hash_info.value
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import pytest
|
||||
|
||||
from dvc.exceptions import InvalidArgumentError
|
||||
from dvc.repo.experiments.base import EXPS_NAMESPACE, ExpRefInfo
|
||||
from dvc.repo.experiments.refs import EXPS_NAMESPACE, ExpRefInfo
|
||||
from dvc.repo.experiments.utils import check_ref_format, resolve_name
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user