mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-22 07:18:28 +01:00
exp: initial executor manager refactor
- introduce ExecutorManager classes - move executor init + execution from experiments.__init__ into the manager classes - make executors serializable via ExecutorInfo
This commit is contained in:
committed by
Peter Rowlands (변기호)
parent
8319cc50c6
commit
37f8010b42
@@ -1,12 +1,8 @@
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
from collections import defaultdict, namedtuple
|
||||
from concurrent.futures import CancelledError, ProcessPoolExecutor, wait
|
||||
from functools import wraps
|
||||
from multiprocessing import Manager
|
||||
from typing import Dict, Iterable, Mapping, Optional
|
||||
from typing import Dict, Iterable, List, Mapping, Optional
|
||||
|
||||
from funcy import cached_property, first
|
||||
|
||||
@@ -20,17 +16,22 @@ from .base import (
|
||||
EXEC_BASELINE,
|
||||
EXEC_BRANCH,
|
||||
EXEC_CHECKPOINT,
|
||||
EXEC_HEAD,
|
||||
EXEC_MERGE,
|
||||
EXEC_NAMESPACE,
|
||||
EXPS_NAMESPACE,
|
||||
EXPS_STASH,
|
||||
BaselineMismatchError,
|
||||
CheckpointExistsError,
|
||||
ExperimentExistsError,
|
||||
ExpRefInfo,
|
||||
ExpStashEntry,
|
||||
MultipleBranchError,
|
||||
)
|
||||
from .executor.base import (
|
||||
EXEC_PID_DIR,
|
||||
EXEC_TMP_DIR,
|
||||
BaseExecutor,
|
||||
ExecutorInfo,
|
||||
)
|
||||
from .executor.manager import BaseExecutorManager, LocalExecutorManager
|
||||
from .utils import exp_refs_by_rev
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -81,12 +82,6 @@ class Experiments:
|
||||
r"^(?P<baseline_rev>[a-f0-9]{7})-(?P<exp_sha>[a-f0-9]+)"
|
||||
r"(?P<checkpoint>-checkpoint)?$"
|
||||
)
|
||||
EXEC_TMP_DIR = "exps"
|
||||
EXEC_PID_DIR = os.path.join(EXEC_TMP_DIR, "run")
|
||||
|
||||
StashEntry = namedtuple(
|
||||
"StashEntry", ["index", "rev", "baseline_rev", "branch", "name"]
|
||||
)
|
||||
|
||||
def __init__(self, repo):
|
||||
from dvc.lock import make_lock
|
||||
@@ -112,8 +107,6 @@ class Experiments:
|
||||
|
||||
@cached_property
|
||||
def args_file(self):
|
||||
from .executor.base import BaseExecutor
|
||||
|
||||
return os.path.join(self.repo.tmp_dir, BaseExecutor.PACKED_ARGS_FILE)
|
||||
|
||||
@cached_property
|
||||
@@ -123,13 +116,13 @@ class Experiments:
|
||||
return Stash(self.scm, EXPS_STASH)
|
||||
|
||||
@property
|
||||
def stash_revs(self):
|
||||
def stash_revs(self) -> Dict[str, ExpStashEntry]:
|
||||
revs = {}
|
||||
for i, entry in enumerate(self.stash):
|
||||
msg = entry.message.decode("utf-8").strip()
|
||||
m = self.STASH_EXPERIMENT_RE.match(msg)
|
||||
if m:
|
||||
revs[entry.new_sha.decode("utf-8")] = self.StashEntry(
|
||||
revs[entry.new_sha.decode("utf-8")] = ExpStashEntry(
|
||||
i,
|
||||
m.group("rev"),
|
||||
m.group("baseline_rev"),
|
||||
@@ -324,8 +317,6 @@ class Experiments:
|
||||
def _pack_args(self, *args, **kwargs):
|
||||
import pickle
|
||||
|
||||
from .executor.base import BaseExecutor
|
||||
|
||||
if os.path.exists(self.args_file) and self.scm.is_tracked(
|
||||
self.args_file
|
||||
):
|
||||
@@ -616,7 +607,7 @@ class Experiments:
|
||||
to_run = {
|
||||
rev: stash_revs[rev]
|
||||
if rev in stash_revs
|
||||
else self.StashEntry(None, rev, rev, None, None)
|
||||
else ExpStashEntry(None, rev, rev, None, None)
|
||||
for rev in revs
|
||||
}
|
||||
|
||||
@@ -625,17 +616,22 @@ class Experiments:
|
||||
", ".join(rev[:7] for rev in to_run),
|
||||
)
|
||||
|
||||
executors = self._init_executors(to_run)
|
||||
manager = LocalExecutorManager.from_stash_entries(
|
||||
self.scm,
|
||||
os.path.join(self.repo.tmp_dir, EXEC_TMP_DIR),
|
||||
self.repo,
|
||||
to_run,
|
||||
)
|
||||
try:
|
||||
exec_results = {}
|
||||
exec_results.update(self._executors_repro(executors, **kwargs))
|
||||
exec_results.update(self._executors_repro(manager, **kwargs))
|
||||
finally:
|
||||
# only drop successfully run stashed experiments
|
||||
to_drop = [
|
||||
entry.index
|
||||
to_drop: List[int] = [
|
||||
entry.stash_index
|
||||
for rev, entry in to_run.items()
|
||||
if (
|
||||
entry.index is not None
|
||||
entry.stash_index is not None
|
||||
and (not keep_stash or rev in exec_results)
|
||||
)
|
||||
]
|
||||
@@ -647,47 +643,11 @@ class Experiments:
|
||||
result.update(exp_result)
|
||||
return result
|
||||
|
||||
def _init_executors(self, to_run):
|
||||
from dvc.utils.fs import makedirs
|
||||
|
||||
from .executor.local import TempDirExecutor
|
||||
|
||||
executors = {}
|
||||
base_tmp_dir = os.path.join(self.repo.tmp_dir, self.EXEC_TMP_DIR)
|
||||
if not os.path.exists(base_tmp_dir):
|
||||
makedirs(base_tmp_dir)
|
||||
pid_dir = os.path.join(self.repo.tmp_dir, self.EXEC_PID_DIR)
|
||||
if not os.path.exists(pid_dir):
|
||||
makedirs(pid_dir)
|
||||
for stash_rev, item in to_run.items():
|
||||
self.scm.set_ref(EXEC_HEAD, item.rev)
|
||||
self.scm.set_ref(EXEC_MERGE, stash_rev)
|
||||
self.scm.set_ref(EXEC_BASELINE, item.baseline_rev)
|
||||
|
||||
# Executor will be initialized with an empty git repo that
|
||||
# we populate by pushing:
|
||||
# EXEC_HEAD - the base commit for this experiment
|
||||
# EXEC_MERGE - the unmerged changes (from our stash)
|
||||
# to be reproduced
|
||||
# EXEC_BASELINE - the baseline commit for this experiment
|
||||
executor = TempDirExecutor(
|
||||
self.scm,
|
||||
self.dvc_dir,
|
||||
name=item.name,
|
||||
branch=item.branch,
|
||||
tmp_dir=base_tmp_dir,
|
||||
)
|
||||
executor.init_cache(self.repo, stash_rev)
|
||||
executors[stash_rev] = executor
|
||||
|
||||
for ref in (EXEC_HEAD, EXEC_MERGE, EXEC_BASELINE):
|
||||
self.scm.remove_ref(ref)
|
||||
|
||||
return executors
|
||||
|
||||
@unlocked_repo
|
||||
def _executors_repro(
|
||||
self, executors: dict, jobs: Optional[int] = 1
|
||||
self,
|
||||
manager: "BaseExecutorManager",
|
||||
**kwargs,
|
||||
) -> Dict[str, Dict[str, str]]:
|
||||
"""Run dvc repro for the specified BaseExecutors in parallel.
|
||||
|
||||
@@ -695,106 +655,7 @@ class Experiments:
|
||||
dict mapping stash revs to the successfully executed experiments
|
||||
for each stash rev.
|
||||
"""
|
||||
from dvc.stage.monitor import CheckpointKilledError
|
||||
|
||||
result: Dict[str, Dict[str, str]] = defaultdict(dict)
|
||||
|
||||
manager = Manager()
|
||||
pid_q = manager.Queue()
|
||||
|
||||
rel_cwd = relpath(os.getcwd(), self.repo.root_dir)
|
||||
with ProcessPoolExecutor(max_workers=jobs) as workers:
|
||||
futures = {}
|
||||
for rev, executor in executors.items():
|
||||
pidfile = os.path.join(
|
||||
self.repo.tmp_dir,
|
||||
self.EXEC_PID_DIR,
|
||||
f"{rev}{executor.PIDFILE_EXT}",
|
||||
)
|
||||
future = workers.submit(
|
||||
executor.reproduce,
|
||||
executor.dvc_dir,
|
||||
rev,
|
||||
queue=pid_q,
|
||||
name=executor.name,
|
||||
rel_cwd=rel_cwd,
|
||||
log_level=logger.getEffectiveLevel(),
|
||||
pidfile=pidfile,
|
||||
git_url=executor.git_url,
|
||||
)
|
||||
futures[future] = (rev, executor)
|
||||
|
||||
try:
|
||||
wait(futures)
|
||||
except KeyboardInterrupt:
|
||||
# forward SIGINT to any running executor processes and
|
||||
# cancel any remaining futures
|
||||
workers.shutdown(wait=False)
|
||||
pids = {}
|
||||
for future, (rev, _) in futures.items():
|
||||
if future.running():
|
||||
# if future has already been started by the scheduler
|
||||
# we still have to wait until it tells us its PID
|
||||
while rev not in pids:
|
||||
rev, pid = pid_q.get()
|
||||
pids[rev] = pid
|
||||
os.kill(pids[rev], signal.SIGINT)
|
||||
elif not future.done():
|
||||
future.cancel()
|
||||
|
||||
for future, (rev, executor) in futures.items():
|
||||
rev, executor = futures[future]
|
||||
|
||||
try:
|
||||
exc = future.exception()
|
||||
if exc is None:
|
||||
exec_result = future.result()
|
||||
result[rev].update(
|
||||
self._collect_executor(executor, exec_result)
|
||||
)
|
||||
elif not isinstance(exc, CheckpointKilledError):
|
||||
logger.error(
|
||||
"Failed to reproduce experiment '%s'", rev[:7]
|
||||
)
|
||||
except CancelledError:
|
||||
logger.error(
|
||||
"Cancelled before attempting to reproduce experiment "
|
||||
"'%s'",
|
||||
rev[:7],
|
||||
)
|
||||
finally:
|
||||
executor.cleanup()
|
||||
|
||||
return result
|
||||
|
||||
def _collect_executor(self, executor, exec_result) -> Mapping[str, str]:
|
||||
# NOTE: GitPython Repo instances cannot be re-used
|
||||
# after process has received SIGINT or SIGTERM, so we
|
||||
# need this hack to re-instantiate git instances after
|
||||
# checkpoint runs. See:
|
||||
# https://github.com/gitpython-developers/GitPython/issues/427
|
||||
del self.repo.scm
|
||||
|
||||
results = {}
|
||||
|
||||
def on_diverged(ref: str, checkpoint: bool):
|
||||
ref_info = ExpRefInfo.from_ref(ref)
|
||||
if checkpoint:
|
||||
raise CheckpointExistsError(ref_info.name)
|
||||
raise ExperimentExistsError(ref_info.name)
|
||||
|
||||
for ref in executor.fetch_exps(
|
||||
self.scm,
|
||||
executor.git_url,
|
||||
force=exec_result.force,
|
||||
on_diverged=on_diverged,
|
||||
):
|
||||
exp_rev = self.scm.get_ref(ref)
|
||||
if exp_rev:
|
||||
logger.debug("Collected experiment '%s'.", exp_rev[:7])
|
||||
results[exp_rev] = exec_result.exp_hash
|
||||
|
||||
return results
|
||||
return manager.exec_queue(**kwargs)
|
||||
|
||||
@unlocked_repo
|
||||
def _workspace_repro(self) -> Mapping[str, str]:
|
||||
@@ -802,17 +663,15 @@ class Experiments:
|
||||
from dvc.stage.monitor import CheckpointKilledError
|
||||
from dvc.utils.fs import makedirs
|
||||
|
||||
from .executor.base import BaseExecutor
|
||||
|
||||
entry = first(self.stash_revs.values())
|
||||
assert entry.index == 0
|
||||
assert entry.stash_index == 0
|
||||
|
||||
# NOTE: the stash commit to be popped already contains all the current
|
||||
# workspace changes plus CLI modified --params changes.
|
||||
# `checkout --force` here will not lose any data (popping stash commit
|
||||
# will result in conflict between workspace params and stashed CLI
|
||||
# params, but we always want the stashed version).
|
||||
with self.scm.detach_head(entry.rev, force=True, client="dvc"):
|
||||
with self.scm.detach_head(entry.head_rev, force=True, client="dvc"):
|
||||
rev = self.stash.pop()
|
||||
self.scm.set_ref(EXEC_BASELINE, entry.baseline_rev)
|
||||
if entry.branch:
|
||||
@@ -821,19 +680,30 @@ class Experiments:
|
||||
self.scm.remove_ref(EXEC_BRANCH)
|
||||
try:
|
||||
orig_checkpoint = self.scm.get_ref(EXEC_CHECKPOINT)
|
||||
pid_dir = os.path.join(self.repo.tmp_dir, self.EXEC_PID_DIR)
|
||||
pid_dir = os.path.join(
|
||||
self.repo.tmp_dir,
|
||||
EXEC_TMP_DIR,
|
||||
EXEC_PID_DIR,
|
||||
)
|
||||
if not os.path.exists(pid_dir):
|
||||
makedirs(pid_dir)
|
||||
pidfile = os.path.join(
|
||||
pid_dir, f"workspace{BaseExecutor.PIDFILE_EXT}"
|
||||
infofile = os.path.join(
|
||||
pid_dir, f"workspace{BaseExecutor.INFOFILE_EXT}"
|
||||
)
|
||||
info = ExecutorInfo(
|
||||
git_url=self.scm.root_dir,
|
||||
baseline_rev=entry.baseline_rev,
|
||||
location="workspace",
|
||||
root_dir=self.scm.root_dir,
|
||||
dvc_dir=self.dvc_dir,
|
||||
name=entry.name,
|
||||
wdir=os.getcwd(),
|
||||
)
|
||||
exec_result = BaseExecutor.reproduce(
|
||||
None,
|
||||
rev,
|
||||
name=entry.name,
|
||||
rel_cwd=relpath(os.getcwd(), self.scm.root_dir),
|
||||
info=info,
|
||||
rev=rev,
|
||||
infofile=infofile,
|
||||
log_errors=False,
|
||||
pidfile=pidfile,
|
||||
)
|
||||
|
||||
if not exec_result.exp_hash:
|
||||
@@ -931,45 +801,55 @@ class Experiments:
|
||||
|
||||
def get_running_exps(self) -> Dict[str, int]:
|
||||
"""Return info for running experiments."""
|
||||
from dvc.utils.serialize import load_yaml
|
||||
|
||||
from .executor.base import BaseExecutor, ExecutorInfo
|
||||
from dvc.scm import InvalidRemoteSCMRepo
|
||||
from dvc.utils.serialize import load_json
|
||||
|
||||
result = {}
|
||||
for pidfile in self.repo.fs.find(
|
||||
os.path.join(self.repo.tmp_dir, self.EXEC_PID_DIR)
|
||||
):
|
||||
rev, _ = os.path.splitext(os.path.basename(pidfile))
|
||||
pid_dir = os.path.join(
|
||||
self.repo.tmp_dir,
|
||||
EXEC_TMP_DIR,
|
||||
EXEC_PID_DIR,
|
||||
)
|
||||
for fname in self.repo.fs.find(pid_dir):
|
||||
rev, ext = os.path.splitext(os.path.basename(fname))
|
||||
if ext != BaseExecutor.INFOFILE_EXT:
|
||||
continue
|
||||
|
||||
try:
|
||||
info = ExecutorInfo.from_dict(load_yaml(pidfile))
|
||||
info = ExecutorInfo.from_dict(load_json(fname))
|
||||
if info.result is not None:
|
||||
continue
|
||||
if rev == "workspace":
|
||||
# If we are appending to a checkpoint branch in a workspace
|
||||
# run, show the latest checkpoint as running.
|
||||
last_rev = self.scm.get_ref(EXEC_BRANCH)
|
||||
if last_rev:
|
||||
result[last_rev] = info.to_dict()
|
||||
result[last_rev] = info.asdict()
|
||||
else:
|
||||
result[rev] = info.to_dict()
|
||||
result[rev] = info.asdict()
|
||||
else:
|
||||
result[rev] = info.to_dict()
|
||||
result[rev] = info.asdict()
|
||||
if info.git_url:
|
||||
|
||||
def on_diverged(_ref: str, _checkpoint: bool):
|
||||
return False
|
||||
|
||||
for ref in BaseExecutor.fetch_exps(
|
||||
self.scm,
|
||||
info.git_url,
|
||||
on_diverged=on_diverged,
|
||||
):
|
||||
logger.debug(
|
||||
"Updated running experiment '%s'.", ref
|
||||
)
|
||||
last_rev = self.scm.get_ref(ref)
|
||||
result[rev]["last"] = last_rev
|
||||
if last_rev:
|
||||
result[last_rev] = info.to_dict()
|
||||
try:
|
||||
for ref in BaseExecutor.fetch_exps(
|
||||
self.scm,
|
||||
info.git_url,
|
||||
on_diverged=on_diverged,
|
||||
):
|
||||
logger.debug(
|
||||
"Updated running experiment '%s'.", ref
|
||||
)
|
||||
last_rev = self.scm.get_ref(ref)
|
||||
result[rev]["last"] = last_rev
|
||||
if last_rev:
|
||||
result[last_rev] = info.asdict()
|
||||
except InvalidRemoteSCMRepo:
|
||||
# ignore stale info files
|
||||
del result[rev]
|
||||
except OSError:
|
||||
pass
|
||||
return result
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from typing import Optional
|
||||
from typing import NamedTuple, Optional
|
||||
|
||||
from dvc.exceptions import DvcException, InvalidArgumentError
|
||||
|
||||
@@ -141,3 +141,20 @@ class ExpRefInfo:
|
||||
baseline_sha = parts[2] + parts[3] if len(parts) >= 4 else None
|
||||
name = parts[4] if len(parts) == 5 else None
|
||||
return cls(baseline_sha, name)
|
||||
|
||||
|
||||
class ExpStashEntry(NamedTuple):
|
||||
"""Experiment stash entry.
|
||||
|
||||
stash_index: EXPS_STASH index for this entry. Can be None if this commit
|
||||
is not pushed onto the stash ref.
|
||||
head_rev: HEAD Git commit to be checked out for this experiment.
|
||||
baseline_rev: Experiment baseline commit.
|
||||
branch: Optional exp (checkpoint) branch name for this experiment.
|
||||
"""
|
||||
|
||||
stash_index: Optional[int]
|
||||
head_rev: str
|
||||
baseline_rev: str
|
||||
branch: Optional[str]
|
||||
name: Optional[str]
|
||||
|
||||
@@ -3,21 +3,28 @@ import os
|
||||
import pickle
|
||||
from abc import ABC, abstractmethod
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import asdict, dataclass
|
||||
from functools import partial
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Callable,
|
||||
Dict,
|
||||
Iterable,
|
||||
NamedTuple,
|
||||
Optional,
|
||||
Type,
|
||||
TypeVar,
|
||||
Union,
|
||||
)
|
||||
|
||||
from dvc.env import DVC_EXP_AUTO_PUSH, DVC_EXP_GIT_REMOTE
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.repo import Repo
|
||||
from dvc.repo.experiments.base import (
|
||||
from dvc.stage.serialize import to_lockfile
|
||||
from dvc.utils import dict_sha256, env2bool, relpath
|
||||
from dvc.utils.fs import remove
|
||||
|
||||
from ..base import (
|
||||
EXEC_BASELINE,
|
||||
EXEC_BRANCH,
|
||||
EXEC_CHECKPOINT,
|
||||
@@ -29,20 +36,24 @@ from dvc.repo.experiments.base import (
|
||||
ExpRefInfo,
|
||||
UnchangedExperimentError,
|
||||
)
|
||||
from dvc.stage import PipelineStage
|
||||
from dvc.stage.monitor import CheckpointKilledError
|
||||
from dvc.stage.serialize import to_lockfile
|
||||
from dvc.utils import dict_sha256, env2bool
|
||||
from dvc.utils.fs import remove
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from multiprocessing import Queue
|
||||
|
||||
from scmrepo.git import Git
|
||||
|
||||
from dvc.repo import Repo
|
||||
from dvc.stage import PipelineStage
|
||||
|
||||
from ..base import ExpStashEntry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
EXEC_TMP_DIR = "exps"
|
||||
EXEC_PID_DIR = "run"
|
||||
|
||||
|
||||
class ExecutorResult(NamedTuple):
|
||||
exp_hash: Optional[str]
|
||||
ref_info: Optional["ExpRefInfo"]
|
||||
@@ -51,68 +62,77 @@ class ExecutorResult(NamedTuple):
|
||||
|
||||
@dataclass
|
||||
class ExecutorInfo:
|
||||
PARAM_PID = "pid"
|
||||
PARAM_GIT_URL = "git"
|
||||
PARAM_BASELINE_REV = "baseline"
|
||||
PARAM_LOCATION = "location"
|
||||
|
||||
pid: Optional[int]
|
||||
git_url: Optional[str]
|
||||
baseline_rev: Optional[str]
|
||||
location: Optional[str]
|
||||
git_url: str
|
||||
baseline_rev: str
|
||||
location: str
|
||||
root_dir: str
|
||||
dvc_dir: str
|
||||
name: Optional[str] = None
|
||||
wdir: Optional[str] = None
|
||||
result_hash: Optional[str] = None
|
||||
result_ref: Optional[str] = None
|
||||
result_force: bool = False
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, d):
|
||||
return cls(
|
||||
d.get(cls.PARAM_PID),
|
||||
d.get(cls.PARAM_GIT_URL),
|
||||
d.get(cls.PARAM_BASELINE_REV),
|
||||
d.get(cls.PARAM_LOCATION),
|
||||
return cls(**d)
|
||||
|
||||
def asdict(self):
|
||||
return asdict(self)
|
||||
|
||||
@property
|
||||
def result(self) -> Optional["ExecutorResult"]:
|
||||
if self.result_hash is None:
|
||||
return None
|
||||
return ExecutorResult(
|
||||
self.result_hash,
|
||||
ExpRefInfo.from_ref(self.result_ref) if self.result_ref else None,
|
||||
self.result_force,
|
||||
)
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
self.PARAM_PID: self.pid,
|
||||
self.PARAM_GIT_URL: self.git_url,
|
||||
self.PARAM_BASELINE_REV: self.baseline_rev,
|
||||
self.PARAM_LOCATION: self.location,
|
||||
}
|
||||
|
||||
_T = TypeVar("_T", bound="BaseExecutor")
|
||||
|
||||
|
||||
class BaseExecutor(ABC):
|
||||
"""Base class for executing experiments in parallel.
|
||||
|
||||
Args:
|
||||
src: source Git SCM instance.
|
||||
dvc_dir: relpath to DVC root from SCM root.
|
||||
|
||||
Optional keyword args:
|
||||
branch: Existing git branch for this experiment.
|
||||
Parameters:
|
||||
root_dir: Path to SCM root.
|
||||
dvc_dir: Path to .dvc dir relative to SCM root.
|
||||
baseline_rev: Experiment baseline revision.
|
||||
wdir: Path to exec working directory relative to SCM root.
|
||||
name: Executor (experiment) name.
|
||||
result: Completed executor result.
|
||||
"""
|
||||
|
||||
PACKED_ARGS_FILE = "repro.dat"
|
||||
WARN_UNTRACKED = False
|
||||
QUIET = False
|
||||
PIDFILE_EXT = ".run"
|
||||
DEFAULT_LOCATION: Optional[str] = "workspace"
|
||||
INFOFILE_EXT = ".run"
|
||||
DEFAULT_LOCATION: str = "workspace"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
src: "Git",
|
||||
root_dir: str,
|
||||
dvc_dir: str,
|
||||
root_dir: Optional[str] = None,
|
||||
branch: Optional[str] = None,
|
||||
baseline_rev: str,
|
||||
wdir: Optional[str] = None,
|
||||
name: Optional[str] = None,
|
||||
location: Optional[str] = None,
|
||||
result: Optional["ExecutorResult"] = None,
|
||||
**kwargs,
|
||||
):
|
||||
assert root_dir is not None
|
||||
self._dvc_dir = dvc_dir
|
||||
self.root_dir = root_dir
|
||||
self._init_git(src, branch)
|
||||
self.wdir = wdir
|
||||
self.name = name
|
||||
self.baseline_rev = baseline_rev
|
||||
self.location: str = location or self.DEFAULT_LOCATION
|
||||
self.result = result
|
||||
|
||||
@abstractmethod
|
||||
def _init_git(self, scm: "Git", branch: Optional[str] = None, **kwargs):
|
||||
def init_git(self, scm: "Git", branch: Optional[str] = None):
|
||||
"""Init git repo and populate it using exp refs from the specified
|
||||
SCM instance.
|
||||
"""
|
||||
@@ -126,12 +146,92 @@ class BaseExecutor(ABC):
|
||||
def init_cache(self, dvc: "Repo", rev: str, run_cache: bool = True):
|
||||
"""Initialize DVC (cache)."""
|
||||
|
||||
@property
|
||||
def info(self) -> "ExecutorInfo":
|
||||
if self.result is not None:
|
||||
result_dict: Dict[str, Any] = {
|
||||
"result_hash": self.result.exp_hash,
|
||||
"result_ref": (
|
||||
str(self.result.ref_info) if self.result.ref_info else None
|
||||
),
|
||||
"result_force": self.result.force,
|
||||
}
|
||||
else:
|
||||
result_dict = {}
|
||||
return ExecutorInfo(
|
||||
git_url=self.git_url,
|
||||
baseline_rev=self.baseline_rev,
|
||||
location=self.location,
|
||||
root_dir=self.root_dir,
|
||||
dvc_dir=self.dvc_dir,
|
||||
name=self.name,
|
||||
wdir=self.wdir,
|
||||
**result_dict,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_info(cls: Type[_T], info: "ExecutorInfo") -> _T:
|
||||
if info.result_hash:
|
||||
result: Optional["ExecutorResult"] = ExecutorResult(
|
||||
info.result_hash,
|
||||
(
|
||||
ExpRefInfo.from_ref(info.result_ref)
|
||||
if info.result_ref
|
||||
else None
|
||||
),
|
||||
info.result_force,
|
||||
)
|
||||
else:
|
||||
result = None
|
||||
return cls(
|
||||
root_dir=info.root_dir,
|
||||
dvc_dir=info.dvc_dir,
|
||||
baseline_rev=info.baseline_rev,
|
||||
name=info.name,
|
||||
wdir=info.wdir,
|
||||
result=result,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def from_stash_entry(
|
||||
cls: Type[_T],
|
||||
repo: "Repo",
|
||||
stash_rev: str,
|
||||
entry: "ExpStashEntry",
|
||||
**kwargs,
|
||||
) -> _T:
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def _from_stash_entry(
|
||||
cls: Type[_T],
|
||||
repo: "Repo",
|
||||
stash_rev: str,
|
||||
entry: "ExpStashEntry",
|
||||
root_dir: str,
|
||||
**kwargs,
|
||||
) -> _T:
|
||||
executor = cls(
|
||||
root_dir=root_dir,
|
||||
dvc_dir=relpath(repo.dvc_dir, repo.scm.root_dir),
|
||||
baseline_rev=entry.baseline_rev,
|
||||
name=entry.name,
|
||||
wdir=relpath(os.getcwd(), repo.scm.root_dir),
|
||||
**kwargs,
|
||||
)
|
||||
executor.init_git(repo.scm, branch=entry.branch)
|
||||
executor.init_cache(repo, stash_rev)
|
||||
return executor
|
||||
|
||||
@property
|
||||
def dvc_dir(self) -> str:
|
||||
return os.path.join(self.root_dir, self._dvc_dir)
|
||||
|
||||
@staticmethod
|
||||
def hash_exp(stages: Iterable["PipelineStage"]) -> str:
|
||||
from dvc.stage import PipelineStage
|
||||
|
||||
exp_data = {}
|
||||
for stage in stages:
|
||||
if isinstance(stage, PipelineStage):
|
||||
@@ -244,11 +344,10 @@ class BaseExecutor(ABC):
|
||||
@classmethod
|
||||
def reproduce(
|
||||
cls,
|
||||
dvc_dir: Optional[str],
|
||||
info: "ExecutorInfo",
|
||||
rev: str,
|
||||
queue: Optional["Queue"] = None,
|
||||
rel_cwd: Optional[str] = None,
|
||||
name: Optional[str] = None,
|
||||
infofile: Optional[str] = None,
|
||||
log_errors: bool = True,
|
||||
log_level: Optional[int] = None,
|
||||
**kwargs,
|
||||
@@ -262,6 +361,7 @@ class BaseExecutor(ABC):
|
||||
"""
|
||||
from dvc.repo.checkout import checkout as dvc_checkout
|
||||
from dvc.repo.reproduce import reproduce as dvc_reproduce
|
||||
from dvc.stage import PipelineStage
|
||||
|
||||
auto_push = env2bool(DVC_EXP_AUTO_PUSH)
|
||||
git_remote = os.getenv(DVC_EXP_GIT_REMOTE, None)
|
||||
@@ -283,9 +383,9 @@ class BaseExecutor(ABC):
|
||||
repro_force: bool = False
|
||||
|
||||
with cls._repro_dvc(
|
||||
dvc_dir,
|
||||
rel_cwd,
|
||||
log_errors,
|
||||
info,
|
||||
log_errors=log_errors,
|
||||
infofile=infofile,
|
||||
**kwargs,
|
||||
) as dvc:
|
||||
if auto_push:
|
||||
@@ -333,7 +433,7 @@ class BaseExecutor(ABC):
|
||||
cls.checkpoint_callback,
|
||||
dvc,
|
||||
dvc.scm,
|
||||
name,
|
||||
info.name,
|
||||
repro_force or checkpoint_reset,
|
||||
)
|
||||
stages = dvc_reproduce(
|
||||
@@ -358,7 +458,7 @@ class BaseExecutor(ABC):
|
||||
cls.commit(
|
||||
dvc.scm,
|
||||
exp_hash,
|
||||
exp_name=name,
|
||||
exp_name=info.name,
|
||||
force=repro_force,
|
||||
checkpoint=is_checkpoint,
|
||||
)
|
||||
@@ -389,38 +489,32 @@ class BaseExecutor(ABC):
|
||||
@contextmanager
|
||||
def _repro_dvc(
|
||||
cls,
|
||||
dvc_dir: Optional[str],
|
||||
rel_cwd: Optional[str],
|
||||
log_errors: bool,
|
||||
pidfile: Optional[str] = None,
|
||||
git_url: Optional[str] = None,
|
||||
info: "ExecutorInfo",
|
||||
log_errors: bool = True,
|
||||
infofile: Optional[str] = None,
|
||||
**kwargs,
|
||||
):
|
||||
from dvc.utils.serialize import modify_yaml
|
||||
from dvc.repo import Repo
|
||||
from dvc.stage.monitor import CheckpointKilledError
|
||||
from dvc.utils.fs import makedirs
|
||||
from dvc.utils.serialize import modify_json
|
||||
|
||||
dvc = Repo(dvc_dir)
|
||||
dvc = Repo(os.path.join(info.root_dir, info.dvc_dir))
|
||||
if cls.QUIET:
|
||||
dvc.scm_context.quiet = cls.QUIET
|
||||
if dvc_dir is not None:
|
||||
old_cwd: Optional[str] = os.getcwd()
|
||||
if rel_cwd:
|
||||
os.chdir(os.path.join(dvc.root_dir, rel_cwd))
|
||||
else:
|
||||
os.chdir(dvc.root_dir)
|
||||
old_cwd = os.getcwd()
|
||||
if info.wdir:
|
||||
os.chdir(os.path.join(dvc.scm.root_dir, info.wdir))
|
||||
else:
|
||||
old_cwd = None
|
||||
if pidfile is not None:
|
||||
info = ExecutorInfo(
|
||||
os.getpid(),
|
||||
git_url,
|
||||
dvc.scm.get_rev(),
|
||||
cls.DEFAULT_LOCATION,
|
||||
)
|
||||
with modify_yaml(pidfile) as d:
|
||||
d.update(info.to_dict())
|
||||
logger.debug("Running repro in '%s'", os.getcwd())
|
||||
os.chdir(dvc.root_dir)
|
||||
|
||||
if infofile is not None:
|
||||
makedirs(os.path.dirname(infofile), exist_ok=True)
|
||||
with modify_json(infofile) as d:
|
||||
d.update(info.asdict())
|
||||
|
||||
try:
|
||||
logger.debug("Running repro in '%s'", os.getcwd())
|
||||
yield dvc
|
||||
except CheckpointKilledError:
|
||||
raise
|
||||
@@ -433,11 +527,10 @@ class BaseExecutor(ABC):
|
||||
logger.exception("unexpected error")
|
||||
raise
|
||||
finally:
|
||||
if pidfile is not None:
|
||||
remove(pidfile)
|
||||
if infofile is not None:
|
||||
remove(infofile)
|
||||
dvc.close()
|
||||
if old_cwd:
|
||||
os.chdir(old_cwd)
|
||||
os.chdir(old_cwd)
|
||||
|
||||
@classmethod
|
||||
def _repro_args(cls, dvc):
|
||||
|
||||
@@ -1,47 +1,32 @@
|
||||
import logging
|
||||
import os
|
||||
from tempfile import TemporaryDirectory
|
||||
from tempfile import mkdtemp
|
||||
from typing import TYPE_CHECKING, Optional
|
||||
|
||||
from funcy import cached_property
|
||||
|
||||
from dvc.repo.experiments.base import (
|
||||
from dvc.scm import SCM
|
||||
from dvc.utils.fs import remove
|
||||
|
||||
from ..base import (
|
||||
EXEC_BRANCH,
|
||||
EXEC_CHECKPOINT,
|
||||
EXEC_HEAD,
|
||||
EXEC_MERGE,
|
||||
EXEC_NAMESPACE,
|
||||
)
|
||||
from dvc.scm import SCM
|
||||
|
||||
from .base import BaseExecutor
|
||||
from .base import EXEC_TMP_DIR, BaseExecutor
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scmrepo.git import Git
|
||||
|
||||
from dvc.repo import Repo
|
||||
|
||||
from ..base import ExpStashEntry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ExpTemporaryDirectory(TemporaryDirectory):
|
||||
# Python's TemporaryDirectory cleanup shutil.rmtree wrapper does not handle
|
||||
# git read-only dirs cleanly in Windows on Python <3.8, so we use our own
|
||||
# remove(). See:
|
||||
# https://github.com/iterative/dvc/pull/5425
|
||||
# https://bugs.python.org/issue26660
|
||||
|
||||
@classmethod
|
||||
def _rmtree(cls, name):
|
||||
from dvc.utils.fs import remove
|
||||
|
||||
remove(name)
|
||||
|
||||
def cleanup(self):
|
||||
if self._finalizer.detach():
|
||||
self._rmtree(self.name)
|
||||
|
||||
|
||||
class BaseLocalExecutor(BaseExecutor):
|
||||
"""Base local machine executor."""
|
||||
|
||||
@@ -70,20 +55,9 @@ class TempDirExecutor(BaseLocalExecutor):
|
||||
# suggestions) that are not applicable outside of workspace runs
|
||||
WARN_UNTRACKED = True
|
||||
QUIET = True
|
||||
DEFAULT_LOCATION: Optional[str] = "temp"
|
||||
DEFAULT_LOCATION = "temp"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*args,
|
||||
tmp_dir: Optional[str] = None,
|
||||
**kwargs,
|
||||
):
|
||||
self._tmp_dir = ExpTemporaryDirectory(dir=tmp_dir)
|
||||
kwargs["root_dir"] = self._tmp_dir.name
|
||||
super().__init__(*args, **kwargs)
|
||||
logger.debug("Init temp dir executor in dir '%s'", self._tmp_dir)
|
||||
|
||||
def _init_git(self, scm: "Git", branch: Optional[str] = None, **kwargs):
|
||||
def init_git(self, scm: "Git", branch: Optional[str] = None):
|
||||
from dulwich.repo import Repo as DulwichRepo
|
||||
|
||||
from ..utils import push_refspec
|
||||
@@ -91,9 +65,9 @@ class TempDirExecutor(BaseLocalExecutor):
|
||||
DulwichRepo.init(os.fspath(self.root_dir))
|
||||
|
||||
refspec = f"{EXEC_NAMESPACE}/"
|
||||
push_refspec(scm, self.git_url, refspec, refspec, **kwargs)
|
||||
push_refspec(scm, self.git_url, refspec, refspec)
|
||||
if branch:
|
||||
push_refspec(scm, self.git_url, branch, branch, **kwargs)
|
||||
push_refspec(scm, self.git_url, branch, branch)
|
||||
self.scm.set_ref(EXEC_BRANCH, branch, symbolic=True)
|
||||
elif self.scm.get_ref(EXEC_BRANCH):
|
||||
self.scm.remove_ref(EXEC_BRANCH)
|
||||
@@ -120,5 +94,26 @@ class TempDirExecutor(BaseLocalExecutor):
|
||||
|
||||
def cleanup(self):
|
||||
super().cleanup()
|
||||
logger.debug("Removing tmpdir '%s'", self._tmp_dir)
|
||||
self._tmp_dir.cleanup()
|
||||
logger.debug("Removing tmpdir '%s'", self.root_dir)
|
||||
remove(self.root_dir)
|
||||
|
||||
@classmethod
|
||||
def from_stash_entry(
|
||||
cls,
|
||||
repo: "Repo",
|
||||
stash_rev: str,
|
||||
entry: "ExpStashEntry",
|
||||
**kwargs,
|
||||
):
|
||||
tmp_dir = mkdtemp(dir=os.path.join(repo.tmp_dir, EXEC_TMP_DIR))
|
||||
try:
|
||||
executor = cls._from_stash_entry(repo, stash_rev, entry, tmp_dir)
|
||||
logger.debug("Init temp dir executor in '%s'", tmp_dir)
|
||||
return executor
|
||||
except Exception:
|
||||
remove(tmp_dir)
|
||||
raise
|
||||
|
||||
|
||||
class WorkspaceExecutor(BaseLocalExecutor):
|
||||
pass
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
import logging
|
||||
import os
|
||||
from abc import ABC
|
||||
from collections import deque
|
||||
from typing import TYPE_CHECKING, Deque, Dict, Optional, Tuple, Type
|
||||
|
||||
from dvc.proc.manager import ProcessManager
|
||||
|
||||
from ..base import (
|
||||
EXEC_BASELINE,
|
||||
EXEC_HEAD,
|
||||
EXEC_MERGE,
|
||||
CheckpointExistsError,
|
||||
ExperimentExistsError,
|
||||
ExpRefInfo,
|
||||
ExpStashEntry,
|
||||
)
|
||||
from .base import EXEC_PID_DIR
|
||||
from .local import TempDirExecutor, WorkspaceExecutor
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scmrepo.git import Git
|
||||
|
||||
from dvc.repo import Repo
|
||||
|
||||
from .base import BaseExecutor
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BaseExecutorManager(ABC):
|
||||
"""Manages executors for a collection of experiments to be run."""
|
||||
|
||||
EXECUTOR_CLS: Type = WorkspaceExecutor
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
scm: "Git",
|
||||
wdir: str,
|
||||
):
|
||||
from dvc.utils.fs import makedirs
|
||||
|
||||
self.scm = scm
|
||||
makedirs(wdir, exist_ok=True)
|
||||
self.wdir = wdir
|
||||
self.executors = self._load_infos()
|
||||
self._queue: Deque[Tuple[str, "BaseExecutor"]] = deque()
|
||||
self.proc = ProcessManager(self.pid_dir)
|
||||
|
||||
@property
|
||||
def pid_dir(self) -> str:
|
||||
return os.path.join(self.wdir, EXEC_PID_DIR)
|
||||
|
||||
def enqueue(self, rev: str, executor: "BaseExecutor"):
|
||||
self._queue.append((rev, executor))
|
||||
|
||||
def _load_infos(self) -> Dict[str, "BaseExecutor"]:
|
||||
# TODO: pending process manager changes
|
||||
# import json
|
||||
# from urllib.parse import urlparse
|
||||
# from .base import ExecutorInfo
|
||||
# from .ssh import SSHExecutor
|
||||
|
||||
# def make_executor(info: "ExecutorInfo"):
|
||||
# if info.git_url:
|
||||
# scheme = urlparse(info.git_url)
|
||||
# if scheme == "file":
|
||||
# cls = TempDirExecutor
|
||||
# elif scheme == "ssh":
|
||||
# cls = SSHExecutor
|
||||
# else:
|
||||
# raise NotImplementedError
|
||||
# else:
|
||||
# cls = WorkspaceExecutor
|
||||
# return cls.from_info(info)
|
||||
|
||||
infos: Dict[str, "BaseExecutor"] = {}
|
||||
# for name in self.proc.processes():
|
||||
# infofile = os.path.join(
|
||||
# name, f"{name}{BaseExecutor.INFOFILE_EXT}"
|
||||
# )
|
||||
# try:
|
||||
# with open(infofile, encoding="utf-8") as fobj:
|
||||
# info = ExecutorInfo.from_dict(json.load(fobj))
|
||||
# infos[name] = make_executor(info)
|
||||
# except OSError:
|
||||
# continue
|
||||
return infos
|
||||
|
||||
@classmethod
|
||||
def from_stash_entries(
|
||||
cls,
|
||||
scm: "Git",
|
||||
wdir: str,
|
||||
repo: "Repo",
|
||||
to_run: Dict[str, ExpStashEntry],
|
||||
**kwargs,
|
||||
):
|
||||
manager = cls(scm, wdir)
|
||||
try:
|
||||
for stash_rev, entry in to_run.items():
|
||||
scm.set_ref(EXEC_HEAD, entry.head_rev)
|
||||
scm.set_ref(EXEC_MERGE, stash_rev)
|
||||
scm.set_ref(EXEC_BASELINE, entry.baseline_rev)
|
||||
|
||||
# Executor will be initialized with an empty git repo that
|
||||
# we populate by pushing:
|
||||
# EXEC_HEAD - the base commit for this experiment
|
||||
# EXEC_MERGE - the unmerged changes (from our stash)
|
||||
# to be reproduced
|
||||
# EXEC_BASELINE - the baseline commit for this experiment
|
||||
executor = cls.EXECUTOR_CLS.from_stash_entry(
|
||||
repo,
|
||||
stash_rev,
|
||||
entry,
|
||||
**kwargs,
|
||||
)
|
||||
manager.enqueue(stash_rev, executor)
|
||||
finally:
|
||||
for ref in (EXEC_HEAD, EXEC_MERGE, EXEC_BASELINE):
|
||||
scm.remove_ref(ref)
|
||||
return manager
|
||||
|
||||
def exec_queue(self, jobs: Optional[int] = 1):
|
||||
"""Run dvc repro for queued executors in parallel."""
|
||||
import signal
|
||||
from collections import defaultdict
|
||||
from concurrent.futures import (
|
||||
CancelledError,
|
||||
ProcessPoolExecutor,
|
||||
wait,
|
||||
)
|
||||
from multiprocessing import Manager
|
||||
|
||||
from dvc.stage.monitor import CheckpointKilledError
|
||||
|
||||
result: Dict[str, Dict[str, str]] = defaultdict(dict)
|
||||
|
||||
manager = Manager()
|
||||
pid_q = manager.Queue()
|
||||
|
||||
with ProcessPoolExecutor(max_workers=jobs) as workers:
|
||||
futures = {}
|
||||
while self._queue:
|
||||
rev, executor = self._queue.popleft()
|
||||
infofile = os.path.join(
|
||||
self.pid_dir,
|
||||
f"{rev}{executor.INFOFILE_EXT}",
|
||||
)
|
||||
future = workers.submit(
|
||||
executor.reproduce,
|
||||
info=executor.info,
|
||||
rev=rev,
|
||||
queue=pid_q,
|
||||
infofile=infofile,
|
||||
log_level=logger.getEffectiveLevel(),
|
||||
)
|
||||
futures[future] = (rev, executor)
|
||||
|
||||
try:
|
||||
wait(futures)
|
||||
except KeyboardInterrupt:
|
||||
# forward SIGINT to any running executor processes and
|
||||
# cancel any remaining futures
|
||||
workers.shutdown(wait=False)
|
||||
pids = {}
|
||||
for future, (rev, _) in futures.items():
|
||||
if future.running():
|
||||
# if future has already been started by the scheduler
|
||||
# we still have to wait until it tells us its PID
|
||||
while rev not in pids:
|
||||
rev, pid = pid_q.get()
|
||||
pids[rev] = pid
|
||||
os.kill(pids[rev], signal.SIGINT)
|
||||
elif not future.done():
|
||||
future.cancel()
|
||||
|
||||
for future, (rev, executor) in futures.items():
|
||||
rev, executor = futures[future]
|
||||
|
||||
try:
|
||||
exc = future.exception()
|
||||
if exc is None:
|
||||
exec_result = future.result()
|
||||
result[rev].update(
|
||||
self._collect_executor(executor, exec_result)
|
||||
)
|
||||
elif not isinstance(exc, CheckpointKilledError):
|
||||
logger.error(
|
||||
"Failed to reproduce experiment '%s'", rev[:7]
|
||||
)
|
||||
except CancelledError:
|
||||
logger.error(
|
||||
"Cancelled before attempting to reproduce experiment "
|
||||
"'%s'",
|
||||
rev[:7],
|
||||
)
|
||||
finally:
|
||||
executor.cleanup()
|
||||
|
||||
return result
|
||||
|
||||
def _collect_executor(self, executor, exec_result) -> Dict[str, str]:
|
||||
# NOTE: GitPython Repo instances cannot be re-used
|
||||
# after process has received SIGINT or SIGTERM, so we
|
||||
# need this hack to re-instantiate git instances after
|
||||
# checkpoint runs. See:
|
||||
# https://github.com/gitpython-developers/GitPython/issues/427
|
||||
# del self.repo.scm
|
||||
|
||||
results = {}
|
||||
|
||||
def on_diverged(ref: str, checkpoint: bool):
|
||||
ref_info = ExpRefInfo.from_ref(ref)
|
||||
if checkpoint:
|
||||
raise CheckpointExistsError(ref_info.name)
|
||||
raise ExperimentExistsError(ref_info.name)
|
||||
|
||||
for ref in executor.fetch_exps(
|
||||
self.scm,
|
||||
executor.git_url,
|
||||
force=exec_result.force,
|
||||
on_diverged=on_diverged,
|
||||
):
|
||||
exp_rev = self.scm.get_ref(ref)
|
||||
if exp_rev:
|
||||
logger.debug("Collected experiment '%s'.", exp_rev[:7])
|
||||
results[exp_rev] = exec_result.exp_hash
|
||||
|
||||
return results
|
||||
|
||||
|
||||
class LocalExecutorManager(BaseExecutorManager):
|
||||
EXECUTOR_CLS = TempDirExecutor
|
||||
@@ -22,6 +22,8 @@ if TYPE_CHECKING:
|
||||
from dvc.machine import MachineManager
|
||||
from dvc.repo import Repo
|
||||
|
||||
from ..base import ExpStashEntry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -51,13 +53,12 @@ class SSHExecutor(BaseExecutor):
|
||||
):
|
||||
assert host
|
||||
|
||||
super().__init__(*args, **kwargs)
|
||||
self.host: str = host
|
||||
self.port = port
|
||||
self.username = username
|
||||
self._fs_factory = fs_factory
|
||||
|
||||
super().__init__(*args, **kwargs)
|
||||
logger.debug("Init SSH executor for host '%s'", self.host)
|
||||
self._repo_abspath = ""
|
||||
|
||||
@classmethod
|
||||
def gen_dirname(cls, name: Optional[str] = None):
|
||||
@@ -66,16 +67,24 @@ class SSHExecutor(BaseExecutor):
|
||||
return "-".join([name or "dvc-exp", "executor", uuid()])
|
||||
|
||||
@classmethod
|
||||
def from_machine(
|
||||
def from_stash_entry(
|
||||
cls,
|
||||
manager: "MachineManager",
|
||||
machine_name: Optional[str],
|
||||
*args,
|
||||
repo: "Repo",
|
||||
stash_rev: str,
|
||||
entry: "ExpStashEntry",
|
||||
**kwargs,
|
||||
):
|
||||
kwargs["root_dir"] = cls.gen_dirname(kwargs.get("name"))
|
||||
kwargs.update(manager.get_executor_kwargs(machine_name))
|
||||
return cls(*args, **kwargs)
|
||||
manager: "MachineManager" = kwargs.pop("manager")
|
||||
machine_name: Optional[str] = kwargs.pop("machine_name", None)
|
||||
executor = cls._from_stash_entry(
|
||||
repo,
|
||||
stash_rev,
|
||||
entry,
|
||||
cls.gen_dirname(entry.name),
|
||||
**manager.get_executor_kwargs(machine_name),
|
||||
)
|
||||
logger.debug("Init SSH executor for host '%s'", executor.host)
|
||||
return executor
|
||||
|
||||
def sshfs(self):
|
||||
return _sshfs(self._fs_factory, host=self.host, port=self.port)
|
||||
@@ -104,7 +113,7 @@ class SSHExecutor(BaseExecutor):
|
||||
}
|
||||
return kwargs
|
||||
|
||||
def _init_git(self, scm: "Git", branch: Optional[str] = None, **kwargs):
|
||||
def init_git(self, scm: "Git", branch: Optional[str] = None):
|
||||
from ..utils import push_refspec
|
||||
|
||||
with self.sshfs() as fs:
|
||||
@@ -121,7 +130,7 @@ class SSHExecutor(BaseExecutor):
|
||||
|
||||
# TODO: support multiple client key retries in git backends
|
||||
# (see https://github.com/iterative/dvc/issues/6508)
|
||||
kwargs.update(self._git_client_args(fs))
|
||||
kwargs = self._git_client_args(fs)
|
||||
refspec = f"{EXEC_NAMESPACE}/"
|
||||
push_refspec(scm, self.git_url, refspec, refspec, **kwargs)
|
||||
if branch:
|
||||
|
||||
@@ -42,7 +42,7 @@ def gc(
|
||||
delete_stashes = []
|
||||
for _, entry in repo.experiments.stash_revs.items():
|
||||
if not queued or entry.baseline_rev not in keep_revs:
|
||||
delete_stashes.append(entry.index)
|
||||
delete_stashes.append(entry.stash_index)
|
||||
for index in sorted(delete_stashes, reverse=True):
|
||||
repo.experiments.stash.drop(index)
|
||||
removed += len(delete_stashes)
|
||||
|
||||
@@ -49,16 +49,16 @@ def _clear_all(repo):
|
||||
|
||||
def _get_exp_stash_index(repo, ref_or_rev: str) -> Optional[int]:
|
||||
stash_revs = repo.experiments.stash_revs
|
||||
for _, ref_info in stash_revs.items():
|
||||
if ref_info.name == ref_or_rev:
|
||||
return ref_info.index
|
||||
for _, entry in stash_revs.items():
|
||||
if entry.name == ref_or_rev:
|
||||
return entry.stash_index
|
||||
|
||||
from dvc.scm import resolve_rev
|
||||
|
||||
try:
|
||||
rev = resolve_rev(repo.scm, ref_or_rev)
|
||||
if rev in stash_revs:
|
||||
return stash_revs.get(rev).index
|
||||
return stash_revs.get(rev).stash_index
|
||||
except RevError:
|
||||
pass
|
||||
return None
|
||||
|
||||
@@ -6,7 +6,6 @@ from typing import Any, Callable, Dict, Optional
|
||||
from dvc.exceptions import InvalidArgumentError
|
||||
from dvc.repo import locked
|
||||
from dvc.repo.experiments.base import ExpRefInfo
|
||||
from dvc.repo.experiments.executor.base import ExecutorInfo
|
||||
from dvc.repo.experiments.utils import fix_exp_head
|
||||
from dvc.repo.metrics.show import _gather_metrics
|
||||
from dvc.repo.params.show import _gather_params
|
||||
@@ -44,7 +43,7 @@ def _collect_experiment_commit(
|
||||
res["queued"] = stash
|
||||
if running is not None and exp_rev in running:
|
||||
res["running"] = True
|
||||
res["executor"] = running[exp_rev].get(ExecutorInfo.PARAM_LOCATION)
|
||||
res["executor"] = running[exp_rev].get("location")
|
||||
else:
|
||||
res["running"] = False
|
||||
res["executor"] = None
|
||||
|
||||
@@ -21,10 +21,19 @@ def _ssh_factory(cloud):
|
||||
)
|
||||
|
||||
|
||||
def test_from_machine(tmp_dir, scm, dvc, machine_instance, mocker):
|
||||
mocker.patch.object(SSHExecutor, "_init_git")
|
||||
executor = SSHExecutor.from_machine(dvc.machine, "foo", scm, "")
|
||||
assert executor.host == machine_instance["instance_ip"]
|
||||
def test_init_from_stash(tmp_dir, scm, dvc, machine_instance, mocker):
|
||||
mock = mocker.patch.object(SSHExecutor, "_from_stash_entry")
|
||||
mock_entry = mocker.Mock()
|
||||
mock_entry.name = ""
|
||||
SSHExecutor.from_stash_entry(
|
||||
None,
|
||||
"",
|
||||
mock_entry,
|
||||
manager=dvc.machine,
|
||||
machine_name="foo",
|
||||
)
|
||||
_args, kwargs = mock.call_args
|
||||
assert kwargs["host"] == machine_instance["instance_ip"]
|
||||
|
||||
|
||||
@pytest.mark.needs_internet
|
||||
|
||||
@@ -8,7 +8,12 @@ from funcy import first, get_in
|
||||
from dvc.exceptions import InvalidArgumentError
|
||||
from dvc.main import main
|
||||
from dvc.repo.experiments.base import EXPS_STASH, ExpRefInfo
|
||||
from dvc.repo.experiments.executor.base import BaseExecutor, ExecutorInfo
|
||||
from dvc.repo.experiments.executor.base import (
|
||||
EXEC_PID_DIR,
|
||||
EXEC_TMP_DIR,
|
||||
BaseExecutor,
|
||||
ExecutorInfo,
|
||||
)
|
||||
from dvc.repo.experiments.utils import exp_refs_by_rev
|
||||
from dvc.utils.fs import makedirs
|
||||
from dvc.utils.serialize import YAMLFileCorruptedError
|
||||
@@ -16,6 +21,20 @@ from tests.func.test_repro_multistage import COPY_SCRIPT
|
||||
from tests.utils import console_width
|
||||
|
||||
|
||||
def make_executor_info(**kwargs):
|
||||
# set default values for required info fields
|
||||
for key in (
|
||||
"git_url",
|
||||
"baseline_rev",
|
||||
"location",
|
||||
"root_dir",
|
||||
"dvc_dir",
|
||||
):
|
||||
if key not in kwargs:
|
||||
kwargs[key] = ""
|
||||
return ExecutorInfo(**kwargs)
|
||||
|
||||
|
||||
def test_show_simple(tmp_dir, scm, dvc, exp_stage):
|
||||
assert dvc.experiments.show()["workspace"] == {
|
||||
"baseline": {
|
||||
@@ -367,11 +386,11 @@ def test_show_sort(tmp_dir, scm, dvc, exp_stage, caplog):
|
||||
|
||||
|
||||
def test_show_running_workspace(tmp_dir, scm, dvc, exp_stage, capsys):
|
||||
pid_dir = os.path.join(dvc.tmp_dir, dvc.experiments.EXEC_PID_DIR)
|
||||
pid_dir = os.path.join(dvc.tmp_dir, EXEC_TMP_DIR, EXEC_PID_DIR)
|
||||
makedirs(pid_dir, True)
|
||||
info = ExecutorInfo(None, None, None, BaseExecutor.DEFAULT_LOCATION)
|
||||
pidfile = os.path.join(pid_dir, f"workspace{BaseExecutor.PIDFILE_EXT}")
|
||||
(tmp_dir / pidfile).dump(info.to_dict())
|
||||
info = make_executor_info(location=BaseExecutor.DEFAULT_LOCATION)
|
||||
pidfile = os.path.join(pid_dir, f"workspace{BaseExecutor.INFOFILE_EXT}")
|
||||
(tmp_dir / pidfile).dump_json(info.asdict())
|
||||
|
||||
assert dvc.experiments.show()["workspace"] == {
|
||||
"baseline": {
|
||||
@@ -398,11 +417,11 @@ def test_show_running_executor(tmp_dir, scm, dvc, exp_stage):
|
||||
dvc.experiments.run(exp_stage.addressing, params=["foo=2"], queue=True)
|
||||
exp_rev = dvc.experiments.scm.resolve_rev(f"{EXPS_STASH}@{{0}}")
|
||||
|
||||
pid_dir = os.path.join(dvc.tmp_dir, dvc.experiments.EXEC_PID_DIR)
|
||||
pid_dir = os.path.join(dvc.tmp_dir, EXEC_TMP_DIR, EXEC_PID_DIR)
|
||||
makedirs(pid_dir, True)
|
||||
info = ExecutorInfo(None, None, None, BaseExecutor.DEFAULT_LOCATION)
|
||||
pidfile = os.path.join(pid_dir, f"{exp_rev}{BaseExecutor.PIDFILE_EXT}")
|
||||
(tmp_dir / pidfile).dump(info.to_dict())
|
||||
info = make_executor_info(location=BaseExecutor.DEFAULT_LOCATION)
|
||||
pidfile = os.path.join(pid_dir, f"{exp_rev}{BaseExecutor.INFOFILE_EXT}")
|
||||
(tmp_dir / pidfile).dump_json(info.asdict())
|
||||
|
||||
results = dvc.experiments.show()
|
||||
exp_data = get_in(results, [baseline_rev, exp_rev, "data"])
|
||||
@@ -430,17 +449,21 @@ def test_show_running_checkpoint(
|
||||
checkpoint_rev = first(run_results)
|
||||
exp_ref = first(exp_refs_by_rev(scm, checkpoint_rev))
|
||||
|
||||
pid_dir = os.path.join(dvc.tmp_dir, dvc.experiments.EXEC_PID_DIR)
|
||||
pid_dir = os.path.join(dvc.tmp_dir, EXEC_TMP_DIR, EXEC_PID_DIR)
|
||||
makedirs(pid_dir, True)
|
||||
executor = (
|
||||
BaseExecutor.DEFAULT_LOCATION
|
||||
if workspace
|
||||
else TempDirExecutor.DEFAULT_LOCATION
|
||||
)
|
||||
info = ExecutorInfo(123, "foo.git", baseline_rev, executor)
|
||||
info = make_executor_info(
|
||||
git_url="foo.git",
|
||||
baseline_rev=baseline_rev,
|
||||
location=executor,
|
||||
)
|
||||
rev = "workspace" if workspace else stash_rev
|
||||
pidfile = os.path.join(pid_dir, f"{rev}{BaseExecutor.PIDFILE_EXT}")
|
||||
(tmp_dir / pidfile).dump(info.to_dict())
|
||||
pidfile = os.path.join(pid_dir, f"{rev}{BaseExecutor.INFOFILE_EXT}")
|
||||
(tmp_dir / pidfile).dump_json(info.asdict())
|
||||
|
||||
mocker.patch.object(
|
||||
BaseExecutor, "fetch_exps", return_value=[str(exp_ref)]
|
||||
@@ -559,7 +582,6 @@ def test_show_only_changed(tmp_dir, dvc, scm, capsys):
|
||||
assert main(["exp", "show"]) == 0
|
||||
cap = capsys.readouterr()
|
||||
|
||||
print(cap)
|
||||
assert "bar" in cap.out
|
||||
|
||||
capsys.readouterr()
|
||||
|
||||
Reference in New Issue
Block a user