exp: initial executor manager refactor

- introduce ExecutorManager classes
- move executor init + execution from experiments.__init__ into the
  manager classes
- make executors serializable via ExecutorInfo
This commit is contained in:
Peter Rowlands
2021-12-15 15:55:32 +09:00
committed by Peter Rowlands (변기호)
parent 8319cc50c6
commit 37f8010b42
11 changed files with 615 additions and 357 deletions
+80 -200
View File
@@ -1,12 +1,8 @@
import logging
import os
import re
import signal
from collections import defaultdict, namedtuple
from concurrent.futures import CancelledError, ProcessPoolExecutor, wait
from functools import wraps
from multiprocessing import Manager
from typing import Dict, Iterable, Mapping, Optional
from typing import Dict, Iterable, List, Mapping, Optional
from funcy import cached_property, first
@@ -20,17 +16,22 @@ from .base import (
EXEC_BASELINE,
EXEC_BRANCH,
EXEC_CHECKPOINT,
EXEC_HEAD,
EXEC_MERGE,
EXEC_NAMESPACE,
EXPS_NAMESPACE,
EXPS_STASH,
BaselineMismatchError,
CheckpointExistsError,
ExperimentExistsError,
ExpRefInfo,
ExpStashEntry,
MultipleBranchError,
)
from .executor.base import (
EXEC_PID_DIR,
EXEC_TMP_DIR,
BaseExecutor,
ExecutorInfo,
)
from .executor.manager import BaseExecutorManager, LocalExecutorManager
from .utils import exp_refs_by_rev
logger = logging.getLogger(__name__)
@@ -81,12 +82,6 @@ class Experiments:
r"^(?P<baseline_rev>[a-f0-9]{7})-(?P<exp_sha>[a-f0-9]+)"
r"(?P<checkpoint>-checkpoint)?$"
)
EXEC_TMP_DIR = "exps"
EXEC_PID_DIR = os.path.join(EXEC_TMP_DIR, "run")
StashEntry = namedtuple(
"StashEntry", ["index", "rev", "baseline_rev", "branch", "name"]
)
def __init__(self, repo):
from dvc.lock import make_lock
@@ -112,8 +107,6 @@ class Experiments:
@cached_property
def args_file(self):
from .executor.base import BaseExecutor
return os.path.join(self.repo.tmp_dir, BaseExecutor.PACKED_ARGS_FILE)
@cached_property
@@ -123,13 +116,13 @@ class Experiments:
return Stash(self.scm, EXPS_STASH)
@property
def stash_revs(self):
def stash_revs(self) -> Dict[str, ExpStashEntry]:
revs = {}
for i, entry in enumerate(self.stash):
msg = entry.message.decode("utf-8").strip()
m = self.STASH_EXPERIMENT_RE.match(msg)
if m:
revs[entry.new_sha.decode("utf-8")] = self.StashEntry(
revs[entry.new_sha.decode("utf-8")] = ExpStashEntry(
i,
m.group("rev"),
m.group("baseline_rev"),
@@ -324,8 +317,6 @@ class Experiments:
def _pack_args(self, *args, **kwargs):
import pickle
from .executor.base import BaseExecutor
if os.path.exists(self.args_file) and self.scm.is_tracked(
self.args_file
):
@@ -616,7 +607,7 @@ class Experiments:
to_run = {
rev: stash_revs[rev]
if rev in stash_revs
else self.StashEntry(None, rev, rev, None, None)
else ExpStashEntry(None, rev, rev, None, None)
for rev in revs
}
@@ -625,17 +616,22 @@ class Experiments:
", ".join(rev[:7] for rev in to_run),
)
executors = self._init_executors(to_run)
manager = LocalExecutorManager.from_stash_entries(
self.scm,
os.path.join(self.repo.tmp_dir, EXEC_TMP_DIR),
self.repo,
to_run,
)
try:
exec_results = {}
exec_results.update(self._executors_repro(executors, **kwargs))
exec_results.update(self._executors_repro(manager, **kwargs))
finally:
# only drop successfully run stashed experiments
to_drop = [
entry.index
to_drop: List[int] = [
entry.stash_index
for rev, entry in to_run.items()
if (
entry.index is not None
entry.stash_index is not None
and (not keep_stash or rev in exec_results)
)
]
@@ -647,47 +643,11 @@ class Experiments:
result.update(exp_result)
return result
def _init_executors(self, to_run):
from dvc.utils.fs import makedirs
from .executor.local import TempDirExecutor
executors = {}
base_tmp_dir = os.path.join(self.repo.tmp_dir, self.EXEC_TMP_DIR)
if not os.path.exists(base_tmp_dir):
makedirs(base_tmp_dir)
pid_dir = os.path.join(self.repo.tmp_dir, self.EXEC_PID_DIR)
if not os.path.exists(pid_dir):
makedirs(pid_dir)
for stash_rev, item in to_run.items():
self.scm.set_ref(EXEC_HEAD, item.rev)
self.scm.set_ref(EXEC_MERGE, stash_rev)
self.scm.set_ref(EXEC_BASELINE, item.baseline_rev)
# Executor will be initialized with an empty git repo that
# we populate by pushing:
# EXEC_HEAD - the base commit for this experiment
# EXEC_MERGE - the unmerged changes (from our stash)
# to be reproduced
# EXEC_BASELINE - the baseline commit for this experiment
executor = TempDirExecutor(
self.scm,
self.dvc_dir,
name=item.name,
branch=item.branch,
tmp_dir=base_tmp_dir,
)
executor.init_cache(self.repo, stash_rev)
executors[stash_rev] = executor
for ref in (EXEC_HEAD, EXEC_MERGE, EXEC_BASELINE):
self.scm.remove_ref(ref)
return executors
@unlocked_repo
def _executors_repro(
self, executors: dict, jobs: Optional[int] = 1
self,
manager: "BaseExecutorManager",
**kwargs,
) -> Dict[str, Dict[str, str]]:
"""Run dvc repro for the specified BaseExecutors in parallel.
@@ -695,106 +655,7 @@ class Experiments:
dict mapping stash revs to the successfully executed experiments
for each stash rev.
"""
from dvc.stage.monitor import CheckpointKilledError
result: Dict[str, Dict[str, str]] = defaultdict(dict)
manager = Manager()
pid_q = manager.Queue()
rel_cwd = relpath(os.getcwd(), self.repo.root_dir)
with ProcessPoolExecutor(max_workers=jobs) as workers:
futures = {}
for rev, executor in executors.items():
pidfile = os.path.join(
self.repo.tmp_dir,
self.EXEC_PID_DIR,
f"{rev}{executor.PIDFILE_EXT}",
)
future = workers.submit(
executor.reproduce,
executor.dvc_dir,
rev,
queue=pid_q,
name=executor.name,
rel_cwd=rel_cwd,
log_level=logger.getEffectiveLevel(),
pidfile=pidfile,
git_url=executor.git_url,
)
futures[future] = (rev, executor)
try:
wait(futures)
except KeyboardInterrupt:
# forward SIGINT to any running executor processes and
# cancel any remaining futures
workers.shutdown(wait=False)
pids = {}
for future, (rev, _) in futures.items():
if future.running():
# if future has already been started by the scheduler
# we still have to wait until it tells us its PID
while rev not in pids:
rev, pid = pid_q.get()
pids[rev] = pid
os.kill(pids[rev], signal.SIGINT)
elif not future.done():
future.cancel()
for future, (rev, executor) in futures.items():
rev, executor = futures[future]
try:
exc = future.exception()
if exc is None:
exec_result = future.result()
result[rev].update(
self._collect_executor(executor, exec_result)
)
elif not isinstance(exc, CheckpointKilledError):
logger.error(
"Failed to reproduce experiment '%s'", rev[:7]
)
except CancelledError:
logger.error(
"Cancelled before attempting to reproduce experiment "
"'%s'",
rev[:7],
)
finally:
executor.cleanup()
return result
def _collect_executor(self, executor, exec_result) -> Mapping[str, str]:
# NOTE: GitPython Repo instances cannot be re-used
# after process has received SIGINT or SIGTERM, so we
# need this hack to re-instantiate git instances after
# checkpoint runs. See:
# https://github.com/gitpython-developers/GitPython/issues/427
del self.repo.scm
results = {}
def on_diverged(ref: str, checkpoint: bool):
ref_info = ExpRefInfo.from_ref(ref)
if checkpoint:
raise CheckpointExistsError(ref_info.name)
raise ExperimentExistsError(ref_info.name)
for ref in executor.fetch_exps(
self.scm,
executor.git_url,
force=exec_result.force,
on_diverged=on_diverged,
):
exp_rev = self.scm.get_ref(ref)
if exp_rev:
logger.debug("Collected experiment '%s'.", exp_rev[:7])
results[exp_rev] = exec_result.exp_hash
return results
return manager.exec_queue(**kwargs)
@unlocked_repo
def _workspace_repro(self) -> Mapping[str, str]:
@@ -802,17 +663,15 @@ class Experiments:
from dvc.stage.monitor import CheckpointKilledError
from dvc.utils.fs import makedirs
from .executor.base import BaseExecutor
entry = first(self.stash_revs.values())
assert entry.index == 0
assert entry.stash_index == 0
# NOTE: the stash commit to be popped already contains all the current
# workspace changes plus CLI modified --params changes.
# `checkout --force` here will not lose any data (popping stash commit
# will result in conflict between workspace params and stashed CLI
# params, but we always want the stashed version).
with self.scm.detach_head(entry.rev, force=True, client="dvc"):
with self.scm.detach_head(entry.head_rev, force=True, client="dvc"):
rev = self.stash.pop()
self.scm.set_ref(EXEC_BASELINE, entry.baseline_rev)
if entry.branch:
@@ -821,19 +680,30 @@ class Experiments:
self.scm.remove_ref(EXEC_BRANCH)
try:
orig_checkpoint = self.scm.get_ref(EXEC_CHECKPOINT)
pid_dir = os.path.join(self.repo.tmp_dir, self.EXEC_PID_DIR)
pid_dir = os.path.join(
self.repo.tmp_dir,
EXEC_TMP_DIR,
EXEC_PID_DIR,
)
if not os.path.exists(pid_dir):
makedirs(pid_dir)
pidfile = os.path.join(
pid_dir, f"workspace{BaseExecutor.PIDFILE_EXT}"
infofile = os.path.join(
pid_dir, f"workspace{BaseExecutor.INFOFILE_EXT}"
)
info = ExecutorInfo(
git_url=self.scm.root_dir,
baseline_rev=entry.baseline_rev,
location="workspace",
root_dir=self.scm.root_dir,
dvc_dir=self.dvc_dir,
name=entry.name,
wdir=os.getcwd(),
)
exec_result = BaseExecutor.reproduce(
None,
rev,
name=entry.name,
rel_cwd=relpath(os.getcwd(), self.scm.root_dir),
info=info,
rev=rev,
infofile=infofile,
log_errors=False,
pidfile=pidfile,
)
if not exec_result.exp_hash:
@@ -931,45 +801,55 @@ class Experiments:
def get_running_exps(self) -> Dict[str, int]:
"""Return info for running experiments."""
from dvc.utils.serialize import load_yaml
from .executor.base import BaseExecutor, ExecutorInfo
from dvc.scm import InvalidRemoteSCMRepo
from dvc.utils.serialize import load_json
result = {}
for pidfile in self.repo.fs.find(
os.path.join(self.repo.tmp_dir, self.EXEC_PID_DIR)
):
rev, _ = os.path.splitext(os.path.basename(pidfile))
pid_dir = os.path.join(
self.repo.tmp_dir,
EXEC_TMP_DIR,
EXEC_PID_DIR,
)
for fname in self.repo.fs.find(pid_dir):
rev, ext = os.path.splitext(os.path.basename(fname))
if ext != BaseExecutor.INFOFILE_EXT:
continue
try:
info = ExecutorInfo.from_dict(load_yaml(pidfile))
info = ExecutorInfo.from_dict(load_json(fname))
if info.result is not None:
continue
if rev == "workspace":
# If we are appending to a checkpoint branch in a workspace
# run, show the latest checkpoint as running.
last_rev = self.scm.get_ref(EXEC_BRANCH)
if last_rev:
result[last_rev] = info.to_dict()
result[last_rev] = info.asdict()
else:
result[rev] = info.to_dict()
result[rev] = info.asdict()
else:
result[rev] = info.to_dict()
result[rev] = info.asdict()
if info.git_url:
def on_diverged(_ref: str, _checkpoint: bool):
return False
for ref in BaseExecutor.fetch_exps(
self.scm,
info.git_url,
on_diverged=on_diverged,
):
logger.debug(
"Updated running experiment '%s'.", ref
)
last_rev = self.scm.get_ref(ref)
result[rev]["last"] = last_rev
if last_rev:
result[last_rev] = info.to_dict()
try:
for ref in BaseExecutor.fetch_exps(
self.scm,
info.git_url,
on_diverged=on_diverged,
):
logger.debug(
"Updated running experiment '%s'.", ref
)
last_rev = self.scm.get_ref(ref)
result[rev]["last"] = last_rev
if last_rev:
result[last_rev] = info.asdict()
except InvalidRemoteSCMRepo:
# ignore stale info files
del result[rev]
except OSError:
pass
return result
+18 -1
View File
@@ -1,4 +1,4 @@
from typing import Optional
from typing import NamedTuple, Optional
from dvc.exceptions import DvcException, InvalidArgumentError
@@ -141,3 +141,20 @@ class ExpRefInfo:
baseline_sha = parts[2] + parts[3] if len(parts) >= 4 else None
name = parts[4] if len(parts) == 5 else None
return cls(baseline_sha, name)
class ExpStashEntry(NamedTuple):
"""Experiment stash entry.
stash_index: EXPS_STASH index for this entry. Can be None if this commit
is not pushed onto the stash ref.
head_rev: HEAD Git commit to be checked out for this experiment.
baseline_rev: Experiment baseline commit.
branch: Optional exp (checkpoint) branch name for this experiment.
"""
stash_index: Optional[int]
head_rev: str
baseline_rev: str
branch: Optional[str]
name: Optional[str]
+172 -79
View File
@@ -3,21 +3,28 @@ import os
import pickle
from abc import ABC, abstractmethod
from contextlib import contextmanager
from dataclasses import dataclass
from dataclasses import asdict, dataclass
from functools import partial
from typing import (
TYPE_CHECKING,
Any,
Callable,
Dict,
Iterable,
NamedTuple,
Optional,
Type,
TypeVar,
Union,
)
from dvc.env import DVC_EXP_AUTO_PUSH, DVC_EXP_GIT_REMOTE
from dvc.exceptions import DvcException
from dvc.repo import Repo
from dvc.repo.experiments.base import (
from dvc.stage.serialize import to_lockfile
from dvc.utils import dict_sha256, env2bool, relpath
from dvc.utils.fs import remove
from ..base import (
EXEC_BASELINE,
EXEC_BRANCH,
EXEC_CHECKPOINT,
@@ -29,20 +36,24 @@ from dvc.repo.experiments.base import (
ExpRefInfo,
UnchangedExperimentError,
)
from dvc.stage import PipelineStage
from dvc.stage.monitor import CheckpointKilledError
from dvc.stage.serialize import to_lockfile
from dvc.utils import dict_sha256, env2bool
from dvc.utils.fs import remove
if TYPE_CHECKING:
from multiprocessing import Queue
from scmrepo.git import Git
from dvc.repo import Repo
from dvc.stage import PipelineStage
from ..base import ExpStashEntry
logger = logging.getLogger(__name__)
EXEC_TMP_DIR = "exps"
EXEC_PID_DIR = "run"
class ExecutorResult(NamedTuple):
exp_hash: Optional[str]
ref_info: Optional["ExpRefInfo"]
@@ -51,68 +62,77 @@ class ExecutorResult(NamedTuple):
@dataclass
class ExecutorInfo:
PARAM_PID = "pid"
PARAM_GIT_URL = "git"
PARAM_BASELINE_REV = "baseline"
PARAM_LOCATION = "location"
pid: Optional[int]
git_url: Optional[str]
baseline_rev: Optional[str]
location: Optional[str]
git_url: str
baseline_rev: str
location: str
root_dir: str
dvc_dir: str
name: Optional[str] = None
wdir: Optional[str] = None
result_hash: Optional[str] = None
result_ref: Optional[str] = None
result_force: bool = False
@classmethod
def from_dict(cls, d):
return cls(
d.get(cls.PARAM_PID),
d.get(cls.PARAM_GIT_URL),
d.get(cls.PARAM_BASELINE_REV),
d.get(cls.PARAM_LOCATION),
return cls(**d)
def asdict(self):
return asdict(self)
@property
def result(self) -> Optional["ExecutorResult"]:
if self.result_hash is None:
return None
return ExecutorResult(
self.result_hash,
ExpRefInfo.from_ref(self.result_ref) if self.result_ref else None,
self.result_force,
)
def to_dict(self):
return {
self.PARAM_PID: self.pid,
self.PARAM_GIT_URL: self.git_url,
self.PARAM_BASELINE_REV: self.baseline_rev,
self.PARAM_LOCATION: self.location,
}
_T = TypeVar("_T", bound="BaseExecutor")
class BaseExecutor(ABC):
"""Base class for executing experiments in parallel.
Args:
src: source Git SCM instance.
dvc_dir: relpath to DVC root from SCM root.
Optional keyword args:
branch: Existing git branch for this experiment.
Parameters:
root_dir: Path to SCM root.
dvc_dir: Path to .dvc dir relative to SCM root.
baseline_rev: Experiment baseline revision.
wdir: Path to exec working directory relative to SCM root.
name: Executor (experiment) name.
result: Completed executor result.
"""
PACKED_ARGS_FILE = "repro.dat"
WARN_UNTRACKED = False
QUIET = False
PIDFILE_EXT = ".run"
DEFAULT_LOCATION: Optional[str] = "workspace"
INFOFILE_EXT = ".run"
DEFAULT_LOCATION: str = "workspace"
def __init__(
self,
src: "Git",
root_dir: str,
dvc_dir: str,
root_dir: Optional[str] = None,
branch: Optional[str] = None,
baseline_rev: str,
wdir: Optional[str] = None,
name: Optional[str] = None,
location: Optional[str] = None,
result: Optional["ExecutorResult"] = None,
**kwargs,
):
assert root_dir is not None
self._dvc_dir = dvc_dir
self.root_dir = root_dir
self._init_git(src, branch)
self.wdir = wdir
self.name = name
self.baseline_rev = baseline_rev
self.location: str = location or self.DEFAULT_LOCATION
self.result = result
@abstractmethod
def _init_git(self, scm: "Git", branch: Optional[str] = None, **kwargs):
def init_git(self, scm: "Git", branch: Optional[str] = None):
"""Init git repo and populate it using exp refs from the specified
SCM instance.
"""
@@ -126,12 +146,92 @@ class BaseExecutor(ABC):
def init_cache(self, dvc: "Repo", rev: str, run_cache: bool = True):
"""Initialize DVC (cache)."""
@property
def info(self) -> "ExecutorInfo":
if self.result is not None:
result_dict: Dict[str, Any] = {
"result_hash": self.result.exp_hash,
"result_ref": (
str(self.result.ref_info) if self.result.ref_info else None
),
"result_force": self.result.force,
}
else:
result_dict = {}
return ExecutorInfo(
git_url=self.git_url,
baseline_rev=self.baseline_rev,
location=self.location,
root_dir=self.root_dir,
dvc_dir=self.dvc_dir,
name=self.name,
wdir=self.wdir,
**result_dict,
)
@classmethod
def from_info(cls: Type[_T], info: "ExecutorInfo") -> _T:
if info.result_hash:
result: Optional["ExecutorResult"] = ExecutorResult(
info.result_hash,
(
ExpRefInfo.from_ref(info.result_ref)
if info.result_ref
else None
),
info.result_force,
)
else:
result = None
return cls(
root_dir=info.root_dir,
dvc_dir=info.dvc_dir,
baseline_rev=info.baseline_rev,
name=info.name,
wdir=info.wdir,
result=result,
)
@classmethod
@abstractmethod
def from_stash_entry(
cls: Type[_T],
repo: "Repo",
stash_rev: str,
entry: "ExpStashEntry",
**kwargs,
) -> _T:
pass
@classmethod
def _from_stash_entry(
cls: Type[_T],
repo: "Repo",
stash_rev: str,
entry: "ExpStashEntry",
root_dir: str,
**kwargs,
) -> _T:
executor = cls(
root_dir=root_dir,
dvc_dir=relpath(repo.dvc_dir, repo.scm.root_dir),
baseline_rev=entry.baseline_rev,
name=entry.name,
wdir=relpath(os.getcwd(), repo.scm.root_dir),
**kwargs,
)
executor.init_git(repo.scm, branch=entry.branch)
executor.init_cache(repo, stash_rev)
return executor
@property
def dvc_dir(self) -> str:
return os.path.join(self.root_dir, self._dvc_dir)
@staticmethod
def hash_exp(stages: Iterable["PipelineStage"]) -> str:
from dvc.stage import PipelineStage
exp_data = {}
for stage in stages:
if isinstance(stage, PipelineStage):
@@ -244,11 +344,10 @@ class BaseExecutor(ABC):
@classmethod
def reproduce(
cls,
dvc_dir: Optional[str],
info: "ExecutorInfo",
rev: str,
queue: Optional["Queue"] = None,
rel_cwd: Optional[str] = None,
name: Optional[str] = None,
infofile: Optional[str] = None,
log_errors: bool = True,
log_level: Optional[int] = None,
**kwargs,
@@ -262,6 +361,7 @@ class BaseExecutor(ABC):
"""
from dvc.repo.checkout import checkout as dvc_checkout
from dvc.repo.reproduce import reproduce as dvc_reproduce
from dvc.stage import PipelineStage
auto_push = env2bool(DVC_EXP_AUTO_PUSH)
git_remote = os.getenv(DVC_EXP_GIT_REMOTE, None)
@@ -283,9 +383,9 @@ class BaseExecutor(ABC):
repro_force: bool = False
with cls._repro_dvc(
dvc_dir,
rel_cwd,
log_errors,
info,
log_errors=log_errors,
infofile=infofile,
**kwargs,
) as dvc:
if auto_push:
@@ -333,7 +433,7 @@ class BaseExecutor(ABC):
cls.checkpoint_callback,
dvc,
dvc.scm,
name,
info.name,
repro_force or checkpoint_reset,
)
stages = dvc_reproduce(
@@ -358,7 +458,7 @@ class BaseExecutor(ABC):
cls.commit(
dvc.scm,
exp_hash,
exp_name=name,
exp_name=info.name,
force=repro_force,
checkpoint=is_checkpoint,
)
@@ -389,38 +489,32 @@ class BaseExecutor(ABC):
@contextmanager
def _repro_dvc(
cls,
dvc_dir: Optional[str],
rel_cwd: Optional[str],
log_errors: bool,
pidfile: Optional[str] = None,
git_url: Optional[str] = None,
info: "ExecutorInfo",
log_errors: bool = True,
infofile: Optional[str] = None,
**kwargs,
):
from dvc.utils.serialize import modify_yaml
from dvc.repo import Repo
from dvc.stage.monitor import CheckpointKilledError
from dvc.utils.fs import makedirs
from dvc.utils.serialize import modify_json
dvc = Repo(dvc_dir)
dvc = Repo(os.path.join(info.root_dir, info.dvc_dir))
if cls.QUIET:
dvc.scm_context.quiet = cls.QUIET
if dvc_dir is not None:
old_cwd: Optional[str] = os.getcwd()
if rel_cwd:
os.chdir(os.path.join(dvc.root_dir, rel_cwd))
else:
os.chdir(dvc.root_dir)
old_cwd = os.getcwd()
if info.wdir:
os.chdir(os.path.join(dvc.scm.root_dir, info.wdir))
else:
old_cwd = None
if pidfile is not None:
info = ExecutorInfo(
os.getpid(),
git_url,
dvc.scm.get_rev(),
cls.DEFAULT_LOCATION,
)
with modify_yaml(pidfile) as d:
d.update(info.to_dict())
logger.debug("Running repro in '%s'", os.getcwd())
os.chdir(dvc.root_dir)
if infofile is not None:
makedirs(os.path.dirname(infofile), exist_ok=True)
with modify_json(infofile) as d:
d.update(info.asdict())
try:
logger.debug("Running repro in '%s'", os.getcwd())
yield dvc
except CheckpointKilledError:
raise
@@ -433,11 +527,10 @@ class BaseExecutor(ABC):
logger.exception("unexpected error")
raise
finally:
if pidfile is not None:
remove(pidfile)
if infofile is not None:
remove(infofile)
dvc.close()
if old_cwd:
os.chdir(old_cwd)
os.chdir(old_cwd)
@classmethod
def _repro_args(cls, dvc):
+35 -40
View File
@@ -1,47 +1,32 @@
import logging
import os
from tempfile import TemporaryDirectory
from tempfile import mkdtemp
from typing import TYPE_CHECKING, Optional
from funcy import cached_property
from dvc.repo.experiments.base import (
from dvc.scm import SCM
from dvc.utils.fs import remove
from ..base import (
EXEC_BRANCH,
EXEC_CHECKPOINT,
EXEC_HEAD,
EXEC_MERGE,
EXEC_NAMESPACE,
)
from dvc.scm import SCM
from .base import BaseExecutor
from .base import EXEC_TMP_DIR, BaseExecutor
if TYPE_CHECKING:
from scmrepo.git import Git
from dvc.repo import Repo
from ..base import ExpStashEntry
logger = logging.getLogger(__name__)
class ExpTemporaryDirectory(TemporaryDirectory):
# Python's TemporaryDirectory cleanup shutil.rmtree wrapper does not handle
# git read-only dirs cleanly in Windows on Python <3.8, so we use our own
# remove(). See:
# https://github.com/iterative/dvc/pull/5425
# https://bugs.python.org/issue26660
@classmethod
def _rmtree(cls, name):
from dvc.utils.fs import remove
remove(name)
def cleanup(self):
if self._finalizer.detach():
self._rmtree(self.name)
class BaseLocalExecutor(BaseExecutor):
"""Base local machine executor."""
@@ -70,20 +55,9 @@ class TempDirExecutor(BaseLocalExecutor):
# suggestions) that are not applicable outside of workspace runs
WARN_UNTRACKED = True
QUIET = True
DEFAULT_LOCATION: Optional[str] = "temp"
DEFAULT_LOCATION = "temp"
def __init__(
self,
*args,
tmp_dir: Optional[str] = None,
**kwargs,
):
self._tmp_dir = ExpTemporaryDirectory(dir=tmp_dir)
kwargs["root_dir"] = self._tmp_dir.name
super().__init__(*args, **kwargs)
logger.debug("Init temp dir executor in dir '%s'", self._tmp_dir)
def _init_git(self, scm: "Git", branch: Optional[str] = None, **kwargs):
def init_git(self, scm: "Git", branch: Optional[str] = None):
from dulwich.repo import Repo as DulwichRepo
from ..utils import push_refspec
@@ -91,9 +65,9 @@ class TempDirExecutor(BaseLocalExecutor):
DulwichRepo.init(os.fspath(self.root_dir))
refspec = f"{EXEC_NAMESPACE}/"
push_refspec(scm, self.git_url, refspec, refspec, **kwargs)
push_refspec(scm, self.git_url, refspec, refspec)
if branch:
push_refspec(scm, self.git_url, branch, branch, **kwargs)
push_refspec(scm, self.git_url, branch, branch)
self.scm.set_ref(EXEC_BRANCH, branch, symbolic=True)
elif self.scm.get_ref(EXEC_BRANCH):
self.scm.remove_ref(EXEC_BRANCH)
@@ -120,5 +94,26 @@ class TempDirExecutor(BaseLocalExecutor):
def cleanup(self):
super().cleanup()
logger.debug("Removing tmpdir '%s'", self._tmp_dir)
self._tmp_dir.cleanup()
logger.debug("Removing tmpdir '%s'", self.root_dir)
remove(self.root_dir)
@classmethod
def from_stash_entry(
cls,
repo: "Repo",
stash_rev: str,
entry: "ExpStashEntry",
**kwargs,
):
tmp_dir = mkdtemp(dir=os.path.join(repo.tmp_dir, EXEC_TMP_DIR))
try:
executor = cls._from_stash_entry(repo, stash_rev, entry, tmp_dir)
logger.debug("Init temp dir executor in '%s'", tmp_dir)
return executor
except Exception:
remove(tmp_dir)
raise
class WorkspaceExecutor(BaseLocalExecutor):
pass
+234
View File
@@ -0,0 +1,234 @@
import logging
import os
from abc import ABC
from collections import deque
from typing import TYPE_CHECKING, Deque, Dict, Optional, Tuple, Type
from dvc.proc.manager import ProcessManager
from ..base import (
EXEC_BASELINE,
EXEC_HEAD,
EXEC_MERGE,
CheckpointExistsError,
ExperimentExistsError,
ExpRefInfo,
ExpStashEntry,
)
from .base import EXEC_PID_DIR
from .local import TempDirExecutor, WorkspaceExecutor
if TYPE_CHECKING:
from scmrepo.git import Git
from dvc.repo import Repo
from .base import BaseExecutor
logger = logging.getLogger(__name__)
class BaseExecutorManager(ABC):
"""Manages executors for a collection of experiments to be run."""
EXECUTOR_CLS: Type = WorkspaceExecutor
def __init__(
self,
scm: "Git",
wdir: str,
):
from dvc.utils.fs import makedirs
self.scm = scm
makedirs(wdir, exist_ok=True)
self.wdir = wdir
self.executors = self._load_infos()
self._queue: Deque[Tuple[str, "BaseExecutor"]] = deque()
self.proc = ProcessManager(self.pid_dir)
@property
def pid_dir(self) -> str:
return os.path.join(self.wdir, EXEC_PID_DIR)
def enqueue(self, rev: str, executor: "BaseExecutor"):
self._queue.append((rev, executor))
def _load_infos(self) -> Dict[str, "BaseExecutor"]:
# TODO: pending process manager changes
# import json
# from urllib.parse import urlparse
# from .base import ExecutorInfo
# from .ssh import SSHExecutor
# def make_executor(info: "ExecutorInfo"):
# if info.git_url:
# scheme = urlparse(info.git_url)
# if scheme == "file":
# cls = TempDirExecutor
# elif scheme == "ssh":
# cls = SSHExecutor
# else:
# raise NotImplementedError
# else:
# cls = WorkspaceExecutor
# return cls.from_info(info)
infos: Dict[str, "BaseExecutor"] = {}
# for name in self.proc.processes():
# infofile = os.path.join(
# name, f"{name}{BaseExecutor.INFOFILE_EXT}"
# )
# try:
# with open(infofile, encoding="utf-8") as fobj:
# info = ExecutorInfo.from_dict(json.load(fobj))
# infos[name] = make_executor(info)
# except OSError:
# continue
return infos
@classmethod
def from_stash_entries(
cls,
scm: "Git",
wdir: str,
repo: "Repo",
to_run: Dict[str, ExpStashEntry],
**kwargs,
):
manager = cls(scm, wdir)
try:
for stash_rev, entry in to_run.items():
scm.set_ref(EXEC_HEAD, entry.head_rev)
scm.set_ref(EXEC_MERGE, stash_rev)
scm.set_ref(EXEC_BASELINE, entry.baseline_rev)
# Executor will be initialized with an empty git repo that
# we populate by pushing:
# EXEC_HEAD - the base commit for this experiment
# EXEC_MERGE - the unmerged changes (from our stash)
# to be reproduced
# EXEC_BASELINE - the baseline commit for this experiment
executor = cls.EXECUTOR_CLS.from_stash_entry(
repo,
stash_rev,
entry,
**kwargs,
)
manager.enqueue(stash_rev, executor)
finally:
for ref in (EXEC_HEAD, EXEC_MERGE, EXEC_BASELINE):
scm.remove_ref(ref)
return manager
def exec_queue(self, jobs: Optional[int] = 1):
"""Run dvc repro for queued executors in parallel."""
import signal
from collections import defaultdict
from concurrent.futures import (
CancelledError,
ProcessPoolExecutor,
wait,
)
from multiprocessing import Manager
from dvc.stage.monitor import CheckpointKilledError
result: Dict[str, Dict[str, str]] = defaultdict(dict)
manager = Manager()
pid_q = manager.Queue()
with ProcessPoolExecutor(max_workers=jobs) as workers:
futures = {}
while self._queue:
rev, executor = self._queue.popleft()
infofile = os.path.join(
self.pid_dir,
f"{rev}{executor.INFOFILE_EXT}",
)
future = workers.submit(
executor.reproduce,
info=executor.info,
rev=rev,
queue=pid_q,
infofile=infofile,
log_level=logger.getEffectiveLevel(),
)
futures[future] = (rev, executor)
try:
wait(futures)
except KeyboardInterrupt:
# forward SIGINT to any running executor processes and
# cancel any remaining futures
workers.shutdown(wait=False)
pids = {}
for future, (rev, _) in futures.items():
if future.running():
# if future has already been started by the scheduler
# we still have to wait until it tells us its PID
while rev not in pids:
rev, pid = pid_q.get()
pids[rev] = pid
os.kill(pids[rev], signal.SIGINT)
elif not future.done():
future.cancel()
for future, (rev, executor) in futures.items():
rev, executor = futures[future]
try:
exc = future.exception()
if exc is None:
exec_result = future.result()
result[rev].update(
self._collect_executor(executor, exec_result)
)
elif not isinstance(exc, CheckpointKilledError):
logger.error(
"Failed to reproduce experiment '%s'", rev[:7]
)
except CancelledError:
logger.error(
"Cancelled before attempting to reproduce experiment "
"'%s'",
rev[:7],
)
finally:
executor.cleanup()
return result
def _collect_executor(self, executor, exec_result) -> Dict[str, str]:
# NOTE: GitPython Repo instances cannot be re-used
# after process has received SIGINT or SIGTERM, so we
# need this hack to re-instantiate git instances after
# checkpoint runs. See:
# https://github.com/gitpython-developers/GitPython/issues/427
# del self.repo.scm
results = {}
def on_diverged(ref: str, checkpoint: bool):
ref_info = ExpRefInfo.from_ref(ref)
if checkpoint:
raise CheckpointExistsError(ref_info.name)
raise ExperimentExistsError(ref_info.name)
for ref in executor.fetch_exps(
self.scm,
executor.git_url,
force=exec_result.force,
on_diverged=on_diverged,
):
exp_rev = self.scm.get_ref(ref)
if exp_rev:
logger.debug("Collected experiment '%s'.", exp_rev[:7])
results[exp_rev] = exec_result.exp_hash
return results
class LocalExecutorManager(BaseExecutorManager):
EXECUTOR_CLS = TempDirExecutor
+21 -12
View File
@@ -22,6 +22,8 @@ if TYPE_CHECKING:
from dvc.machine import MachineManager
from dvc.repo import Repo
from ..base import ExpStashEntry
logger = logging.getLogger(__name__)
@@ -51,13 +53,12 @@ class SSHExecutor(BaseExecutor):
):
assert host
super().__init__(*args, **kwargs)
self.host: str = host
self.port = port
self.username = username
self._fs_factory = fs_factory
super().__init__(*args, **kwargs)
logger.debug("Init SSH executor for host '%s'", self.host)
self._repo_abspath = ""
@classmethod
def gen_dirname(cls, name: Optional[str] = None):
@@ -66,16 +67,24 @@ class SSHExecutor(BaseExecutor):
return "-".join([name or "dvc-exp", "executor", uuid()])
@classmethod
def from_machine(
def from_stash_entry(
cls,
manager: "MachineManager",
machine_name: Optional[str],
*args,
repo: "Repo",
stash_rev: str,
entry: "ExpStashEntry",
**kwargs,
):
kwargs["root_dir"] = cls.gen_dirname(kwargs.get("name"))
kwargs.update(manager.get_executor_kwargs(machine_name))
return cls(*args, **kwargs)
manager: "MachineManager" = kwargs.pop("manager")
machine_name: Optional[str] = kwargs.pop("machine_name", None)
executor = cls._from_stash_entry(
repo,
stash_rev,
entry,
cls.gen_dirname(entry.name),
**manager.get_executor_kwargs(machine_name),
)
logger.debug("Init SSH executor for host '%s'", executor.host)
return executor
def sshfs(self):
return _sshfs(self._fs_factory, host=self.host, port=self.port)
@@ -104,7 +113,7 @@ class SSHExecutor(BaseExecutor):
}
return kwargs
def _init_git(self, scm: "Git", branch: Optional[str] = None, **kwargs):
def init_git(self, scm: "Git", branch: Optional[str] = None):
from ..utils import push_refspec
with self.sshfs() as fs:
@@ -121,7 +130,7 @@ class SSHExecutor(BaseExecutor):
# TODO: support multiple client key retries in git backends
# (see https://github.com/iterative/dvc/issues/6508)
kwargs.update(self._git_client_args(fs))
kwargs = self._git_client_args(fs)
refspec = f"{EXEC_NAMESPACE}/"
push_refspec(scm, self.git_url, refspec, refspec, **kwargs)
if branch:
+1 -1
View File
@@ -42,7 +42,7 @@ def gc(
delete_stashes = []
for _, entry in repo.experiments.stash_revs.items():
if not queued or entry.baseline_rev not in keep_revs:
delete_stashes.append(entry.index)
delete_stashes.append(entry.stash_index)
for index in sorted(delete_stashes, reverse=True):
repo.experiments.stash.drop(index)
removed += len(delete_stashes)
+4 -4
View File
@@ -49,16 +49,16 @@ def _clear_all(repo):
def _get_exp_stash_index(repo, ref_or_rev: str) -> Optional[int]:
stash_revs = repo.experiments.stash_revs
for _, ref_info in stash_revs.items():
if ref_info.name == ref_or_rev:
return ref_info.index
for _, entry in stash_revs.items():
if entry.name == ref_or_rev:
return entry.stash_index
from dvc.scm import resolve_rev
try:
rev = resolve_rev(repo.scm, ref_or_rev)
if rev in stash_revs:
return stash_revs.get(rev).index
return stash_revs.get(rev).stash_index
except RevError:
pass
return None
+1 -2
View File
@@ -6,7 +6,6 @@ from typing import Any, Callable, Dict, Optional
from dvc.exceptions import InvalidArgumentError
from dvc.repo import locked
from dvc.repo.experiments.base import ExpRefInfo
from dvc.repo.experiments.executor.base import ExecutorInfo
from dvc.repo.experiments.utils import fix_exp_head
from dvc.repo.metrics.show import _gather_metrics
from dvc.repo.params.show import _gather_params
@@ -44,7 +43,7 @@ def _collect_experiment_commit(
res["queued"] = stash
if running is not None and exp_rev in running:
res["running"] = True
res["executor"] = running[exp_rev].get(ExecutorInfo.PARAM_LOCATION)
res["executor"] = running[exp_rev].get("location")
else:
res["running"] = False
res["executor"] = None
+13 -4
View File
@@ -21,10 +21,19 @@ def _ssh_factory(cloud):
)
def test_from_machine(tmp_dir, scm, dvc, machine_instance, mocker):
mocker.patch.object(SSHExecutor, "_init_git")
executor = SSHExecutor.from_machine(dvc.machine, "foo", scm, "")
assert executor.host == machine_instance["instance_ip"]
def test_init_from_stash(tmp_dir, scm, dvc, machine_instance, mocker):
mock = mocker.patch.object(SSHExecutor, "_from_stash_entry")
mock_entry = mocker.Mock()
mock_entry.name = ""
SSHExecutor.from_stash_entry(
None,
"",
mock_entry,
manager=dvc.machine,
machine_name="foo",
)
_args, kwargs = mock.call_args
assert kwargs["host"] == machine_instance["instance_ip"]
@pytest.mark.needs_internet
+36 -14
View File
@@ -8,7 +8,12 @@ from funcy import first, get_in
from dvc.exceptions import InvalidArgumentError
from dvc.main import main
from dvc.repo.experiments.base import EXPS_STASH, ExpRefInfo
from dvc.repo.experiments.executor.base import BaseExecutor, ExecutorInfo
from dvc.repo.experiments.executor.base import (
EXEC_PID_DIR,
EXEC_TMP_DIR,
BaseExecutor,
ExecutorInfo,
)
from dvc.repo.experiments.utils import exp_refs_by_rev
from dvc.utils.fs import makedirs
from dvc.utils.serialize import YAMLFileCorruptedError
@@ -16,6 +21,20 @@ from tests.func.test_repro_multistage import COPY_SCRIPT
from tests.utils import console_width
def make_executor_info(**kwargs):
# set default values for required info fields
for key in (
"git_url",
"baseline_rev",
"location",
"root_dir",
"dvc_dir",
):
if key not in kwargs:
kwargs[key] = ""
return ExecutorInfo(**kwargs)
def test_show_simple(tmp_dir, scm, dvc, exp_stage):
assert dvc.experiments.show()["workspace"] == {
"baseline": {
@@ -367,11 +386,11 @@ def test_show_sort(tmp_dir, scm, dvc, exp_stage, caplog):
def test_show_running_workspace(tmp_dir, scm, dvc, exp_stage, capsys):
pid_dir = os.path.join(dvc.tmp_dir, dvc.experiments.EXEC_PID_DIR)
pid_dir = os.path.join(dvc.tmp_dir, EXEC_TMP_DIR, EXEC_PID_DIR)
makedirs(pid_dir, True)
info = ExecutorInfo(None, None, None, BaseExecutor.DEFAULT_LOCATION)
pidfile = os.path.join(pid_dir, f"workspace{BaseExecutor.PIDFILE_EXT}")
(tmp_dir / pidfile).dump(info.to_dict())
info = make_executor_info(location=BaseExecutor.DEFAULT_LOCATION)
pidfile = os.path.join(pid_dir, f"workspace{BaseExecutor.INFOFILE_EXT}")
(tmp_dir / pidfile).dump_json(info.asdict())
assert dvc.experiments.show()["workspace"] == {
"baseline": {
@@ -398,11 +417,11 @@ def test_show_running_executor(tmp_dir, scm, dvc, exp_stage):
dvc.experiments.run(exp_stage.addressing, params=["foo=2"], queue=True)
exp_rev = dvc.experiments.scm.resolve_rev(f"{EXPS_STASH}@{{0}}")
pid_dir = os.path.join(dvc.tmp_dir, dvc.experiments.EXEC_PID_DIR)
pid_dir = os.path.join(dvc.tmp_dir, EXEC_TMP_DIR, EXEC_PID_DIR)
makedirs(pid_dir, True)
info = ExecutorInfo(None, None, None, BaseExecutor.DEFAULT_LOCATION)
pidfile = os.path.join(pid_dir, f"{exp_rev}{BaseExecutor.PIDFILE_EXT}")
(tmp_dir / pidfile).dump(info.to_dict())
info = make_executor_info(location=BaseExecutor.DEFAULT_LOCATION)
pidfile = os.path.join(pid_dir, f"{exp_rev}{BaseExecutor.INFOFILE_EXT}")
(tmp_dir / pidfile).dump_json(info.asdict())
results = dvc.experiments.show()
exp_data = get_in(results, [baseline_rev, exp_rev, "data"])
@@ -430,17 +449,21 @@ def test_show_running_checkpoint(
checkpoint_rev = first(run_results)
exp_ref = first(exp_refs_by_rev(scm, checkpoint_rev))
pid_dir = os.path.join(dvc.tmp_dir, dvc.experiments.EXEC_PID_DIR)
pid_dir = os.path.join(dvc.tmp_dir, EXEC_TMP_DIR, EXEC_PID_DIR)
makedirs(pid_dir, True)
executor = (
BaseExecutor.DEFAULT_LOCATION
if workspace
else TempDirExecutor.DEFAULT_LOCATION
)
info = ExecutorInfo(123, "foo.git", baseline_rev, executor)
info = make_executor_info(
git_url="foo.git",
baseline_rev=baseline_rev,
location=executor,
)
rev = "workspace" if workspace else stash_rev
pidfile = os.path.join(pid_dir, f"{rev}{BaseExecutor.PIDFILE_EXT}")
(tmp_dir / pidfile).dump(info.to_dict())
pidfile = os.path.join(pid_dir, f"{rev}{BaseExecutor.INFOFILE_EXT}")
(tmp_dir / pidfile).dump_json(info.asdict())
mocker.patch.object(
BaseExecutor, "fetch_exps", return_value=[str(exp_ref)]
@@ -559,7 +582,6 @@ def test_show_only_changed(tmp_dir, dvc, scm, capsys):
assert main(["exp", "show"]) == 0
cap = capsys.readouterr()
print(cap)
assert "bar" in cap.out
capsys.readouterr()