diff --git a/dvc/repo/experiments/__init__.py b/dvc/repo/experiments/__init__.py index e2a6da324..523e120ae 100644 --- a/dvc/repo/experiments/__init__.py +++ b/dvc/repo/experiments/__init__.py @@ -1,12 +1,8 @@ import logging import os import re -import signal -from collections import defaultdict, namedtuple -from concurrent.futures import CancelledError, ProcessPoolExecutor, wait from functools import wraps -from multiprocessing import Manager -from typing import Dict, Iterable, Mapping, Optional +from typing import Dict, Iterable, List, Mapping, Optional from funcy import cached_property, first @@ -20,17 +16,22 @@ from .base import ( EXEC_BASELINE, EXEC_BRANCH, EXEC_CHECKPOINT, - EXEC_HEAD, - EXEC_MERGE, EXEC_NAMESPACE, EXPS_NAMESPACE, EXPS_STASH, BaselineMismatchError, - CheckpointExistsError, ExperimentExistsError, ExpRefInfo, + ExpStashEntry, MultipleBranchError, ) +from .executor.base import ( + EXEC_PID_DIR, + EXEC_TMP_DIR, + BaseExecutor, + ExecutorInfo, +) +from .executor.manager import BaseExecutorManager, LocalExecutorManager from .utils import exp_refs_by_rev logger = logging.getLogger(__name__) @@ -81,12 +82,6 @@ class Experiments: r"^(?P[a-f0-9]{7})-(?P[a-f0-9]+)" r"(?P-checkpoint)?$" ) - EXEC_TMP_DIR = "exps" - EXEC_PID_DIR = os.path.join(EXEC_TMP_DIR, "run") - - StashEntry = namedtuple( - "StashEntry", ["index", "rev", "baseline_rev", "branch", "name"] - ) def __init__(self, repo): from dvc.lock import make_lock @@ -112,8 +107,6 @@ class Experiments: @cached_property def args_file(self): - from .executor.base import BaseExecutor - return os.path.join(self.repo.tmp_dir, BaseExecutor.PACKED_ARGS_FILE) @cached_property @@ -123,13 +116,13 @@ class Experiments: return Stash(self.scm, EXPS_STASH) @property - def stash_revs(self): + def stash_revs(self) -> Dict[str, ExpStashEntry]: revs = {} for i, entry in enumerate(self.stash): msg = entry.message.decode("utf-8").strip() m = self.STASH_EXPERIMENT_RE.match(msg) if m: - revs[entry.new_sha.decode("utf-8")] = self.StashEntry( + revs[entry.new_sha.decode("utf-8")] = ExpStashEntry( i, m.group("rev"), m.group("baseline_rev"), @@ -324,8 +317,6 @@ class Experiments: def _pack_args(self, *args, **kwargs): import pickle - from .executor.base import BaseExecutor - if os.path.exists(self.args_file) and self.scm.is_tracked( self.args_file ): @@ -616,7 +607,7 @@ class Experiments: to_run = { rev: stash_revs[rev] if rev in stash_revs - else self.StashEntry(None, rev, rev, None, None) + else ExpStashEntry(None, rev, rev, None, None) for rev in revs } @@ -625,17 +616,22 @@ class Experiments: ", ".join(rev[:7] for rev in to_run), ) - executors = self._init_executors(to_run) + manager = LocalExecutorManager.from_stash_entries( + self.scm, + os.path.join(self.repo.tmp_dir, EXEC_TMP_DIR), + self.repo, + to_run, + ) try: exec_results = {} - exec_results.update(self._executors_repro(executors, **kwargs)) + exec_results.update(self._executors_repro(manager, **kwargs)) finally: # only drop successfully run stashed experiments - to_drop = [ - entry.index + to_drop: List[int] = [ + entry.stash_index for rev, entry in to_run.items() if ( - entry.index is not None + entry.stash_index is not None and (not keep_stash or rev in exec_results) ) ] @@ -647,47 +643,11 @@ class Experiments: result.update(exp_result) return result - def _init_executors(self, to_run): - from dvc.utils.fs import makedirs - - from .executor.local import TempDirExecutor - - executors = {} - base_tmp_dir = os.path.join(self.repo.tmp_dir, self.EXEC_TMP_DIR) - if not os.path.exists(base_tmp_dir): - makedirs(base_tmp_dir) - pid_dir = os.path.join(self.repo.tmp_dir, self.EXEC_PID_DIR) - if not os.path.exists(pid_dir): - makedirs(pid_dir) - for stash_rev, item in to_run.items(): - self.scm.set_ref(EXEC_HEAD, item.rev) - self.scm.set_ref(EXEC_MERGE, stash_rev) - self.scm.set_ref(EXEC_BASELINE, item.baseline_rev) - - # Executor will be initialized with an empty git repo that - # we populate by pushing: - # EXEC_HEAD - the base commit for this experiment - # EXEC_MERGE - the unmerged changes (from our stash) - # to be reproduced - # EXEC_BASELINE - the baseline commit for this experiment - executor = TempDirExecutor( - self.scm, - self.dvc_dir, - name=item.name, - branch=item.branch, - tmp_dir=base_tmp_dir, - ) - executor.init_cache(self.repo, stash_rev) - executors[stash_rev] = executor - - for ref in (EXEC_HEAD, EXEC_MERGE, EXEC_BASELINE): - self.scm.remove_ref(ref) - - return executors - @unlocked_repo def _executors_repro( - self, executors: dict, jobs: Optional[int] = 1 + self, + manager: "BaseExecutorManager", + **kwargs, ) -> Dict[str, Dict[str, str]]: """Run dvc repro for the specified BaseExecutors in parallel. @@ -695,106 +655,7 @@ class Experiments: dict mapping stash revs to the successfully executed experiments for each stash rev. """ - from dvc.stage.monitor import CheckpointKilledError - - result: Dict[str, Dict[str, str]] = defaultdict(dict) - - manager = Manager() - pid_q = manager.Queue() - - rel_cwd = relpath(os.getcwd(), self.repo.root_dir) - with ProcessPoolExecutor(max_workers=jobs) as workers: - futures = {} - for rev, executor in executors.items(): - pidfile = os.path.join( - self.repo.tmp_dir, - self.EXEC_PID_DIR, - f"{rev}{executor.PIDFILE_EXT}", - ) - future = workers.submit( - executor.reproduce, - executor.dvc_dir, - rev, - queue=pid_q, - name=executor.name, - rel_cwd=rel_cwd, - log_level=logger.getEffectiveLevel(), - pidfile=pidfile, - git_url=executor.git_url, - ) - futures[future] = (rev, executor) - - try: - wait(futures) - except KeyboardInterrupt: - # forward SIGINT to any running executor processes and - # cancel any remaining futures - workers.shutdown(wait=False) - pids = {} - for future, (rev, _) in futures.items(): - if future.running(): - # if future has already been started by the scheduler - # we still have to wait until it tells us its PID - while rev not in pids: - rev, pid = pid_q.get() - pids[rev] = pid - os.kill(pids[rev], signal.SIGINT) - elif not future.done(): - future.cancel() - - for future, (rev, executor) in futures.items(): - rev, executor = futures[future] - - try: - exc = future.exception() - if exc is None: - exec_result = future.result() - result[rev].update( - self._collect_executor(executor, exec_result) - ) - elif not isinstance(exc, CheckpointKilledError): - logger.error( - "Failed to reproduce experiment '%s'", rev[:7] - ) - except CancelledError: - logger.error( - "Cancelled before attempting to reproduce experiment " - "'%s'", - rev[:7], - ) - finally: - executor.cleanup() - - return result - - def _collect_executor(self, executor, exec_result) -> Mapping[str, str]: - # NOTE: GitPython Repo instances cannot be re-used - # after process has received SIGINT or SIGTERM, so we - # need this hack to re-instantiate git instances after - # checkpoint runs. See: - # https://github.com/gitpython-developers/GitPython/issues/427 - del self.repo.scm - - results = {} - - def on_diverged(ref: str, checkpoint: bool): - ref_info = ExpRefInfo.from_ref(ref) - if checkpoint: - raise CheckpointExistsError(ref_info.name) - raise ExperimentExistsError(ref_info.name) - - for ref in executor.fetch_exps( - self.scm, - executor.git_url, - force=exec_result.force, - on_diverged=on_diverged, - ): - exp_rev = self.scm.get_ref(ref) - if exp_rev: - logger.debug("Collected experiment '%s'.", exp_rev[:7]) - results[exp_rev] = exec_result.exp_hash - - return results + return manager.exec_queue(**kwargs) @unlocked_repo def _workspace_repro(self) -> Mapping[str, str]: @@ -802,17 +663,15 @@ class Experiments: from dvc.stage.monitor import CheckpointKilledError from dvc.utils.fs import makedirs - from .executor.base import BaseExecutor - entry = first(self.stash_revs.values()) - assert entry.index == 0 + assert entry.stash_index == 0 # NOTE: the stash commit to be popped already contains all the current # workspace changes plus CLI modified --params changes. # `checkout --force` here will not lose any data (popping stash commit # will result in conflict between workspace params and stashed CLI # params, but we always want the stashed version). - with self.scm.detach_head(entry.rev, force=True, client="dvc"): + with self.scm.detach_head(entry.head_rev, force=True, client="dvc"): rev = self.stash.pop() self.scm.set_ref(EXEC_BASELINE, entry.baseline_rev) if entry.branch: @@ -821,19 +680,30 @@ class Experiments: self.scm.remove_ref(EXEC_BRANCH) try: orig_checkpoint = self.scm.get_ref(EXEC_CHECKPOINT) - pid_dir = os.path.join(self.repo.tmp_dir, self.EXEC_PID_DIR) + pid_dir = os.path.join( + self.repo.tmp_dir, + EXEC_TMP_DIR, + EXEC_PID_DIR, + ) if not os.path.exists(pid_dir): makedirs(pid_dir) - pidfile = os.path.join( - pid_dir, f"workspace{BaseExecutor.PIDFILE_EXT}" + infofile = os.path.join( + pid_dir, f"workspace{BaseExecutor.INFOFILE_EXT}" + ) + info = ExecutorInfo( + git_url=self.scm.root_dir, + baseline_rev=entry.baseline_rev, + location="workspace", + root_dir=self.scm.root_dir, + dvc_dir=self.dvc_dir, + name=entry.name, + wdir=os.getcwd(), ) exec_result = BaseExecutor.reproduce( - None, - rev, - name=entry.name, - rel_cwd=relpath(os.getcwd(), self.scm.root_dir), + info=info, + rev=rev, + infofile=infofile, log_errors=False, - pidfile=pidfile, ) if not exec_result.exp_hash: @@ -931,45 +801,55 @@ class Experiments: def get_running_exps(self) -> Dict[str, int]: """Return info for running experiments.""" - from dvc.utils.serialize import load_yaml - - from .executor.base import BaseExecutor, ExecutorInfo + from dvc.scm import InvalidRemoteSCMRepo + from dvc.utils.serialize import load_json result = {} - for pidfile in self.repo.fs.find( - os.path.join(self.repo.tmp_dir, self.EXEC_PID_DIR) - ): - rev, _ = os.path.splitext(os.path.basename(pidfile)) + pid_dir = os.path.join( + self.repo.tmp_dir, + EXEC_TMP_DIR, + EXEC_PID_DIR, + ) + for fname in self.repo.fs.find(pid_dir): + rev, ext = os.path.splitext(os.path.basename(fname)) + if ext != BaseExecutor.INFOFILE_EXT: + continue try: - info = ExecutorInfo.from_dict(load_yaml(pidfile)) + info = ExecutorInfo.from_dict(load_json(fname)) + if info.result is not None: + continue if rev == "workspace": # If we are appending to a checkpoint branch in a workspace # run, show the latest checkpoint as running. last_rev = self.scm.get_ref(EXEC_BRANCH) if last_rev: - result[last_rev] = info.to_dict() + result[last_rev] = info.asdict() else: - result[rev] = info.to_dict() + result[rev] = info.asdict() else: - result[rev] = info.to_dict() + result[rev] = info.asdict() if info.git_url: def on_diverged(_ref: str, _checkpoint: bool): return False - for ref in BaseExecutor.fetch_exps( - self.scm, - info.git_url, - on_diverged=on_diverged, - ): - logger.debug( - "Updated running experiment '%s'.", ref - ) - last_rev = self.scm.get_ref(ref) - result[rev]["last"] = last_rev - if last_rev: - result[last_rev] = info.to_dict() + try: + for ref in BaseExecutor.fetch_exps( + self.scm, + info.git_url, + on_diverged=on_diverged, + ): + logger.debug( + "Updated running experiment '%s'.", ref + ) + last_rev = self.scm.get_ref(ref) + result[rev]["last"] = last_rev + if last_rev: + result[last_rev] = info.asdict() + except InvalidRemoteSCMRepo: + # ignore stale info files + del result[rev] except OSError: pass return result diff --git a/dvc/repo/experiments/base.py b/dvc/repo/experiments/base.py index 69a7f9853..9a23a9289 100644 --- a/dvc/repo/experiments/base.py +++ b/dvc/repo/experiments/base.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import NamedTuple, Optional from dvc.exceptions import DvcException, InvalidArgumentError @@ -141,3 +141,20 @@ class ExpRefInfo: baseline_sha = parts[2] + parts[3] if len(parts) >= 4 else None name = parts[4] if len(parts) == 5 else None return cls(baseline_sha, name) + + +class ExpStashEntry(NamedTuple): + """Experiment stash entry. + + stash_index: EXPS_STASH index for this entry. Can be None if this commit + is not pushed onto the stash ref. + head_rev: HEAD Git commit to be checked out for this experiment. + baseline_rev: Experiment baseline commit. + branch: Optional exp (checkpoint) branch name for this experiment. + """ + + stash_index: Optional[int] + head_rev: str + baseline_rev: str + branch: Optional[str] + name: Optional[str] diff --git a/dvc/repo/experiments/executor/base.py b/dvc/repo/experiments/executor/base.py index b025c9b37..1d9d3858d 100644 --- a/dvc/repo/experiments/executor/base.py +++ b/dvc/repo/experiments/executor/base.py @@ -3,21 +3,28 @@ import os import pickle from abc import ABC, abstractmethod from contextlib import contextmanager -from dataclasses import dataclass +from dataclasses import asdict, dataclass from functools import partial from typing import ( TYPE_CHECKING, + Any, Callable, + Dict, Iterable, NamedTuple, Optional, + Type, + TypeVar, Union, ) from dvc.env import DVC_EXP_AUTO_PUSH, DVC_EXP_GIT_REMOTE from dvc.exceptions import DvcException -from dvc.repo import Repo -from dvc.repo.experiments.base import ( +from dvc.stage.serialize import to_lockfile +from dvc.utils import dict_sha256, env2bool, relpath +from dvc.utils.fs import remove + +from ..base import ( EXEC_BASELINE, EXEC_BRANCH, EXEC_CHECKPOINT, @@ -29,20 +36,24 @@ from dvc.repo.experiments.base import ( ExpRefInfo, UnchangedExperimentError, ) -from dvc.stage import PipelineStage -from dvc.stage.monitor import CheckpointKilledError -from dvc.stage.serialize import to_lockfile -from dvc.utils import dict_sha256, env2bool -from dvc.utils.fs import remove if TYPE_CHECKING: from multiprocessing import Queue from scmrepo.git import Git + from dvc.repo import Repo + from dvc.stage import PipelineStage + + from ..base import ExpStashEntry + logger = logging.getLogger(__name__) +EXEC_TMP_DIR = "exps" +EXEC_PID_DIR = "run" + + class ExecutorResult(NamedTuple): exp_hash: Optional[str] ref_info: Optional["ExpRefInfo"] @@ -51,68 +62,77 @@ class ExecutorResult(NamedTuple): @dataclass class ExecutorInfo: - PARAM_PID = "pid" - PARAM_GIT_URL = "git" - PARAM_BASELINE_REV = "baseline" - PARAM_LOCATION = "location" - - pid: Optional[int] - git_url: Optional[str] - baseline_rev: Optional[str] - location: Optional[str] + git_url: str + baseline_rev: str + location: str + root_dir: str + dvc_dir: str + name: Optional[str] = None + wdir: Optional[str] = None + result_hash: Optional[str] = None + result_ref: Optional[str] = None + result_force: bool = False @classmethod def from_dict(cls, d): - return cls( - d.get(cls.PARAM_PID), - d.get(cls.PARAM_GIT_URL), - d.get(cls.PARAM_BASELINE_REV), - d.get(cls.PARAM_LOCATION), + return cls(**d) + + def asdict(self): + return asdict(self) + + @property + def result(self) -> Optional["ExecutorResult"]: + if self.result_hash is None: + return None + return ExecutorResult( + self.result_hash, + ExpRefInfo.from_ref(self.result_ref) if self.result_ref else None, + self.result_force, ) - def to_dict(self): - return { - self.PARAM_PID: self.pid, - self.PARAM_GIT_URL: self.git_url, - self.PARAM_BASELINE_REV: self.baseline_rev, - self.PARAM_LOCATION: self.location, - } + +_T = TypeVar("_T", bound="BaseExecutor") class BaseExecutor(ABC): """Base class for executing experiments in parallel. - Args: - src: source Git SCM instance. - dvc_dir: relpath to DVC root from SCM root. - - Optional keyword args: - branch: Existing git branch for this experiment. + Parameters: + root_dir: Path to SCM root. + dvc_dir: Path to .dvc dir relative to SCM root. + baseline_rev: Experiment baseline revision. + wdir: Path to exec working directory relative to SCM root. + name: Executor (experiment) name. + result: Completed executor result. """ PACKED_ARGS_FILE = "repro.dat" WARN_UNTRACKED = False QUIET = False - PIDFILE_EXT = ".run" - DEFAULT_LOCATION: Optional[str] = "workspace" + INFOFILE_EXT = ".run" + DEFAULT_LOCATION: str = "workspace" def __init__( self, - src: "Git", + root_dir: str, dvc_dir: str, - root_dir: Optional[str] = None, - branch: Optional[str] = None, + baseline_rev: str, + wdir: Optional[str] = None, name: Optional[str] = None, + location: Optional[str] = None, + result: Optional["ExecutorResult"] = None, **kwargs, ): - assert root_dir is not None self._dvc_dir = dvc_dir self.root_dir = root_dir - self._init_git(src, branch) + self.wdir = wdir self.name = name + self.baseline_rev = baseline_rev + self.location: str = location or self.DEFAULT_LOCATION + self.result = result @abstractmethod - def _init_git(self, scm: "Git", branch: Optional[str] = None, **kwargs): + def init_git(self, scm: "Git", branch: Optional[str] = None): """Init git repo and populate it using exp refs from the specified SCM instance. """ @@ -126,12 +146,92 @@ class BaseExecutor(ABC): def init_cache(self, dvc: "Repo", rev: str, run_cache: bool = True): """Initialize DVC (cache).""" + @property + def info(self) -> "ExecutorInfo": + if self.result is not None: + result_dict: Dict[str, Any] = { + "result_hash": self.result.exp_hash, + "result_ref": ( + str(self.result.ref_info) if self.result.ref_info else None + ), + "result_force": self.result.force, + } + else: + result_dict = {} + return ExecutorInfo( + git_url=self.git_url, + baseline_rev=self.baseline_rev, + location=self.location, + root_dir=self.root_dir, + dvc_dir=self.dvc_dir, + name=self.name, + wdir=self.wdir, + **result_dict, + ) + + @classmethod + def from_info(cls: Type[_T], info: "ExecutorInfo") -> _T: + if info.result_hash: + result: Optional["ExecutorResult"] = ExecutorResult( + info.result_hash, + ( + ExpRefInfo.from_ref(info.result_ref) + if info.result_ref + else None + ), + info.result_force, + ) + else: + result = None + return cls( + root_dir=info.root_dir, + dvc_dir=info.dvc_dir, + baseline_rev=info.baseline_rev, + name=info.name, + wdir=info.wdir, + result=result, + ) + + @classmethod + @abstractmethod + def from_stash_entry( + cls: Type[_T], + repo: "Repo", + stash_rev: str, + entry: "ExpStashEntry", + **kwargs, + ) -> _T: + pass + + @classmethod + def _from_stash_entry( + cls: Type[_T], + repo: "Repo", + stash_rev: str, + entry: "ExpStashEntry", + root_dir: str, + **kwargs, + ) -> _T: + executor = cls( + root_dir=root_dir, + dvc_dir=relpath(repo.dvc_dir, repo.scm.root_dir), + baseline_rev=entry.baseline_rev, + name=entry.name, + wdir=relpath(os.getcwd(), repo.scm.root_dir), + **kwargs, + ) + executor.init_git(repo.scm, branch=entry.branch) + executor.init_cache(repo, stash_rev) + return executor + @property def dvc_dir(self) -> str: return os.path.join(self.root_dir, self._dvc_dir) @staticmethod def hash_exp(stages: Iterable["PipelineStage"]) -> str: + from dvc.stage import PipelineStage + exp_data = {} for stage in stages: if isinstance(stage, PipelineStage): @@ -244,11 +344,10 @@ class BaseExecutor(ABC): @classmethod def reproduce( cls, - dvc_dir: Optional[str], + info: "ExecutorInfo", rev: str, queue: Optional["Queue"] = None, - rel_cwd: Optional[str] = None, - name: Optional[str] = None, + infofile: Optional[str] = None, log_errors: bool = True, log_level: Optional[int] = None, **kwargs, @@ -262,6 +361,7 @@ class BaseExecutor(ABC): """ from dvc.repo.checkout import checkout as dvc_checkout from dvc.repo.reproduce import reproduce as dvc_reproduce + from dvc.stage import PipelineStage auto_push = env2bool(DVC_EXP_AUTO_PUSH) git_remote = os.getenv(DVC_EXP_GIT_REMOTE, None) @@ -283,9 +383,9 @@ class BaseExecutor(ABC): repro_force: bool = False with cls._repro_dvc( - dvc_dir, - rel_cwd, - log_errors, + info, + log_errors=log_errors, + infofile=infofile, **kwargs, ) as dvc: if auto_push: @@ -333,7 +433,7 @@ class BaseExecutor(ABC): cls.checkpoint_callback, dvc, dvc.scm, - name, + info.name, repro_force or checkpoint_reset, ) stages = dvc_reproduce( @@ -358,7 +458,7 @@ class BaseExecutor(ABC): cls.commit( dvc.scm, exp_hash, - exp_name=name, + exp_name=info.name, force=repro_force, checkpoint=is_checkpoint, ) @@ -389,38 +489,32 @@ class BaseExecutor(ABC): @contextmanager def _repro_dvc( cls, - dvc_dir: Optional[str], - rel_cwd: Optional[str], - log_errors: bool, - pidfile: Optional[str] = None, - git_url: Optional[str] = None, + info: "ExecutorInfo", + log_errors: bool = True, + infofile: Optional[str] = None, **kwargs, ): - from dvc.utils.serialize import modify_yaml + from dvc.repo import Repo + from dvc.stage.monitor import CheckpointKilledError + from dvc.utils.fs import makedirs + from dvc.utils.serialize import modify_json - dvc = Repo(dvc_dir) + dvc = Repo(os.path.join(info.root_dir, info.dvc_dir)) if cls.QUIET: dvc.scm_context.quiet = cls.QUIET - if dvc_dir is not None: - old_cwd: Optional[str] = os.getcwd() - if rel_cwd: - os.chdir(os.path.join(dvc.root_dir, rel_cwd)) - else: - os.chdir(dvc.root_dir) + old_cwd = os.getcwd() + if info.wdir: + os.chdir(os.path.join(dvc.scm.root_dir, info.wdir)) else: - old_cwd = None - if pidfile is not None: - info = ExecutorInfo( - os.getpid(), - git_url, - dvc.scm.get_rev(), - cls.DEFAULT_LOCATION, - ) - with modify_yaml(pidfile) as d: - d.update(info.to_dict()) - logger.debug("Running repro in '%s'", os.getcwd()) + os.chdir(dvc.root_dir) + + if infofile is not None: + makedirs(os.path.dirname(infofile), exist_ok=True) + with modify_json(infofile) as d: + d.update(info.asdict()) try: + logger.debug("Running repro in '%s'", os.getcwd()) yield dvc except CheckpointKilledError: raise @@ -433,11 +527,10 @@ class BaseExecutor(ABC): logger.exception("unexpected error") raise finally: - if pidfile is not None: - remove(pidfile) + if infofile is not None: + remove(infofile) dvc.close() - if old_cwd: - os.chdir(old_cwd) + os.chdir(old_cwd) @classmethod def _repro_args(cls, dvc): diff --git a/dvc/repo/experiments/executor/local.py b/dvc/repo/experiments/executor/local.py index ab86ba3d7..f7ab240c8 100644 --- a/dvc/repo/experiments/executor/local.py +++ b/dvc/repo/experiments/executor/local.py @@ -1,47 +1,32 @@ import logging import os -from tempfile import TemporaryDirectory +from tempfile import mkdtemp from typing import TYPE_CHECKING, Optional from funcy import cached_property -from dvc.repo.experiments.base import ( +from dvc.scm import SCM +from dvc.utils.fs import remove + +from ..base import ( EXEC_BRANCH, EXEC_CHECKPOINT, EXEC_HEAD, EXEC_MERGE, EXEC_NAMESPACE, ) -from dvc.scm import SCM - -from .base import BaseExecutor +from .base import EXEC_TMP_DIR, BaseExecutor if TYPE_CHECKING: from scmrepo.git import Git from dvc.repo import Repo + from ..base import ExpStashEntry + logger = logging.getLogger(__name__) -class ExpTemporaryDirectory(TemporaryDirectory): - # Python's TemporaryDirectory cleanup shutil.rmtree wrapper does not handle - # git read-only dirs cleanly in Windows on Python <3.8, so we use our own - # remove(). See: - # https://github.com/iterative/dvc/pull/5425 - # https://bugs.python.org/issue26660 - - @classmethod - def _rmtree(cls, name): - from dvc.utils.fs import remove - - remove(name) - - def cleanup(self): - if self._finalizer.detach(): - self._rmtree(self.name) - - class BaseLocalExecutor(BaseExecutor): """Base local machine executor.""" @@ -70,20 +55,9 @@ class TempDirExecutor(BaseLocalExecutor): # suggestions) that are not applicable outside of workspace runs WARN_UNTRACKED = True QUIET = True - DEFAULT_LOCATION: Optional[str] = "temp" + DEFAULT_LOCATION = "temp" - def __init__( - self, - *args, - tmp_dir: Optional[str] = None, - **kwargs, - ): - self._tmp_dir = ExpTemporaryDirectory(dir=tmp_dir) - kwargs["root_dir"] = self._tmp_dir.name - super().__init__(*args, **kwargs) - logger.debug("Init temp dir executor in dir '%s'", self._tmp_dir) - - def _init_git(self, scm: "Git", branch: Optional[str] = None, **kwargs): + def init_git(self, scm: "Git", branch: Optional[str] = None): from dulwich.repo import Repo as DulwichRepo from ..utils import push_refspec @@ -91,9 +65,9 @@ class TempDirExecutor(BaseLocalExecutor): DulwichRepo.init(os.fspath(self.root_dir)) refspec = f"{EXEC_NAMESPACE}/" - push_refspec(scm, self.git_url, refspec, refspec, **kwargs) + push_refspec(scm, self.git_url, refspec, refspec) if branch: - push_refspec(scm, self.git_url, branch, branch, **kwargs) + push_refspec(scm, self.git_url, branch, branch) self.scm.set_ref(EXEC_BRANCH, branch, symbolic=True) elif self.scm.get_ref(EXEC_BRANCH): self.scm.remove_ref(EXEC_BRANCH) @@ -120,5 +94,26 @@ class TempDirExecutor(BaseLocalExecutor): def cleanup(self): super().cleanup() - logger.debug("Removing tmpdir '%s'", self._tmp_dir) - self._tmp_dir.cleanup() + logger.debug("Removing tmpdir '%s'", self.root_dir) + remove(self.root_dir) + + @classmethod + def from_stash_entry( + cls, + repo: "Repo", + stash_rev: str, + entry: "ExpStashEntry", + **kwargs, + ): + tmp_dir = mkdtemp(dir=os.path.join(repo.tmp_dir, EXEC_TMP_DIR)) + try: + executor = cls._from_stash_entry(repo, stash_rev, entry, tmp_dir) + logger.debug("Init temp dir executor in '%s'", tmp_dir) + return executor + except Exception: + remove(tmp_dir) + raise + + +class WorkspaceExecutor(BaseLocalExecutor): + pass diff --git a/dvc/repo/experiments/executor/manager.py b/dvc/repo/experiments/executor/manager.py new file mode 100644 index 000000000..217d3a1d2 --- /dev/null +++ b/dvc/repo/experiments/executor/manager.py @@ -0,0 +1,234 @@ +import logging +import os +from abc import ABC +from collections import deque +from typing import TYPE_CHECKING, Deque, Dict, Optional, Tuple, Type + +from dvc.proc.manager import ProcessManager + +from ..base import ( + EXEC_BASELINE, + EXEC_HEAD, + EXEC_MERGE, + CheckpointExistsError, + ExperimentExistsError, + ExpRefInfo, + ExpStashEntry, +) +from .base import EXEC_PID_DIR +from .local import TempDirExecutor, WorkspaceExecutor + +if TYPE_CHECKING: + from scmrepo.git import Git + + from dvc.repo import Repo + + from .base import BaseExecutor + +logger = logging.getLogger(__name__) + + +class BaseExecutorManager(ABC): + """Manages executors for a collection of experiments to be run.""" + + EXECUTOR_CLS: Type = WorkspaceExecutor + + def __init__( + self, + scm: "Git", + wdir: str, + ): + from dvc.utils.fs import makedirs + + self.scm = scm + makedirs(wdir, exist_ok=True) + self.wdir = wdir + self.executors = self._load_infos() + self._queue: Deque[Tuple[str, "BaseExecutor"]] = deque() + self.proc = ProcessManager(self.pid_dir) + + @property + def pid_dir(self) -> str: + return os.path.join(self.wdir, EXEC_PID_DIR) + + def enqueue(self, rev: str, executor: "BaseExecutor"): + self._queue.append((rev, executor)) + + def _load_infos(self) -> Dict[str, "BaseExecutor"]: + # TODO: pending process manager changes + # import json + # from urllib.parse import urlparse + # from .base import ExecutorInfo + # from .ssh import SSHExecutor + + # def make_executor(info: "ExecutorInfo"): + # if info.git_url: + # scheme = urlparse(info.git_url) + # if scheme == "file": + # cls = TempDirExecutor + # elif scheme == "ssh": + # cls = SSHExecutor + # else: + # raise NotImplementedError + # else: + # cls = WorkspaceExecutor + # return cls.from_info(info) + + infos: Dict[str, "BaseExecutor"] = {} + # for name in self.proc.processes(): + # infofile = os.path.join( + # name, f"{name}{BaseExecutor.INFOFILE_EXT}" + # ) + # try: + # with open(infofile, encoding="utf-8") as fobj: + # info = ExecutorInfo.from_dict(json.load(fobj)) + # infos[name] = make_executor(info) + # except OSError: + # continue + return infos + + @classmethod + def from_stash_entries( + cls, + scm: "Git", + wdir: str, + repo: "Repo", + to_run: Dict[str, ExpStashEntry], + **kwargs, + ): + manager = cls(scm, wdir) + try: + for stash_rev, entry in to_run.items(): + scm.set_ref(EXEC_HEAD, entry.head_rev) + scm.set_ref(EXEC_MERGE, stash_rev) + scm.set_ref(EXEC_BASELINE, entry.baseline_rev) + + # Executor will be initialized with an empty git repo that + # we populate by pushing: + # EXEC_HEAD - the base commit for this experiment + # EXEC_MERGE - the unmerged changes (from our stash) + # to be reproduced + # EXEC_BASELINE - the baseline commit for this experiment + executor = cls.EXECUTOR_CLS.from_stash_entry( + repo, + stash_rev, + entry, + **kwargs, + ) + manager.enqueue(stash_rev, executor) + finally: + for ref in (EXEC_HEAD, EXEC_MERGE, EXEC_BASELINE): + scm.remove_ref(ref) + return manager + + def exec_queue(self, jobs: Optional[int] = 1): + """Run dvc repro for queued executors in parallel.""" + import signal + from collections import defaultdict + from concurrent.futures import ( + CancelledError, + ProcessPoolExecutor, + wait, + ) + from multiprocessing import Manager + + from dvc.stage.monitor import CheckpointKilledError + + result: Dict[str, Dict[str, str]] = defaultdict(dict) + + manager = Manager() + pid_q = manager.Queue() + + with ProcessPoolExecutor(max_workers=jobs) as workers: + futures = {} + while self._queue: + rev, executor = self._queue.popleft() + infofile = os.path.join( + self.pid_dir, + f"{rev}{executor.INFOFILE_EXT}", + ) + future = workers.submit( + executor.reproduce, + info=executor.info, + rev=rev, + queue=pid_q, + infofile=infofile, + log_level=logger.getEffectiveLevel(), + ) + futures[future] = (rev, executor) + + try: + wait(futures) + except KeyboardInterrupt: + # forward SIGINT to any running executor processes and + # cancel any remaining futures + workers.shutdown(wait=False) + pids = {} + for future, (rev, _) in futures.items(): + if future.running(): + # if future has already been started by the scheduler + # we still have to wait until it tells us its PID + while rev not in pids: + rev, pid = pid_q.get() + pids[rev] = pid + os.kill(pids[rev], signal.SIGINT) + elif not future.done(): + future.cancel() + + for future, (rev, executor) in futures.items(): + rev, executor = futures[future] + + try: + exc = future.exception() + if exc is None: + exec_result = future.result() + result[rev].update( + self._collect_executor(executor, exec_result) + ) + elif not isinstance(exc, CheckpointKilledError): + logger.error( + "Failed to reproduce experiment '%s'", rev[:7] + ) + except CancelledError: + logger.error( + "Cancelled before attempting to reproduce experiment " + "'%s'", + rev[:7], + ) + finally: + executor.cleanup() + + return result + + def _collect_executor(self, executor, exec_result) -> Dict[str, str]: + # NOTE: GitPython Repo instances cannot be re-used + # after process has received SIGINT or SIGTERM, so we + # need this hack to re-instantiate git instances after + # checkpoint runs. See: + # https://github.com/gitpython-developers/GitPython/issues/427 + # del self.repo.scm + + results = {} + + def on_diverged(ref: str, checkpoint: bool): + ref_info = ExpRefInfo.from_ref(ref) + if checkpoint: + raise CheckpointExistsError(ref_info.name) + raise ExperimentExistsError(ref_info.name) + + for ref in executor.fetch_exps( + self.scm, + executor.git_url, + force=exec_result.force, + on_diverged=on_diverged, + ): + exp_rev = self.scm.get_ref(ref) + if exp_rev: + logger.debug("Collected experiment '%s'.", exp_rev[:7]) + results[exp_rev] = exec_result.exp_hash + + return results + + +class LocalExecutorManager(BaseExecutorManager): + EXECUTOR_CLS = TempDirExecutor diff --git a/dvc/repo/experiments/executor/ssh.py b/dvc/repo/experiments/executor/ssh.py index 29f7f422d..5aafb954c 100644 --- a/dvc/repo/experiments/executor/ssh.py +++ b/dvc/repo/experiments/executor/ssh.py @@ -22,6 +22,8 @@ if TYPE_CHECKING: from dvc.machine import MachineManager from dvc.repo import Repo + from ..base import ExpStashEntry + logger = logging.getLogger(__name__) @@ -51,13 +53,12 @@ class SSHExecutor(BaseExecutor): ): assert host + super().__init__(*args, **kwargs) self.host: str = host self.port = port self.username = username self._fs_factory = fs_factory - - super().__init__(*args, **kwargs) - logger.debug("Init SSH executor for host '%s'", self.host) + self._repo_abspath = "" @classmethod def gen_dirname(cls, name: Optional[str] = None): @@ -66,16 +67,24 @@ class SSHExecutor(BaseExecutor): return "-".join([name or "dvc-exp", "executor", uuid()]) @classmethod - def from_machine( + def from_stash_entry( cls, - manager: "MachineManager", - machine_name: Optional[str], - *args, + repo: "Repo", + stash_rev: str, + entry: "ExpStashEntry", **kwargs, ): - kwargs["root_dir"] = cls.gen_dirname(kwargs.get("name")) - kwargs.update(manager.get_executor_kwargs(machine_name)) - return cls(*args, **kwargs) + manager: "MachineManager" = kwargs.pop("manager") + machine_name: Optional[str] = kwargs.pop("machine_name", None) + executor = cls._from_stash_entry( + repo, + stash_rev, + entry, + cls.gen_dirname(entry.name), + **manager.get_executor_kwargs(machine_name), + ) + logger.debug("Init SSH executor for host '%s'", executor.host) + return executor def sshfs(self): return _sshfs(self._fs_factory, host=self.host, port=self.port) @@ -104,7 +113,7 @@ class SSHExecutor(BaseExecutor): } return kwargs - def _init_git(self, scm: "Git", branch: Optional[str] = None, **kwargs): + def init_git(self, scm: "Git", branch: Optional[str] = None): from ..utils import push_refspec with self.sshfs() as fs: @@ -121,7 +130,7 @@ class SSHExecutor(BaseExecutor): # TODO: support multiple client key retries in git backends # (see https://github.com/iterative/dvc/issues/6508) - kwargs.update(self._git_client_args(fs)) + kwargs = self._git_client_args(fs) refspec = f"{EXEC_NAMESPACE}/" push_refspec(scm, self.git_url, refspec, refspec, **kwargs) if branch: diff --git a/dvc/repo/experiments/gc.py b/dvc/repo/experiments/gc.py index f12e607ba..ce7c71cb1 100644 --- a/dvc/repo/experiments/gc.py +++ b/dvc/repo/experiments/gc.py @@ -42,7 +42,7 @@ def gc( delete_stashes = [] for _, entry in repo.experiments.stash_revs.items(): if not queued or entry.baseline_rev not in keep_revs: - delete_stashes.append(entry.index) + delete_stashes.append(entry.stash_index) for index in sorted(delete_stashes, reverse=True): repo.experiments.stash.drop(index) removed += len(delete_stashes) diff --git a/dvc/repo/experiments/remove.py b/dvc/repo/experiments/remove.py index 13138bb20..08a7396a1 100644 --- a/dvc/repo/experiments/remove.py +++ b/dvc/repo/experiments/remove.py @@ -49,16 +49,16 @@ def _clear_all(repo): def _get_exp_stash_index(repo, ref_or_rev: str) -> Optional[int]: stash_revs = repo.experiments.stash_revs - for _, ref_info in stash_revs.items(): - if ref_info.name == ref_or_rev: - return ref_info.index + for _, entry in stash_revs.items(): + if entry.name == ref_or_rev: + return entry.stash_index from dvc.scm import resolve_rev try: rev = resolve_rev(repo.scm, ref_or_rev) if rev in stash_revs: - return stash_revs.get(rev).index + return stash_revs.get(rev).stash_index except RevError: pass return None diff --git a/dvc/repo/experiments/show.py b/dvc/repo/experiments/show.py index 7ad6b6d6f..b65d7016c 100644 --- a/dvc/repo/experiments/show.py +++ b/dvc/repo/experiments/show.py @@ -6,7 +6,6 @@ from typing import Any, Callable, Dict, Optional from dvc.exceptions import InvalidArgumentError from dvc.repo import locked from dvc.repo.experiments.base import ExpRefInfo -from dvc.repo.experiments.executor.base import ExecutorInfo from dvc.repo.experiments.utils import fix_exp_head from dvc.repo.metrics.show import _gather_metrics from dvc.repo.params.show import _gather_params @@ -44,7 +43,7 @@ def _collect_experiment_commit( res["queued"] = stash if running is not None and exp_rev in running: res["running"] = True - res["executor"] = running[exp_rev].get(ExecutorInfo.PARAM_LOCATION) + res["executor"] = running[exp_rev].get("location") else: res["running"] = False res["executor"] = None diff --git a/tests/func/experiments/executor/test_ssh.py b/tests/func/experiments/executor/test_ssh.py index bc29f8b94..a2128de0f 100644 --- a/tests/func/experiments/executor/test_ssh.py +++ b/tests/func/experiments/executor/test_ssh.py @@ -21,10 +21,19 @@ def _ssh_factory(cloud): ) -def test_from_machine(tmp_dir, scm, dvc, machine_instance, mocker): - mocker.patch.object(SSHExecutor, "_init_git") - executor = SSHExecutor.from_machine(dvc.machine, "foo", scm, "") - assert executor.host == machine_instance["instance_ip"] +def test_init_from_stash(tmp_dir, scm, dvc, machine_instance, mocker): + mock = mocker.patch.object(SSHExecutor, "_from_stash_entry") + mock_entry = mocker.Mock() + mock_entry.name = "" + SSHExecutor.from_stash_entry( + None, + "", + mock_entry, + manager=dvc.machine, + machine_name="foo", + ) + _args, kwargs = mock.call_args + assert kwargs["host"] == machine_instance["instance_ip"] @pytest.mark.needs_internet diff --git a/tests/func/experiments/test_show.py b/tests/func/experiments/test_show.py index 1aa6cdcd8..6a50fb4ad 100644 --- a/tests/func/experiments/test_show.py +++ b/tests/func/experiments/test_show.py @@ -8,7 +8,12 @@ from funcy import first, get_in from dvc.exceptions import InvalidArgumentError from dvc.main import main from dvc.repo.experiments.base import EXPS_STASH, ExpRefInfo -from dvc.repo.experiments.executor.base import BaseExecutor, ExecutorInfo +from dvc.repo.experiments.executor.base import ( + EXEC_PID_DIR, + EXEC_TMP_DIR, + BaseExecutor, + ExecutorInfo, +) from dvc.repo.experiments.utils import exp_refs_by_rev from dvc.utils.fs import makedirs from dvc.utils.serialize import YAMLFileCorruptedError @@ -16,6 +21,20 @@ from tests.func.test_repro_multistage import COPY_SCRIPT from tests.utils import console_width +def make_executor_info(**kwargs): + # set default values for required info fields + for key in ( + "git_url", + "baseline_rev", + "location", + "root_dir", + "dvc_dir", + ): + if key not in kwargs: + kwargs[key] = "" + return ExecutorInfo(**kwargs) + + def test_show_simple(tmp_dir, scm, dvc, exp_stage): assert dvc.experiments.show()["workspace"] == { "baseline": { @@ -367,11 +386,11 @@ def test_show_sort(tmp_dir, scm, dvc, exp_stage, caplog): def test_show_running_workspace(tmp_dir, scm, dvc, exp_stage, capsys): - pid_dir = os.path.join(dvc.tmp_dir, dvc.experiments.EXEC_PID_DIR) + pid_dir = os.path.join(dvc.tmp_dir, EXEC_TMP_DIR, EXEC_PID_DIR) makedirs(pid_dir, True) - info = ExecutorInfo(None, None, None, BaseExecutor.DEFAULT_LOCATION) - pidfile = os.path.join(pid_dir, f"workspace{BaseExecutor.PIDFILE_EXT}") - (tmp_dir / pidfile).dump(info.to_dict()) + info = make_executor_info(location=BaseExecutor.DEFAULT_LOCATION) + pidfile = os.path.join(pid_dir, f"workspace{BaseExecutor.INFOFILE_EXT}") + (tmp_dir / pidfile).dump_json(info.asdict()) assert dvc.experiments.show()["workspace"] == { "baseline": { @@ -398,11 +417,11 @@ def test_show_running_executor(tmp_dir, scm, dvc, exp_stage): dvc.experiments.run(exp_stage.addressing, params=["foo=2"], queue=True) exp_rev = dvc.experiments.scm.resolve_rev(f"{EXPS_STASH}@{{0}}") - pid_dir = os.path.join(dvc.tmp_dir, dvc.experiments.EXEC_PID_DIR) + pid_dir = os.path.join(dvc.tmp_dir, EXEC_TMP_DIR, EXEC_PID_DIR) makedirs(pid_dir, True) - info = ExecutorInfo(None, None, None, BaseExecutor.DEFAULT_LOCATION) - pidfile = os.path.join(pid_dir, f"{exp_rev}{BaseExecutor.PIDFILE_EXT}") - (tmp_dir / pidfile).dump(info.to_dict()) + info = make_executor_info(location=BaseExecutor.DEFAULT_LOCATION) + pidfile = os.path.join(pid_dir, f"{exp_rev}{BaseExecutor.INFOFILE_EXT}") + (tmp_dir / pidfile).dump_json(info.asdict()) results = dvc.experiments.show() exp_data = get_in(results, [baseline_rev, exp_rev, "data"]) @@ -430,17 +449,21 @@ def test_show_running_checkpoint( checkpoint_rev = first(run_results) exp_ref = first(exp_refs_by_rev(scm, checkpoint_rev)) - pid_dir = os.path.join(dvc.tmp_dir, dvc.experiments.EXEC_PID_DIR) + pid_dir = os.path.join(dvc.tmp_dir, EXEC_TMP_DIR, EXEC_PID_DIR) makedirs(pid_dir, True) executor = ( BaseExecutor.DEFAULT_LOCATION if workspace else TempDirExecutor.DEFAULT_LOCATION ) - info = ExecutorInfo(123, "foo.git", baseline_rev, executor) + info = make_executor_info( + git_url="foo.git", + baseline_rev=baseline_rev, + location=executor, + ) rev = "workspace" if workspace else stash_rev - pidfile = os.path.join(pid_dir, f"{rev}{BaseExecutor.PIDFILE_EXT}") - (tmp_dir / pidfile).dump(info.to_dict()) + pidfile = os.path.join(pid_dir, f"{rev}{BaseExecutor.INFOFILE_EXT}") + (tmp_dir / pidfile).dump_json(info.asdict()) mocker.patch.object( BaseExecutor, "fetch_exps", return_value=[str(exp_ref)] @@ -559,7 +582,6 @@ def test_show_only_changed(tmp_dir, dvc, scm, capsys): assert main(["exp", "show"]) == 0 cap = capsys.readouterr() - print(cap) assert "bar" in cap.out capsys.readouterr()