From 84da78a62b170f7899aa5c8285f5e81713338824 Mon Sep 17 00:00:00 2001 From: Peter Rowlands Date: Mon, 20 Dec 2021 17:34:27 +0900 Subject: [PATCH] exp: initial SSHExecutorManager implementation --- dvc/repo/experiments/__init__.py | 13 +- .../experiments/executor/manager/__init__.py | 0 .../executor/{manager.py => manager/base.py} | 116 +++--------------- .../experiments/executor/manager/local.py | 109 ++++++++++++++++ dvc/repo/experiments/executor/manager/ssh.py | 109 ++++++++++++++++ tests/func/experiments/executor/test_ssh.py | 13 +- 6 files changed, 253 insertions(+), 107 deletions(-) create mode 100644 dvc/repo/experiments/executor/manager/__init__.py rename dvc/repo/experiments/executor/{manager.py => manager/base.py} (73%) create mode 100644 dvc/repo/experiments/executor/manager/local.py create mode 100644 dvc/repo/experiments/executor/manager/ssh.py diff --git a/dvc/repo/experiments/__init__.py b/dvc/repo/experiments/__init__.py index ca32f4785..a2447df1d 100644 --- a/dvc/repo/experiments/__init__.py +++ b/dvc/repo/experiments/__init__.py @@ -31,11 +31,12 @@ from .executor.base import ( BaseExecutor, ExecutorInfo, ) -from .executor.manager import ( - BaseExecutorManager, +from .executor.manager.base import BaseExecutorManager +from .executor.manager.local import ( TempDirExecutorManager, WorkspaceExecutorManager, ) +from .executor.manager.ssh import SSHExecutorManager from .utils import exp_refs_by_rev logger = logging.getLogger(__name__) @@ -386,6 +387,7 @@ class Experiments: tmp_dir: bool = False, checkpoint_resume: Optional[str] = None, reset: bool = False, + machine: Optional[str] = None, **kwargs, ): """Reproduce and checkout a single experiment.""" @@ -395,7 +397,7 @@ class Experiments: if reset: self.reset_checkpoints() - if not (queue or tmp_dir): + if not (queue or tmp_dir or machine): staged, _, _ = self.scm.status() if staged: logger.warning( @@ -429,12 +431,15 @@ class Experiments: return [stash_rev] if tmp_dir or queue: manager_cls: Type = TempDirExecutorManager + elif machine: + manager_cls = SSHExecutorManager else: manager_cls = WorkspaceExecutorManager results = self._reproduce_revs( revs=[stash_rev], keep_stash=False, manager_cls=manager_cls, + machine=machine, ) exp_rev = first(results) if exp_rev is not None: @@ -590,6 +595,7 @@ class Experiments: revs: Optional[Iterable] = None, keep_stash: Optional[bool] = True, manager_cls: Type = TempDirExecutorManager, + machine: Optional[str] = None, **kwargs, ) -> Mapping[str, str]: """Reproduce the specified experiments. @@ -631,6 +637,7 @@ class Experiments: os.path.join(self.repo.tmp_dir, EXEC_TMP_DIR), self.repo, to_run, + machine_name=machine, ) try: exec_results = {} diff --git a/dvc/repo/experiments/executor/manager/__init__.py b/dvc/repo/experiments/executor/manager/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dvc/repo/experiments/executor/manager.py b/dvc/repo/experiments/executor/manager/base.py similarity index 73% rename from dvc/repo/experiments/executor/manager.py rename to dvc/repo/experiments/executor/manager/base.py index 02c182921..e5c621fbf 100644 --- a/dvc/repo/experiments/executor/manager.py +++ b/dvc/repo/experiments/executor/manager/base.py @@ -7,9 +7,8 @@ from typing import TYPE_CHECKING, Deque, Dict, Generator, Optional, Tuple, Type from dvc.proc.manager import ProcessManager -from ..base import ( +from ...base import ( EXEC_BASELINE, - EXEC_BRANCH, EXEC_HEAD, EXEC_MERGE, CheckpointExistsError, @@ -17,8 +16,8 @@ from ..base import ( ExpRefInfo, ExpStashEntry, ) -from .base import EXEC_PID_DIR, BaseExecutor -from .local import TempDirExecutor, WorkspaceExecutor +from ..base import EXEC_PID_DIR, BaseExecutor +from ..local import TempDirExecutor, WorkspaceExecutor if TYPE_CHECKING: from scmrepo.git import Git @@ -37,6 +36,7 @@ class BaseExecutorManager(ABC, Mapping): self, scm: "Git", wdir: str, + **kwargs, ): from dvc.utils.fs import makedirs @@ -74,8 +74,8 @@ class BaseExecutorManager(ABC, Mapping): import json from urllib.parse import urlparse - from .base import ExecutorInfo - from .ssh import SSHExecutor + from ..base import ExecutorInfo + from ..ssh import SSHExecutor def make_executor(info: "ExecutorInfo"): if info.git_url: @@ -116,6 +116,16 @@ class BaseExecutorManager(ABC, Mapping): **kwargs, ): manager = cls(scm, wdir) + manager._enqueue_stash_entries(scm, repo, to_run, **kwargs) + return manager + + def _enqueue_stash_entries( + self, + scm: "Git", + repo: "Repo", + to_run: Dict[str, ExpStashEntry], + **kwargs, + ): try: for stash_rev, entry in to_run.items(): scm.set_ref(EXEC_HEAD, entry.head_rev) @@ -128,17 +138,16 @@ class BaseExecutorManager(ABC, Mapping): # EXEC_MERGE - the unmerged changes (from our stash) # to be reproduced # EXEC_BASELINE - the baseline commit for this experiment - executor = cls.EXECUTOR_CLS.from_stash_entry( + executor = self.EXECUTOR_CLS.from_stash_entry( repo, stash_rev, entry, **kwargs, ) - manager.enqueue(stash_rev, executor) + self.enqueue(stash_rev, executor) finally: for ref in (EXEC_HEAD, EXEC_MERGE, EXEC_BASELINE): scm.remove_ref(ref) - return manager def exec_queue(self, jobs: Optional[int] = 1, detach: bool = False): """Run dvc repro for queued executors in parallel.""" @@ -260,92 +269,3 @@ class BaseExecutorManager(ABC, Mapping): except KeyError: pass remove(os.path.join(self.pid_dir, rev)) - - -class TempDirExecutorManager(BaseExecutorManager): - EXECUTOR_CLS = TempDirExecutor - - -class WorkspaceExecutorManager(BaseExecutorManager): - EXECUTOR_CLS = WorkspaceExecutor - - @classmethod - def from_stash_entries( - cls, - scm: "Git", - wdir: str, - repo: "Repo", - to_run: Dict[str, ExpStashEntry], - **kwargs, - ): - manager = cls(scm, wdir) - try: - assert len(to_run) == 1 - for stash_rev, entry in to_run.items(): - scm.set_ref(EXEC_HEAD, entry.head_rev) - scm.set_ref(EXEC_MERGE, stash_rev) - scm.set_ref(EXEC_BASELINE, entry.baseline_rev) - - executor = cls.EXECUTOR_CLS.from_stash_entry( - repo, - stash_rev, - entry, - **kwargs, - ) - manager.enqueue(stash_rev, executor) - finally: - for ref in (EXEC_MERGE,): - scm.remove_ref(ref) - return manager - - def _collect_executor(self, executor, exec_result) -> Dict[str, str]: - results = {} - exp_rev = self.scm.get_ref(EXEC_BRANCH) - if exp_rev: - logger.debug("Collected experiment '%s'.", exp_rev[:7]) - results[exp_rev] = exec_result.exp_hash - return results - - def exec_queue(self, jobs: Optional[int] = 1, detach: bool = False): - """Run a single WorkspaceExecutor. - - Workspace execution is done within the main DVC process - (rather than in multiprocessing context) - """ - from dvc.exceptions import DvcException - from dvc.stage.monitor import CheckpointKilledError - - assert len(self._queue) == 1 - assert not detach - result: Dict[str, Dict[str, str]] = defaultdict(dict) - rev, executor = self._queue.popleft() - - exec_name = "workspace" - infofile = self.get_infofile_path(exec_name) - try: - exec_result = executor.reproduce( - info=executor.info, - rev=rev, - infofile=infofile, - log_level=logger.getEffectiveLevel(), - ) - if not exec_result.exp_hash: - raise DvcException( - f"Failed to reproduce experiment '{rev[:7]}'" - ) - if exec_result.ref_info: - result[rev].update( - self._collect_executor(executor, exec_result) - ) - except CheckpointKilledError: - # Checkpoint errors have already been logged - return {} - except DvcException: - raise - except Exception as exc: - raise DvcException( - f"Failed to reproduce experiment '{rev[:7]}'" - ) from exc - finally: - self.cleanup_executor(exec_name, executor) - return result diff --git a/dvc/repo/experiments/executor/manager/local.py b/dvc/repo/experiments/executor/manager/local.py new file mode 100644 index 000000000..95daa2eaa --- /dev/null +++ b/dvc/repo/experiments/executor/manager/local.py @@ -0,0 +1,109 @@ +import logging +from collections import defaultdict +from typing import TYPE_CHECKING, Dict, Optional + +from ...base import ( + EXEC_BASELINE, + EXEC_BRANCH, + EXEC_HEAD, + EXEC_MERGE, + ExpStashEntry, +) +from ..local import TempDirExecutor, WorkspaceExecutor +from .base import BaseExecutorManager + +if TYPE_CHECKING: + from scmrepo.git import Git + + from dvc.repo import Repo + +logger = logging.getLogger(__name__) + + +class TempDirExecutorManager(BaseExecutorManager): + EXECUTOR_CLS = TempDirExecutor + + +class WorkspaceExecutorManager(BaseExecutorManager): + EXECUTOR_CLS = WorkspaceExecutor + + @classmethod + def from_stash_entries( + cls, + scm: "Git", + wdir: str, + repo: "Repo", + to_run: Dict[str, ExpStashEntry], + **kwargs, + ): + manager = cls(scm, wdir) + try: + assert len(to_run) == 1 + for stash_rev, entry in to_run.items(): + scm.set_ref(EXEC_HEAD, entry.head_rev) + scm.set_ref(EXEC_MERGE, stash_rev) + scm.set_ref(EXEC_BASELINE, entry.baseline_rev) + + executor = cls.EXECUTOR_CLS.from_stash_entry( + repo, + stash_rev, + entry, + **kwargs, + ) + manager.enqueue(stash_rev, executor) + finally: + for ref in (EXEC_MERGE,): + scm.remove_ref(ref) + return manager + + def _collect_executor(self, executor, exec_result) -> Dict[str, str]: + results = {} + exp_rev = self.scm.get_ref(EXEC_BRANCH) + if exp_rev: + logger.debug("Collected experiment '%s'.", exp_rev[:7]) + results[exp_rev] = exec_result.exp_hash + return results + + def exec_queue(self, jobs: Optional[int] = 1, detach: bool = False): + """Run a single WorkspaceExecutor. + + Workspace execution is done within the main DVC process + (rather than in multiprocessing context) + """ + from dvc.exceptions import DvcException + from dvc.stage.monitor import CheckpointKilledError + + assert len(self._queue) == 1 + assert not detach + result: Dict[str, Dict[str, str]] = defaultdict(dict) + rev, executor = self._queue.popleft() + + exec_name = "workspace" + infofile = self.get_infofile_path(exec_name) + try: + exec_result = executor.reproduce( + info=executor.info, + rev=rev, + infofile=infofile, + log_level=logger.getEffectiveLevel(), + ) + if not exec_result.exp_hash: + raise DvcException( + f"Failed to reproduce experiment '{rev[:7]}'" + ) + if exec_result.ref_info: + result[rev].update( + self._collect_executor(executor, exec_result) + ) + except CheckpointKilledError: + # Checkpoint errors have already been logged + return {} + except DvcException: + raise + except Exception as exc: + raise DvcException( + f"Failed to reproduce experiment '{rev[:7]}'" + ) from exc + finally: + self.cleanup_executor(exec_name, executor) + return result diff --git a/dvc/repo/experiments/executor/manager/ssh.py b/dvc/repo/experiments/executor/manager/ssh.py new file mode 100644 index 000000000..5f1982651 --- /dev/null +++ b/dvc/repo/experiments/executor/manager/ssh.py @@ -0,0 +1,109 @@ +import logging +import posixpath +from collections import defaultdict +from typing import TYPE_CHECKING, Callable, Dict, Generator, Optional, Tuple + +from ...base import ExpStashEntry +from ..base import BaseExecutor +from ..ssh import SSHExecutor, _sshfs +from .base import BaseExecutorManager + +if TYPE_CHECKING: + from scmrepo.git import Git + + from dvc.repo import Repo + +logger = logging.getLogger(__name__) + + +class SSHExecutorManager(BaseExecutorManager): + EXECUTOR_CLS = SSHExecutor + + def __init__( + self, + scm: "Git", + wdir: str, + host: Optional[str] = None, + port: Optional[int] = None, + username: Optional[str] = None, + fs_factory: Optional[Callable] = None, + **kwargs, + ): + assert host + super().__init__(scm, wdir, **kwargs) + self.host = host + self.port = port + self.username = username + self._fs_factory = fs_factory + + def _load_infos(self) -> Generator[Tuple[str, "BaseExecutor"], None, None]: + # TODO: load existing infos using sshfs + yield from [] + + @classmethod + def from_stash_entries( + cls, + scm: "Git", + wdir: str, + repo: "Repo", + to_run: Dict[str, ExpStashEntry], + **kwargs, + ): + machine_name: Optional[str] = kwargs.get("machine_name", None) + manager = cls( + scm, wdir, **repo.machine.get_executor_kwargs(machine_name) + ) + manager._enqueue_stash_entries(scm, repo, to_run, **kwargs) + return manager + + def sshfs(self): + return _sshfs(self._fs_factory, host=self.host, port=self.port) + + def get_infofile_path(self, name: str) -> str: + return f"{name}{BaseExecutor.INFOFILE_EXT}" + + def _exec_attached(self, jobs: Optional[int] = 1): + from dvc.exceptions import DvcException + from dvc.stage.monitor import CheckpointKilledError + + assert len(self._queue) == 1 + result: Dict[str, Dict[str, str]] = defaultdict(dict) + rev, executor = self._queue.popleft() + info = executor.info + infofile = posixpath.join( + info.root_dir, + info.dvc_dir, + "tmp", + self.get_infofile_path(rev), + ) + try: + exec_result = executor.reproduce( + info=executor.info, + rev=rev, + infofile=infofile, + log_level=logger.getEffectiveLevel(), + fs_factory=self._fs_factory, + ) + if not exec_result.exp_hash: + raise DvcException( + f"Failed to reproduce experiment '{rev[:7]}'" + ) + if exec_result.ref_info: + result[rev].update( + self._collect_executor(executor, exec_result) + ) + except CheckpointKilledError: + # Checkpoint errors have already been logged + return {} + except DvcException: + raise + except Exception as exc: + raise DvcException( + f"Failed to reproduce experiment '{rev[:7]}'" + ) from exc + finally: + self.cleanup_executor(rev, executor) + return result + + def cleanup_executor(self, rev: str, executor: "BaseExecutor"): + executor.cleanup() diff --git a/tests/func/experiments/executor/test_ssh.py b/tests/func/experiments/executor/test_ssh.py index a2128de0f..0f0271e65 100644 --- a/tests/func/experiments/executor/test_ssh.py +++ b/tests/func/experiments/executor/test_ssh.py @@ -40,8 +40,8 @@ def test_init_from_stash(tmp_dir, scm, dvc, machine_instance, mocker): @pytest.mark.parametrize("cloud", [pytest.lazy_fixture("git_ssh")]) def test_init_git(tmp_dir, scm, cloud): tmp_dir.scm_gen({"foo": "foo", "dir": {"bar": "bar"}}, commit="init") - rev = scm.get_rev() - scm.set_ref(EXEC_HEAD, rev) + baseline_rev = scm.get_rev() + scm.set_ref(EXEC_HEAD, baseline_rev) tmp_dir.gen("foo", "stashed") scm.gitpython.git.stash() rev = scm.resolve_rev("stash@{0}") @@ -50,14 +50,15 @@ def test_init_git(tmp_dir, scm, cloud): root_url = cloud / SSHExecutor.gen_dirname() executor = SSHExecutor( - scm, - ".", root_dir=root_url.path, + dvc_dir=".dvc", + baseline_rev=baseline_rev, host=root_url.host, port=root_url.port, username=TEST_SSH_USER, fs_factory=partial(_ssh_factory, cloud), ) + executor.init_git(scm) assert root_url.path == executor._repo_abspath fs = cloud._ssh @@ -76,9 +77,9 @@ def test_init_cache(tmp_dir, dvc, scm, cloud): root_url = cloud / SSHExecutor.gen_dirname() executor = SSHExecutor( - scm, - ".", root_dir=root_url.path, + dvc_dir=".dvc", + baseline_rev=rev, host=root_url.host, port=root_url.port, username=TEST_SSH_USER,