exp: initial SSHExecutorManager implementation

This commit is contained in:
Peter Rowlands
2022-01-13 16:34:20 +09:00
committed by Peter Rowlands (변기호)
parent 4b9d39c901
commit 84da78a62b
6 changed files with 253 additions and 107 deletions
+10 -3
View File
@@ -31,11 +31,12 @@ from .executor.base import (
BaseExecutor,
ExecutorInfo,
)
from .executor.manager import (
BaseExecutorManager,
from .executor.manager.base import BaseExecutorManager
from .executor.manager.local import (
TempDirExecutorManager,
WorkspaceExecutorManager,
)
from .executor.manager.ssh import SSHExecutorManager
from .utils import exp_refs_by_rev
logger = logging.getLogger(__name__)
@@ -386,6 +387,7 @@ class Experiments:
tmp_dir: bool = False,
checkpoint_resume: Optional[str] = None,
reset: bool = False,
machine: Optional[str] = None,
**kwargs,
):
"""Reproduce and checkout a single experiment."""
@@ -395,7 +397,7 @@ class Experiments:
if reset:
self.reset_checkpoints()
if not (queue or tmp_dir):
if not (queue or tmp_dir or machine):
staged, _, _ = self.scm.status()
if staged:
logger.warning(
@@ -429,12 +431,15 @@ class Experiments:
return [stash_rev]
if tmp_dir or queue:
manager_cls: Type = TempDirExecutorManager
elif machine:
manager_cls = SSHExecutorManager
else:
manager_cls = WorkspaceExecutorManager
results = self._reproduce_revs(
revs=[stash_rev],
keep_stash=False,
manager_cls=manager_cls,
machine=machine,
)
exp_rev = first(results)
if exp_rev is not None:
@@ -590,6 +595,7 @@ class Experiments:
revs: Optional[Iterable] = None,
keep_stash: Optional[bool] = True,
manager_cls: Type = TempDirExecutorManager,
machine: Optional[str] = None,
**kwargs,
) -> Mapping[str, str]:
"""Reproduce the specified experiments.
@@ -631,6 +637,7 @@ class Experiments:
os.path.join(self.repo.tmp_dir, EXEC_TMP_DIR),
self.repo,
to_run,
machine_name=machine,
)
try:
exec_results = {}
@@ -7,9 +7,8 @@ from typing import TYPE_CHECKING, Deque, Dict, Generator, Optional, Tuple, Type
from dvc.proc.manager import ProcessManager
from ..base import (
from ...base import (
EXEC_BASELINE,
EXEC_BRANCH,
EXEC_HEAD,
EXEC_MERGE,
CheckpointExistsError,
@@ -17,8 +16,8 @@ from ..base import (
ExpRefInfo,
ExpStashEntry,
)
from .base import EXEC_PID_DIR, BaseExecutor
from .local import TempDirExecutor, WorkspaceExecutor
from ..base import EXEC_PID_DIR, BaseExecutor
from ..local import TempDirExecutor, WorkspaceExecutor
if TYPE_CHECKING:
from scmrepo.git import Git
@@ -37,6 +36,7 @@ class BaseExecutorManager(ABC, Mapping):
self,
scm: "Git",
wdir: str,
**kwargs,
):
from dvc.utils.fs import makedirs
@@ -74,8 +74,8 @@ class BaseExecutorManager(ABC, Mapping):
import json
from urllib.parse import urlparse
from .base import ExecutorInfo
from .ssh import SSHExecutor
from ..base import ExecutorInfo
from ..ssh import SSHExecutor
def make_executor(info: "ExecutorInfo"):
if info.git_url:
@@ -116,6 +116,16 @@ class BaseExecutorManager(ABC, Mapping):
**kwargs,
):
manager = cls(scm, wdir)
manager._enqueue_stash_entries(scm, repo, to_run, **kwargs)
return manager
def _enqueue_stash_entries(
self,
scm: "Git",
repo: "Repo",
to_run: Dict[str, ExpStashEntry],
**kwargs,
):
try:
for stash_rev, entry in to_run.items():
scm.set_ref(EXEC_HEAD, entry.head_rev)
@@ -128,17 +138,16 @@ class BaseExecutorManager(ABC, Mapping):
# EXEC_MERGE - the unmerged changes (from our stash)
# to be reproduced
# EXEC_BASELINE - the baseline commit for this experiment
executor = cls.EXECUTOR_CLS.from_stash_entry(
executor = self.EXECUTOR_CLS.from_stash_entry(
repo,
stash_rev,
entry,
**kwargs,
)
manager.enqueue(stash_rev, executor)
self.enqueue(stash_rev, executor)
finally:
for ref in (EXEC_HEAD, EXEC_MERGE, EXEC_BASELINE):
scm.remove_ref(ref)
return manager
def exec_queue(self, jobs: Optional[int] = 1, detach: bool = False):
"""Run dvc repro for queued executors in parallel."""
@@ -260,92 +269,3 @@ class BaseExecutorManager(ABC, Mapping):
except KeyError:
pass
remove(os.path.join(self.pid_dir, rev))
class TempDirExecutorManager(BaseExecutorManager):
EXECUTOR_CLS = TempDirExecutor
class WorkspaceExecutorManager(BaseExecutorManager):
EXECUTOR_CLS = WorkspaceExecutor
@classmethod
def from_stash_entries(
cls,
scm: "Git",
wdir: str,
repo: "Repo",
to_run: Dict[str, ExpStashEntry],
**kwargs,
):
manager = cls(scm, wdir)
try:
assert len(to_run) == 1
for stash_rev, entry in to_run.items():
scm.set_ref(EXEC_HEAD, entry.head_rev)
scm.set_ref(EXEC_MERGE, stash_rev)
scm.set_ref(EXEC_BASELINE, entry.baseline_rev)
executor = cls.EXECUTOR_CLS.from_stash_entry(
repo,
stash_rev,
entry,
**kwargs,
)
manager.enqueue(stash_rev, executor)
finally:
for ref in (EXEC_MERGE,):
scm.remove_ref(ref)
return manager
def _collect_executor(self, executor, exec_result) -> Dict[str, str]:
results = {}
exp_rev = self.scm.get_ref(EXEC_BRANCH)
if exp_rev:
logger.debug("Collected experiment '%s'.", exp_rev[:7])
results[exp_rev] = exec_result.exp_hash
return results
def exec_queue(self, jobs: Optional[int] = 1, detach: bool = False):
"""Run a single WorkspaceExecutor.
Workspace execution is done within the main DVC process
(rather than in multiprocessing context)
"""
from dvc.exceptions import DvcException
from dvc.stage.monitor import CheckpointKilledError
assert len(self._queue) == 1
assert not detach
result: Dict[str, Dict[str, str]] = defaultdict(dict)
rev, executor = self._queue.popleft()
exec_name = "workspace"
infofile = self.get_infofile_path(exec_name)
try:
exec_result = executor.reproduce(
info=executor.info,
rev=rev,
infofile=infofile,
log_level=logger.getEffectiveLevel(),
)
if not exec_result.exp_hash:
raise DvcException(
f"Failed to reproduce experiment '{rev[:7]}'"
)
if exec_result.ref_info:
result[rev].update(
self._collect_executor(executor, exec_result)
)
except CheckpointKilledError:
# Checkpoint errors have already been logged
return {}
except DvcException:
raise
except Exception as exc:
raise DvcException(
f"Failed to reproduce experiment '{rev[:7]}'"
) from exc
finally:
self.cleanup_executor(exec_name, executor)
return result
@@ -0,0 +1,109 @@
import logging
from collections import defaultdict
from typing import TYPE_CHECKING, Dict, Optional
from ...base import (
EXEC_BASELINE,
EXEC_BRANCH,
EXEC_HEAD,
EXEC_MERGE,
ExpStashEntry,
)
from ..local import TempDirExecutor, WorkspaceExecutor
from .base import BaseExecutorManager
if TYPE_CHECKING:
from scmrepo.git import Git
from dvc.repo import Repo
logger = logging.getLogger(__name__)
class TempDirExecutorManager(BaseExecutorManager):
EXECUTOR_CLS = TempDirExecutor
class WorkspaceExecutorManager(BaseExecutorManager):
EXECUTOR_CLS = WorkspaceExecutor
@classmethod
def from_stash_entries(
cls,
scm: "Git",
wdir: str,
repo: "Repo",
to_run: Dict[str, ExpStashEntry],
**kwargs,
):
manager = cls(scm, wdir)
try:
assert len(to_run) == 1
for stash_rev, entry in to_run.items():
scm.set_ref(EXEC_HEAD, entry.head_rev)
scm.set_ref(EXEC_MERGE, stash_rev)
scm.set_ref(EXEC_BASELINE, entry.baseline_rev)
executor = cls.EXECUTOR_CLS.from_stash_entry(
repo,
stash_rev,
entry,
**kwargs,
)
manager.enqueue(stash_rev, executor)
finally:
for ref in (EXEC_MERGE,):
scm.remove_ref(ref)
return manager
def _collect_executor(self, executor, exec_result) -> Dict[str, str]:
results = {}
exp_rev = self.scm.get_ref(EXEC_BRANCH)
if exp_rev:
logger.debug("Collected experiment '%s'.", exp_rev[:7])
results[exp_rev] = exec_result.exp_hash
return results
def exec_queue(self, jobs: Optional[int] = 1, detach: bool = False):
"""Run a single WorkspaceExecutor.
Workspace execution is done within the main DVC process
(rather than in multiprocessing context)
"""
from dvc.exceptions import DvcException
from dvc.stage.monitor import CheckpointKilledError
assert len(self._queue) == 1
assert not detach
result: Dict[str, Dict[str, str]] = defaultdict(dict)
rev, executor = self._queue.popleft()
exec_name = "workspace"
infofile = self.get_infofile_path(exec_name)
try:
exec_result = executor.reproduce(
info=executor.info,
rev=rev,
infofile=infofile,
log_level=logger.getEffectiveLevel(),
)
if not exec_result.exp_hash:
raise DvcException(
f"Failed to reproduce experiment '{rev[:7]}'"
)
if exec_result.ref_info:
result[rev].update(
self._collect_executor(executor, exec_result)
)
except CheckpointKilledError:
# Checkpoint errors have already been logged
return {}
except DvcException:
raise
except Exception as exc:
raise DvcException(
f"Failed to reproduce experiment '{rev[:7]}'"
) from exc
finally:
self.cleanup_executor(exec_name, executor)
return result
@@ -0,0 +1,109 @@
import logging
import posixpath
from collections import defaultdict
from typing import TYPE_CHECKING, Callable, Dict, Generator, Optional, Tuple
from ...base import ExpStashEntry
from ..base import BaseExecutor
from ..ssh import SSHExecutor, _sshfs
from .base import BaseExecutorManager
if TYPE_CHECKING:
from scmrepo.git import Git
from dvc.repo import Repo
logger = logging.getLogger(__name__)
class SSHExecutorManager(BaseExecutorManager):
EXECUTOR_CLS = SSHExecutor
def __init__(
self,
scm: "Git",
wdir: str,
host: Optional[str] = None,
port: Optional[int] = None,
username: Optional[str] = None,
fs_factory: Optional[Callable] = None,
**kwargs,
):
assert host
super().__init__(scm, wdir, **kwargs)
self.host = host
self.port = port
self.username = username
self._fs_factory = fs_factory
def _load_infos(self) -> Generator[Tuple[str, "BaseExecutor"], None, None]:
# TODO: load existing infos using sshfs
yield from []
@classmethod
def from_stash_entries(
cls,
scm: "Git",
wdir: str,
repo: "Repo",
to_run: Dict[str, ExpStashEntry],
**kwargs,
):
machine_name: Optional[str] = kwargs.get("machine_name", None)
manager = cls(
scm, wdir, **repo.machine.get_executor_kwargs(machine_name)
)
manager._enqueue_stash_entries(scm, repo, to_run, **kwargs)
return manager
def sshfs(self):
return _sshfs(self._fs_factory, host=self.host, port=self.port)
def get_infofile_path(self, name: str) -> str:
return f"{name}{BaseExecutor.INFOFILE_EXT}"
def _exec_attached(self, jobs: Optional[int] = 1):
from dvc.exceptions import DvcException
from dvc.stage.monitor import CheckpointKilledError
assert len(self._queue) == 1
result: Dict[str, Dict[str, str]] = defaultdict(dict)
rev, executor = self._queue.popleft()
info = executor.info
infofile = posixpath.join(
info.root_dir,
info.dvc_dir,
"tmp",
self.get_infofile_path(rev),
)
try:
exec_result = executor.reproduce(
info=executor.info,
rev=rev,
infofile=infofile,
log_level=logger.getEffectiveLevel(),
fs_factory=self._fs_factory,
)
if not exec_result.exp_hash:
raise DvcException(
f"Failed to reproduce experiment '{rev[:7]}'"
)
if exec_result.ref_info:
result[rev].update(
self._collect_executor(executor, exec_result)
)
except CheckpointKilledError:
# Checkpoint errors have already been logged
return {}
except DvcException:
raise
except Exception as exc:
raise DvcException(
f"Failed to reproduce experiment '{rev[:7]}'"
) from exc
finally:
self.cleanup_executor(rev, executor)
return result
def cleanup_executor(self, rev: str, executor: "BaseExecutor"):
executor.cleanup()
+7 -6
View File
@@ -40,8 +40,8 @@ def test_init_from_stash(tmp_dir, scm, dvc, machine_instance, mocker):
@pytest.mark.parametrize("cloud", [pytest.lazy_fixture("git_ssh")])
def test_init_git(tmp_dir, scm, cloud):
tmp_dir.scm_gen({"foo": "foo", "dir": {"bar": "bar"}}, commit="init")
rev = scm.get_rev()
scm.set_ref(EXEC_HEAD, rev)
baseline_rev = scm.get_rev()
scm.set_ref(EXEC_HEAD, baseline_rev)
tmp_dir.gen("foo", "stashed")
scm.gitpython.git.stash()
rev = scm.resolve_rev("stash@{0}")
@@ -50,14 +50,15 @@ def test_init_git(tmp_dir, scm, cloud):
root_url = cloud / SSHExecutor.gen_dirname()
executor = SSHExecutor(
scm,
".",
root_dir=root_url.path,
dvc_dir=".dvc",
baseline_rev=baseline_rev,
host=root_url.host,
port=root_url.port,
username=TEST_SSH_USER,
fs_factory=partial(_ssh_factory, cloud),
)
executor.init_git(scm)
assert root_url.path == executor._repo_abspath
fs = cloud._ssh
@@ -76,9 +77,9 @@ def test_init_cache(tmp_dir, dvc, scm, cloud):
root_url = cloud / SSHExecutor.gen_dirname()
executor = SSHExecutor(
scm,
".",
root_dir=root_url.path,
dvc_dir=".dvc",
baseline_rev=rev,
host=root_url.host,
port=root_url.port,
username=TEST_SSH_USER,