mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-21 15:28:29 +01:00
exp: initial SSHExecutorManager implementation
This commit is contained in:
committed by
Peter Rowlands (변기호)
parent
4b9d39c901
commit
84da78a62b
@@ -31,11 +31,12 @@ from .executor.base import (
|
||||
BaseExecutor,
|
||||
ExecutorInfo,
|
||||
)
|
||||
from .executor.manager import (
|
||||
BaseExecutorManager,
|
||||
from .executor.manager.base import BaseExecutorManager
|
||||
from .executor.manager.local import (
|
||||
TempDirExecutorManager,
|
||||
WorkspaceExecutorManager,
|
||||
)
|
||||
from .executor.manager.ssh import SSHExecutorManager
|
||||
from .utils import exp_refs_by_rev
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -386,6 +387,7 @@ class Experiments:
|
||||
tmp_dir: bool = False,
|
||||
checkpoint_resume: Optional[str] = None,
|
||||
reset: bool = False,
|
||||
machine: Optional[str] = None,
|
||||
**kwargs,
|
||||
):
|
||||
"""Reproduce and checkout a single experiment."""
|
||||
@@ -395,7 +397,7 @@ class Experiments:
|
||||
if reset:
|
||||
self.reset_checkpoints()
|
||||
|
||||
if not (queue or tmp_dir):
|
||||
if not (queue or tmp_dir or machine):
|
||||
staged, _, _ = self.scm.status()
|
||||
if staged:
|
||||
logger.warning(
|
||||
@@ -429,12 +431,15 @@ class Experiments:
|
||||
return [stash_rev]
|
||||
if tmp_dir or queue:
|
||||
manager_cls: Type = TempDirExecutorManager
|
||||
elif machine:
|
||||
manager_cls = SSHExecutorManager
|
||||
else:
|
||||
manager_cls = WorkspaceExecutorManager
|
||||
results = self._reproduce_revs(
|
||||
revs=[stash_rev],
|
||||
keep_stash=False,
|
||||
manager_cls=manager_cls,
|
||||
machine=machine,
|
||||
)
|
||||
exp_rev = first(results)
|
||||
if exp_rev is not None:
|
||||
@@ -590,6 +595,7 @@ class Experiments:
|
||||
revs: Optional[Iterable] = None,
|
||||
keep_stash: Optional[bool] = True,
|
||||
manager_cls: Type = TempDirExecutorManager,
|
||||
machine: Optional[str] = None,
|
||||
**kwargs,
|
||||
) -> Mapping[str, str]:
|
||||
"""Reproduce the specified experiments.
|
||||
@@ -631,6 +637,7 @@ class Experiments:
|
||||
os.path.join(self.repo.tmp_dir, EXEC_TMP_DIR),
|
||||
self.repo,
|
||||
to_run,
|
||||
machine_name=machine,
|
||||
)
|
||||
try:
|
||||
exec_results = {}
|
||||
|
||||
+18
-98
@@ -7,9 +7,8 @@ from typing import TYPE_CHECKING, Deque, Dict, Generator, Optional, Tuple, Type
|
||||
|
||||
from dvc.proc.manager import ProcessManager
|
||||
|
||||
from ..base import (
|
||||
from ...base import (
|
||||
EXEC_BASELINE,
|
||||
EXEC_BRANCH,
|
||||
EXEC_HEAD,
|
||||
EXEC_MERGE,
|
||||
CheckpointExistsError,
|
||||
@@ -17,8 +16,8 @@ from ..base import (
|
||||
ExpRefInfo,
|
||||
ExpStashEntry,
|
||||
)
|
||||
from .base import EXEC_PID_DIR, BaseExecutor
|
||||
from .local import TempDirExecutor, WorkspaceExecutor
|
||||
from ..base import EXEC_PID_DIR, BaseExecutor
|
||||
from ..local import TempDirExecutor, WorkspaceExecutor
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scmrepo.git import Git
|
||||
@@ -37,6 +36,7 @@ class BaseExecutorManager(ABC, Mapping):
|
||||
self,
|
||||
scm: "Git",
|
||||
wdir: str,
|
||||
**kwargs,
|
||||
):
|
||||
from dvc.utils.fs import makedirs
|
||||
|
||||
@@ -74,8 +74,8 @@ class BaseExecutorManager(ABC, Mapping):
|
||||
import json
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from .base import ExecutorInfo
|
||||
from .ssh import SSHExecutor
|
||||
from ..base import ExecutorInfo
|
||||
from ..ssh import SSHExecutor
|
||||
|
||||
def make_executor(info: "ExecutorInfo"):
|
||||
if info.git_url:
|
||||
@@ -116,6 +116,16 @@ class BaseExecutorManager(ABC, Mapping):
|
||||
**kwargs,
|
||||
):
|
||||
manager = cls(scm, wdir)
|
||||
manager._enqueue_stash_entries(scm, repo, to_run, **kwargs)
|
||||
return manager
|
||||
|
||||
def _enqueue_stash_entries(
|
||||
self,
|
||||
scm: "Git",
|
||||
repo: "Repo",
|
||||
to_run: Dict[str, ExpStashEntry],
|
||||
**kwargs,
|
||||
):
|
||||
try:
|
||||
for stash_rev, entry in to_run.items():
|
||||
scm.set_ref(EXEC_HEAD, entry.head_rev)
|
||||
@@ -128,17 +138,16 @@ class BaseExecutorManager(ABC, Mapping):
|
||||
# EXEC_MERGE - the unmerged changes (from our stash)
|
||||
# to be reproduced
|
||||
# EXEC_BASELINE - the baseline commit for this experiment
|
||||
executor = cls.EXECUTOR_CLS.from_stash_entry(
|
||||
executor = self.EXECUTOR_CLS.from_stash_entry(
|
||||
repo,
|
||||
stash_rev,
|
||||
entry,
|
||||
**kwargs,
|
||||
)
|
||||
manager.enqueue(stash_rev, executor)
|
||||
self.enqueue(stash_rev, executor)
|
||||
finally:
|
||||
for ref in (EXEC_HEAD, EXEC_MERGE, EXEC_BASELINE):
|
||||
scm.remove_ref(ref)
|
||||
return manager
|
||||
|
||||
def exec_queue(self, jobs: Optional[int] = 1, detach: bool = False):
|
||||
"""Run dvc repro for queued executors in parallel."""
|
||||
@@ -260,92 +269,3 @@ class BaseExecutorManager(ABC, Mapping):
|
||||
except KeyError:
|
||||
pass
|
||||
remove(os.path.join(self.pid_dir, rev))
|
||||
|
||||
|
||||
class TempDirExecutorManager(BaseExecutorManager):
|
||||
EXECUTOR_CLS = TempDirExecutor
|
||||
|
||||
|
||||
class WorkspaceExecutorManager(BaseExecutorManager):
|
||||
EXECUTOR_CLS = WorkspaceExecutor
|
||||
|
||||
@classmethod
|
||||
def from_stash_entries(
|
||||
cls,
|
||||
scm: "Git",
|
||||
wdir: str,
|
||||
repo: "Repo",
|
||||
to_run: Dict[str, ExpStashEntry],
|
||||
**kwargs,
|
||||
):
|
||||
manager = cls(scm, wdir)
|
||||
try:
|
||||
assert len(to_run) == 1
|
||||
for stash_rev, entry in to_run.items():
|
||||
scm.set_ref(EXEC_HEAD, entry.head_rev)
|
||||
scm.set_ref(EXEC_MERGE, stash_rev)
|
||||
scm.set_ref(EXEC_BASELINE, entry.baseline_rev)
|
||||
|
||||
executor = cls.EXECUTOR_CLS.from_stash_entry(
|
||||
repo,
|
||||
stash_rev,
|
||||
entry,
|
||||
**kwargs,
|
||||
)
|
||||
manager.enqueue(stash_rev, executor)
|
||||
finally:
|
||||
for ref in (EXEC_MERGE,):
|
||||
scm.remove_ref(ref)
|
||||
return manager
|
||||
|
||||
def _collect_executor(self, executor, exec_result) -> Dict[str, str]:
|
||||
results = {}
|
||||
exp_rev = self.scm.get_ref(EXEC_BRANCH)
|
||||
if exp_rev:
|
||||
logger.debug("Collected experiment '%s'.", exp_rev[:7])
|
||||
results[exp_rev] = exec_result.exp_hash
|
||||
return results
|
||||
|
||||
def exec_queue(self, jobs: Optional[int] = 1, detach: bool = False):
|
||||
"""Run a single WorkspaceExecutor.
|
||||
|
||||
Workspace execution is done within the main DVC process
|
||||
(rather than in multiprocessing context)
|
||||
"""
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.stage.monitor import CheckpointKilledError
|
||||
|
||||
assert len(self._queue) == 1
|
||||
assert not detach
|
||||
result: Dict[str, Dict[str, str]] = defaultdict(dict)
|
||||
rev, executor = self._queue.popleft()
|
||||
|
||||
exec_name = "workspace"
|
||||
infofile = self.get_infofile_path(exec_name)
|
||||
try:
|
||||
exec_result = executor.reproduce(
|
||||
info=executor.info,
|
||||
rev=rev,
|
||||
infofile=infofile,
|
||||
log_level=logger.getEffectiveLevel(),
|
||||
)
|
||||
if not exec_result.exp_hash:
|
||||
raise DvcException(
|
||||
f"Failed to reproduce experiment '{rev[:7]}'"
|
||||
)
|
||||
if exec_result.ref_info:
|
||||
result[rev].update(
|
||||
self._collect_executor(executor, exec_result)
|
||||
)
|
||||
except CheckpointKilledError:
|
||||
# Checkpoint errors have already been logged
|
||||
return {}
|
||||
except DvcException:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise DvcException(
|
||||
f"Failed to reproduce experiment '{rev[:7]}'"
|
||||
) from exc
|
||||
finally:
|
||||
self.cleanup_executor(exec_name, executor)
|
||||
return result
|
||||
@@ -0,0 +1,109 @@
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from typing import TYPE_CHECKING, Dict, Optional
|
||||
|
||||
from ...base import (
|
||||
EXEC_BASELINE,
|
||||
EXEC_BRANCH,
|
||||
EXEC_HEAD,
|
||||
EXEC_MERGE,
|
||||
ExpStashEntry,
|
||||
)
|
||||
from ..local import TempDirExecutor, WorkspaceExecutor
|
||||
from .base import BaseExecutorManager
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scmrepo.git import Git
|
||||
|
||||
from dvc.repo import Repo
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TempDirExecutorManager(BaseExecutorManager):
|
||||
EXECUTOR_CLS = TempDirExecutor
|
||||
|
||||
|
||||
class WorkspaceExecutorManager(BaseExecutorManager):
|
||||
EXECUTOR_CLS = WorkspaceExecutor
|
||||
|
||||
@classmethod
|
||||
def from_stash_entries(
|
||||
cls,
|
||||
scm: "Git",
|
||||
wdir: str,
|
||||
repo: "Repo",
|
||||
to_run: Dict[str, ExpStashEntry],
|
||||
**kwargs,
|
||||
):
|
||||
manager = cls(scm, wdir)
|
||||
try:
|
||||
assert len(to_run) == 1
|
||||
for stash_rev, entry in to_run.items():
|
||||
scm.set_ref(EXEC_HEAD, entry.head_rev)
|
||||
scm.set_ref(EXEC_MERGE, stash_rev)
|
||||
scm.set_ref(EXEC_BASELINE, entry.baseline_rev)
|
||||
|
||||
executor = cls.EXECUTOR_CLS.from_stash_entry(
|
||||
repo,
|
||||
stash_rev,
|
||||
entry,
|
||||
**kwargs,
|
||||
)
|
||||
manager.enqueue(stash_rev, executor)
|
||||
finally:
|
||||
for ref in (EXEC_MERGE,):
|
||||
scm.remove_ref(ref)
|
||||
return manager
|
||||
|
||||
def _collect_executor(self, executor, exec_result) -> Dict[str, str]:
|
||||
results = {}
|
||||
exp_rev = self.scm.get_ref(EXEC_BRANCH)
|
||||
if exp_rev:
|
||||
logger.debug("Collected experiment '%s'.", exp_rev[:7])
|
||||
results[exp_rev] = exec_result.exp_hash
|
||||
return results
|
||||
|
||||
def exec_queue(self, jobs: Optional[int] = 1, detach: bool = False):
|
||||
"""Run a single WorkspaceExecutor.
|
||||
|
||||
Workspace execution is done within the main DVC process
|
||||
(rather than in multiprocessing context)
|
||||
"""
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.stage.monitor import CheckpointKilledError
|
||||
|
||||
assert len(self._queue) == 1
|
||||
assert not detach
|
||||
result: Dict[str, Dict[str, str]] = defaultdict(dict)
|
||||
rev, executor = self._queue.popleft()
|
||||
|
||||
exec_name = "workspace"
|
||||
infofile = self.get_infofile_path(exec_name)
|
||||
try:
|
||||
exec_result = executor.reproduce(
|
||||
info=executor.info,
|
||||
rev=rev,
|
||||
infofile=infofile,
|
||||
log_level=logger.getEffectiveLevel(),
|
||||
)
|
||||
if not exec_result.exp_hash:
|
||||
raise DvcException(
|
||||
f"Failed to reproduce experiment '{rev[:7]}'"
|
||||
)
|
||||
if exec_result.ref_info:
|
||||
result[rev].update(
|
||||
self._collect_executor(executor, exec_result)
|
||||
)
|
||||
except CheckpointKilledError:
|
||||
# Checkpoint errors have already been logged
|
||||
return {}
|
||||
except DvcException:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise DvcException(
|
||||
f"Failed to reproduce experiment '{rev[:7]}'"
|
||||
) from exc
|
||||
finally:
|
||||
self.cleanup_executor(exec_name, executor)
|
||||
return result
|
||||
@@ -0,0 +1,109 @@
|
||||
import logging
|
||||
import posixpath
|
||||
from collections import defaultdict
|
||||
from typing import TYPE_CHECKING, Callable, Dict, Generator, Optional, Tuple
|
||||
|
||||
from ...base import ExpStashEntry
|
||||
from ..base import BaseExecutor
|
||||
from ..ssh import SSHExecutor, _sshfs
|
||||
from .base import BaseExecutorManager
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scmrepo.git import Git
|
||||
|
||||
from dvc.repo import Repo
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class SSHExecutorManager(BaseExecutorManager):
|
||||
EXECUTOR_CLS = SSHExecutor
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
scm: "Git",
|
||||
wdir: str,
|
||||
host: Optional[str] = None,
|
||||
port: Optional[int] = None,
|
||||
username: Optional[str] = None,
|
||||
fs_factory: Optional[Callable] = None,
|
||||
**kwargs,
|
||||
):
|
||||
assert host
|
||||
super().__init__(scm, wdir, **kwargs)
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.username = username
|
||||
self._fs_factory = fs_factory
|
||||
|
||||
def _load_infos(self) -> Generator[Tuple[str, "BaseExecutor"], None, None]:
|
||||
# TODO: load existing infos using sshfs
|
||||
yield from []
|
||||
|
||||
@classmethod
|
||||
def from_stash_entries(
|
||||
cls,
|
||||
scm: "Git",
|
||||
wdir: str,
|
||||
repo: "Repo",
|
||||
to_run: Dict[str, ExpStashEntry],
|
||||
**kwargs,
|
||||
):
|
||||
machine_name: Optional[str] = kwargs.get("machine_name", None)
|
||||
manager = cls(
|
||||
scm, wdir, **repo.machine.get_executor_kwargs(machine_name)
|
||||
)
|
||||
manager._enqueue_stash_entries(scm, repo, to_run, **kwargs)
|
||||
return manager
|
||||
|
||||
def sshfs(self):
|
||||
return _sshfs(self._fs_factory, host=self.host, port=self.port)
|
||||
|
||||
def get_infofile_path(self, name: str) -> str:
|
||||
return f"{name}{BaseExecutor.INFOFILE_EXT}"
|
||||
|
||||
def _exec_attached(self, jobs: Optional[int] = 1):
|
||||
from dvc.exceptions import DvcException
|
||||
from dvc.stage.monitor import CheckpointKilledError
|
||||
|
||||
assert len(self._queue) == 1
|
||||
result: Dict[str, Dict[str, str]] = defaultdict(dict)
|
||||
rev, executor = self._queue.popleft()
|
||||
info = executor.info
|
||||
infofile = posixpath.join(
|
||||
info.root_dir,
|
||||
info.dvc_dir,
|
||||
"tmp",
|
||||
self.get_infofile_path(rev),
|
||||
)
|
||||
try:
|
||||
exec_result = executor.reproduce(
|
||||
info=executor.info,
|
||||
rev=rev,
|
||||
infofile=infofile,
|
||||
log_level=logger.getEffectiveLevel(),
|
||||
fs_factory=self._fs_factory,
|
||||
)
|
||||
if not exec_result.exp_hash:
|
||||
raise DvcException(
|
||||
f"Failed to reproduce experiment '{rev[:7]}'"
|
||||
)
|
||||
if exec_result.ref_info:
|
||||
result[rev].update(
|
||||
self._collect_executor(executor, exec_result)
|
||||
)
|
||||
except CheckpointKilledError:
|
||||
# Checkpoint errors have already been logged
|
||||
return {}
|
||||
except DvcException:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise DvcException(
|
||||
f"Failed to reproduce experiment '{rev[:7]}'"
|
||||
) from exc
|
||||
finally:
|
||||
self.cleanup_executor(rev, executor)
|
||||
return result
|
||||
|
||||
def cleanup_executor(self, rev: str, executor: "BaseExecutor"):
|
||||
executor.cleanup()
|
||||
@@ -40,8 +40,8 @@ def test_init_from_stash(tmp_dir, scm, dvc, machine_instance, mocker):
|
||||
@pytest.mark.parametrize("cloud", [pytest.lazy_fixture("git_ssh")])
|
||||
def test_init_git(tmp_dir, scm, cloud):
|
||||
tmp_dir.scm_gen({"foo": "foo", "dir": {"bar": "bar"}}, commit="init")
|
||||
rev = scm.get_rev()
|
||||
scm.set_ref(EXEC_HEAD, rev)
|
||||
baseline_rev = scm.get_rev()
|
||||
scm.set_ref(EXEC_HEAD, baseline_rev)
|
||||
tmp_dir.gen("foo", "stashed")
|
||||
scm.gitpython.git.stash()
|
||||
rev = scm.resolve_rev("stash@{0}")
|
||||
@@ -50,14 +50,15 @@ def test_init_git(tmp_dir, scm, cloud):
|
||||
root_url = cloud / SSHExecutor.gen_dirname()
|
||||
|
||||
executor = SSHExecutor(
|
||||
scm,
|
||||
".",
|
||||
root_dir=root_url.path,
|
||||
dvc_dir=".dvc",
|
||||
baseline_rev=baseline_rev,
|
||||
host=root_url.host,
|
||||
port=root_url.port,
|
||||
username=TEST_SSH_USER,
|
||||
fs_factory=partial(_ssh_factory, cloud),
|
||||
)
|
||||
executor.init_git(scm)
|
||||
assert root_url.path == executor._repo_abspath
|
||||
|
||||
fs = cloud._ssh
|
||||
@@ -76,9 +77,9 @@ def test_init_cache(tmp_dir, dvc, scm, cloud):
|
||||
root_url = cloud / SSHExecutor.gen_dirname()
|
||||
|
||||
executor = SSHExecutor(
|
||||
scm,
|
||||
".",
|
||||
root_dir=root_url.path,
|
||||
dvc_dir=".dvc",
|
||||
baseline_rev=rev,
|
||||
host=root_url.host,
|
||||
port=root_url.port,
|
||||
username=TEST_SSH_USER,
|
||||
|
||||
Reference in New Issue
Block a user