mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-19 15:28:28 +01:00
Now exp remove can clean up failed tasks and associated successful tasks
fix: #8018 fix: #8071 1. Add one line to make `exp remove` can remove failed tasks, and will also delete associated successful tasks 2. Add a new unit test for this behaviour. 3. Solve the unstable test_queue problem.
This commit is contained in:
committed by
Peter Rowlands (변기호)
parent
f68c4e9a4b
commit
ae83e3c787
@@ -14,15 +14,8 @@ from dvc.repo.scm_context import scm_context
|
||||
from dvc.scm import iter_revs
|
||||
|
||||
from .exceptions import UnresolvedExpNamesError
|
||||
from .queue.remove import remove_tasks
|
||||
from .refs import ExpRefInfo
|
||||
from .utils import (
|
||||
exp_refs,
|
||||
exp_refs_by_baseline,
|
||||
push_refspec,
|
||||
remove_exp_refs,
|
||||
resolve_name,
|
||||
)
|
||||
from .utils import exp_refs, exp_refs_by_baseline, push_refspec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.repo import Repo
|
||||
@@ -47,6 +40,8 @@ def _get_ref_and_entry_by_names(
|
||||
celery_queue: "LocalCeleryQueue",
|
||||
git_remote: Optional[str],
|
||||
) -> ExpRefAndQueueEntry:
|
||||
from .utils import resolve_name
|
||||
|
||||
exp_ref_list: List["ExpRefInfo"] = []
|
||||
queue_entry_list: List["QueueEntry"] = []
|
||||
removed: List[str] = []
|
||||
@@ -59,7 +54,7 @@ def _get_ref_and_entry_by_names(
|
||||
queue_entry_match: Dict[
|
||||
str, Optional["QueueEntry"]
|
||||
] = celery_queue.match_queue_entry_by_name(
|
||||
exp_names, celery_queue.iter_queued()
|
||||
exp_names, celery_queue.iter_queued(), celery_queue.iter_done()
|
||||
)
|
||||
|
||||
remained = []
|
||||
@@ -111,7 +106,7 @@ def remove(
|
||||
result = _get_ref_and_entry_by_names(
|
||||
exp_names, repo.scm, celery_queue, git_remote
|
||||
)
|
||||
removed.extend(exp_names)
|
||||
removed.extend(result.removed)
|
||||
exp_ref_list.extend(result.exp_ref_list)
|
||||
queue_entry_list.extend(result.queue_entry_list)
|
||||
elif rev:
|
||||
@@ -123,6 +118,8 @@ def remove(
|
||||
_remove_commited_exps(repo.scm, exp_ref_list, git_remote)
|
||||
|
||||
if queue_entry_list:
|
||||
from .queue.remove import remove_tasks
|
||||
|
||||
remove_tasks(celery_queue, queue_entry_list)
|
||||
|
||||
return removed
|
||||
@@ -160,5 +157,7 @@ def _remove_commited_exps(
|
||||
progress=pbar.update_git,
|
||||
)
|
||||
else:
|
||||
from .utils import remove_exp_refs
|
||||
|
||||
remove_exp_refs(scm, exp_refs_list)
|
||||
return [exp_ref.name for exp_ref in exp_refs_list]
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
from dvc.repo.experiments.queue.base import QueueDoneResult
|
||||
|
||||
|
||||
def test_remove_done_tasks(dvc, test_queue, scm, mocker):
|
||||
from funcy import concat
|
||||
|
||||
failed_test = ["failed1", "failed2"]
|
||||
success_test = ["success1", "success2"]
|
||||
|
||||
# create mock ref info
|
||||
ref_info_dict = {}
|
||||
for name in success_test:
|
||||
ref_info_dict[name] = mocker.Mock()
|
||||
for name in failed_test:
|
||||
ref_info_dict[name] = None
|
||||
|
||||
# create mock queue entry
|
||||
entry_dict = {}
|
||||
for name in concat(failed_test, success_test):
|
||||
entry_dict[name] = mocker.Mock(stash_rev=name)
|
||||
entry_dict[name].name = name
|
||||
|
||||
done_iter = [
|
||||
QueueDoneResult(entry_dict[name], None)
|
||||
for name in concat(failed_test, success_test)
|
||||
]
|
||||
|
||||
mocker.patch.object(test_queue, "iter_done", return_value=done_iter)
|
||||
|
||||
mocker.patch(
|
||||
"dvc.repo.experiments.utils.resolve_name",
|
||||
autospec=True,
|
||||
return_value=ref_info_dict,
|
||||
)
|
||||
|
||||
remove_exp_refs = mocker.patch(
|
||||
"dvc.repo.experiments.utils.remove_exp_refs",
|
||||
)
|
||||
remove_tasks_mocker = mocker.patch(
|
||||
"dvc.repo.experiments.queue.remove.remove_tasks",
|
||||
)
|
||||
|
||||
assert (
|
||||
dvc.experiments.remove(failed_test + success_test)
|
||||
== failed_test + success_test
|
||||
)
|
||||
|
||||
remove_tasks_mocker.assert_called_once_with(
|
||||
test_queue,
|
||||
[entry_dict[name] for name in failed_test + success_test],
|
||||
)
|
||||
|
||||
remove_exp_refs.assert_called_once_with(
|
||||
dvc.scm, [ref_info_dict[name] for name in success_test]
|
||||
)
|
||||
Reference in New Issue
Block a user