Now exp remove can clean up failed tasks and associated successful tasks

fix: #8018
fix: #8071

1. Add one line to make `exp remove` can remove failed tasks, and will
   also delete associated successful tasks
2. Add a new unit test for this behaviour.
3. Solve the unstable test_queue problem.
This commit is contained in:
karajan1001
2022-08-05 16:09:51 +09:00
committed by Peter Rowlands (변기호)
parent f68c4e9a4b
commit ae83e3c787
2 changed files with 64 additions and 10 deletions
+9 -10
View File
@@ -14,15 +14,8 @@ from dvc.repo.scm_context import scm_context
from dvc.scm import iter_revs
from .exceptions import UnresolvedExpNamesError
from .queue.remove import remove_tasks
from .refs import ExpRefInfo
from .utils import (
exp_refs,
exp_refs_by_baseline,
push_refspec,
remove_exp_refs,
resolve_name,
)
from .utils import exp_refs, exp_refs_by_baseline, push_refspec
if TYPE_CHECKING:
from dvc.repo import Repo
@@ -47,6 +40,8 @@ def _get_ref_and_entry_by_names(
celery_queue: "LocalCeleryQueue",
git_remote: Optional[str],
) -> ExpRefAndQueueEntry:
from .utils import resolve_name
exp_ref_list: List["ExpRefInfo"] = []
queue_entry_list: List["QueueEntry"] = []
removed: List[str] = []
@@ -59,7 +54,7 @@ def _get_ref_and_entry_by_names(
queue_entry_match: Dict[
str, Optional["QueueEntry"]
] = celery_queue.match_queue_entry_by_name(
exp_names, celery_queue.iter_queued()
exp_names, celery_queue.iter_queued(), celery_queue.iter_done()
)
remained = []
@@ -111,7 +106,7 @@ def remove(
result = _get_ref_and_entry_by_names(
exp_names, repo.scm, celery_queue, git_remote
)
removed.extend(exp_names)
removed.extend(result.removed)
exp_ref_list.extend(result.exp_ref_list)
queue_entry_list.extend(result.queue_entry_list)
elif rev:
@@ -123,6 +118,8 @@ def remove(
_remove_commited_exps(repo.scm, exp_ref_list, git_remote)
if queue_entry_list:
from .queue.remove import remove_tasks
remove_tasks(celery_queue, queue_entry_list)
return removed
@@ -160,5 +157,7 @@ def _remove_commited_exps(
progress=pbar.update_git,
)
else:
from .utils import remove_exp_refs
remove_exp_refs(scm, exp_refs_list)
return [exp_ref.name for exp_ref in exp_refs_list]
@@ -0,0 +1,55 @@
from dvc.repo.experiments.queue.base import QueueDoneResult
def test_remove_done_tasks(dvc, test_queue, scm, mocker):
from funcy import concat
failed_test = ["failed1", "failed2"]
success_test = ["success1", "success2"]
# create mock ref info
ref_info_dict = {}
for name in success_test:
ref_info_dict[name] = mocker.Mock()
for name in failed_test:
ref_info_dict[name] = None
# create mock queue entry
entry_dict = {}
for name in concat(failed_test, success_test):
entry_dict[name] = mocker.Mock(stash_rev=name)
entry_dict[name].name = name
done_iter = [
QueueDoneResult(entry_dict[name], None)
for name in concat(failed_test, success_test)
]
mocker.patch.object(test_queue, "iter_done", return_value=done_iter)
mocker.patch(
"dvc.repo.experiments.utils.resolve_name",
autospec=True,
return_value=ref_info_dict,
)
remove_exp_refs = mocker.patch(
"dvc.repo.experiments.utils.remove_exp_refs",
)
remove_tasks_mocker = mocker.patch(
"dvc.repo.experiments.queue.remove.remove_tasks",
)
assert (
dvc.experiments.remove(failed_test + success_test)
== failed_test + success_test
)
remove_tasks_mocker.assert_called_once_with(
test_queue,
[entry_dict[name] for name in failed_test + success_test],
)
remove_exp_refs.assert_called_once_with(
dvc.scm, [ref_info_dict[name] for name in success_test]
)