From ae83e3c78720b1e989d0f22352aaf67d8ef761ad Mon Sep 17 00:00:00 2001 From: karajan1001 Date: Tue, 2 Aug 2022 10:47:24 +0800 Subject: [PATCH] Now `exp remove` can clean up failed tasks and associated successful tasks fix: #8018 fix: #8071 1. Add one line to make `exp remove` can remove failed tasks, and will also delete associated successful tasks 2. Add a new unit test for this behaviour. 3. Solve the unstable test_queue problem. --- dvc/repo/experiments/remove.py | 19 ++++---- tests/unit/repo/experiments/test_remove.py | 55 ++++++++++++++++++++++ 2 files changed, 64 insertions(+), 10 deletions(-) create mode 100644 tests/unit/repo/experiments/test_remove.py diff --git a/dvc/repo/experiments/remove.py b/dvc/repo/experiments/remove.py index a64503e13..2d6693a2d 100644 --- a/dvc/repo/experiments/remove.py +++ b/dvc/repo/experiments/remove.py @@ -14,15 +14,8 @@ from dvc.repo.scm_context import scm_context from dvc.scm import iter_revs from .exceptions import UnresolvedExpNamesError -from .queue.remove import remove_tasks from .refs import ExpRefInfo -from .utils import ( - exp_refs, - exp_refs_by_baseline, - push_refspec, - remove_exp_refs, - resolve_name, -) +from .utils import exp_refs, exp_refs_by_baseline, push_refspec if TYPE_CHECKING: from dvc.repo import Repo @@ -47,6 +40,8 @@ def _get_ref_and_entry_by_names( celery_queue: "LocalCeleryQueue", git_remote: Optional[str], ) -> ExpRefAndQueueEntry: + from .utils import resolve_name + exp_ref_list: List["ExpRefInfo"] = [] queue_entry_list: List["QueueEntry"] = [] removed: List[str] = [] @@ -59,7 +54,7 @@ def _get_ref_and_entry_by_names( queue_entry_match: Dict[ str, Optional["QueueEntry"] ] = celery_queue.match_queue_entry_by_name( - exp_names, celery_queue.iter_queued() + exp_names, celery_queue.iter_queued(), celery_queue.iter_done() ) remained = [] @@ -111,7 +106,7 @@ def remove( result = _get_ref_and_entry_by_names( exp_names, repo.scm, celery_queue, git_remote ) - removed.extend(exp_names) + removed.extend(result.removed) exp_ref_list.extend(result.exp_ref_list) queue_entry_list.extend(result.queue_entry_list) elif rev: @@ -123,6 +118,8 @@ def remove( _remove_commited_exps(repo.scm, exp_ref_list, git_remote) if queue_entry_list: + from .queue.remove import remove_tasks + remove_tasks(celery_queue, queue_entry_list) return removed @@ -160,5 +157,7 @@ def _remove_commited_exps( progress=pbar.update_git, ) else: + from .utils import remove_exp_refs + remove_exp_refs(scm, exp_refs_list) return [exp_ref.name for exp_ref in exp_refs_list] diff --git a/tests/unit/repo/experiments/test_remove.py b/tests/unit/repo/experiments/test_remove.py new file mode 100644 index 000000000..c9dd3861a --- /dev/null +++ b/tests/unit/repo/experiments/test_remove.py @@ -0,0 +1,55 @@ +from dvc.repo.experiments.queue.base import QueueDoneResult + + +def test_remove_done_tasks(dvc, test_queue, scm, mocker): + from funcy import concat + + failed_test = ["failed1", "failed2"] + success_test = ["success1", "success2"] + + # create mock ref info + ref_info_dict = {} + for name in success_test: + ref_info_dict[name] = mocker.Mock() + for name in failed_test: + ref_info_dict[name] = None + + # create mock queue entry + entry_dict = {} + for name in concat(failed_test, success_test): + entry_dict[name] = mocker.Mock(stash_rev=name) + entry_dict[name].name = name + + done_iter = [ + QueueDoneResult(entry_dict[name], None) + for name in concat(failed_test, success_test) + ] + + mocker.patch.object(test_queue, "iter_done", return_value=done_iter) + + mocker.patch( + "dvc.repo.experiments.utils.resolve_name", + autospec=True, + return_value=ref_info_dict, + ) + + remove_exp_refs = mocker.patch( + "dvc.repo.experiments.utils.remove_exp_refs", + ) + remove_tasks_mocker = mocker.patch( + "dvc.repo.experiments.queue.remove.remove_tasks", + ) + + assert ( + dvc.experiments.remove(failed_test + success_test) + == failed_test + success_test + ) + + remove_tasks_mocker.assert_called_once_with( + test_queue, + [entry_dict[name] for name in failed_test + success_test], + ) + + remove_exp_refs.assert_called_once_with( + dvc.scm, [ref_info_dict[name] for name in success_test] + )