From f245567613e84a57ad137703ab13d92ac61e75bb Mon Sep 17 00:00:00 2001 From: David de la Iglesia Castro Date: Mon, 22 May 2023 12:23:25 +0200 Subject: [PATCH] exp: Drop `gc`. (#9273) Closes #7991 --- dvc/commands/experiments/__init__.py | 2 - dvc/commands/experiments/gc.py | 146 ------------------------- dvc/repo/experiments/__init__.py | 5 - dvc/repo/experiments/gc.py | 53 --------- tests/func/experiments/test_gc.py | 114 ------------------- tests/unit/command/test_experiments.py | 39 ------- 6 files changed, 359 deletions(-) delete mode 100644 dvc/commands/experiments/gc.py delete mode 100644 dvc/repo/experiments/gc.py delete mode 100644 tests/func/experiments/test_gc.py diff --git a/dvc/commands/experiments/__init__.py b/dvc/commands/experiments/__init__.py index 2f06cc15d..3d4b86875 100644 --- a/dvc/commands/experiments/__init__.py +++ b/dvc/commands/experiments/__init__.py @@ -7,7 +7,6 @@ from dvc.commands.experiments import ( clean, diff, exec_run, - gc, ls, pull, push, @@ -24,7 +23,6 @@ SUB_COMMANDS = [ clean, diff, exec_run, - gc, ls, pull, push, diff --git a/dvc/commands/experiments/gc.py b/dvc/commands/experiments/gc.py deleted file mode 100644 index b610670f8..000000000 --- a/dvc/commands/experiments/gc.py +++ /dev/null @@ -1,146 +0,0 @@ -import argparse -import logging - -from dvc.cli.command import CmdBase -from dvc.cli.utils import append_doc_link -from dvc.exceptions import InvalidArgumentError -from dvc.ui import ui - -logger = logging.getLogger(__name__) - - -def _raise_error_if_all_disabled(**kwargs): - if not any(kwargs.values()): - raise InvalidArgumentError( - "Either of `-w|--workspace`, `-a|--all-branches`, `-T|--all-tags`," - " `--all-commits` or `--date` needs to be set." - ) - - -class CmdExperimentsGC(CmdBase): - def run(self): - _raise_error_if_all_disabled( - all_branches=self.args.all_branches, - all_tags=self.args.all_tags, - all_commits=self.args.all_commits, - commit_date=self.args.commit_date, - workspace=self.args.workspace, - ) - - msg = "This will remove all experiments except those derived from " - - msg += "the workspace" - if self.args.all_commits: - msg += " and all git commits" - else: - if self.args.all_branches and self.args.all_tags: - msg += " and all git branches and tags" - elif self.args.all_branches: - msg += " and all git branches" - elif self.args.all_tags: - msg += " and all git tags" - - if self.args.commit_date: - msg += f" and all git commits before date {self.args.commit_date}" - msg += " of the current repo." - - if self.args.queued: - msg += " Run queued experiments will be preserved." - else: - msg += " Run queued experiments will be removed." - - logger.warning(msg) - - msg = "Are you sure you want to proceed?" - if not self.args.force and not ui.confirm(msg): - return 1 - - removed = self.repo.experiments.gc( - all_branches=self.args.all_branches, - all_tags=self.args.all_tags, - all_commits=self.args.all_commits, - workspace=self.args.workspace, - commit_date=self.args.commit_date, - queued=self.args.queued, - ) - - if removed: - ui.write( - f"Removed {removed} experiments.", - "To remove unused cache files", - "use 'dvc gc'.", - ) - else: - ui.write("No experiments to remove.") - return 0 - - -def add_parser(experiments_subparsers, parent_parser): - EXPERIMENTS_GC_HELP = "Garbage collect unneeded experiments." - EXPERIMENTS_GC_DESCRIPTION = ( - "Removes all experiments which are not derived" - " from the specified Git revisions." - ) - experiments_gc_parser = experiments_subparsers.add_parser( - "gc", - parents=[parent_parser], - description=append_doc_link(EXPERIMENTS_GC_DESCRIPTION, "exp/gc"), - help=EXPERIMENTS_GC_HELP, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - experiments_gc_parser.add_argument( - "-w", - "--workspace", - action="store_true", - default=False, - help="Keep experiments derived from the current workspace.", - ) - experiments_gc_parser.add_argument( - "-a", - "--all-branches", - action="store_true", - default=False, - help="Keep experiments derived from the tips of all Git branches.", - ) - experiments_gc_parser.add_argument( - "-T", - "--all-tags", - action="store_true", - default=False, - help="Keep experiments derived from all Git tags.", - ) - experiments_gc_parser.add_argument( - "-A", - "--all-commits", - action="store_true", - default=False, - help="Keep experiments derived from all Git commits.", - ) - experiments_gc_parser.add_argument( - "--date", - type=str, - dest="commit_date", - metavar="", - default=None, - help=( - "Keep experiments from the commits after (inclusive) a certain " - "date. Date must match the extended ISO 8601 format (YYYY-MM-DD)." - ), - ) - experiments_gc_parser.add_argument( - "--queued", - action="store_true", - default=False, - help=( - "Keep queued experiments (experiments run queue will be cleared " - "by default)." - ), - ) - experiments_gc_parser.add_argument( - "-f", - "--force", - action="store_true", - default=False, - help="Force garbage collection - automatically agree to all prompts.", - ) - experiments_gc_parser.set_defaults(func=CmdExperimentsGC) diff --git a/dvc/repo/experiments/__init__.py b/dvc/repo/experiments/__init__.py index 959ce6fc5..53571dbae 100644 --- a/dvc/repo/experiments/__init__.py +++ b/dvc/repo/experiments/__init__.py @@ -490,11 +490,6 @@ class Experiments: return save(self.repo, *args, **kwargs) - def gc(self, *args, **kwargs): - from dvc.repo.experiments.gc import gc - - return gc(self.repo, *args, **kwargs) - def push(self, *args, **kwargs): from dvc.repo.experiments.push import push diff --git a/dvc/repo/experiments/gc.py b/dvc/repo/experiments/gc.py deleted file mode 100644 index edb57a1e3..000000000 --- a/dvc/repo/experiments/gc.py +++ /dev/null @@ -1,53 +0,0 @@ -import logging -from typing import Optional - -from dvc.repo import locked - -from .utils import exp_refs, remove_exp_refs - -logger = logging.getLogger(__name__) - - -@locked -def gc( - repo, - all_branches: Optional[bool] = False, - all_tags: Optional[bool] = False, - all_commits: Optional[bool] = False, - workspace: Optional[bool] = False, - commit_date: Optional[str] = None, - queued: Optional[bool] = False, -): - keep_revs = set( - repo.brancher( - all_branches=all_branches, - all_tags=all_tags, - all_commits=all_commits, - commit_date=commit_date, - sha_only=True, - ) - ) - if workspace: - keep_revs.add(repo.scm.get_rev()) - - if not keep_revs: - return 0 - - to_remove = [ - ref_info - for ref_info in exp_refs(repo.scm) - if ref_info.baseline_sha not in keep_revs - ] - remove_exp_refs(repo.scm, to_remove) - removed = len(to_remove) - - delete_stashes = [] - stash = repo.experiments.celery_queue.stash - for stash_rev, entry in stash.stash_revs.items(): - if not queued or entry.baseline_rev not in keep_revs: - delete_stashes.append(stash_rev) - if delete_stashes: - repo.experiments.celery_queue.remove(delete_stashes) - removed += len(delete_stashes) - - return removed diff --git a/tests/func/experiments/test_gc.py b/tests/func/experiments/test_gc.py deleted file mode 100644 index 4837952bb..000000000 --- a/tests/func/experiments/test_gc.py +++ /dev/null @@ -1,114 +0,0 @@ -import pytest -from funcy import first - - -@pytest.mark.parametrize("queued, expected", [(True, 0), (False, 1)]) -def test_workspace(tmp_dir, scm, dvc, queued, expected, copy_script): - tmp_dir.gen("params.yaml", "foo: 1") - - stage = dvc.run( - cmd="python copy.py params.yaml metrics.yaml", - metrics_no_cache=["metrics.yaml"], - params=["foo"], - name="foo", - ) - scm.add(["dvc.yaml", "dvc.lock", "copy.py", "params.yaml", "metrics.yaml"]) - scm.commit("v1") - - dvc.experiments.run(stage.addressing, params=["foo=2"]) - dvc.experiments.run(stage.addressing, params=["foo=3"], queue=True) - - removed = dvc.experiments.gc(workspace=True, queued=queued) - assert removed == expected - - -@pytest.mark.parametrize("queued, expected", [(True, 0), (False, 2)]) -def test_all_commits(tmp_dir, scm, dvc, queued, expected, copy_script): - tmp_dir.gen("params.yaml", "foo: 1") - - stage = dvc.run( - cmd="python copy.py params.yaml metrics.yaml", - metrics_no_cache=["metrics.yaml"], - params=["foo"], - name="foo", - ) - scm.add(["dvc.yaml", "dvc.lock", "copy.py", "params.yaml", "metrics.yaml"]) - scm.commit("v1") - - results = dvc.experiments.run(stage.addressing, params=["foo=2"]) - exp_rev = first(results) - dvc.experiments.run(stage.addressing, params=["foo=3"], queue=True) - - dvc.experiments.apply(exp_rev) - scm.add(["dvc.yaml", "dvc.lock", "copy.py", "params.yaml", "metrics.yaml"]) - scm.commit("v2") - - dvc.experiments.run(stage.addressing, params=["foo=4"]) - dvc.experiments.run(stage.addressing, params=["foo=5"], queue=True) - - removed = dvc.experiments.gc(all_commits=True, queued=queued) - assert removed == expected - - -@pytest.mark.parametrize("queued, expected", [(True, 2), (False, 3)]) -def test_all_branches(tmp_dir, scm, dvc, queued, expected, copy_script): - tmp_dir.gen("params.yaml", "foo: 1") - - stage = dvc.run( - cmd="python copy.py params.yaml metrics.yaml", - metrics_no_cache=["metrics.yaml"], - params=["foo"], - name="foo", - ) - scm.add(["dvc.yaml", "dvc.lock", "copy.py", "params.yaml", "metrics.yaml"]) - scm.commit("v1") - - results = dvc.experiments.run(stage.addressing, params=["foo=2"]) - exp_rev = first(results) - dvc.experiments.run(stage.addressing, params=["foo=3"], queue=True) - - dvc.experiments.apply(exp_rev) - scm.add(["dvc.yaml", "dvc.lock", "copy.py", "params.yaml", "metrics.yaml"]) - scm.commit("v2") - - scm.branch("branch") - - results = dvc.experiments.run(stage.addressing, params=["foo=4"]) - dvc.experiments.run(stage.addressing, params=["foo=5"], queue=True) - exp_rev = first(results) - - dvc.experiments.apply(exp_rev) - scm.add(["dvc.yaml", "dvc.lock", "copy.py", "params.yaml", "metrics.yaml"]) - scm.commit("v3") - - removed = dvc.experiments.gc(all_branches=True, queued=queued) - assert removed == expected - - -@pytest.mark.parametrize("queued, expected", [(True, 2), (False, 3)]) -def test_all_tags(tmp_dir, scm, dvc, queued, expected, copy_script): - tmp_dir.gen("params.yaml", "foo: 1") - - stage = dvc.run( - cmd="python copy.py params.yaml metrics.yaml", - metrics_no_cache=["metrics.yaml"], - params=["foo"], - name="foo", - ) - scm.add(["dvc.yaml", "dvc.lock", "copy.py", "params.yaml", "metrics.yaml"]) - scm.commit("v1") - - results = dvc.experiments.run(stage.addressing, params=["foo=2"]) - exp_rev = first(results) - dvc.experiments.run(stage.addressing, params=["foo=3"], queue=True) - - dvc.experiments.apply(exp_rev) - scm.add(["dvc.yaml", "dvc.lock", "copy.py", "params.yaml", "metrics.yaml"]) - scm.commit("v2") - scm.tag("tag-v2") - - dvc.experiments.run(stage.addressing, params=["foo=4"]) - dvc.experiments.run(stage.addressing, params=["foo=5"], queue=True) - - removed = dvc.experiments.gc(all_tags=True, queued=queued) - assert removed == expected diff --git a/tests/unit/command/test_experiments.py b/tests/unit/command/test_experiments.py index 98065a743..4b7898574 100644 --- a/tests/unit/command/test_experiments.py +++ b/tests/unit/command/test_experiments.py @@ -5,7 +5,6 @@ from dvc.commands.experiments.apply import CmdExperimentsApply from dvc.commands.experiments.branch import CmdExperimentsBranch from dvc.commands.experiments.clean import CmdExperimentsClean from dvc.commands.experiments.diff import CmdExperimentsDiff -from dvc.commands.experiments.gc import CmdExperimentsGC from dvc.commands.experiments.ls import CmdExperimentsList from dvc.commands.experiments.pull import CmdExperimentsPull from dvc.commands.experiments.push import CmdExperimentsPush @@ -141,44 +140,6 @@ def test_experiments_run(dvc, scm, mocker): cmd.repo.experiments.run.assert_called_with(**default_arguments) -def test_experiments_gc(dvc, scm, mocker): - cli_args = parse_args( - [ - "exp", - "gc", - "--workspace", - "--all-tags", - "--all-branches", - "--all-commits", - "--queued", - "--date", - "2022-07-04", - "--force", - ] - ) - assert cli_args.func == CmdExperimentsGC - - cmd = cli_args.func(cli_args) - m = mocker.patch("dvc.repo.experiments.gc.gc", return_value={}) - - assert cmd.run() == 0 - - m.assert_called_once_with( - cmd.repo, - workspace=True, - all_tags=True, - all_branches=True, - all_commits=True, - commit_date="2022-07-04", - queued=True, - ) - - cli_args = parse_args(["exp", "gc"]) - cmd = cli_args.func(cli_args) - with pytest.raises(InvalidArgumentError): - cmd.run() - - def test_experiments_branch(dvc, scm, mocker): m = mocker.patch("dvc.repo.experiments.branch.branch", return_value={})