Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

exp: Drop gc. (#9273)

Browse Source 
Closes #7991
This commit is contained in:
David de la Iglesia Castro
2023-05-22 16:08:25 +05:45
committed by GitHub
parent 4163c8f57d
commit f245567613
6 changed files with 0 additions and 359 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/commands/experiments/__init__.py
-2
View File
@@ -7,7 +7,6 @@ from dvc.commands.experiments import (
clean,
diff,
exec_run,
gc,
ls,
pull,
push,
@@ -24,7 +23,6 @@ SUB_COMMANDS = [
clean,
diff,
exec_run,
gc,
ls,
pull,
push,
dvc/commands/experiments/gc.py
-146
View File
@@ -1,146 +0,0 @@
import argparse
import logging
from dvc.cli.command import CmdBase
from dvc.cli.utils import append_doc_link
from dvc.exceptions import InvalidArgumentError
from dvc.ui import ui
logger = logging.getLogger(__name__)
def _raise_error_if_all_disabled(**kwargs):
if not any(kwargs.values()):
raise InvalidArgumentError(
"Either of `-w|--workspace`, `-a|--all-branches`, `-T|--all-tags`,"
" `--all-commits` or `--date` needs to be set."
)
class CmdExperimentsGC(CmdBase):
def run(self):
_raise_error_if_all_disabled(
all_branches=self.args.all_branches,
all_tags=self.args.all_tags,
all_commits=self.args.all_commits,
commit_date=self.args.commit_date,
workspace=self.args.workspace,
)
msg = "This will remove all experiments except those derived from "
msg += "the workspace"
if self.args.all_commits:
msg += " and all git commits"
else:
if self.args.all_branches and self.args.all_tags:
msg += " and all git branches and tags"
elif self.args.all_branches:
msg += " and all git branches"
elif self.args.all_tags:
msg += " and all git tags"
if self.args.commit_date:
msg += f" and all git commits before date {self.args.commit_date}"
msg += " of the current repo."
if self.args.queued:
msg += " Run queued experiments will be preserved."
else:
msg += " Run queued experiments will be removed."
logger.warning(msg)
msg = "Are you sure you want to proceed?"
if not self.args.force and not ui.confirm(msg):
return 1
removed = self.repo.experiments.gc(
all_branches=self.args.all_branches,
all_tags=self.args.all_tags,
all_commits=self.args.all_commits,
workspace=self.args.workspace,
commit_date=self.args.commit_date,
queued=self.args.queued,
)
if removed:
ui.write(
f"Removed {removed} experiments.",
"To remove unused cache files",
"use 'dvc gc'.",
)
else:
ui.write("No experiments to remove.")
return 0
def add_parser(experiments_subparsers, parent_parser):
EXPERIMENTS_GC_HELP = "Garbage collect unneeded experiments."
EXPERIMENTS_GC_DESCRIPTION = (
"Removes all experiments which are not derived"
" from the specified Git revisions."
)
experiments_gc_parser = experiments_subparsers.add_parser(
"gc",
parents=[parent_parser],
description=append_doc_link(EXPERIMENTS_GC_DESCRIPTION, "exp/gc"),
help=EXPERIMENTS_GC_HELP,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
experiments_gc_parser.add_argument(
"-w",
"--workspace",
action="store_true",
default=False,
help="Keep experiments derived from the current workspace.",
)
experiments_gc_parser.add_argument(
"-a",
"--all-branches",
action="store_true",
default=False,
help="Keep experiments derived from the tips of all Git branches.",
)
experiments_gc_parser.add_argument(
"-T",
"--all-tags",
action="store_true",
default=False,
help="Keep experiments derived from all Git tags.",
)
experiments_gc_parser.add_argument(
"-A",
"--all-commits",
action="store_true",
default=False,
help="Keep experiments derived from all Git commits.",
)
experiments_gc_parser.add_argument(
"--date",
type=str,
dest="commit_date",
metavar="<YYYY-MM-DD>",
default=None,
help=(
"Keep experiments from the commits after (inclusive) a certain "
"date. Date must match the extended ISO 8601 format (YYYY-MM-DD)."
),
)
experiments_gc_parser.add_argument(
"--queued",
action="store_true",
default=False,
help=(
"Keep queued experiments (experiments run queue will be cleared "
"by default)."
),
)
experiments_gc_parser.add_argument(
"-f",
"--force",
action="store_true",
default=False,
help="Force garbage collection - automatically agree to all prompts.",
)
experiments_gc_parser.set_defaults(func=CmdExperimentsGC)
dvc/repo/experiments/__init__.py
-5
View File
@@ -490,11 +490,6 @@ class Experiments:
return save(self.repo, *args, **kwargs)
def gc(self, *args, **kwargs):
from dvc.repo.experiments.gc import gc
return gc(self.repo, *args, **kwargs)
def push(self, *args, **kwargs):
from dvc.repo.experiments.push import push
dvc/repo/experiments/gc.py
-53
View File
@@ -1,53 +0,0 @@
import logging
from typing import Optional
from dvc.repo import locked
from .utils import exp_refs, remove_exp_refs
logger = logging.getLogger(__name__)
@locked
def gc(
repo,
all_branches: Optional[bool] = False,
all_tags: Optional[bool] = False,
all_commits: Optional[bool] = False,
workspace: Optional[bool] = False,
commit_date: Optional[str] = None,
queued: Optional[bool] = False,
):
keep_revs = set(
repo.brancher(
all_branches=all_branches,
all_tags=all_tags,
all_commits=all_commits,
commit_date=commit_date,
sha_only=True,
)
)
if workspace:
keep_revs.add(repo.scm.get_rev())
if not keep_revs:
return 0
to_remove = [
ref_info
for ref_info in exp_refs(repo.scm)
if ref_info.baseline_sha not in keep_revs
]
remove_exp_refs(repo.scm, to_remove)
removed = len(to_remove)
delete_stashes = []
stash = repo.experiments.celery_queue.stash
for stash_rev, entry in stash.stash_revs.items():
if not queued or entry.baseline_rev not in keep_revs:
delete_stashes.append(stash_rev)
if delete_stashes:
repo.experiments.celery_queue.remove(delete_stashes)
removed += len(delete_stashes)
return removed
tests/func/experiments/test_gc.py
-114
View File
@@ -1,114 +0,0 @@
import pytest
from funcy import first
@pytest.mark.parametrize("queued, expected", [(True, 0), (False, 1)])
def test_workspace(tmp_dir, scm, dvc, queued, expected, copy_script):
tmp_dir.gen("params.yaml", "foo: 1")
stage = dvc.run(
cmd="python copy.py params.yaml metrics.yaml",
metrics_no_cache=["metrics.yaml"],
params=["foo"],
name="foo",
)
scm.add(["dvc.yaml", "dvc.lock", "copy.py", "params.yaml", "metrics.yaml"])
scm.commit("v1")
dvc.experiments.run(stage.addressing, params=["foo=2"])
dvc.experiments.run(stage.addressing, params=["foo=3"], queue=True)
removed = dvc.experiments.gc(workspace=True, queued=queued)
assert removed == expected
@pytest.mark.parametrize("queued, expected", [(True, 0), (False, 2)])
def test_all_commits(tmp_dir, scm, dvc, queued, expected, copy_script):
tmp_dir.gen("params.yaml", "foo: 1")
stage = dvc.run(
cmd="python copy.py params.yaml metrics.yaml",
metrics_no_cache=["metrics.yaml"],
params=["foo"],
name="foo",
)
scm.add(["dvc.yaml", "dvc.lock", "copy.py", "params.yaml", "metrics.yaml"])
scm.commit("v1")
results = dvc.experiments.run(stage.addressing, params=["foo=2"])
exp_rev = first(results)
dvc.experiments.run(stage.addressing, params=["foo=3"], queue=True)
dvc.experiments.apply(exp_rev)
scm.add(["dvc.yaml", "dvc.lock", "copy.py", "params.yaml", "metrics.yaml"])
scm.commit("v2")
dvc.experiments.run(stage.addressing, params=["foo=4"])
dvc.experiments.run(stage.addressing, params=["foo=5"], queue=True)
removed = dvc.experiments.gc(all_commits=True, queued=queued)
assert removed == expected
@pytest.mark.parametrize("queued, expected", [(True, 2), (False, 3)])
def test_all_branches(tmp_dir, scm, dvc, queued, expected, copy_script):
tmp_dir.gen("params.yaml", "foo: 1")
stage = dvc.run(
cmd="python copy.py params.yaml metrics.yaml",
metrics_no_cache=["metrics.yaml"],
params=["foo"],
name="foo",
)
scm.add(["dvc.yaml", "dvc.lock", "copy.py", "params.yaml", "metrics.yaml"])
scm.commit("v1")
results = dvc.experiments.run(stage.addressing, params=["foo=2"])
exp_rev = first(results)
dvc.experiments.run(stage.addressing, params=["foo=3"], queue=True)
dvc.experiments.apply(exp_rev)
scm.add(["dvc.yaml", "dvc.lock", "copy.py", "params.yaml", "metrics.yaml"])
scm.commit("v2")
scm.branch("branch")
results = dvc.experiments.run(stage.addressing, params=["foo=4"])
dvc.experiments.run(stage.addressing, params=["foo=5"], queue=True)
exp_rev = first(results)
dvc.experiments.apply(exp_rev)
scm.add(["dvc.yaml", "dvc.lock", "copy.py", "params.yaml", "metrics.yaml"])
scm.commit("v3")
removed = dvc.experiments.gc(all_branches=True, queued=queued)
assert removed == expected
@pytest.mark.parametrize("queued, expected", [(True, 2), (False, 3)])
def test_all_tags(tmp_dir, scm, dvc, queued, expected, copy_script):
tmp_dir.gen("params.yaml", "foo: 1")
stage = dvc.run(
cmd="python copy.py params.yaml metrics.yaml",
metrics_no_cache=["metrics.yaml"],
params=["foo"],
name="foo",
)
scm.add(["dvc.yaml", "dvc.lock", "copy.py", "params.yaml", "metrics.yaml"])
scm.commit("v1")
results = dvc.experiments.run(stage.addressing, params=["foo=2"])
exp_rev = first(results)
dvc.experiments.run(stage.addressing, params=["foo=3"], queue=True)
dvc.experiments.apply(exp_rev)
scm.add(["dvc.yaml", "dvc.lock", "copy.py", "params.yaml", "metrics.yaml"])
scm.commit("v2")
scm.tag("tag-v2")
dvc.experiments.run(stage.addressing, params=["foo=4"])
dvc.experiments.run(stage.addressing, params=["foo=5"], queue=True)
removed = dvc.experiments.gc(all_tags=True, queued=queued)
assert removed == expected
tests/unit/command/test_experiments.py
-39
View File
@@ -5,7 +5,6 @@ from dvc.commands.experiments.apply import CmdExperimentsApply
from dvc.commands.experiments.branch import CmdExperimentsBranch
from dvc.commands.experiments.clean import CmdExperimentsClean
from dvc.commands.experiments.diff import CmdExperimentsDiff
from dvc.commands.experiments.gc import CmdExperimentsGC
from dvc.commands.experiments.ls import CmdExperimentsList
from dvc.commands.experiments.pull import CmdExperimentsPull
from dvc.commands.experiments.push import CmdExperimentsPush
@@ -141,44 +140,6 @@ def test_experiments_run(dvc, scm, mocker):
cmd.repo.experiments.run.assert_called_with(**default_arguments)
def test_experiments_gc(dvc, scm, mocker):
cli_args = parse_args(
[
"exp",
"gc",
"--workspace",
"--all-tags",
"--all-branches",
"--all-commits",
"--queued",
"--date",
"2022-07-04",
"--force",
]
)
assert cli_args.func == CmdExperimentsGC
cmd = cli_args.func(cli_args)
m = mocker.patch("dvc.repo.experiments.gc.gc", return_value={})
assert cmd.run() == 0
m.assert_called_once_with(
cmd.repo,
workspace=True,
all_tags=True,
all_branches=True,
all_commits=True,
commit_date="2022-07-04",
queued=True,
)
cli_args = parse_args(["exp", "gc"])
cmd = cli_args.func(cli_args)
with pytest.raises(InvalidArgumentError):
cmd.run()
def test_experiments_branch(dvc, scm, mocker):
m = mocker.patch("dvc.repo.experiments.branch.branch", return_value={})