Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

Implement --date to exp gc

Browse Source 
1. Implement `--time` to `exp gc`
2. add a unit test for it.
3. Rename `--time` to `--date` and change the warning msg.
4. To get a better msg for `gc` and `exp gc`

Co-authored-by: Peter Rowlands (변기호) <[email protected]>
Co-authored-by: Dave Berenbaum <[email protected]>
This commit is contained in:
karajan1001
2022-07-12 13:53:46 +02:00
committed by David de la Iglesia Castro
co-authored by Peter Rowlands Dave Berenbaum
parent 217eb13495
commit 15ada9a1c3
11 changed files with 75 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dvc/commands/experiments/gc.py
+29 -8
View File
@@ -12,8 +12,8 @@ logger = logging.getLogger(__name__)
def _raise_error_if_all_disabled(**kwargs):
if not any(kwargs.values()):
raise InvalidArgumentError(
"Either of `-w|--workspace`, `-a|--all-branches`, `-T|--all-tags` "
"or `--all-commits` needs to be set."
"Either of `-w|--workspace`, `-a|--all-branches`, `-T|--all-tags`,"
" `--all-commits` or `--date` needs to be set."
)
@@ -23,6 +23,7 @@ class CmdExperimentsGC(CmdBase):
all_branches=self.args.all_branches,
all_tags=self.args.all_tags,
all_commits=self.args.all_commits,
commit_date=self.args.commit_date,
workspace=self.args.workspace,
)
@@ -31,13 +32,21 @@ class CmdExperimentsGC(CmdBase):
msg += "the workspace"
if self.args.all_commits:
msg += " and all git commits"
elif self.args.all_branches and self.args.all_tags:
msg += " and all git branches and tags"
elif self.args.all_branches:
msg += " and all git branches"
elif self.args.all_tags:
msg += " and all git tags"
else:
if self.args.all_branches and self.args.all_tags:
msg += " and all git branches and tags"
elif self.args.all_branches:
msg += " and all git branches"
elif self.args.all_tags:
msg += " and all git tags"
if self.args.commit_date:
msg += (
" and all git commits before date "
f"{self.args.commit_date}"
)
msg += " of the current repo."
if self.args.queued:
msg += " Run queued experiments will be preserved."
else:
@@ -54,6 +63,7 @@ class CmdExperimentsGC(CmdBase):
all_tags=self.args.all_tags,
all_commits=self.args.all_commits,
workspace=self.args.workspace,
commit_date=self.args.commit_date,
queued=self.args.queued,
)
@@ -110,6 +120,17 @@ def add_parser(experiments_subparsers, parent_parser):
default=False,
help="Keep experiments derived from all Git commits.",
)
experiments_gc_parser.add_argument(
"--date",
type=str,
dest="commit_date",
metavar="<YYYY-MM-DD>",
default=None,
help=(
"Keep experiments from the commits after (inclusive) a certain "
"date. Date must match the extended ISO 8601 format (YYYY-MM-DD)."
),
)
experiments_gc_parser.add_argument(
"--queued",
action="store_true",
dvc/commands/gc.py
+18 -11
View File
@@ -18,7 +18,7 @@ class CmdGC(CmdBase):
all_tags=self.args.all_tags,
all_commits=self.args.all_commits,
all_experiments=self.args.all_experiments,
commit_time=self.args.commit_time,
commit_date=self.args.commit_date,
workspace=self.args.workspace,
)
@@ -27,13 +27,20 @@ class CmdGC(CmdBase):
msg += "the workspace"
if self.args.all_commits:
msg += " and all git commits"
elif self.args.all_branches and self.args.all_tags:
msg += " and all git branches and tags"
elif self.args.all_branches:
msg += " and all git branches"
elif self.args.all_tags:
msg += " and all git tags"
elif self.args.all_experiments:
else:
if self.args.all_branches and self.args.all_tags:
msg += " and all git branches and tags"
elif self.args.all_branches:
msg += " and all git branches"
elif self.args.all_tags:
msg += " and all git tags"
if self.args.commit_date:
msg += (
" and all git commits before date "
f"{self.args.commit_date}"
)
if self.args.all_experiments:
msg += " and all experiments"
if self.args.repos:
@@ -55,7 +62,7 @@ class CmdGC(CmdBase):
all_tags=self.args.all_tags,
all_commits=self.args.all_commits,
all_experiments=self.args.all_experiments,
commit_time=self.args.commit_time,
commit_date=self.args.commit_date,
cloud=self.args.cloud,
remote=self.args.remote,
force=self.args.force,
@@ -108,9 +115,9 @@ def add_parser(subparsers, parent_parser):
help="Keep data files for all Git commits.",
)
gc_parser.add_argument(
"--time",
"--date",
type=str,
dest="commit_time",
dest="commit_date",
metavar="<YYYY-MM-DD>",
default=None,
help=(
dvc/repo/__init__.py
+2 -2
View File
@@ -386,7 +386,7 @@ class Repo:
all_tags=False,
all_commits=False,
all_experiments=False,
commit_time: Optional[str] = None,
commit_date: Optional[str] = None,
remote=None,
force=False,
jobs=None,
@@ -417,7 +417,7 @@ class Repo:
all_tags=all_tags,
all_commits=all_commits,
all_experiments=all_experiments,
commit_time=commit_time,
commit_date=commit_date,
):
for odb, objs in self.index.used_objs(
targets,
dvc/repo/brancher.py
+2 -2
View File
@@ -10,7 +10,7 @@ def brancher( # noqa: E302
all_tags=False,
all_commits=False,
all_experiments=False,
commit_time: Optional[str] = None,
commit_date: Optional[str] = None,
sha_only=False,
):
"""Generator that iterates over specified revisions.
@@ -68,7 +68,7 @@ def brancher( # noqa: E302
all_tags=all_tags,
all_commits=all_commits,
all_experiments=all_experiments,
commit_time=commit_time,
commit_date=commit_date,
)
try:
dvc/repo/experiments/gc.py
+2
View File
@@ -15,6 +15,7 @@ def gc(
all_tags: Optional[bool] = False,
all_commits: Optional[bool] = False,
workspace: Optional[bool] = False,
commit_date: Optional[str] = None,
queued: Optional[bool] = False,
):
keep_revs = set(
@@ -22,6 +23,7 @@ def gc(
all_branches=all_branches,
all_tags=all_tags,
all_commits=all_commits,
commit_date=commit_date,
sha_only=True,
)
)
dvc/repo/gc.py
+4 -4
View File
@@ -12,7 +12,7 @@ def _raise_error_if_all_disabled(**kwargs):
if not any(kwargs.values()):
raise InvalidArgumentError(
"Either of `-w|--workspace`, `-a|--all-branches`, `-T|--all-tags` "
"`--all-experiments`, `--all-commits` or `--time` needs to be set."
"`--all-experiments`, `--all-commits` or `--date` needs to be set."
)
@@ -26,7 +26,7 @@ def gc(
all_tags=False,
all_commits=False,
all_experiments=False,
commit_time: Optional[str] = None,
commit_date: Optional[str] = None,
force=False,
jobs=None,
repos=None,
@@ -42,7 +42,7 @@ def gc(
all_commits=all_commits,
all_branches=all_branches,
all_experiments=all_experiments,
commit_time=commit_time,
commit_date=commit_date,
)
from contextlib import ExitStack
@@ -67,7 +67,7 @@ def gc(
all_tags=all_tags,
all_commits=all_commits,
all_experiments=all_experiments,
commit_time=commit_time,
commit_date=commit_date,
remote=remote,
force=force,
jobs=jobs,
dvc/scm.py
+6 -6
View File
@@ -203,7 +203,7 @@ def iter_revs(
all_tags: bool = False,
all_commits: bool = False,
all_experiments: bool = False,
commit_time: Optional[str] = None,
commit_date: Optional[str] = None,
) -> Mapping[str, List[str]]:
from scmrepo.exceptions import SCMError as _SCMError
@@ -216,7 +216,7 @@ def iter_revs(
all_tags,
all_commits,
all_experiments,
commit_time,
commit_date,
]
):
return {}
@@ -233,16 +233,16 @@ def iter_revs(
if all_tags:
results.extend(scm.list_tags())
if commit_time:
if commit_date:
from datetime import datetime
commit_timestamp = datetime.strptime(
commit_time, "%Y-%m-%d"
commit_datestamp = datetime.strptime(
commit_date, "%Y-%m-%d"
).timestamp()
def _time_filter(rev):
try:
if scm.resolve_commit(rev).commit_time >= commit_timestamp:
if scm.resolve_commit(rev).commit_date >= commit_datestamp:
return True
return False
except _SCMError:
tests/func/test_gc.py
+4 -13
View File
@@ -268,23 +268,14 @@ def test_gc_without_workspace_on_tags_branches_commits(tmp_dir, dvc):
dvc.gc(force=False, all_branches=True, all_commits=False, workspace=False)
def test_gc_without_workspace(tmp_dir, dvc, caplog):
@pytest.mark.parametrize("cloud", ["c", ""])
def test_gc_without_workspace(tmp_dir, dvc, caplog, cloud):
with caplog.at_level(logging.WARNING, logger="dvc"):
assert main(["gc", "-vf"]) == 255
assert main(["gc", f"-{cloud}vf"]) == 255
assert (
"Either of `-w|--workspace`, `-a|--all-branches`, `-T|--all-tags` "
"`--all-experiments` or `--all-commits` needs to be set."
) in caplog.text
def test_gc_cloud_without_any_specifier(tmp_dir, dvc, caplog):
with caplog.at_level(logging.WARNING, logger="dvc"):
assert main(["gc", "-cvf"]) == 255
assert (
"Either of `-w|--workspace`, `-a|--all-branches`, `-T|--all-tags` "
"`--all-experiments` or `--all-commits` needs to be set."
"`--all-experiments`, `--all-commits` or `--date` needs to be set."
) in caplog.text
tests/unit/command/test_experiments.py
+3
View File
@@ -155,6 +155,8 @@ def test_experiments_gc(dvc, scm, mocker):
"--all-branches",
"--all-commits",
"--queued",
"--date",
"2022-07-04",
"--force",
]
)
@@ -171,6 +173,7 @@ def test_experiments_gc(dvc, scm, mocker):
all_tags=True,
all_branches=True,
all_commits=True,
commit_date="2022-07-04",
queued=True,
)
tests/unit/command/test_gc.py
+2 -2
View File
@@ -14,7 +14,7 @@ def test_(dvc, scm, mocker):
"--all-branches",
"--all-commits",
"--all-experiments",
"--time",
"--date",
"2022-06-30",
"--cloud",
"--remote",
@@ -40,7 +40,7 @@ def test_(dvc, scm, mocker):
all_branches=True,
all_commits=True,
all_experiments=True,
commit_time="2022-06-30",
commit_date="2022-06-30",
cloud=True,
remote="origin",
force=True,
tests/unit/scm/test_scm.py
+3 -3
View File
@@ -70,16 +70,16 @@ def test_iter_revs(
def _func(rev):
if rev == rev_root:
return mocker.Mock(commit_time=datetime(2022, 6, 28).timestamp())
return mocker.Mock(commit_date=datetime(2022, 6, 28).timestamp())
if rev == rev_old:
raise SCMError
return mocker.Mock(commit_time=datetime(2022, 6, 30).timestamp())
return mocker.Mock(commit_date=datetime(2022, 6, 30).timestamp())
mocker.patch(
"scmrepo.git.Git.resolve_commit", mocker.MagicMock(side_effect=_func)
)
gen = iter_revs(scm, commit_time="2022-06-29")
gen = iter_revs(scm, commit_date="2022-06-29")
assert gen == {
rev_new: [rev_new],
rev_old: [rev_old],