From 217eb134958b84513f3ad9c175d6590827cf796d Mon Sep 17 00:00:00 2001 From: karajan1001 Date: Mon, 4 Jul 2022 11:09:29 +0800 Subject: [PATCH] Add argument `commit_time` to `gc`/`brancher` and `iter_revs` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Add argument `commit_time` to `gc`/`brancher` and `iter_revs` 2. Add unit test for iter_revs. Co-authored-by: Peter Rowlands (변기호) --- dvc/repo/__init__.py | 2 ++ dvc/repo/brancher.py | 11 ++++++- dvc/repo/gc.py | 1 + dvc/scm.py | 65 ++++++++++++++++++++++++++++---------- tests/unit/scm/test_scm.py | 24 ++++++++++++++ 5 files changed, 86 insertions(+), 17 deletions(-) diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index 9a8b62647..df724be07 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -386,6 +386,7 @@ class Repo: all_tags=False, all_commits=False, all_experiments=False, + commit_time: Optional[str] = None, remote=None, force=False, jobs=None, @@ -416,6 +417,7 @@ class Repo: all_tags=all_tags, all_commits=all_commits, all_experiments=all_experiments, + commit_time=commit_time, ): for odb, objs in self.index.used_objs( targets, diff --git a/dvc/repo/brancher.py b/dvc/repo/brancher.py index db24c4fe4..4d27c7f98 100644 --- a/dvc/repo/brancher.py +++ b/dvc/repo/brancher.py @@ -1,3 +1,5 @@ +from typing import Optional + from dvc.scm import iter_revs @@ -8,6 +10,7 @@ def brancher( # noqa: E302 all_tags=False, all_commits=False, all_experiments=False, + commit_time: Optional[str] = None, sha_only=False, ): """Generator that iterates over specified revisions. @@ -17,6 +20,9 @@ def brancher( # noqa: E302 all_branches (bool): iterate over all available branches. all_commits (bool): iterate over all commits. all_tags (bool): iterate over all available tags. + commit_date (str): Keep experiments from the commits after(include) + a certain date. Date must match the extended + ISO 8601 format (YYYY-MM-DD). sha_only (bool): only return git SHA for a revision. Yields: @@ -62,6 +68,7 @@ def brancher( # noqa: E302 all_tags=all_tags, all_commits=all_commits, all_experiments=all_experiments, + commit_time=commit_time, ) try: @@ -72,7 +79,9 @@ def brancher( # noqa: E302 self.root_dir = self.fs.path.join("/", *repo_root_parts) if cwd_parts: - cwd = self.fs.path.join("/", *cwd_parts) + cwd = self.fs.path.join( # type: ignore[unreachable] + "/", *cwd_parts + ) self.fs.path.chdir(cwd) # ignore revs that don't contain repo root diff --git a/dvc/repo/gc.py b/dvc/repo/gc.py index ca024e977..db2fa861c 100644 --- a/dvc/repo/gc.py +++ b/dvc/repo/gc.py @@ -67,6 +67,7 @@ def gc( all_tags=all_tags, all_commits=all_commits, all_experiments=all_experiments, + commit_time=commit_time, remote=remote, force=force, jobs=jobs, diff --git a/dvc/scm.py b/dvc/scm.py index 9c81579ad..a45a4266f 100644 --- a/dvc/scm.py +++ b/dvc/scm.py @@ -176,20 +176,7 @@ def resolve_rev(scm: "Git", rev: str) -> str: raise RevError(str(exc)) -def iter_revs( - scm: "Git", - revs: Optional[List[str]] = None, - num: int = 1, - all_branches: bool = False, - all_tags: bool = False, - all_commits: bool = False, - all_experiments: bool = False, -) -> Mapping[str, List[str]]: - - if not any([revs, all_branches, all_tags, all_commits, all_experiments]): - return {} - - revs = revs or [] +def _get_n_commits(scm: "Git", revs: List[str], num: int) -> List[str]: results = [] for rev in revs: if num == 0: @@ -205,6 +192,37 @@ def iter_revs( except RevError: break n += 1 + return results + + +def iter_revs( + scm: "Git", + revs: Optional[List[str]] = None, + num: int = 1, + all_branches: bool = False, + all_tags: bool = False, + all_commits: bool = False, + all_experiments: bool = False, + commit_time: Optional[str] = None, +) -> Mapping[str, List[str]]: + from scmrepo.exceptions import SCMError as _SCMError + + from dvc.repo.experiments.utils import exp_commits + + if not any( + [ + revs, + all_branches, + all_tags, + all_commits, + all_experiments, + commit_time, + ] + ): + return {} + + revs = revs or [] + results: List[str] = _get_n_commits(scm, revs, num) if all_commits: results.extend(scm.list_all_commits()) @@ -215,9 +233,24 @@ def iter_revs( if all_tags: results.extend(scm.list_tags()) - if all_experiments: - from dvc.repo.experiments.utils import exp_commits + if commit_time: + from datetime import datetime + commit_timestamp = datetime.strptime( + commit_time, "%Y-%m-%d" + ).timestamp() + + def _time_filter(rev): + try: + if scm.resolve_commit(rev).commit_time >= commit_timestamp: + return True + return False + except _SCMError: + return True + + results.extend(filter(_time_filter, scm.list_all_commits())) + + if all_experiments: results.extend(exp_commits(scm)) rev_resolver = partial(resolve_rev, scm) diff --git a/tests/unit/scm/test_scm.py b/tests/unit/scm/test_scm.py index e078021fc..d53b2f659 100644 --- a/tests/unit/scm/test_scm.py +++ b/tests/unit/scm/test_scm.py @@ -1,3 +1,7 @@ +from datetime import datetime + +from scmrepo.exceptions import SCMError + from dvc.repo.experiments import ExpRefInfo from dvc.scm import GitMergeError, iter_revs @@ -5,6 +9,7 @@ from dvc.scm import GitMergeError, iter_revs def test_iter_revs( tmp_dir, scm, + mocker, ): """ new other @@ -62,6 +67,25 @@ def test_iter_revs( rev_root: [rev_root], } + def _func(rev): + + if rev == rev_root: + return mocker.Mock(commit_time=datetime(2022, 6, 28).timestamp()) + if rev == rev_old: + raise SCMError + return mocker.Mock(commit_time=datetime(2022, 6, 30).timestamp()) + + mocker.patch( + "scmrepo.git.Git.resolve_commit", mocker.MagicMock(side_effect=_func) + ) + + gen = iter_revs(scm, commit_time="2022-06-29") + assert gen == { + rev_new: [rev_new], + rev_old: [rev_old], + rev_other: [rev_other], + } + def test_merge_error(tmp_dir, scm): exc = GitMergeError("Merge failed")