diff --git a/dvc/repo/experiments/__init__.py b/dvc/repo/experiments/__init__.py index 49c7db0cb..19b0f51da 100644 --- a/dvc/repo/experiments/__init__.py +++ b/dvc/repo/experiments/__init__.py @@ -761,7 +761,7 @@ class Experiments: # if we can't tell from branch name, fall back to parent commit exp_commit = self.scm.resolve_commit(exp_rev) if exp_commit: - exp_baseline = first(exp_commit.parents).hexsha + exp_baseline = first(exp_commit.parents) if exp_baseline == baseline_sha: return exp_baseline raise BaselineMismatchError(exp_baseline, baseline_sha) diff --git a/dvc/repo/experiments/show.py b/dvc/repo/experiments/show.py index 0da395397..6a2556b69 100644 --- a/dvc/repo/experiments/show.py +++ b/dvc/repo/experiments/show.py @@ -19,7 +19,7 @@ def _collect_experiment_commit(repo, rev, stash=False, sha_only=True): res["timestamp"] = None else: commit = repo.scm.resolve_commit(rev) - res["timestamp"] = datetime.fromtimestamp(commit.committed_date) + res["timestamp"] = datetime.fromtimestamp(commit.commit_time) configs = _collect_configs(repo, rev=rev) params = _read_params(repo, configs, rev) @@ -121,7 +121,7 @@ def show( for ref in repo.scm.iter_refs(base=str(ref_info)) ] for exp_ref, _ in sorted( - commits, key=lambda x: x[1].committed_date, reverse=True, + commits, key=lambda x: x[1].commit_time, reverse=True, ): ref_info = ExpRefInfo.from_ref(exp_ref) assert ref_info.baseline_sha == rev diff --git a/dvc/repo/experiments/utils.py b/dvc/repo/experiments/utils.py index 97b7d8138..5cc0aab10 100644 --- a/dvc/repo/experiments/utils.py +++ b/dvc/repo/experiments/utils.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Generator, Iterable +from typing import TYPE_CHECKING, Generator, Iterable, Set from .base import EXEC_NAMESPACE, EXPS_NAMESPACE, EXPS_STASH, ExpRefInfo @@ -75,13 +75,14 @@ def remote_exp_refs_by_baseline( def exp_commits( scm: "Git", ref_infos: Iterable["ExpRefInfo"] = None -) -> Generator[str, None, None]: +) -> Iterable[str]: """Iterate over all experiment commits.""" - shas = set() + shas: Set["str"] = set() refs = ref_infos if ref_infos else exp_refs(scm) for ref_info in refs: shas.update(scm.branch_revs(str(ref_info), ref_info.baseline_sha)) - shas.add(ref_info.baseline_sha) + if ref_info.baseline_sha: + shas.add(ref_info.baseline_sha) yield from shas diff --git a/dvc/scm/git/__init__.py b/dvc/scm/git/__init__.py index 65e05d2b3..bc8fe5283 100644 --- a/dvc/scm/git/__init__.py +++ b/dvc/scm/git/__init__.py @@ -365,7 +365,6 @@ class Git(Base): get_rev = partialmethod(_backend_func, "get_rev") _resolve_rev = partialmethod(_backend_func, "resolve_rev") resolve_commit = partialmethod(_backend_func, "resolve_commit") - branch_revs = partialmethod(_backend_func, "branch_revs") set_ref = partialmethod(_backend_func, "set_ref") get_ref = partialmethod(_backend_func, "get_ref") @@ -402,6 +401,22 @@ class Git(Base): raise RevError(f"ambiguous Git revision '{rev}'") raise + def branch_revs( + self, branch: str, end_rev: Optional[str] = None + ) -> Iterable[str]: + """Iterate over revisions in a given branch (from newest to oldest). + + If end_rev is set, iterator will stop when the specified revision is + reached. + """ + commit = self.resolve_commit(branch) + while commit is not None: + yield commit.hexsha + parent = first(commit.parents) + if parent is None or parent == end_rev: + return + commit = self.resolve_commit(parent) + @contextmanager def detach_head(self, rev: Optional[str] = None): """Context manager for performing detached HEAD SCM operations. diff --git a/dvc/scm/git/backend/base.py b/dvc/scm/git/backend/base.py index 63ac4457e..f0c71f458 100644 --- a/dvc/scm/git/backend/base.py +++ b/dvc/scm/git/backend/base.py @@ -1,11 +1,22 @@ import os from abc import ABC, abstractmethod -from typing import Callable, Iterable, Mapping, Optional, Tuple, Union +from typing import ( + TYPE_CHECKING, + Callable, + Iterable, + Mapping, + Optional, + Tuple, + Union, +) from dvc.scm.base import SCMError from ..objects import GitObject +if TYPE_CHECKING: + from ..objects import GitCommit + class NoGitBackendError(SCMError): def __init__(self, func): @@ -117,13 +128,7 @@ class BaseGitBackend(ABC): pass @abstractmethod - def resolve_commit(self, rev: str) -> str: - pass - - @abstractmethod - def branch_revs( - self, branch: str, end_rev: Optional[str] = None - ) -> Iterable[str]: + def resolve_commit(self, rev: str) -> "GitCommit": pass @abstractmethod diff --git a/dvc/scm/git/backend/dulwich.py b/dvc/scm/git/backend/dulwich.py index 9f496ab45..865aa50a9 100644 --- a/dvc/scm/git/backend/dulwich.py +++ b/dvc/scm/git/backend/dulwich.py @@ -5,6 +5,7 @@ import os import stat from io import BytesIO, StringIO from typing import ( + TYPE_CHECKING, Callable, Dict, Iterable, @@ -25,6 +26,9 @@ from dvc.utils import relpath from ..objects import GitObject from .base import BaseGitBackend +if TYPE_CHECKING: + from ..objects import GitCommit + logger = logging.getLogger(__name__) @@ -240,10 +244,7 @@ class DulwichBackend(BaseGitBackend): # pylint:disable=abstract-method def resolve_rev(self, rev: str) -> str: raise NotImplementedError - def resolve_commit(self, rev: str) -> str: - raise NotImplementedError - - def branch_revs(self, branch: str, end_rev: Optional[str] = None): + def resolve_commit(self, rev: str) -> "GitCommit": raise NotImplementedError def _get_stash(self, ref: str): diff --git a/dvc/scm/git/backend/gitpython.py b/dvc/scm/git/backend/gitpython.py index 59061f396..c7f439657 100644 --- a/dvc/scm/git/backend/gitpython.py +++ b/dvc/scm/git/backend/gitpython.py @@ -5,13 +5,13 @@ import os from functools import partial from typing import Callable, Iterable, List, Mapping, Optional, Tuple, Union -from funcy import first, ignore +from funcy import ignore from dvc.progress import Tqdm from dvc.scm.base import CloneError, MergeConflictError, RevError, SCMError from dvc.utils import fix_env, is_binary, relpath -from ..objects import GitObject +from ..objects import GitCommit, GitObject from .base import BaseGitBackend logger = logging.getLogger(__name__) @@ -315,7 +315,7 @@ class GitPythonBackend(BaseGitBackend): # pylint:disable=abstract-method raise RevError(f"unknown Git revision '{rev}'") - def resolve_commit(self, rev): + def resolve_commit(self, rev: str) -> "GitCommit": """Return Commit object for the specified revision.""" from git.exc import BadName, GitCommandError from git.objects.tag import TagObject @@ -323,25 +323,16 @@ class GitPythonBackend(BaseGitBackend): # pylint:disable=abstract-method try: commit = self.repo.rev_parse(rev) except (BadName, GitCommandError): - commit = None + raise SCMError(f"Invalid commit '{rev}'") if isinstance(commit, TagObject): commit = commit.object - return commit - - def branch_revs( - self, branch: str, end_rev: Optional[str] = None - ) -> Iterable[str]: - """Iterate over revisions in a given branch (from newest to oldest). - - If end_rev is set, iterator will stop when the specified revision is - reached. - """ - commit = self.resolve_commit(branch) - while commit is not None: - yield commit.hexsha - commit = first(commit.parents) - if commit and commit.hexsha == end_rev: - return + return GitCommit( + commit.hexsha, + commit.committed_date, + commit.committer_tz_offset, + commit.message, + [str(parent) for parent in commit.parents], + ) def set_ref( self, diff --git a/dvc/scm/git/backend/pygit2.py b/dvc/scm/git/backend/pygit2.py index d7c816ed8..7f26eb6d0 100644 --- a/dvc/scm/git/backend/pygit2.py +++ b/dvc/scm/git/backend/pygit2.py @@ -7,7 +7,7 @@ from typing import Callable, Iterable, List, Mapping, Optional, Tuple, Union from dvc.scm.base import MergeConflictError, RevError, SCMError from dvc.utils import relpath -from ..objects import GitObject +from ..objects import GitCommit, GitObject from .base import BaseGitBackend logger = logging.getLogger(__name__) @@ -169,11 +169,20 @@ class Pygit2Backend(BaseGitBackend): # pylint:disable=abstract-method return shas.pop() # type: ignore raise RevError(f"unknown Git revision '{rev}'") - def resolve_commit(self, rev: str) -> str: - raise NotImplementedError + def resolve_commit(self, rev: str) -> "GitCommit": + from pygit2 import GitError - def branch_revs(self, branch: str, end_rev: Optional[str] = None): - raise NotImplementedError + try: + commit, _ref = self.repo.resolve_refish(rev) + except (KeyError, GitError): + raise SCMError(f"Invalid commit '{rev}'") + return GitCommit( + str(commit.id), + commit.commit_time, + commit.commit_time_offset, + commit.message, + [str(parent) for parent in commit.parent_ids], + ) def _get_stash(self, ref: str): raise NotImplementedError diff --git a/dvc/scm/git/objects.py b/dvc/scm/git/objects.py index 990e5ffd2..3b8ad2a3b 100644 --- a/dvc/scm/git/objects.py +++ b/dvc/scm/git/objects.py @@ -1,7 +1,8 @@ import os import stat from abc import ABC, abstractmethod -from typing import Iterable, Optional +from dataclasses import dataclass +from typing import Iterable, List, Optional from pygtrie import Trie @@ -118,3 +119,12 @@ class GitTrie: def stat(self, key: tuple) -> os.stat_result: obj = self.trie[key] return os.stat_result((obj.mode, 0, 0, 0, 0, 0, 0, 0, 0, 0)) + + +@dataclass +class GitCommit: + hexsha: str + commit_time: int + commit_time_offset: int + message: str + parents: List[str]