scm: move logic to gitpython backend (#5022)

This commit is contained in:
Ruslan Kuprieiev
2020-12-03 20:25:20 +02:00
committed by GitHub
parent f49feddb7f
commit 5152866634
19 changed files with 610 additions and 336 deletions
+3 -3
View File
@@ -369,13 +369,13 @@ def _clone_default_branch(url, rev, for_write=False):
def _unshallow(git):
if git.repo.head.is_detached:
if git.gitpython.repo.head.is_detached:
# If this is a detached head (i.e. we shallow cloned a tag) switch to
# the default branch
origin_refs = git.repo.remotes["origin"].refs
origin_refs = git.gitpython.repo.remotes["origin"].refs
ref = origin_refs["HEAD"].reference
branch_name = ref.name.split("/")[-1]
branch = git.repo.create_head(branch_name, ref)
branch = git.gitpython.repo.create_head(branch_name, ref)
branch.set_tracking_branch(ref)
branch.checkout()
git.pull(unshallow=True)
+4 -4
View File
@@ -205,8 +205,8 @@ class Experiments:
)
# Reset/clean any changes before prior workspace is unstashed
self.scm.repo.git.reset(hard=True)
self.scm.repo.git.clean(force=True)
self.scm.gitpython.repo.git.reset(hard=True)
self.scm.gitpython.repo.git.clean(force=True)
return stash_rev
@@ -447,8 +447,8 @@ class Experiments:
for ref in (EXEC_HEAD, EXEC_MERGE, EXEC_BASELINE):
self.scm.remove_ref(ref)
self.scm.repo.git.reset(hard=True)
self.scm.repo.git.clean(force=True)
self.scm.gitpython.repo.git.reset(hard=True)
self.scm.gitpython.repo.git.clean(force=True)
return executors
def _reproduce(
+3 -3
View File
@@ -48,7 +48,7 @@ def apply(repo, rev, *args, **kwargs):
else:
workspace = None
repo.scm.repo.git.merge(branch, squash=True, no_commit=True)
repo.scm.gitpython.repo.git.merge(branch, squash=True, no_commit=True)
if workspace:
try:
@@ -56,9 +56,9 @@ def apply(repo, rev, *args, **kwargs):
except GitCommandError:
# if stash apply returns merge conflicts, prefer experiment
# changes over prior stashed changes
repo.scm.repo.git.checkout("--ours", "--", ".")
repo.scm.gitpython.repo.git.checkout("--ours", "--", ".")
repo.scm.stash.drop()
repo.scm.repo.git.reset()
repo.scm.gitpython.repo.git.reset()
if stash_rev:
args_path = os.path.join(repo.tmp_dir, BaseExecutor.PACKED_ARGS_FILE)
+10 -6
View File
@@ -85,8 +85,10 @@ class BaseExecutor:
# committing
head = EXEC_BRANCH if branch else EXEC_HEAD
self.scm.checkout(head, detach=True)
self.scm.repo.git.merge(EXEC_MERGE, squash=True, no_commit=True)
self.scm.repo.git.reset()
self.scm.gitpython.repo.git.merge(
EXEC_MERGE, squash=True, no_commit=True
)
self.scm.gitpython.repo.git.reset()
self._prune_lockfiles()
finally:
os.chdir(cwd)
@@ -94,11 +96,13 @@ class BaseExecutor:
def _prune_lockfiles(self):
# NOTE: dirty DVC lock files must be restored to index state to
# avoid checking out incorrect persist or checkpoint outs
dirty = [diff.a_path for diff in self.scm.repo.index.diff(None)]
dirty = [
diff.a_path for diff in self.scm.gitpython.repo.index.diff(None)
]
to_checkout = [fname for fname in dirty if is_lock_file(fname)]
self.scm.repo.index.checkout(paths=to_checkout, force=True)
self.scm.gitpython.repo.index.checkout(paths=to_checkout, force=True)
untracked = self.scm.repo.untracked_files
untracked = self.scm.gitpython.repo.untracked_files
to_remove = [fname for fname in untracked if is_lock_file(fname)]
for fname in to_remove:
remove(fname)
@@ -320,7 +324,7 @@ class BaseExecutor:
old_ref = None
logger.debug("Commit to new experiment branch '%s'", branch)
scm.repo.git.add(A=True)
scm.gitpython.repo.git.add(A=True)
scm.commit(f"dvc: commit experiment {exp_hash}")
new_rev = scm.get_rev()
scm.set_ref(branch, new_rev, old_ref=old_ref)
+67 -268
View File
@@ -3,23 +3,17 @@
import logging
import os
import shlex
from collections import OrderedDict
from contextlib import contextmanager
from functools import partial, partialmethod
from functools import partialmethod
from typing import Optional
from funcy import cached_property, first
from funcy import cached_property
from pathspec.patterns import GitWildMatchPattern
from dvc.exceptions import GitHookAlreadyExistsError
from dvc.progress import Tqdm
from dvc.scm.base import (
Base,
CloneError,
FileNotInRepoError,
RevError,
SCMError,
)
from dvc.utils import fix_env, is_binary, relpath
from dvc.scm.base import Base, FileNotInRepoError, RevError
from dvc.utils import relpath
from dvc.utils.fs import path_isin
from dvc.utils.serialize import modify_yaml
@@ -31,150 +25,68 @@ from .stash import Stash
logger = logging.getLogger(__name__)
class TqdmGit(Tqdm):
def update_git(self, op_code, cur_count, max_count=None, message=""):
op_code = self.code2desc(op_code)
if op_code:
message = (op_code + " | " + message) if message else op_code
if message:
self.postfix["info"] = f" {message} | "
self.update_to(cur_count, max_count)
@staticmethod
def code2desc(op_code):
from git import RootUpdateProgress as OP
ops = {
OP.COUNTING: "Counting",
OP.COMPRESSING: "Compressing",
OP.WRITING: "Writing",
OP.RECEIVING: "Receiving",
OP.RESOLVING: "Resolving",
OP.FINDING_SOURCES: "Finding sources",
OP.CHECKING_OUT: "Checking out",
OP.CLONE: "Cloning",
OP.FETCH: "Fetching",
OP.UPDWKTREE: "Updating working tree",
OP.REMOVE: "Removing",
OP.PATHCHANGE: "Changing path",
OP.URLCHANGE: "Changing URL",
OP.BRANCHCHANGE: "Changing branch",
}
return ops.get(op_code & OP.OP_MASK, "")
class Git(Base):
"""Class for managing Git."""
GITIGNORE = ".gitignore"
GIT_DIR = ".git"
DEFAULT_BACKENDS = (DulwichBackend, GitPythonBackend)
DEFAULT_BACKENDS = OrderedDict(
[("dulwich", DulwichBackend), ("gitpython", GitPythonBackend)]
)
LOCAL_BRANCH_PREFIX = "refs/heads/"
def __init__(self, root_dir=os.curdir, search_parent_directories=True):
"""Git class constructor.
Requires `Repo` class from `git` module (from gitpython package).
"""
super().__init__(root_dir)
import git
from git.exc import InvalidGitRepositoryError
try:
self.repo = git.Repo(
root_dir, search_parent_directories=search_parent_directories
)
except InvalidGitRepositoryError:
msg = "{} is not a git repository"
raise SCMError(msg.format(root_dir))
# NOTE: fixing LD_LIBRARY_PATH for binary built by PyInstaller.
# http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
env = fix_env(None)
libpath = env.get("LD_LIBRARY_PATH", None)
self.repo.git.update_environment(LD_LIBRARY_PATH=libpath)
def __init__(self, *args, **kwargs):
self.ignored_paths = []
self.files_to_track = set()
self.backends = [backend(self) for backend in self.DEFAULT_BACKENDS]
self.backends = OrderedDict(
[
(name, cls(self, *args, **kwargs))
for name, cls in self.DEFAULT_BACKENDS.items()
]
)
super().__init__(self.gitpython.root_dir)
@property
def root_dir(self) -> str:
return self.repo.working_tree_dir
def dir(self):
return self.gitpython.repo.git_dir
@property
def gitpython(self):
return self.backends["gitpython"]
@property
def dulwich(self):
return self.backends["dulwich"]
@cached_property
def stash(self):
return Stash(self)
@staticmethod
def clone(url, to_path, rev=None, shallow_branch=None):
import git
ld_key = "LD_LIBRARY_PATH"
env = fix_env(None)
if is_binary() and ld_key not in env.keys():
# In fix_env, we delete LD_LIBRARY_PATH key if it was empty before
# PyInstaller modified it. GitPython, in git.Repo.clone_from, uses
# env to update its own internal state. When there is no key in
# env, this value is not updated and GitPython re-uses
# LD_LIBRARY_PATH that has been set by PyInstaller.
# See [1] for more info.
# [1] https://github.com/gitpython-developers/GitPython/issues/924
env[ld_key] = ""
try:
if shallow_branch is not None and os.path.exists(url):
# git disables --depth for local clones unless file:// url
# scheme is used
url = f"file://{url}"
with TqdmGit(desc="Cloning", unit="obj") as pbar:
clone_from = partial(
git.Repo.clone_from,
url,
to_path,
env=env, # needed before we can fix it in __init__
no_single_branch=True,
progress=pbar.update_git,
)
if shallow_branch is None:
tmp_repo = clone_from()
else:
tmp_repo = clone_from(branch=shallow_branch, depth=1)
tmp_repo.close()
except git.exc.GitCommandError as exc: # pylint: disable=no-member
raise CloneError(url, to_path) from exc
# NOTE: using our wrapper to make sure that env is fixed in __init__
repo = Git(to_path)
if rev:
@classmethod
def clone(cls, url, to_path, **kwargs):
for _, backend in cls.DEFAULT_BACKENDS.items():
try:
repo.checkout(rev)
except git.exc.GitCommandError as exc: # pylint: disable=no-member
raise RevError(
"failed to access revision '{}' for repo '{}'".format(
rev, url
)
) from exc
backend.clone(url, to_path, **kwargs)
return Git(to_path)
except NotImplementedError:
pass
raise NoGitBackendError("clone")
return repo
@staticmethod
def is_sha(rev):
import git
return rev and git.Repo.re_hexsha_shortened.search(rev)
@classmethod
def is_sha(cls, rev):
for _, backend in cls.DEFAULT_BACKENDS.items():
try:
return backend.is_sha(rev)
except NotImplementedError:
pass
raise NoGitBackendError("is_sha")
@staticmethod
def _get_git_dir(root_dir):
return os.path.join(root_dir, Git.GIT_DIR)
@property
def dir(self):
return self.repo.git_dir
@property
def ignore_file(self):
return self.GITIGNORE
@@ -245,70 +157,6 @@ class Git(Base):
self.track_file(relpath(gitignore))
def add(self, paths):
# NOTE: GitPython is not currently able to handle index version >= 3.
# See https://github.com/iterative/dvc/issues/610 for more details.
try:
self.repo.index.add(paths)
except AssertionError:
msg = (
"failed to add '{}' to git. You can add those files "
"manually using `git add`. See "
"https://github.com/iterative/dvc/issues/610 for more "
"details.".format(str(paths))
)
logger.exception(msg)
def commit(self, msg):
self.repo.index.commit(msg)
def checkout(self, branch, create_new=False, **kwargs):
if create_new:
self.repo.git.checkout("HEAD", b=branch, **kwargs)
else:
self.repo.git.checkout(branch, **kwargs)
def pull(self, **kwargs):
infos = self.repo.remote().pull(**kwargs)
for info in infos:
if info.flags & info.ERROR:
raise SCMError(f"pull failed: {info.note}")
def push(self):
infos = self.repo.remote().push()
for info in infos:
if info.flags & info.ERROR:
raise SCMError(f"push failed: {info.summary}")
def branch(self, branch):
self.repo.git.branch(branch)
def tag(self, tag):
self.repo.git.tag(tag)
def untracked_files(self):
files = self.repo.untracked_files
return [os.path.join(self.repo.working_dir, fname) for fname in files]
def is_tracked(self, path):
return bool(self.repo.git.ls_files(path))
def is_dirty(self, **kwargs):
return self.repo.is_dirty(**kwargs)
def active_branch(self):
return self.repo.active_branch.name
def list_branches(self):
return [h.name for h in self.repo.heads]
def list_tags(self):
return [t.name for t in self.repo.tags]
def list_all_commits(self):
return [c.hexsha for c in self.repo.iter_commits("--all")]
def _install_hook(self, name):
hook = self._hook_path(name)
with open(hook, "w+") as fobj:
@@ -317,8 +165,8 @@ class Git(Base):
os.chmod(hook, 0o777)
def _install_merge_driver(self):
self.repo.git.config("merge.dvc.name", "DVC merge driver")
self.repo.git.config(
self.gitpython.repo.git.config("merge.dvc.name", "DVC merge driver")
self.gitpython.repo.git.config(
"merge.dvc.driver",
(
"dvc git-hook merge-driver "
@@ -404,48 +252,6 @@ class Git(Base):
path_parts = os.path.normpath(path).split(os.path.sep)
return basename == self.ignore_file or Git.GIT_DIR in path_parts
def get_tree(self, rev, **kwargs):
from dvc.tree.git import GitTree
return GitTree(self.repo, self.resolve_rev(rev), **kwargs)
def get_rev(self):
return self.repo.rev_parse("HEAD").hexsha
def resolve_rev(self, rev):
from contextlib import suppress
from git.exc import BadName, GitCommandError
def _resolve_rev(name):
with suppress(BadName, GitCommandError):
try:
# Try python implementation of rev-parse first, it's faster
return self.repo.rev_parse(name).hexsha
except NotImplementedError:
# Fall back to `git rev-parse` for advanced features
return self.repo.git.rev_parse(name)
except ValueError:
raise RevError(f"unknown Git revision '{name}'")
# Resolve across local names
sha = _resolve_rev(rev)
if sha:
return sha
# Try all the remotes and if it resolves unambiguously then take it
if not Git.is_sha(rev):
shas = {
_resolve_rev(f"{remote.name}/{rev}")
for remote in self.repo.remotes
} - {None}
if len(shas) > 1:
raise RevError(f"ambiguous Git revision '{rev}'")
if len(shas) == 1:
return shas.pop()
raise RevError(f"unknown Git revision '{rev}'")
def has_rev(self, rev):
try:
self.resolve_rev(rev)
@@ -454,9 +260,8 @@ class Git(Base):
return False
def close(self):
for backend in self.backends:
for backend in self.backends.values():
backend.close()
self.repo.close()
@cached_property
def _hooks_home(self):
@@ -478,34 +283,8 @@ class Git(Base):
def no_commits(self):
return not self.list_all_commits()
def branch_revs(self, branch: str, end_rev: Optional[str] = None):
"""Iterate over revisions in a given branch (from newest to oldest).
If end_rev is set, iterator will stop when the specified revision is
reached.
"""
commit = self.resolve_commit(branch)
while commit is not None:
yield commit.hexsha
commit = first(commit.parents)
if commit and commit.hexsha == end_rev:
return
def resolve_commit(self, rev):
"""Return Commit object for the specified revision."""
from git.exc import BadName, GitCommandError
from git.objects.tag import TagObject
try:
commit = self.repo.rev_parse(rev)
except (BadName, GitCommandError):
commit = None
if isinstance(commit, TagObject):
commit = commit.object
return commit
def _backend_func(self, name, *args, **kwargs):
for backend in self.backends:
for backend in self.backends.values():
try:
func = getattr(backend, name)
return func(*args, **kwargs)
@@ -514,6 +293,26 @@ class Git(Base):
raise NoGitBackendError(name)
is_ignored = partialmethod(_backend_func, "is_ignored")
add = partialmethod(_backend_func, "add")
commit = partialmethod(_backend_func, "commit")
checkout = partialmethod(_backend_func, "checkout")
pull = partialmethod(_backend_func, "pull")
push = partialmethod(_backend_func, "push")
branch = partialmethod(_backend_func, "branch")
tag = partialmethod(_backend_func, "tag")
untracked_files = partialmethod(_backend_func, "untracked_files")
is_tracked = partialmethod(_backend_func, "is_tracked")
is_dirty = partialmethod(_backend_func, "is_dirty")
active_branch = partialmethod(_backend_func, "active_branch")
list_branches = partialmethod(_backend_func, "list_branches")
list_tags = partialmethod(_backend_func, "list_tags")
list_all_commits = partialmethod(_backend_func, "list_all_commits")
get_tree = partialmethod(_backend_func, "get_tree")
get_rev = partialmethod(_backend_func, "get_rev")
resolve_rev = partialmethod(_backend_func, "resolve_rev")
resolve_commit = partialmethod(_backend_func, "resolve_commit")
branch_revs = partialmethod(_backend_func, "branch_revs")
set_ref = partialmethod(_backend_func, "set_ref")
get_ref = partialmethod(_backend_func, "get_ref")
remove_ref = partialmethod(_backend_func, "remove_ref")
+112 -7
View File
@@ -1,11 +1,9 @@
import os
from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, Callable, Iterable, Optional, Tuple
from typing import Callable, Iterable, Optional, Tuple
from dvc.scm.base import SCMError
if TYPE_CHECKING:
from dvc.scm.git import Git
from dvc.tree.base import BaseTree
class NoGitBackendError(SCMError):
@@ -16,9 +14,11 @@ class NoGitBackendError(SCMError):
class BaseGitBackend(ABC):
"""Base Git backend class."""
def __init__(self, scm: "Git", **kwargs):
self.scm = scm
self.root_dir = os.fspath(scm.root_dir)
@abstractmethod
def __init__(
self, scm, root_dir=os.curdir, search_parent_directories=True
):
pass
def close(self):
pass
@@ -27,6 +27,111 @@ class BaseGitBackend(ABC):
def is_ignored(self, path):
"""Return True if the specified path is gitignored."""
@property
@abstractmethod
def root_dir(self) -> str:
pass
@staticmethod
@abstractmethod
def clone(
url: str,
to_path: str,
rev: Optional[str] = None,
shallow_branch: Optional[str] = None,
):
pass
@staticmethod
@abstractmethod
def is_sha(rev: str) -> bool:
pass
@property
@abstractmethod
def dir(self) -> str:
pass
@abstractmethod
def add(self, paths: Iterable[str]):
pass
@abstractmethod
def commit(self, msg: str):
pass
@abstractmethod
def checkout(
self, branch: str, create_new: Optional[bool] = False, **kwargs,
):
pass
@abstractmethod
def pull(self, **kwargs):
pass
@abstractmethod
def push(self):
pass
@abstractmethod
def branch(self, branch: str):
pass
@abstractmethod
def tag(self, tag: str):
pass
@abstractmethod
def untracked_files(self) -> Iterable[str]:
pass
@abstractmethod
def is_tracked(self, path: str) -> bool:
pass
@abstractmethod
def is_dirty(self, **kwargs) -> bool:
pass
@abstractmethod
def active_branch(self) -> str:
pass
@abstractmethod
def list_branches(self) -> Iterable[str]:
pass
@abstractmethod
def list_tags(self) -> Iterable[str]:
pass
@abstractmethod
def list_all_commits(self) -> Iterable[str]:
pass
@abstractmethod
def get_tree(self, rev: str, **kwargs) -> BaseTree:
pass
@abstractmethod
def get_rev(self) -> str:
pass
@abstractmethod
def resolve_rev(self, rev: str) -> str:
pass
@abstractmethod
def resolve_commit(self, rev: str) -> str:
pass
@abstractmethod
def branch_revs(
self, branch: str, end_rev: Optional[str] = None
) -> Iterable[str]:
pass
@abstractmethod
def set_ref(
self,
+96 -7
View File
@@ -2,32 +2,121 @@ import fnmatch
import logging
import os
from io import BytesIO
from typing import TYPE_CHECKING, Callable, Iterable, Optional, Tuple
from typing import Callable, Iterable, Optional, Tuple
from dvc.scm.base import SCMError
from dvc.tree.base import BaseTree
from dvc.utils import relpath
from .base import BaseGitBackend
if TYPE_CHECKING:
from dvc.scm.git import Git
logger = logging.getLogger(__name__)
class DulwichBackend(BaseGitBackend): # pylint:disable=abstract-method
"""Dulwich Git backend."""
def __init__(self, scm: "Git", **kwargs):
def __init__( # pylint:disable=W0231
self, scm, root_dir=os.curdir, search_parent_directories=True
):
from dulwich.errors import NotGitRepository
from dulwich.repo import Repo
super().__init__(scm, **kwargs)
self.repo = Repo(self.root_dir)
self.scm = scm
try:
if search_parent_directories:
self.repo = Repo.discover(start=root_dir)
else:
self.repo = Repo(root_dir)
except NotGitRepository as exc:
raise SCMError(f"{root_dir} is not a git repository") from exc
self._stashes: dict = {}
def close(self):
self.repo.close()
@property
def root_dir(self) -> str:
return self.repo.path
@staticmethod
def clone(
url: str,
to_path: str,
rev: Optional[str] = None,
shallow_branch: Optional[str] = None,
):
raise NotImplementedError
@staticmethod
def is_sha(rev: str) -> bool:
raise NotImplementedError
@property
def dir(self) -> str:
raise NotImplementedError
def add(self, paths: Iterable[str]):
raise NotImplementedError
def commit(self, msg: str):
raise NotImplementedError
def checkout(
self, branch: str, create_new: Optional[bool] = False, **kwargs,
):
raise NotImplementedError
def pull(self, **kwargs):
raise NotImplementedError
def push(self):
raise NotImplementedError
def branch(self, branch: str):
raise NotImplementedError
def tag(self, tag: str):
raise NotImplementedError
def untracked_files(self) -> Iterable[str]:
raise NotImplementedError
def is_tracked(self, path: str) -> bool:
raise NotImplementedError
def is_dirty(self, **kwargs) -> bool:
raise NotImplementedError
def active_branch(self) -> str:
raise NotImplementedError
def list_branches(self) -> Iterable[str]:
raise NotImplementedError
def list_tags(self) -> Iterable[str]:
raise NotImplementedError
def list_all_commits(self) -> Iterable[str]:
raise NotImplementedError
def get_tree(self, rev: str, **kwargs) -> BaseTree:
raise NotImplementedError
def get_rev(self) -> str:
raise NotImplementedError
def resolve_rev(self, rev: str) -> str:
raise NotImplementedError
def resolve_commit(self, rev: str) -> str:
raise NotImplementedError
def branch_revs(self, branch: str, end_rev: Optional[str] = None):
raise NotImplementedError
def _get_stash(self, ref: str):
from dulwich.stash import Stash as DulwichStash
+279 -4
View File
@@ -1,21 +1,294 @@
import logging
import os
from functools import partial
from typing import Callable, Iterable, Optional, Tuple
from funcy import first
from dvc.progress import Tqdm
from dvc.scm.base import CloneError, RevError, SCMError
from dvc.tree.base import BaseTree
from dvc.utils import fix_env, is_binary
from .base import BaseGitBackend
logger = logging.getLogger(__name__)
class TqdmGit(Tqdm):
def update_git(self, op_code, cur_count, max_count=None, message=""):
op_code = self.code2desc(op_code)
if op_code:
message = (op_code + " | " + message) if message else op_code
if message:
self.postfix["info"] = f" {message} | "
self.update_to(cur_count, max_count)
@staticmethod
def code2desc(op_code):
from git import RootUpdateProgress as OP
ops = {
OP.COUNTING: "Counting",
OP.COMPRESSING: "Compressing",
OP.WRITING: "Writing",
OP.RECEIVING: "Receiving",
OP.RESOLVING: "Resolving",
OP.FINDING_SOURCES: "Finding sources",
OP.CHECKING_OUT: "Checking out",
OP.CLONE: "Cloning",
OP.FETCH: "Fetching",
OP.UPDWKTREE: "Updating working tree",
OP.REMOVE: "Removing",
OP.PATHCHANGE: "Changing path",
OP.URLCHANGE: "Changing URL",
OP.BRANCHCHANGE: "Changing branch",
}
return ops.get(op_code & OP.OP_MASK, "")
class GitPythonBackend(BaseGitBackend): # pylint:disable=abstract-method
"""git-python Git backend."""
def __init__( # pylint:disable=W0231
self, scm, root_dir=os.curdir, search_parent_directories=True
):
import git
from git.exc import InvalidGitRepositoryError
self.scm = scm
try:
self.repo = git.Repo(
root_dir, search_parent_directories=search_parent_directories
)
except InvalidGitRepositoryError:
msg = "{} is not a git repository"
raise SCMError(msg.format(root_dir))
# NOTE: fixing LD_LIBRARY_PATH for binary built by PyInstaller.
# http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
env = fix_env(None)
libpath = env.get("LD_LIBRARY_PATH", None)
self.repo.git.update_environment(LD_LIBRARY_PATH=libpath)
def close(self):
self.repo.close()
@property
def git(self):
return self.scm.repo.git
return self.repo.git
def is_ignored(self, path):
raise NotImplementedError
@property
def root_dir(self) -> str:
return self.repo.working_tree_dir
@staticmethod
def clone(
url: str,
to_path: str,
rev: Optional[str] = None,
shallow_branch: Optional[str] = None,
):
import git
ld_key = "LD_LIBRARY_PATH"
env = fix_env(None)
if is_binary() and ld_key not in env.keys():
# In fix_env, we delete LD_LIBRARY_PATH key if it was empty before
# PyInstaller modified it. GitPython, in git.Repo.clone_from, uses
# env to update its own internal state. When there is no key in
# env, this value is not updated and GitPython re-uses
# LD_LIBRARY_PATH that has been set by PyInstaller.
# See [1] for more info.
# [1] https://github.com/gitpython-developers/GitPython/issues/924
env[ld_key] = ""
try:
if shallow_branch is not None and os.path.exists(url):
# git disables --depth for local clones unless file:// url
# scheme is used
url = f"file://{url}"
with TqdmGit(desc="Cloning", unit="obj") as pbar:
clone_from = partial(
git.Repo.clone_from,
url,
to_path,
env=env, # needed before we can fix it in __init__
no_single_branch=True,
progress=pbar.update_git,
)
if shallow_branch is None:
tmp_repo = clone_from()
else:
tmp_repo = clone_from(branch=shallow_branch, depth=1)
tmp_repo.close()
except git.exc.GitCommandError as exc: # pylint: disable=no-member
raise CloneError(url, to_path) from exc
# NOTE: using our wrapper to make sure that env is fixed in __init__
repo = GitPythonBackend(None, to_path)
if rev:
try:
repo.checkout(rev)
except git.exc.GitCommandError as exc: # pylint: disable=no-member
raise RevError(
"failed to access revision '{}' for repo '{}'".format(
rev, url
)
) from exc
@staticmethod
def is_sha(rev):
import git
return rev and git.Repo.re_hexsha_shortened.search(rev)
@property
def dir(self) -> str:
return self.repo.git_dir
def add(self, paths: Iterable[str]):
# NOTE: GitPython is not currently able to handle index version >= 3.
# See https://github.com/iterative/dvc/issues/610 for more details.
try:
self.repo.index.add(paths)
except AssertionError:
msg = (
"failed to add '{}' to git. You can add those files "
"manually using `git add`. See "
"https://github.com/iterative/dvc/issues/610 for more "
"details.".format(str(paths))
)
logger.exception(msg)
def commit(self, msg: str):
self.repo.index.commit(msg)
def checkout(
self, branch: str, create_new: Optional[bool] = False, **kwargs
):
if create_new:
self.repo.git.checkout("HEAD", b=branch, **kwargs)
else:
self.repo.git.checkout(branch, **kwargs)
def pull(self, **kwargs):
infos = self.repo.remote().pull(**kwargs)
for info in infos:
if info.flags & info.ERROR:
raise SCMError(f"pull failed: {info.note}")
def push(self):
infos = self.repo.remote().push()
for info in infos:
if info.flags & info.ERROR:
raise SCMError(f"push failed: {info.summary}")
def branch(self, branch):
self.repo.git.branch(branch)
def tag(self, tag):
self.repo.git.tag(tag)
def untracked_files(self):
files = self.repo.untracked_files
return [os.path.join(self.repo.working_dir, fname) for fname in files]
def is_tracked(self, path):
return bool(self.repo.git.ls_files(path))
def is_dirty(self, **kwargs):
return self.repo.is_dirty(**kwargs)
def active_branch(self):
return self.repo.active_branch.name
def list_branches(self):
return [h.name for h in self.repo.heads]
def list_tags(self):
return [t.name for t in self.repo.tags]
def list_all_commits(self):
return [c.hexsha for c in self.repo.iter_commits("--all")]
def get_tree(self, rev: str, **kwargs) -> BaseTree:
from dvc.tree.git import GitTree
return GitTree(self.repo, self.resolve_rev(rev), **kwargs)
def get_rev(self):
return self.repo.rev_parse("HEAD").hexsha
def resolve_rev(self, rev):
from contextlib import suppress
from git.exc import BadName, GitCommandError
def _resolve_rev(name):
with suppress(BadName, GitCommandError):
try:
# Try python implementation of rev-parse first, it's faster
return self.repo.rev_parse(name).hexsha
except NotImplementedError:
# Fall back to `git rev-parse` for advanced features
return self.repo.git.rev_parse(name)
except ValueError:
raise RevError(f"unknown Git revision '{name}'")
# Resolve across local names
sha = _resolve_rev(rev)
if sha:
return sha
# Try all the remotes and if it resolves unambiguously then take it
if not self.is_sha(rev):
shas = {
_resolve_rev(f"{remote.name}/{rev}")
for remote in self.repo.remotes
} - {None}
if len(shas) > 1:
raise RevError(f"ambiguous Git revision '{rev}'")
if len(shas) == 1:
return shas.pop()
raise RevError(f"unknown Git revision '{rev}'")
def resolve_commit(self, rev):
"""Return Commit object for the specified revision."""
from git.exc import BadName, GitCommandError
from git.objects.tag import TagObject
try:
commit = self.repo.rev_parse(rev)
except (BadName, GitCommandError):
commit = None
if isinstance(commit, TagObject):
commit = commit.object
return commit
def branch_revs(
self, branch: str, end_rev: Optional[str] = None
) -> Iterable[str]:
"""Iterate over revisions in a given branch (from newest to oldest).
If end_rev is set, iterator will stop when the specified revision is
reached.
"""
commit = self.resolve_commit(branch)
while commit is not None:
yield commit.hexsha
commit = first(commit.parents)
if commit and commit.hexsha == end_rev:
return
def set_ref(
self,
name: str,
@@ -35,7 +308,7 @@ class GitPythonBackend(BaseGitBackend): # pylint:disable=abstract-method
def iter_refs(self, base: Optional[str] = None):
from git import Reference
for ref in Reference.iter_items(self.scm.repo, common_path=base):
for ref in Reference.iter_items(self.repo, common_path=base):
if base and base.endswith("/"):
base = base[:-1]
yield "/".join([base, ref.name])
@@ -86,7 +359,7 @@ class GitPythonBackend(BaseGitBackend): # pylint:disable=abstract-method
if include_untracked:
args.append("--include-untracked")
self.git.stash(*args)
commit = self.scm.resolve_commit("stash@{0}")
commit = self.resolve_commit("stash@{0}")
if ref != Stash.DEFAULT_STASH:
# `git stash` CLI doesn't support using custom refspecs,
# so we push a commit onto refs/stash, make our refspec
@@ -94,7 +367,9 @@ class GitPythonBackend(BaseGitBackend): # pylint:disable=abstract-method
# `git stash create` is intended to be used for this kind of
# behavior but it doesn't support --include-untracked so we need to
# use push
self.scm.set_ref(ref, commit.hexsha, message=commit.message)
self.scm.dulwich.set_ref(
ref, commit.hexsha, message=commit.message
)
self.git.stash("drop")
return commit.hexsha, False
+1 -5
View File
@@ -19,10 +19,6 @@ class Stash:
self.ref = ref if ref else self.DEFAULT_STASH
self.scm = scm
@property
def git(self):
return self.scm.repo.git
def __iter__(self):
yield from self.scm._stash_iter(self.ref)
@@ -49,7 +45,7 @@ class Stash:
self.ref, message=message, include_untracked=include_untracked
)
if reset:
self.git.reset(hard=True)
self.scm.gitpython.repo.git.reset(hard=True)
return rev
def pop(self):
+3 -3
View File
@@ -60,8 +60,8 @@ def test_reset_checkpoint(tmp_dir, scm, dvc, checkpoint_stage, caplog):
from dvc.repo.experiments.base import CheckpointExistsError
dvc.experiments.run(checkpoint_stage.addressing, name="foo")
scm.repo.git.reset(hard=True)
scm.repo.git.clean(force=True)
scm.gitpython.repo.git.reset(hard=True)
scm.gitpython.repo.git.clean(force=True)
with pytest.raises(CheckpointExistsError):
dvc.experiments.run(
@@ -113,6 +113,6 @@ def test_resume_branch(tmp_dir, scm, dvc, checkpoint_stage):
with pytest.raises(MultipleBranchError):
dvc.experiments.get_branch_containing(branch_rev)
assert branch_rev == dvc.experiments.scm.repo.git.merge_base(
assert branch_rev == dvc.experiments.scm.gitpython.repo.git.merge_base(
checkpoint_a, checkpoint_b
)
+1 -1
View File
@@ -258,7 +258,7 @@ def test_detached_parent(tmp_dir, scm, dvc, exp_stage, mocker):
scm.commit("v2")
scm.checkout(detached_rev)
assert scm.repo.head.is_detached
assert scm.gitpython.repo.head.is_detached
results = dvc.experiments.run(exp_stage.addressing, params=["foo=3"])
exp_rev = first(results)
+1 -1
View File
@@ -41,7 +41,7 @@ def test_show_experiment(tmp_dir, scm, dvc):
scm.commit("baseline")
baseline_rev = scm.get_rev()
timestamp = datetime.fromtimestamp(
scm.repo.rev_parse(baseline_rev).committed_date
scm.gitpython.repo.rev_parse(baseline_rev).committed_date
)
dvc.experiments.run(PIPELINE_FILE, params=["foo=2"])
+1 -1
View File
@@ -580,7 +580,7 @@ def test_stats_on_checkout(tmp_dir, dvc, scm):
tmp_dir.gen({"lorem": "lorem", "bar": "new bar", "dir2": {"file": "file"}})
(tmp_dir / "foo").unlink()
scm.repo.git.rm("foo.dvc")
scm.gitpython.repo.git.rm("foo.dvc")
tmp_dir.dvc_add(["bar", "lorem", "dir2"], commit="second")
scm.checkout("HEAD~")
+10 -10
View File
@@ -25,7 +25,7 @@ def test_import(tmp_dir, scm, dvc, erepo_dir):
assert os.path.isfile("foo_imported")
assert (tmp_dir / "foo_imported").read_text() == "foo content"
assert scm.repo.git.check_ignore("foo_imported")
assert scm.is_ignored("foo_imported")
assert stage.deps[0].def_repo == {
"url": os.fspath(erepo_dir),
"rev_lock": erepo_dir.scm.get_rev(),
@@ -42,7 +42,7 @@ def test_import_git_file(tmp_dir, scm, dvc, git_dir, src_is_dvc):
stage = tmp_dir.dvc.imp(os.fspath(git_dir), "src", "dst")
assert (tmp_dir / "dst").read_text() == "hello"
assert tmp_dir.scm.repo.git.check_ignore(os.fspath(tmp_dir / "dst"))
assert tmp_dir.scm.is_ignored(os.fspath(tmp_dir / "dst"))
assert stage.deps[0].def_repo == {
"url": os.fspath(git_dir),
"rev_lock": git_dir.scm.get_rev(),
@@ -77,7 +77,7 @@ def test_import_git_dir(tmp_dir, scm, dvc, git_dir, src_is_dvc):
stage = dvc.imp(os.fspath(git_dir), "src", "dst")
assert (tmp_dir / "dst").read_text() == {"file.txt": "hello"}
assert tmp_dir.scm.repo.git.check_ignore(os.fspath(tmp_dir / "dst"))
assert tmp_dir.scm.is_ignored(os.fspath(tmp_dir / "dst"))
assert stage.deps[0].def_repo == {
"url": os.fspath(git_dir),
"rev_lock": git_dir.scm.get_rev(),
@@ -91,7 +91,7 @@ def test_import_dir(tmp_dir, scm, dvc, erepo_dir):
stage = dvc.imp(os.fspath(erepo_dir), "dir", "dir_imported")
assert (tmp_dir / "dir_imported").read_text() == {"foo": "foo content"}
assert scm.repo.git.check_ignore("dir_imported")
assert scm.is_ignored("dir_imported")
assert stage.deps[0].def_repo == {
"url": os.fspath(erepo_dir),
"rev_lock": erepo_dir.scm.get_rev(),
@@ -114,7 +114,7 @@ def test_import_file_from_dir(tmp_dir, scm, dvc, erepo_dir):
stage = dvc.imp(os.fspath(erepo_dir), os.path.join("dir", "1"))
assert (tmp_dir / "1").read_text() == "1"
assert scm.repo.git.check_ignore("1")
assert scm.is_ignored("1")
assert stage.deps[0].def_repo == {
"url": os.fspath(erepo_dir),
"rev_lock": erepo_dir.scm.get_rev(),
@@ -175,7 +175,7 @@ def test_import_non_cached(erepo_dir, tmp_dir, dvc, scm):
assert (tmp_dir / dst).is_file()
assert filecmp.cmp(erepo_dir / src, tmp_dir / dst, shallow=False)
assert tmp_dir.scm.repo.git.check_ignore(dst)
assert tmp_dir.scm.is_ignored(dst)
assert stage.deps[0].def_repo == {
"url": os.fspath(erepo_dir),
"rev_lock": erepo_dir.scm.get_rev(),
@@ -191,7 +191,7 @@ def test_import_rev(tmp_dir, scm, dvc, erepo_dir):
stage = dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported", rev="branch")
assert (tmp_dir / "foo_imported").read_text() == "foo content"
assert scm.repo.git.check_ignore("foo_imported")
assert scm.is_ignored("foo_imported")
assert stage.deps[0].def_repo == {
"url": os.fspath(erepo_dir),
"rev": "branch",
@@ -231,7 +231,7 @@ def test_cache_type_is_properly_overridden(tmp_dir, scm, dvc, erepo_dir):
assert not System.is_symlink("foo_imported")
assert (tmp_dir / "foo_imported").read_text() == "foo content"
assert scm.repo.git.check_ignore("foo_imported")
assert scm.is_ignored("foo_imported")
def test_pull_imported_directory_stage(tmp_dir, dvc, erepo_dir):
@@ -340,8 +340,8 @@ def test_import_from_bare_git_repo(
erepo_dir.dvc_gen({"foo": "foo"}, commit="initial")
erepo_dir.dvc.push()
erepo_dir.scm.repo.create_remote("origin", os.fspath(tmp_dir))
erepo_dir.scm.repo.remote("origin").push("master")
erepo_dir.scm.gitpython.repo.create_remote("origin", os.fspath(tmp_dir))
erepo_dir.scm.gitpython.repo.remote("origin").push("master")
dvc_repo = make_tmp_dir("dvc-repo", scm=True, dvc=True)
with dvc_repo.chdir():
+10 -6
View File
@@ -43,7 +43,7 @@ class TestInstall:
# scm.commit bypasses hooks
with pytest.raises(GitCommandError, match=r"modified:\s*file"):
scm.repo.git.commit(m="file modified")
scm.gitpython.repo.git.commit(m="file modified")
def test_post_checkout(self, tmp_dir, scm, dvc):
tmp_dir.dvc_gen({"file": "file content"}, commit="add")
@@ -69,13 +69,13 @@ class TestInstall:
storage_path / file_checksum[:2] / file_checksum[2:]
)
scm.repo.clone(os.fspath(git_remote))
scm.repo.create_remote("origin", os.fspath(git_remote))
scm.gitpython.repo.clone(os.fspath(git_remote))
scm.gitpython.repo.create_remote("origin", os.fspath(git_remote))
scm.install()
assert not expected_storage_path.is_file()
scm.repo.git.push("origin", "master")
scm.gitpython.repo.git.push("origin", "master")
assert expected_storage_path.is_file()
assert expected_storage_path.read_text() == "file_content"
@@ -97,7 +97,9 @@ def test_merge_driver_no_ancestor(tmp_dir, scm, dvc):
scm.install()
(tmp_dir / ".gitattributes").write_text("*.dvc merge=dvc")
scm.repo.git.merge("one", m="merged", no_gpg_sign=True, no_signoff=True)
scm.gitpython.repo.git.merge(
"one", m="merged", no_gpg_sign=True, no_signoff=True
)
# NOTE: dvc shouldn't checkout automatically as it might take a long time
assert (tmp_dir / "data").read_text() == {"bar": "bar"}
@@ -132,7 +134,9 @@ def test_merge_driver(tmp_dir, scm, dvc):
scm.install()
(tmp_dir / ".gitattributes").write_text("*.dvc merge=dvc")
scm.repo.git.merge("one", m="merged", no_gpg_sign=True, no_signoff=True)
scm.gitpython.repo.git.merge(
"one", m="merged", no_gpg_sign=True, no_signoff=True
)
# NOTE: dvc shouldn't checkout automatically as it might take a long time
assert (tmp_dir / "data").read_text() == {"master": "master", "two": "two"}
+3 -3
View File
@@ -50,7 +50,7 @@ class TestSCMGit(TestGit):
G.commit("add")
self.assertTrue(G.is_tracked(foo))
self.assertTrue(G.is_tracked(self.FOO))
G.repo.index.remove([self.FOO], working_tree=True)
G.gitpython.repo.index.remove([self.FOO], working_tree=True)
self.assertFalse(G.is_tracked(foo))
self.assertFalse(G.is_tracked(self.FOO))
self.assertFalse(G.is_tracked("not-existing-file"))
@@ -183,9 +183,9 @@ def test_git_stash_workspace(tmp_dir, scm):
tmp_dir.gen("file", "1")
with scm.stash_workspace():
assert not scm.repo.is_dirty()
assert not scm.is_dirty()
assert "0" == (tmp_dir / "file").read_text()
assert scm.repo.is_dirty()
assert scm.is_dirty()
assert "1" == (tmp_dir / "file").read_text()
+1 -1
View File
@@ -58,7 +58,7 @@ def test_status_before_and_after_dvc_init(tmp_dir, dvc, git_dir):
with git_dir.chdir():
git_dir.init(dvc=True)
git_dir.scm.repo.index.remove(["file"])
git_dir.scm.gitpython.repo.index.remove(["file"])
os.remove("file")
git_dir.dvc_gen("file", "second version", commit="with dvc")
new_rev = git_dir.scm.get_rev()
+2 -2
View File
@@ -97,7 +97,7 @@ def test_update_import_after_remote_updates_to_dvc(tmp_dir, dvc, erepo_dir):
new_rev = None
with erepo_dir.branch("branch", new=False), erepo_dir.chdir():
erepo_dir.scm.repo.index.remove(["version"])
erepo_dir.scm.gitpython.repo.index.remove(["version"])
erepo_dir.dvc_gen(
"version", "updated", commit="upgrade to DVC tracking"
)
@@ -134,7 +134,7 @@ def test_update_before_and_after_dvc_init(tmp_dir, dvc, git_dir):
with git_dir.chdir():
git_dir.init(dvc=True)
git_dir.scm.repo.index.remove(["file"])
git_dir.scm.gitpython.repo.index.remove(["file"])
os.remove("file")
git_dir.dvc_gen("file", "second version", commit="with dvc")
new_rev = git_dir.scm.get_rev()
+3 -1
View File
@@ -25,7 +25,9 @@ def test_walk_with_submodules(tmp_dir, scm, git_dir):
{"foo": "foo", "bar": "bar", "dir": {"data": "data"}},
commit="add dir and files",
)
scm.repo.create_submodule("submodule", "submodule", url=os.fspath(git_dir))
scm.gitpython.repo.create_submodule(
"submodule", "submodule", url=os.fspath(git_dir)
)
scm.commit("added submodule")
files = []