mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-17 07:18:26 +01:00
scm: move logic to gitpython backend (#5022)
This commit is contained in:
@@ -369,13 +369,13 @@ def _clone_default_branch(url, rev, for_write=False):
|
||||
|
||||
|
||||
def _unshallow(git):
|
||||
if git.repo.head.is_detached:
|
||||
if git.gitpython.repo.head.is_detached:
|
||||
# If this is a detached head (i.e. we shallow cloned a tag) switch to
|
||||
# the default branch
|
||||
origin_refs = git.repo.remotes["origin"].refs
|
||||
origin_refs = git.gitpython.repo.remotes["origin"].refs
|
||||
ref = origin_refs["HEAD"].reference
|
||||
branch_name = ref.name.split("/")[-1]
|
||||
branch = git.repo.create_head(branch_name, ref)
|
||||
branch = git.gitpython.repo.create_head(branch_name, ref)
|
||||
branch.set_tracking_branch(ref)
|
||||
branch.checkout()
|
||||
git.pull(unshallow=True)
|
||||
|
||||
@@ -205,8 +205,8 @@ class Experiments:
|
||||
)
|
||||
|
||||
# Reset/clean any changes before prior workspace is unstashed
|
||||
self.scm.repo.git.reset(hard=True)
|
||||
self.scm.repo.git.clean(force=True)
|
||||
self.scm.gitpython.repo.git.reset(hard=True)
|
||||
self.scm.gitpython.repo.git.clean(force=True)
|
||||
|
||||
return stash_rev
|
||||
|
||||
@@ -447,8 +447,8 @@ class Experiments:
|
||||
for ref in (EXEC_HEAD, EXEC_MERGE, EXEC_BASELINE):
|
||||
self.scm.remove_ref(ref)
|
||||
|
||||
self.scm.repo.git.reset(hard=True)
|
||||
self.scm.repo.git.clean(force=True)
|
||||
self.scm.gitpython.repo.git.reset(hard=True)
|
||||
self.scm.gitpython.repo.git.clean(force=True)
|
||||
return executors
|
||||
|
||||
def _reproduce(
|
||||
|
||||
@@ -48,7 +48,7 @@ def apply(repo, rev, *args, **kwargs):
|
||||
else:
|
||||
workspace = None
|
||||
|
||||
repo.scm.repo.git.merge(branch, squash=True, no_commit=True)
|
||||
repo.scm.gitpython.repo.git.merge(branch, squash=True, no_commit=True)
|
||||
|
||||
if workspace:
|
||||
try:
|
||||
@@ -56,9 +56,9 @@ def apply(repo, rev, *args, **kwargs):
|
||||
except GitCommandError:
|
||||
# if stash apply returns merge conflicts, prefer experiment
|
||||
# changes over prior stashed changes
|
||||
repo.scm.repo.git.checkout("--ours", "--", ".")
|
||||
repo.scm.gitpython.repo.git.checkout("--ours", "--", ".")
|
||||
repo.scm.stash.drop()
|
||||
repo.scm.repo.git.reset()
|
||||
repo.scm.gitpython.repo.git.reset()
|
||||
|
||||
if stash_rev:
|
||||
args_path = os.path.join(repo.tmp_dir, BaseExecutor.PACKED_ARGS_FILE)
|
||||
|
||||
@@ -85,8 +85,10 @@ class BaseExecutor:
|
||||
# committing
|
||||
head = EXEC_BRANCH if branch else EXEC_HEAD
|
||||
self.scm.checkout(head, detach=True)
|
||||
self.scm.repo.git.merge(EXEC_MERGE, squash=True, no_commit=True)
|
||||
self.scm.repo.git.reset()
|
||||
self.scm.gitpython.repo.git.merge(
|
||||
EXEC_MERGE, squash=True, no_commit=True
|
||||
)
|
||||
self.scm.gitpython.repo.git.reset()
|
||||
self._prune_lockfiles()
|
||||
finally:
|
||||
os.chdir(cwd)
|
||||
@@ -94,11 +96,13 @@ class BaseExecutor:
|
||||
def _prune_lockfiles(self):
|
||||
# NOTE: dirty DVC lock files must be restored to index state to
|
||||
# avoid checking out incorrect persist or checkpoint outs
|
||||
dirty = [diff.a_path for diff in self.scm.repo.index.diff(None)]
|
||||
dirty = [
|
||||
diff.a_path for diff in self.scm.gitpython.repo.index.diff(None)
|
||||
]
|
||||
to_checkout = [fname for fname in dirty if is_lock_file(fname)]
|
||||
self.scm.repo.index.checkout(paths=to_checkout, force=True)
|
||||
self.scm.gitpython.repo.index.checkout(paths=to_checkout, force=True)
|
||||
|
||||
untracked = self.scm.repo.untracked_files
|
||||
untracked = self.scm.gitpython.repo.untracked_files
|
||||
to_remove = [fname for fname in untracked if is_lock_file(fname)]
|
||||
for fname in to_remove:
|
||||
remove(fname)
|
||||
@@ -320,7 +324,7 @@ class BaseExecutor:
|
||||
old_ref = None
|
||||
logger.debug("Commit to new experiment branch '%s'", branch)
|
||||
|
||||
scm.repo.git.add(A=True)
|
||||
scm.gitpython.repo.git.add(A=True)
|
||||
scm.commit(f"dvc: commit experiment {exp_hash}")
|
||||
new_rev = scm.get_rev()
|
||||
scm.set_ref(branch, new_rev, old_ref=old_ref)
|
||||
|
||||
+67
-268
@@ -3,23 +3,17 @@
|
||||
import logging
|
||||
import os
|
||||
import shlex
|
||||
from collections import OrderedDict
|
||||
from contextlib import contextmanager
|
||||
from functools import partial, partialmethod
|
||||
from functools import partialmethod
|
||||
from typing import Optional
|
||||
|
||||
from funcy import cached_property, first
|
||||
from funcy import cached_property
|
||||
from pathspec.patterns import GitWildMatchPattern
|
||||
|
||||
from dvc.exceptions import GitHookAlreadyExistsError
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.scm.base import (
|
||||
Base,
|
||||
CloneError,
|
||||
FileNotInRepoError,
|
||||
RevError,
|
||||
SCMError,
|
||||
)
|
||||
from dvc.utils import fix_env, is_binary, relpath
|
||||
from dvc.scm.base import Base, FileNotInRepoError, RevError
|
||||
from dvc.utils import relpath
|
||||
from dvc.utils.fs import path_isin
|
||||
from dvc.utils.serialize import modify_yaml
|
||||
|
||||
@@ -31,150 +25,68 @@ from .stash import Stash
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TqdmGit(Tqdm):
|
||||
def update_git(self, op_code, cur_count, max_count=None, message=""):
|
||||
op_code = self.code2desc(op_code)
|
||||
if op_code:
|
||||
message = (op_code + " | " + message) if message else op_code
|
||||
if message:
|
||||
self.postfix["info"] = f" {message} | "
|
||||
self.update_to(cur_count, max_count)
|
||||
|
||||
@staticmethod
|
||||
def code2desc(op_code):
|
||||
from git import RootUpdateProgress as OP
|
||||
|
||||
ops = {
|
||||
OP.COUNTING: "Counting",
|
||||
OP.COMPRESSING: "Compressing",
|
||||
OP.WRITING: "Writing",
|
||||
OP.RECEIVING: "Receiving",
|
||||
OP.RESOLVING: "Resolving",
|
||||
OP.FINDING_SOURCES: "Finding sources",
|
||||
OP.CHECKING_OUT: "Checking out",
|
||||
OP.CLONE: "Cloning",
|
||||
OP.FETCH: "Fetching",
|
||||
OP.UPDWKTREE: "Updating working tree",
|
||||
OP.REMOVE: "Removing",
|
||||
OP.PATHCHANGE: "Changing path",
|
||||
OP.URLCHANGE: "Changing URL",
|
||||
OP.BRANCHCHANGE: "Changing branch",
|
||||
}
|
||||
return ops.get(op_code & OP.OP_MASK, "")
|
||||
|
||||
|
||||
class Git(Base):
|
||||
"""Class for managing Git."""
|
||||
|
||||
GITIGNORE = ".gitignore"
|
||||
GIT_DIR = ".git"
|
||||
DEFAULT_BACKENDS = (DulwichBackend, GitPythonBackend)
|
||||
DEFAULT_BACKENDS = OrderedDict(
|
||||
[("dulwich", DulwichBackend), ("gitpython", GitPythonBackend)]
|
||||
)
|
||||
LOCAL_BRANCH_PREFIX = "refs/heads/"
|
||||
|
||||
def __init__(self, root_dir=os.curdir, search_parent_directories=True):
|
||||
"""Git class constructor.
|
||||
Requires `Repo` class from `git` module (from gitpython package).
|
||||
"""
|
||||
super().__init__(root_dir)
|
||||
|
||||
import git
|
||||
from git.exc import InvalidGitRepositoryError
|
||||
|
||||
try:
|
||||
self.repo = git.Repo(
|
||||
root_dir, search_parent_directories=search_parent_directories
|
||||
)
|
||||
except InvalidGitRepositoryError:
|
||||
msg = "{} is not a git repository"
|
||||
raise SCMError(msg.format(root_dir))
|
||||
|
||||
# NOTE: fixing LD_LIBRARY_PATH for binary built by PyInstaller.
|
||||
# http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
|
||||
env = fix_env(None)
|
||||
libpath = env.get("LD_LIBRARY_PATH", None)
|
||||
self.repo.git.update_environment(LD_LIBRARY_PATH=libpath)
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.ignored_paths = []
|
||||
self.files_to_track = set()
|
||||
|
||||
self.backends = [backend(self) for backend in self.DEFAULT_BACKENDS]
|
||||
self.backends = OrderedDict(
|
||||
[
|
||||
(name, cls(self, *args, **kwargs))
|
||||
for name, cls in self.DEFAULT_BACKENDS.items()
|
||||
]
|
||||
)
|
||||
|
||||
super().__init__(self.gitpython.root_dir)
|
||||
|
||||
@property
|
||||
def root_dir(self) -> str:
|
||||
return self.repo.working_tree_dir
|
||||
def dir(self):
|
||||
return self.gitpython.repo.git_dir
|
||||
|
||||
@property
|
||||
def gitpython(self):
|
||||
return self.backends["gitpython"]
|
||||
|
||||
@property
|
||||
def dulwich(self):
|
||||
return self.backends["dulwich"]
|
||||
|
||||
@cached_property
|
||||
def stash(self):
|
||||
return Stash(self)
|
||||
|
||||
@staticmethod
|
||||
def clone(url, to_path, rev=None, shallow_branch=None):
|
||||
import git
|
||||
|
||||
ld_key = "LD_LIBRARY_PATH"
|
||||
|
||||
env = fix_env(None)
|
||||
if is_binary() and ld_key not in env.keys():
|
||||
# In fix_env, we delete LD_LIBRARY_PATH key if it was empty before
|
||||
# PyInstaller modified it. GitPython, in git.Repo.clone_from, uses
|
||||
# env to update its own internal state. When there is no key in
|
||||
# env, this value is not updated and GitPython re-uses
|
||||
# LD_LIBRARY_PATH that has been set by PyInstaller.
|
||||
# See [1] for more info.
|
||||
# [1] https://github.com/gitpython-developers/GitPython/issues/924
|
||||
env[ld_key] = ""
|
||||
|
||||
try:
|
||||
if shallow_branch is not None and os.path.exists(url):
|
||||
# git disables --depth for local clones unless file:// url
|
||||
# scheme is used
|
||||
url = f"file://{url}"
|
||||
with TqdmGit(desc="Cloning", unit="obj") as pbar:
|
||||
clone_from = partial(
|
||||
git.Repo.clone_from,
|
||||
url,
|
||||
to_path,
|
||||
env=env, # needed before we can fix it in __init__
|
||||
no_single_branch=True,
|
||||
progress=pbar.update_git,
|
||||
)
|
||||
if shallow_branch is None:
|
||||
tmp_repo = clone_from()
|
||||
else:
|
||||
tmp_repo = clone_from(branch=shallow_branch, depth=1)
|
||||
tmp_repo.close()
|
||||
except git.exc.GitCommandError as exc: # pylint: disable=no-member
|
||||
raise CloneError(url, to_path) from exc
|
||||
|
||||
# NOTE: using our wrapper to make sure that env is fixed in __init__
|
||||
repo = Git(to_path)
|
||||
|
||||
if rev:
|
||||
@classmethod
|
||||
def clone(cls, url, to_path, **kwargs):
|
||||
for _, backend in cls.DEFAULT_BACKENDS.items():
|
||||
try:
|
||||
repo.checkout(rev)
|
||||
except git.exc.GitCommandError as exc: # pylint: disable=no-member
|
||||
raise RevError(
|
||||
"failed to access revision '{}' for repo '{}'".format(
|
||||
rev, url
|
||||
)
|
||||
) from exc
|
||||
backend.clone(url, to_path, **kwargs)
|
||||
return Git(to_path)
|
||||
except NotImplementedError:
|
||||
pass
|
||||
raise NoGitBackendError("clone")
|
||||
|
||||
return repo
|
||||
|
||||
@staticmethod
|
||||
def is_sha(rev):
|
||||
import git
|
||||
|
||||
return rev and git.Repo.re_hexsha_shortened.search(rev)
|
||||
@classmethod
|
||||
def is_sha(cls, rev):
|
||||
for _, backend in cls.DEFAULT_BACKENDS.items():
|
||||
try:
|
||||
return backend.is_sha(rev)
|
||||
except NotImplementedError:
|
||||
pass
|
||||
raise NoGitBackendError("is_sha")
|
||||
|
||||
@staticmethod
|
||||
def _get_git_dir(root_dir):
|
||||
return os.path.join(root_dir, Git.GIT_DIR)
|
||||
|
||||
@property
|
||||
def dir(self):
|
||||
return self.repo.git_dir
|
||||
|
||||
@property
|
||||
def ignore_file(self):
|
||||
return self.GITIGNORE
|
||||
@@ -245,70 +157,6 @@ class Git(Base):
|
||||
|
||||
self.track_file(relpath(gitignore))
|
||||
|
||||
def add(self, paths):
|
||||
# NOTE: GitPython is not currently able to handle index version >= 3.
|
||||
# See https://github.com/iterative/dvc/issues/610 for more details.
|
||||
try:
|
||||
self.repo.index.add(paths)
|
||||
except AssertionError:
|
||||
msg = (
|
||||
"failed to add '{}' to git. You can add those files "
|
||||
"manually using `git add`. See "
|
||||
"https://github.com/iterative/dvc/issues/610 for more "
|
||||
"details.".format(str(paths))
|
||||
)
|
||||
|
||||
logger.exception(msg)
|
||||
|
||||
def commit(self, msg):
|
||||
self.repo.index.commit(msg)
|
||||
|
||||
def checkout(self, branch, create_new=False, **kwargs):
|
||||
if create_new:
|
||||
self.repo.git.checkout("HEAD", b=branch, **kwargs)
|
||||
else:
|
||||
self.repo.git.checkout(branch, **kwargs)
|
||||
|
||||
def pull(self, **kwargs):
|
||||
infos = self.repo.remote().pull(**kwargs)
|
||||
for info in infos:
|
||||
if info.flags & info.ERROR:
|
||||
raise SCMError(f"pull failed: {info.note}")
|
||||
|
||||
def push(self):
|
||||
infos = self.repo.remote().push()
|
||||
for info in infos:
|
||||
if info.flags & info.ERROR:
|
||||
raise SCMError(f"push failed: {info.summary}")
|
||||
|
||||
def branch(self, branch):
|
||||
self.repo.git.branch(branch)
|
||||
|
||||
def tag(self, tag):
|
||||
self.repo.git.tag(tag)
|
||||
|
||||
def untracked_files(self):
|
||||
files = self.repo.untracked_files
|
||||
return [os.path.join(self.repo.working_dir, fname) for fname in files]
|
||||
|
||||
def is_tracked(self, path):
|
||||
return bool(self.repo.git.ls_files(path))
|
||||
|
||||
def is_dirty(self, **kwargs):
|
||||
return self.repo.is_dirty(**kwargs)
|
||||
|
||||
def active_branch(self):
|
||||
return self.repo.active_branch.name
|
||||
|
||||
def list_branches(self):
|
||||
return [h.name for h in self.repo.heads]
|
||||
|
||||
def list_tags(self):
|
||||
return [t.name for t in self.repo.tags]
|
||||
|
||||
def list_all_commits(self):
|
||||
return [c.hexsha for c in self.repo.iter_commits("--all")]
|
||||
|
||||
def _install_hook(self, name):
|
||||
hook = self._hook_path(name)
|
||||
with open(hook, "w+") as fobj:
|
||||
@@ -317,8 +165,8 @@ class Git(Base):
|
||||
os.chmod(hook, 0o777)
|
||||
|
||||
def _install_merge_driver(self):
|
||||
self.repo.git.config("merge.dvc.name", "DVC merge driver")
|
||||
self.repo.git.config(
|
||||
self.gitpython.repo.git.config("merge.dvc.name", "DVC merge driver")
|
||||
self.gitpython.repo.git.config(
|
||||
"merge.dvc.driver",
|
||||
(
|
||||
"dvc git-hook merge-driver "
|
||||
@@ -404,48 +252,6 @@ class Git(Base):
|
||||
path_parts = os.path.normpath(path).split(os.path.sep)
|
||||
return basename == self.ignore_file or Git.GIT_DIR in path_parts
|
||||
|
||||
def get_tree(self, rev, **kwargs):
|
||||
from dvc.tree.git import GitTree
|
||||
|
||||
return GitTree(self.repo, self.resolve_rev(rev), **kwargs)
|
||||
|
||||
def get_rev(self):
|
||||
return self.repo.rev_parse("HEAD").hexsha
|
||||
|
||||
def resolve_rev(self, rev):
|
||||
from contextlib import suppress
|
||||
|
||||
from git.exc import BadName, GitCommandError
|
||||
|
||||
def _resolve_rev(name):
|
||||
with suppress(BadName, GitCommandError):
|
||||
try:
|
||||
# Try python implementation of rev-parse first, it's faster
|
||||
return self.repo.rev_parse(name).hexsha
|
||||
except NotImplementedError:
|
||||
# Fall back to `git rev-parse` for advanced features
|
||||
return self.repo.git.rev_parse(name)
|
||||
except ValueError:
|
||||
raise RevError(f"unknown Git revision '{name}'")
|
||||
|
||||
# Resolve across local names
|
||||
sha = _resolve_rev(rev)
|
||||
if sha:
|
||||
return sha
|
||||
|
||||
# Try all the remotes and if it resolves unambiguously then take it
|
||||
if not Git.is_sha(rev):
|
||||
shas = {
|
||||
_resolve_rev(f"{remote.name}/{rev}")
|
||||
for remote in self.repo.remotes
|
||||
} - {None}
|
||||
if len(shas) > 1:
|
||||
raise RevError(f"ambiguous Git revision '{rev}'")
|
||||
if len(shas) == 1:
|
||||
return shas.pop()
|
||||
|
||||
raise RevError(f"unknown Git revision '{rev}'")
|
||||
|
||||
def has_rev(self, rev):
|
||||
try:
|
||||
self.resolve_rev(rev)
|
||||
@@ -454,9 +260,8 @@ class Git(Base):
|
||||
return False
|
||||
|
||||
def close(self):
|
||||
for backend in self.backends:
|
||||
for backend in self.backends.values():
|
||||
backend.close()
|
||||
self.repo.close()
|
||||
|
||||
@cached_property
|
||||
def _hooks_home(self):
|
||||
@@ -478,34 +283,8 @@ class Git(Base):
|
||||
def no_commits(self):
|
||||
return not self.list_all_commits()
|
||||
|
||||
def branch_revs(self, branch: str, end_rev: Optional[str] = None):
|
||||
"""Iterate over revisions in a given branch (from newest to oldest).
|
||||
|
||||
If end_rev is set, iterator will stop when the specified revision is
|
||||
reached.
|
||||
"""
|
||||
commit = self.resolve_commit(branch)
|
||||
while commit is not None:
|
||||
yield commit.hexsha
|
||||
commit = first(commit.parents)
|
||||
if commit and commit.hexsha == end_rev:
|
||||
return
|
||||
|
||||
def resolve_commit(self, rev):
|
||||
"""Return Commit object for the specified revision."""
|
||||
from git.exc import BadName, GitCommandError
|
||||
from git.objects.tag import TagObject
|
||||
|
||||
try:
|
||||
commit = self.repo.rev_parse(rev)
|
||||
except (BadName, GitCommandError):
|
||||
commit = None
|
||||
if isinstance(commit, TagObject):
|
||||
commit = commit.object
|
||||
return commit
|
||||
|
||||
def _backend_func(self, name, *args, **kwargs):
|
||||
for backend in self.backends:
|
||||
for backend in self.backends.values():
|
||||
try:
|
||||
func = getattr(backend, name)
|
||||
return func(*args, **kwargs)
|
||||
@@ -514,6 +293,26 @@ class Git(Base):
|
||||
raise NoGitBackendError(name)
|
||||
|
||||
is_ignored = partialmethod(_backend_func, "is_ignored")
|
||||
add = partialmethod(_backend_func, "add")
|
||||
commit = partialmethod(_backend_func, "commit")
|
||||
checkout = partialmethod(_backend_func, "checkout")
|
||||
pull = partialmethod(_backend_func, "pull")
|
||||
push = partialmethod(_backend_func, "push")
|
||||
branch = partialmethod(_backend_func, "branch")
|
||||
tag = partialmethod(_backend_func, "tag")
|
||||
untracked_files = partialmethod(_backend_func, "untracked_files")
|
||||
is_tracked = partialmethod(_backend_func, "is_tracked")
|
||||
is_dirty = partialmethod(_backend_func, "is_dirty")
|
||||
active_branch = partialmethod(_backend_func, "active_branch")
|
||||
list_branches = partialmethod(_backend_func, "list_branches")
|
||||
list_tags = partialmethod(_backend_func, "list_tags")
|
||||
list_all_commits = partialmethod(_backend_func, "list_all_commits")
|
||||
get_tree = partialmethod(_backend_func, "get_tree")
|
||||
get_rev = partialmethod(_backend_func, "get_rev")
|
||||
resolve_rev = partialmethod(_backend_func, "resolve_rev")
|
||||
resolve_commit = partialmethod(_backend_func, "resolve_commit")
|
||||
branch_revs = partialmethod(_backend_func, "branch_revs")
|
||||
|
||||
set_ref = partialmethod(_backend_func, "set_ref")
|
||||
get_ref = partialmethod(_backend_func, "get_ref")
|
||||
remove_ref = partialmethod(_backend_func, "remove_ref")
|
||||
|
||||
+112
-7
@@ -1,11 +1,9 @@
|
||||
import os
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import TYPE_CHECKING, Callable, Iterable, Optional, Tuple
|
||||
from typing import Callable, Iterable, Optional, Tuple
|
||||
|
||||
from dvc.scm.base import SCMError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.scm.git import Git
|
||||
from dvc.tree.base import BaseTree
|
||||
|
||||
|
||||
class NoGitBackendError(SCMError):
|
||||
@@ -16,9 +14,11 @@ class NoGitBackendError(SCMError):
|
||||
class BaseGitBackend(ABC):
|
||||
"""Base Git backend class."""
|
||||
|
||||
def __init__(self, scm: "Git", **kwargs):
|
||||
self.scm = scm
|
||||
self.root_dir = os.fspath(scm.root_dir)
|
||||
@abstractmethod
|
||||
def __init__(
|
||||
self, scm, root_dir=os.curdir, search_parent_directories=True
|
||||
):
|
||||
pass
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
@@ -27,6 +27,111 @@ class BaseGitBackend(ABC):
|
||||
def is_ignored(self, path):
|
||||
"""Return True if the specified path is gitignored."""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def root_dir(self) -> str:
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def clone(
|
||||
url: str,
|
||||
to_path: str,
|
||||
rev: Optional[str] = None,
|
||||
shallow_branch: Optional[str] = None,
|
||||
):
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def is_sha(rev: str) -> bool:
|
||||
pass
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def dir(self) -> str:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def add(self, paths: Iterable[str]):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def commit(self, msg: str):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def checkout(
|
||||
self, branch: str, create_new: Optional[bool] = False, **kwargs,
|
||||
):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def pull(self, **kwargs):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def push(self):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def branch(self, branch: str):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def tag(self, tag: str):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def untracked_files(self) -> Iterable[str]:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def is_tracked(self, path: str) -> bool:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def is_dirty(self, **kwargs) -> bool:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def active_branch(self) -> str:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def list_branches(self) -> Iterable[str]:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def list_tags(self) -> Iterable[str]:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def list_all_commits(self) -> Iterable[str]:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_tree(self, rev: str, **kwargs) -> BaseTree:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_rev(self) -> str:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def resolve_rev(self, rev: str) -> str:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def resolve_commit(self, rev: str) -> str:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def branch_revs(
|
||||
self, branch: str, end_rev: Optional[str] = None
|
||||
) -> Iterable[str]:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def set_ref(
|
||||
self,
|
||||
|
||||
@@ -2,32 +2,121 @@ import fnmatch
|
||||
import logging
|
||||
import os
|
||||
from io import BytesIO
|
||||
from typing import TYPE_CHECKING, Callable, Iterable, Optional, Tuple
|
||||
from typing import Callable, Iterable, Optional, Tuple
|
||||
|
||||
from dvc.scm.base import SCMError
|
||||
from dvc.tree.base import BaseTree
|
||||
from dvc.utils import relpath
|
||||
|
||||
from .base import BaseGitBackend
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from dvc.scm.git import Git
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class DulwichBackend(BaseGitBackend): # pylint:disable=abstract-method
|
||||
"""Dulwich Git backend."""
|
||||
|
||||
def __init__(self, scm: "Git", **kwargs):
|
||||
def __init__( # pylint:disable=W0231
|
||||
self, scm, root_dir=os.curdir, search_parent_directories=True
|
||||
):
|
||||
from dulwich.errors import NotGitRepository
|
||||
from dulwich.repo import Repo
|
||||
|
||||
super().__init__(scm, **kwargs)
|
||||
self.repo = Repo(self.root_dir)
|
||||
self.scm = scm
|
||||
|
||||
try:
|
||||
if search_parent_directories:
|
||||
self.repo = Repo.discover(start=root_dir)
|
||||
else:
|
||||
self.repo = Repo(root_dir)
|
||||
except NotGitRepository as exc:
|
||||
raise SCMError(f"{root_dir} is not a git repository") from exc
|
||||
|
||||
self._stashes: dict = {}
|
||||
|
||||
def close(self):
|
||||
self.repo.close()
|
||||
|
||||
@property
|
||||
def root_dir(self) -> str:
|
||||
return self.repo.path
|
||||
|
||||
@staticmethod
|
||||
def clone(
|
||||
url: str,
|
||||
to_path: str,
|
||||
rev: Optional[str] = None,
|
||||
shallow_branch: Optional[str] = None,
|
||||
):
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
def is_sha(rev: str) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
def dir(self) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def add(self, paths: Iterable[str]):
|
||||
raise NotImplementedError
|
||||
|
||||
def commit(self, msg: str):
|
||||
raise NotImplementedError
|
||||
|
||||
def checkout(
|
||||
self, branch: str, create_new: Optional[bool] = False, **kwargs,
|
||||
):
|
||||
raise NotImplementedError
|
||||
|
||||
def pull(self, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
def push(self):
|
||||
raise NotImplementedError
|
||||
|
||||
def branch(self, branch: str):
|
||||
raise NotImplementedError
|
||||
|
||||
def tag(self, tag: str):
|
||||
raise NotImplementedError
|
||||
|
||||
def untracked_files(self) -> Iterable[str]:
|
||||
raise NotImplementedError
|
||||
|
||||
def is_tracked(self, path: str) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
def is_dirty(self, **kwargs) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
def active_branch(self) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def list_branches(self) -> Iterable[str]:
|
||||
raise NotImplementedError
|
||||
|
||||
def list_tags(self) -> Iterable[str]:
|
||||
raise NotImplementedError
|
||||
|
||||
def list_all_commits(self) -> Iterable[str]:
|
||||
raise NotImplementedError
|
||||
|
||||
def get_tree(self, rev: str, **kwargs) -> BaseTree:
|
||||
raise NotImplementedError
|
||||
|
||||
def get_rev(self) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def resolve_rev(self, rev: str) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def resolve_commit(self, rev: str) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def branch_revs(self, branch: str, end_rev: Optional[str] = None):
|
||||
raise NotImplementedError
|
||||
|
||||
def _get_stash(self, ref: str):
|
||||
from dulwich.stash import Stash as DulwichStash
|
||||
|
||||
|
||||
@@ -1,21 +1,294 @@
|
||||
import logging
|
||||
import os
|
||||
from functools import partial
|
||||
from typing import Callable, Iterable, Optional, Tuple
|
||||
|
||||
from funcy import first
|
||||
|
||||
from dvc.progress import Tqdm
|
||||
from dvc.scm.base import CloneError, RevError, SCMError
|
||||
from dvc.tree.base import BaseTree
|
||||
from dvc.utils import fix_env, is_binary
|
||||
|
||||
from .base import BaseGitBackend
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TqdmGit(Tqdm):
|
||||
def update_git(self, op_code, cur_count, max_count=None, message=""):
|
||||
op_code = self.code2desc(op_code)
|
||||
if op_code:
|
||||
message = (op_code + " | " + message) if message else op_code
|
||||
if message:
|
||||
self.postfix["info"] = f" {message} | "
|
||||
self.update_to(cur_count, max_count)
|
||||
|
||||
@staticmethod
|
||||
def code2desc(op_code):
|
||||
from git import RootUpdateProgress as OP
|
||||
|
||||
ops = {
|
||||
OP.COUNTING: "Counting",
|
||||
OP.COMPRESSING: "Compressing",
|
||||
OP.WRITING: "Writing",
|
||||
OP.RECEIVING: "Receiving",
|
||||
OP.RESOLVING: "Resolving",
|
||||
OP.FINDING_SOURCES: "Finding sources",
|
||||
OP.CHECKING_OUT: "Checking out",
|
||||
OP.CLONE: "Cloning",
|
||||
OP.FETCH: "Fetching",
|
||||
OP.UPDWKTREE: "Updating working tree",
|
||||
OP.REMOVE: "Removing",
|
||||
OP.PATHCHANGE: "Changing path",
|
||||
OP.URLCHANGE: "Changing URL",
|
||||
OP.BRANCHCHANGE: "Changing branch",
|
||||
}
|
||||
return ops.get(op_code & OP.OP_MASK, "")
|
||||
|
||||
|
||||
class GitPythonBackend(BaseGitBackend): # pylint:disable=abstract-method
|
||||
"""git-python Git backend."""
|
||||
|
||||
def __init__( # pylint:disable=W0231
|
||||
self, scm, root_dir=os.curdir, search_parent_directories=True
|
||||
):
|
||||
import git
|
||||
from git.exc import InvalidGitRepositoryError
|
||||
|
||||
self.scm = scm
|
||||
|
||||
try:
|
||||
self.repo = git.Repo(
|
||||
root_dir, search_parent_directories=search_parent_directories
|
||||
)
|
||||
except InvalidGitRepositoryError:
|
||||
msg = "{} is not a git repository"
|
||||
raise SCMError(msg.format(root_dir))
|
||||
|
||||
# NOTE: fixing LD_LIBRARY_PATH for binary built by PyInstaller.
|
||||
# http://pyinstaller.readthedocs.io/en/stable/runtime-information.html
|
||||
env = fix_env(None)
|
||||
libpath = env.get("LD_LIBRARY_PATH", None)
|
||||
self.repo.git.update_environment(LD_LIBRARY_PATH=libpath)
|
||||
|
||||
def close(self):
|
||||
self.repo.close()
|
||||
|
||||
@property
|
||||
def git(self):
|
||||
return self.scm.repo.git
|
||||
return self.repo.git
|
||||
|
||||
def is_ignored(self, path):
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
def root_dir(self) -> str:
|
||||
return self.repo.working_tree_dir
|
||||
|
||||
@staticmethod
|
||||
def clone(
|
||||
url: str,
|
||||
to_path: str,
|
||||
rev: Optional[str] = None,
|
||||
shallow_branch: Optional[str] = None,
|
||||
):
|
||||
import git
|
||||
|
||||
ld_key = "LD_LIBRARY_PATH"
|
||||
|
||||
env = fix_env(None)
|
||||
if is_binary() and ld_key not in env.keys():
|
||||
# In fix_env, we delete LD_LIBRARY_PATH key if it was empty before
|
||||
# PyInstaller modified it. GitPython, in git.Repo.clone_from, uses
|
||||
# env to update its own internal state. When there is no key in
|
||||
# env, this value is not updated and GitPython re-uses
|
||||
# LD_LIBRARY_PATH that has been set by PyInstaller.
|
||||
# See [1] for more info.
|
||||
# [1] https://github.com/gitpython-developers/GitPython/issues/924
|
||||
env[ld_key] = ""
|
||||
|
||||
try:
|
||||
if shallow_branch is not None and os.path.exists(url):
|
||||
# git disables --depth for local clones unless file:// url
|
||||
# scheme is used
|
||||
url = f"file://{url}"
|
||||
with TqdmGit(desc="Cloning", unit="obj") as pbar:
|
||||
clone_from = partial(
|
||||
git.Repo.clone_from,
|
||||
url,
|
||||
to_path,
|
||||
env=env, # needed before we can fix it in __init__
|
||||
no_single_branch=True,
|
||||
progress=pbar.update_git,
|
||||
)
|
||||
if shallow_branch is None:
|
||||
tmp_repo = clone_from()
|
||||
else:
|
||||
tmp_repo = clone_from(branch=shallow_branch, depth=1)
|
||||
tmp_repo.close()
|
||||
except git.exc.GitCommandError as exc: # pylint: disable=no-member
|
||||
raise CloneError(url, to_path) from exc
|
||||
|
||||
# NOTE: using our wrapper to make sure that env is fixed in __init__
|
||||
repo = GitPythonBackend(None, to_path)
|
||||
|
||||
if rev:
|
||||
try:
|
||||
repo.checkout(rev)
|
||||
except git.exc.GitCommandError as exc: # pylint: disable=no-member
|
||||
raise RevError(
|
||||
"failed to access revision '{}' for repo '{}'".format(
|
||||
rev, url
|
||||
)
|
||||
) from exc
|
||||
|
||||
@staticmethod
|
||||
def is_sha(rev):
|
||||
import git
|
||||
|
||||
return rev and git.Repo.re_hexsha_shortened.search(rev)
|
||||
|
||||
@property
|
||||
def dir(self) -> str:
|
||||
return self.repo.git_dir
|
||||
|
||||
def add(self, paths: Iterable[str]):
|
||||
# NOTE: GitPython is not currently able to handle index version >= 3.
|
||||
# See https://github.com/iterative/dvc/issues/610 for more details.
|
||||
try:
|
||||
self.repo.index.add(paths)
|
||||
except AssertionError:
|
||||
msg = (
|
||||
"failed to add '{}' to git. You can add those files "
|
||||
"manually using `git add`. See "
|
||||
"https://github.com/iterative/dvc/issues/610 for more "
|
||||
"details.".format(str(paths))
|
||||
)
|
||||
|
||||
logger.exception(msg)
|
||||
|
||||
def commit(self, msg: str):
|
||||
self.repo.index.commit(msg)
|
||||
|
||||
def checkout(
|
||||
self, branch: str, create_new: Optional[bool] = False, **kwargs
|
||||
):
|
||||
if create_new:
|
||||
self.repo.git.checkout("HEAD", b=branch, **kwargs)
|
||||
else:
|
||||
self.repo.git.checkout(branch, **kwargs)
|
||||
|
||||
def pull(self, **kwargs):
|
||||
infos = self.repo.remote().pull(**kwargs)
|
||||
for info in infos:
|
||||
if info.flags & info.ERROR:
|
||||
raise SCMError(f"pull failed: {info.note}")
|
||||
|
||||
def push(self):
|
||||
infos = self.repo.remote().push()
|
||||
for info in infos:
|
||||
if info.flags & info.ERROR:
|
||||
raise SCMError(f"push failed: {info.summary}")
|
||||
|
||||
def branch(self, branch):
|
||||
self.repo.git.branch(branch)
|
||||
|
||||
def tag(self, tag):
|
||||
self.repo.git.tag(tag)
|
||||
|
||||
def untracked_files(self):
|
||||
files = self.repo.untracked_files
|
||||
return [os.path.join(self.repo.working_dir, fname) for fname in files]
|
||||
|
||||
def is_tracked(self, path):
|
||||
return bool(self.repo.git.ls_files(path))
|
||||
|
||||
def is_dirty(self, **kwargs):
|
||||
return self.repo.is_dirty(**kwargs)
|
||||
|
||||
def active_branch(self):
|
||||
return self.repo.active_branch.name
|
||||
|
||||
def list_branches(self):
|
||||
return [h.name for h in self.repo.heads]
|
||||
|
||||
def list_tags(self):
|
||||
return [t.name for t in self.repo.tags]
|
||||
|
||||
def list_all_commits(self):
|
||||
return [c.hexsha for c in self.repo.iter_commits("--all")]
|
||||
|
||||
def get_tree(self, rev: str, **kwargs) -> BaseTree:
|
||||
from dvc.tree.git import GitTree
|
||||
|
||||
return GitTree(self.repo, self.resolve_rev(rev), **kwargs)
|
||||
|
||||
def get_rev(self):
|
||||
return self.repo.rev_parse("HEAD").hexsha
|
||||
|
||||
def resolve_rev(self, rev):
|
||||
from contextlib import suppress
|
||||
|
||||
from git.exc import BadName, GitCommandError
|
||||
|
||||
def _resolve_rev(name):
|
||||
with suppress(BadName, GitCommandError):
|
||||
try:
|
||||
# Try python implementation of rev-parse first, it's faster
|
||||
return self.repo.rev_parse(name).hexsha
|
||||
except NotImplementedError:
|
||||
# Fall back to `git rev-parse` for advanced features
|
||||
return self.repo.git.rev_parse(name)
|
||||
except ValueError:
|
||||
raise RevError(f"unknown Git revision '{name}'")
|
||||
|
||||
# Resolve across local names
|
||||
sha = _resolve_rev(rev)
|
||||
if sha:
|
||||
return sha
|
||||
|
||||
# Try all the remotes and if it resolves unambiguously then take it
|
||||
if not self.is_sha(rev):
|
||||
shas = {
|
||||
_resolve_rev(f"{remote.name}/{rev}")
|
||||
for remote in self.repo.remotes
|
||||
} - {None}
|
||||
if len(shas) > 1:
|
||||
raise RevError(f"ambiguous Git revision '{rev}'")
|
||||
if len(shas) == 1:
|
||||
return shas.pop()
|
||||
|
||||
raise RevError(f"unknown Git revision '{rev}'")
|
||||
|
||||
def resolve_commit(self, rev):
|
||||
"""Return Commit object for the specified revision."""
|
||||
from git.exc import BadName, GitCommandError
|
||||
from git.objects.tag import TagObject
|
||||
|
||||
try:
|
||||
commit = self.repo.rev_parse(rev)
|
||||
except (BadName, GitCommandError):
|
||||
commit = None
|
||||
if isinstance(commit, TagObject):
|
||||
commit = commit.object
|
||||
return commit
|
||||
|
||||
def branch_revs(
|
||||
self, branch: str, end_rev: Optional[str] = None
|
||||
) -> Iterable[str]:
|
||||
"""Iterate over revisions in a given branch (from newest to oldest).
|
||||
|
||||
If end_rev is set, iterator will stop when the specified revision is
|
||||
reached.
|
||||
"""
|
||||
commit = self.resolve_commit(branch)
|
||||
while commit is not None:
|
||||
yield commit.hexsha
|
||||
commit = first(commit.parents)
|
||||
if commit and commit.hexsha == end_rev:
|
||||
return
|
||||
|
||||
def set_ref(
|
||||
self,
|
||||
name: str,
|
||||
@@ -35,7 +308,7 @@ class GitPythonBackend(BaseGitBackend): # pylint:disable=abstract-method
|
||||
def iter_refs(self, base: Optional[str] = None):
|
||||
from git import Reference
|
||||
|
||||
for ref in Reference.iter_items(self.scm.repo, common_path=base):
|
||||
for ref in Reference.iter_items(self.repo, common_path=base):
|
||||
if base and base.endswith("/"):
|
||||
base = base[:-1]
|
||||
yield "/".join([base, ref.name])
|
||||
@@ -86,7 +359,7 @@ class GitPythonBackend(BaseGitBackend): # pylint:disable=abstract-method
|
||||
if include_untracked:
|
||||
args.append("--include-untracked")
|
||||
self.git.stash(*args)
|
||||
commit = self.scm.resolve_commit("stash@{0}")
|
||||
commit = self.resolve_commit("stash@{0}")
|
||||
if ref != Stash.DEFAULT_STASH:
|
||||
# `git stash` CLI doesn't support using custom refspecs,
|
||||
# so we push a commit onto refs/stash, make our refspec
|
||||
@@ -94,7 +367,9 @@ class GitPythonBackend(BaseGitBackend): # pylint:disable=abstract-method
|
||||
# `git stash create` is intended to be used for this kind of
|
||||
# behavior but it doesn't support --include-untracked so we need to
|
||||
# use push
|
||||
self.scm.set_ref(ref, commit.hexsha, message=commit.message)
|
||||
self.scm.dulwich.set_ref(
|
||||
ref, commit.hexsha, message=commit.message
|
||||
)
|
||||
self.git.stash("drop")
|
||||
return commit.hexsha, False
|
||||
|
||||
|
||||
@@ -19,10 +19,6 @@ class Stash:
|
||||
self.ref = ref if ref else self.DEFAULT_STASH
|
||||
self.scm = scm
|
||||
|
||||
@property
|
||||
def git(self):
|
||||
return self.scm.repo.git
|
||||
|
||||
def __iter__(self):
|
||||
yield from self.scm._stash_iter(self.ref)
|
||||
|
||||
@@ -49,7 +45,7 @@ class Stash:
|
||||
self.ref, message=message, include_untracked=include_untracked
|
||||
)
|
||||
if reset:
|
||||
self.git.reset(hard=True)
|
||||
self.scm.gitpython.repo.git.reset(hard=True)
|
||||
return rev
|
||||
|
||||
def pop(self):
|
||||
|
||||
@@ -60,8 +60,8 @@ def test_reset_checkpoint(tmp_dir, scm, dvc, checkpoint_stage, caplog):
|
||||
from dvc.repo.experiments.base import CheckpointExistsError
|
||||
|
||||
dvc.experiments.run(checkpoint_stage.addressing, name="foo")
|
||||
scm.repo.git.reset(hard=True)
|
||||
scm.repo.git.clean(force=True)
|
||||
scm.gitpython.repo.git.reset(hard=True)
|
||||
scm.gitpython.repo.git.clean(force=True)
|
||||
|
||||
with pytest.raises(CheckpointExistsError):
|
||||
dvc.experiments.run(
|
||||
@@ -113,6 +113,6 @@ def test_resume_branch(tmp_dir, scm, dvc, checkpoint_stage):
|
||||
with pytest.raises(MultipleBranchError):
|
||||
dvc.experiments.get_branch_containing(branch_rev)
|
||||
|
||||
assert branch_rev == dvc.experiments.scm.repo.git.merge_base(
|
||||
assert branch_rev == dvc.experiments.scm.gitpython.repo.git.merge_base(
|
||||
checkpoint_a, checkpoint_b
|
||||
)
|
||||
|
||||
@@ -258,7 +258,7 @@ def test_detached_parent(tmp_dir, scm, dvc, exp_stage, mocker):
|
||||
scm.commit("v2")
|
||||
|
||||
scm.checkout(detached_rev)
|
||||
assert scm.repo.head.is_detached
|
||||
assert scm.gitpython.repo.head.is_detached
|
||||
results = dvc.experiments.run(exp_stage.addressing, params=["foo=3"])
|
||||
|
||||
exp_rev = first(results)
|
||||
|
||||
@@ -41,7 +41,7 @@ def test_show_experiment(tmp_dir, scm, dvc):
|
||||
scm.commit("baseline")
|
||||
baseline_rev = scm.get_rev()
|
||||
timestamp = datetime.fromtimestamp(
|
||||
scm.repo.rev_parse(baseline_rev).committed_date
|
||||
scm.gitpython.repo.rev_parse(baseline_rev).committed_date
|
||||
)
|
||||
|
||||
dvc.experiments.run(PIPELINE_FILE, params=["foo=2"])
|
||||
|
||||
@@ -580,7 +580,7 @@ def test_stats_on_checkout(tmp_dir, dvc, scm):
|
||||
|
||||
tmp_dir.gen({"lorem": "lorem", "bar": "new bar", "dir2": {"file": "file"}})
|
||||
(tmp_dir / "foo").unlink()
|
||||
scm.repo.git.rm("foo.dvc")
|
||||
scm.gitpython.repo.git.rm("foo.dvc")
|
||||
tmp_dir.dvc_add(["bar", "lorem", "dir2"], commit="second")
|
||||
|
||||
scm.checkout("HEAD~")
|
||||
|
||||
+10
-10
@@ -25,7 +25,7 @@ def test_import(tmp_dir, scm, dvc, erepo_dir):
|
||||
|
||||
assert os.path.isfile("foo_imported")
|
||||
assert (tmp_dir / "foo_imported").read_text() == "foo content"
|
||||
assert scm.repo.git.check_ignore("foo_imported")
|
||||
assert scm.is_ignored("foo_imported")
|
||||
assert stage.deps[0].def_repo == {
|
||||
"url": os.fspath(erepo_dir),
|
||||
"rev_lock": erepo_dir.scm.get_rev(),
|
||||
@@ -42,7 +42,7 @@ def test_import_git_file(tmp_dir, scm, dvc, git_dir, src_is_dvc):
|
||||
stage = tmp_dir.dvc.imp(os.fspath(git_dir), "src", "dst")
|
||||
|
||||
assert (tmp_dir / "dst").read_text() == "hello"
|
||||
assert tmp_dir.scm.repo.git.check_ignore(os.fspath(tmp_dir / "dst"))
|
||||
assert tmp_dir.scm.is_ignored(os.fspath(tmp_dir / "dst"))
|
||||
assert stage.deps[0].def_repo == {
|
||||
"url": os.fspath(git_dir),
|
||||
"rev_lock": git_dir.scm.get_rev(),
|
||||
@@ -77,7 +77,7 @@ def test_import_git_dir(tmp_dir, scm, dvc, git_dir, src_is_dvc):
|
||||
stage = dvc.imp(os.fspath(git_dir), "src", "dst")
|
||||
|
||||
assert (tmp_dir / "dst").read_text() == {"file.txt": "hello"}
|
||||
assert tmp_dir.scm.repo.git.check_ignore(os.fspath(tmp_dir / "dst"))
|
||||
assert tmp_dir.scm.is_ignored(os.fspath(tmp_dir / "dst"))
|
||||
assert stage.deps[0].def_repo == {
|
||||
"url": os.fspath(git_dir),
|
||||
"rev_lock": git_dir.scm.get_rev(),
|
||||
@@ -91,7 +91,7 @@ def test_import_dir(tmp_dir, scm, dvc, erepo_dir):
|
||||
stage = dvc.imp(os.fspath(erepo_dir), "dir", "dir_imported")
|
||||
|
||||
assert (tmp_dir / "dir_imported").read_text() == {"foo": "foo content"}
|
||||
assert scm.repo.git.check_ignore("dir_imported")
|
||||
assert scm.is_ignored("dir_imported")
|
||||
assert stage.deps[0].def_repo == {
|
||||
"url": os.fspath(erepo_dir),
|
||||
"rev_lock": erepo_dir.scm.get_rev(),
|
||||
@@ -114,7 +114,7 @@ def test_import_file_from_dir(tmp_dir, scm, dvc, erepo_dir):
|
||||
stage = dvc.imp(os.fspath(erepo_dir), os.path.join("dir", "1"))
|
||||
|
||||
assert (tmp_dir / "1").read_text() == "1"
|
||||
assert scm.repo.git.check_ignore("1")
|
||||
assert scm.is_ignored("1")
|
||||
assert stage.deps[0].def_repo == {
|
||||
"url": os.fspath(erepo_dir),
|
||||
"rev_lock": erepo_dir.scm.get_rev(),
|
||||
@@ -175,7 +175,7 @@ def test_import_non_cached(erepo_dir, tmp_dir, dvc, scm):
|
||||
|
||||
assert (tmp_dir / dst).is_file()
|
||||
assert filecmp.cmp(erepo_dir / src, tmp_dir / dst, shallow=False)
|
||||
assert tmp_dir.scm.repo.git.check_ignore(dst)
|
||||
assert tmp_dir.scm.is_ignored(dst)
|
||||
assert stage.deps[0].def_repo == {
|
||||
"url": os.fspath(erepo_dir),
|
||||
"rev_lock": erepo_dir.scm.get_rev(),
|
||||
@@ -191,7 +191,7 @@ def test_import_rev(tmp_dir, scm, dvc, erepo_dir):
|
||||
stage = dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported", rev="branch")
|
||||
|
||||
assert (tmp_dir / "foo_imported").read_text() == "foo content"
|
||||
assert scm.repo.git.check_ignore("foo_imported")
|
||||
assert scm.is_ignored("foo_imported")
|
||||
assert stage.deps[0].def_repo == {
|
||||
"url": os.fspath(erepo_dir),
|
||||
"rev": "branch",
|
||||
@@ -231,7 +231,7 @@ def test_cache_type_is_properly_overridden(tmp_dir, scm, dvc, erepo_dir):
|
||||
|
||||
assert not System.is_symlink("foo_imported")
|
||||
assert (tmp_dir / "foo_imported").read_text() == "foo content"
|
||||
assert scm.repo.git.check_ignore("foo_imported")
|
||||
assert scm.is_ignored("foo_imported")
|
||||
|
||||
|
||||
def test_pull_imported_directory_stage(tmp_dir, dvc, erepo_dir):
|
||||
@@ -340,8 +340,8 @@ def test_import_from_bare_git_repo(
|
||||
erepo_dir.dvc_gen({"foo": "foo"}, commit="initial")
|
||||
erepo_dir.dvc.push()
|
||||
|
||||
erepo_dir.scm.repo.create_remote("origin", os.fspath(tmp_dir))
|
||||
erepo_dir.scm.repo.remote("origin").push("master")
|
||||
erepo_dir.scm.gitpython.repo.create_remote("origin", os.fspath(tmp_dir))
|
||||
erepo_dir.scm.gitpython.repo.remote("origin").push("master")
|
||||
|
||||
dvc_repo = make_tmp_dir("dvc-repo", scm=True, dvc=True)
|
||||
with dvc_repo.chdir():
|
||||
|
||||
@@ -43,7 +43,7 @@ class TestInstall:
|
||||
|
||||
# scm.commit bypasses hooks
|
||||
with pytest.raises(GitCommandError, match=r"modified:\s*file"):
|
||||
scm.repo.git.commit(m="file modified")
|
||||
scm.gitpython.repo.git.commit(m="file modified")
|
||||
|
||||
def test_post_checkout(self, tmp_dir, scm, dvc):
|
||||
tmp_dir.dvc_gen({"file": "file content"}, commit="add")
|
||||
@@ -69,13 +69,13 @@ class TestInstall:
|
||||
storage_path / file_checksum[:2] / file_checksum[2:]
|
||||
)
|
||||
|
||||
scm.repo.clone(os.fspath(git_remote))
|
||||
scm.repo.create_remote("origin", os.fspath(git_remote))
|
||||
scm.gitpython.repo.clone(os.fspath(git_remote))
|
||||
scm.gitpython.repo.create_remote("origin", os.fspath(git_remote))
|
||||
|
||||
scm.install()
|
||||
|
||||
assert not expected_storage_path.is_file()
|
||||
scm.repo.git.push("origin", "master")
|
||||
scm.gitpython.repo.git.push("origin", "master")
|
||||
assert expected_storage_path.is_file()
|
||||
assert expected_storage_path.read_text() == "file_content"
|
||||
|
||||
@@ -97,7 +97,9 @@ def test_merge_driver_no_ancestor(tmp_dir, scm, dvc):
|
||||
scm.install()
|
||||
(tmp_dir / ".gitattributes").write_text("*.dvc merge=dvc")
|
||||
|
||||
scm.repo.git.merge("one", m="merged", no_gpg_sign=True, no_signoff=True)
|
||||
scm.gitpython.repo.git.merge(
|
||||
"one", m="merged", no_gpg_sign=True, no_signoff=True
|
||||
)
|
||||
|
||||
# NOTE: dvc shouldn't checkout automatically as it might take a long time
|
||||
assert (tmp_dir / "data").read_text() == {"bar": "bar"}
|
||||
@@ -132,7 +134,9 @@ def test_merge_driver(tmp_dir, scm, dvc):
|
||||
scm.install()
|
||||
(tmp_dir / ".gitattributes").write_text("*.dvc merge=dvc")
|
||||
|
||||
scm.repo.git.merge("one", m="merged", no_gpg_sign=True, no_signoff=True)
|
||||
scm.gitpython.repo.git.merge(
|
||||
"one", m="merged", no_gpg_sign=True, no_signoff=True
|
||||
)
|
||||
|
||||
# NOTE: dvc shouldn't checkout automatically as it might take a long time
|
||||
assert (tmp_dir / "data").read_text() == {"master": "master", "two": "two"}
|
||||
|
||||
@@ -50,7 +50,7 @@ class TestSCMGit(TestGit):
|
||||
G.commit("add")
|
||||
self.assertTrue(G.is_tracked(foo))
|
||||
self.assertTrue(G.is_tracked(self.FOO))
|
||||
G.repo.index.remove([self.FOO], working_tree=True)
|
||||
G.gitpython.repo.index.remove([self.FOO], working_tree=True)
|
||||
self.assertFalse(G.is_tracked(foo))
|
||||
self.assertFalse(G.is_tracked(self.FOO))
|
||||
self.assertFalse(G.is_tracked("not-existing-file"))
|
||||
@@ -183,9 +183,9 @@ def test_git_stash_workspace(tmp_dir, scm):
|
||||
tmp_dir.gen("file", "1")
|
||||
|
||||
with scm.stash_workspace():
|
||||
assert not scm.repo.is_dirty()
|
||||
assert not scm.is_dirty()
|
||||
assert "0" == (tmp_dir / "file").read_text()
|
||||
assert scm.repo.is_dirty()
|
||||
assert scm.is_dirty()
|
||||
assert "1" == (tmp_dir / "file").read_text()
|
||||
|
||||
|
||||
|
||||
@@ -58,7 +58,7 @@ def test_status_before_and_after_dvc_init(tmp_dir, dvc, git_dir):
|
||||
|
||||
with git_dir.chdir():
|
||||
git_dir.init(dvc=True)
|
||||
git_dir.scm.repo.index.remove(["file"])
|
||||
git_dir.scm.gitpython.repo.index.remove(["file"])
|
||||
os.remove("file")
|
||||
git_dir.dvc_gen("file", "second version", commit="with dvc")
|
||||
new_rev = git_dir.scm.get_rev()
|
||||
|
||||
@@ -97,7 +97,7 @@ def test_update_import_after_remote_updates_to_dvc(tmp_dir, dvc, erepo_dir):
|
||||
|
||||
new_rev = None
|
||||
with erepo_dir.branch("branch", new=False), erepo_dir.chdir():
|
||||
erepo_dir.scm.repo.index.remove(["version"])
|
||||
erepo_dir.scm.gitpython.repo.index.remove(["version"])
|
||||
erepo_dir.dvc_gen(
|
||||
"version", "updated", commit="upgrade to DVC tracking"
|
||||
)
|
||||
@@ -134,7 +134,7 @@ def test_update_before_and_after_dvc_init(tmp_dir, dvc, git_dir):
|
||||
|
||||
with git_dir.chdir():
|
||||
git_dir.init(dvc=True)
|
||||
git_dir.scm.repo.index.remove(["file"])
|
||||
git_dir.scm.gitpython.repo.index.remove(["file"])
|
||||
os.remove("file")
|
||||
git_dir.dvc_gen("file", "second version", commit="with dvc")
|
||||
new_rev = git_dir.scm.get_rev()
|
||||
|
||||
@@ -25,7 +25,9 @@ def test_walk_with_submodules(tmp_dir, scm, git_dir):
|
||||
{"foo": "foo", "bar": "bar", "dir": {"data": "data"}},
|
||||
commit="add dir and files",
|
||||
)
|
||||
scm.repo.create_submodule("submodule", "submodule", url=os.fspath(git_dir))
|
||||
scm.gitpython.repo.create_submodule(
|
||||
"submodule", "submodule", url=os.fspath(git_dir)
|
||||
)
|
||||
scm.commit("added submodule")
|
||||
|
||||
files = []
|
||||
|
||||
Reference in New Issue
Block a user