From 515286663434569f7b5b2e48ec8ed5654ce5d776 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 3 Dec 2020 20:25:20 +0200 Subject: [PATCH] scm: move logic to gitpython backend (#5022) --- dvc/external_repo.py | 6 +- dvc/repo/experiments/__init__.py | 8 +- dvc/repo/experiments/apply.py | 6 +- dvc/repo/experiments/executor.py | 16 +- dvc/scm/git/__init__.py | 335 +++++---------------- dvc/scm/git/backend/base.py | 119 +++++++- dvc/scm/git/backend/dulwich.py | 103 ++++++- dvc/scm/git/backend/gitpython.py | 283 ++++++++++++++++- dvc/scm/git/stash.py | 6 +- tests/func/experiments/test_checkpoints.py | 6 +- tests/func/experiments/test_experiments.py | 2 +- tests/func/experiments/test_show.py | 2 +- tests/func/test_checkout.py | 2 +- tests/func/test_import.py | 20 +- tests/func/test_install.py | 16 +- tests/func/test_scm.py | 6 +- tests/func/test_status.py | 2 +- tests/func/test_update.py | 4 +- tests/unit/scm/test_git.py | 4 +- 19 files changed, 610 insertions(+), 336 deletions(-) diff --git a/dvc/external_repo.py b/dvc/external_repo.py index 7debd028c..9d0004438 100644 --- a/dvc/external_repo.py +++ b/dvc/external_repo.py @@ -369,13 +369,13 @@ def _clone_default_branch(url, rev, for_write=False): def _unshallow(git): - if git.repo.head.is_detached: + if git.gitpython.repo.head.is_detached: # If this is a detached head (i.e. we shallow cloned a tag) switch to # the default branch - origin_refs = git.repo.remotes["origin"].refs + origin_refs = git.gitpython.repo.remotes["origin"].refs ref = origin_refs["HEAD"].reference branch_name = ref.name.split("/")[-1] - branch = git.repo.create_head(branch_name, ref) + branch = git.gitpython.repo.create_head(branch_name, ref) branch.set_tracking_branch(ref) branch.checkout() git.pull(unshallow=True) diff --git a/dvc/repo/experiments/__init__.py b/dvc/repo/experiments/__init__.py index f44ac1904..4571d77a4 100644 --- a/dvc/repo/experiments/__init__.py +++ b/dvc/repo/experiments/__init__.py @@ -205,8 +205,8 @@ class Experiments: ) # Reset/clean any changes before prior workspace is unstashed - self.scm.repo.git.reset(hard=True) - self.scm.repo.git.clean(force=True) + self.scm.gitpython.repo.git.reset(hard=True) + self.scm.gitpython.repo.git.clean(force=True) return stash_rev @@ -447,8 +447,8 @@ class Experiments: for ref in (EXEC_HEAD, EXEC_MERGE, EXEC_BASELINE): self.scm.remove_ref(ref) - self.scm.repo.git.reset(hard=True) - self.scm.repo.git.clean(force=True) + self.scm.gitpython.repo.git.reset(hard=True) + self.scm.gitpython.repo.git.clean(force=True) return executors def _reproduce( diff --git a/dvc/repo/experiments/apply.py b/dvc/repo/experiments/apply.py index d08b3ada7..b86beb152 100644 --- a/dvc/repo/experiments/apply.py +++ b/dvc/repo/experiments/apply.py @@ -48,7 +48,7 @@ def apply(repo, rev, *args, **kwargs): else: workspace = None - repo.scm.repo.git.merge(branch, squash=True, no_commit=True) + repo.scm.gitpython.repo.git.merge(branch, squash=True, no_commit=True) if workspace: try: @@ -56,9 +56,9 @@ def apply(repo, rev, *args, **kwargs): except GitCommandError: # if stash apply returns merge conflicts, prefer experiment # changes over prior stashed changes - repo.scm.repo.git.checkout("--ours", "--", ".") + repo.scm.gitpython.repo.git.checkout("--ours", "--", ".") repo.scm.stash.drop() - repo.scm.repo.git.reset() + repo.scm.gitpython.repo.git.reset() if stash_rev: args_path = os.path.join(repo.tmp_dir, BaseExecutor.PACKED_ARGS_FILE) diff --git a/dvc/repo/experiments/executor.py b/dvc/repo/experiments/executor.py index 83642cf68..32b74aac6 100644 --- a/dvc/repo/experiments/executor.py +++ b/dvc/repo/experiments/executor.py @@ -85,8 +85,10 @@ class BaseExecutor: # committing head = EXEC_BRANCH if branch else EXEC_HEAD self.scm.checkout(head, detach=True) - self.scm.repo.git.merge(EXEC_MERGE, squash=True, no_commit=True) - self.scm.repo.git.reset() + self.scm.gitpython.repo.git.merge( + EXEC_MERGE, squash=True, no_commit=True + ) + self.scm.gitpython.repo.git.reset() self._prune_lockfiles() finally: os.chdir(cwd) @@ -94,11 +96,13 @@ class BaseExecutor: def _prune_lockfiles(self): # NOTE: dirty DVC lock files must be restored to index state to # avoid checking out incorrect persist or checkpoint outs - dirty = [diff.a_path for diff in self.scm.repo.index.diff(None)] + dirty = [ + diff.a_path for diff in self.scm.gitpython.repo.index.diff(None) + ] to_checkout = [fname for fname in dirty if is_lock_file(fname)] - self.scm.repo.index.checkout(paths=to_checkout, force=True) + self.scm.gitpython.repo.index.checkout(paths=to_checkout, force=True) - untracked = self.scm.repo.untracked_files + untracked = self.scm.gitpython.repo.untracked_files to_remove = [fname for fname in untracked if is_lock_file(fname)] for fname in to_remove: remove(fname) @@ -320,7 +324,7 @@ class BaseExecutor: old_ref = None logger.debug("Commit to new experiment branch '%s'", branch) - scm.repo.git.add(A=True) + scm.gitpython.repo.git.add(A=True) scm.commit(f"dvc: commit experiment {exp_hash}") new_rev = scm.get_rev() scm.set_ref(branch, new_rev, old_ref=old_ref) diff --git a/dvc/scm/git/__init__.py b/dvc/scm/git/__init__.py index 3c40856a1..d6fbbd9b0 100644 --- a/dvc/scm/git/__init__.py +++ b/dvc/scm/git/__init__.py @@ -3,23 +3,17 @@ import logging import os import shlex +from collections import OrderedDict from contextlib import contextmanager -from functools import partial, partialmethod +from functools import partialmethod from typing import Optional -from funcy import cached_property, first +from funcy import cached_property from pathspec.patterns import GitWildMatchPattern from dvc.exceptions import GitHookAlreadyExistsError -from dvc.progress import Tqdm -from dvc.scm.base import ( - Base, - CloneError, - FileNotInRepoError, - RevError, - SCMError, -) -from dvc.utils import fix_env, is_binary, relpath +from dvc.scm.base import Base, FileNotInRepoError, RevError +from dvc.utils import relpath from dvc.utils.fs import path_isin from dvc.utils.serialize import modify_yaml @@ -31,150 +25,68 @@ from .stash import Stash logger = logging.getLogger(__name__) -class TqdmGit(Tqdm): - def update_git(self, op_code, cur_count, max_count=None, message=""): - op_code = self.code2desc(op_code) - if op_code: - message = (op_code + " | " + message) if message else op_code - if message: - self.postfix["info"] = f" {message} | " - self.update_to(cur_count, max_count) - - @staticmethod - def code2desc(op_code): - from git import RootUpdateProgress as OP - - ops = { - OP.COUNTING: "Counting", - OP.COMPRESSING: "Compressing", - OP.WRITING: "Writing", - OP.RECEIVING: "Receiving", - OP.RESOLVING: "Resolving", - OP.FINDING_SOURCES: "Finding sources", - OP.CHECKING_OUT: "Checking out", - OP.CLONE: "Cloning", - OP.FETCH: "Fetching", - OP.UPDWKTREE: "Updating working tree", - OP.REMOVE: "Removing", - OP.PATHCHANGE: "Changing path", - OP.URLCHANGE: "Changing URL", - OP.BRANCHCHANGE: "Changing branch", - } - return ops.get(op_code & OP.OP_MASK, "") - - class Git(Base): """Class for managing Git.""" GITIGNORE = ".gitignore" GIT_DIR = ".git" - DEFAULT_BACKENDS = (DulwichBackend, GitPythonBackend) + DEFAULT_BACKENDS = OrderedDict( + [("dulwich", DulwichBackend), ("gitpython", GitPythonBackend)] + ) LOCAL_BRANCH_PREFIX = "refs/heads/" - def __init__(self, root_dir=os.curdir, search_parent_directories=True): - """Git class constructor. - Requires `Repo` class from `git` module (from gitpython package). - """ - super().__init__(root_dir) - - import git - from git.exc import InvalidGitRepositoryError - - try: - self.repo = git.Repo( - root_dir, search_parent_directories=search_parent_directories - ) - except InvalidGitRepositoryError: - msg = "{} is not a git repository" - raise SCMError(msg.format(root_dir)) - - # NOTE: fixing LD_LIBRARY_PATH for binary built by PyInstaller. - # http://pyinstaller.readthedocs.io/en/stable/runtime-information.html - env = fix_env(None) - libpath = env.get("LD_LIBRARY_PATH", None) - self.repo.git.update_environment(LD_LIBRARY_PATH=libpath) - + def __init__(self, *args, **kwargs): self.ignored_paths = [] self.files_to_track = set() - self.backends = [backend(self) for backend in self.DEFAULT_BACKENDS] + self.backends = OrderedDict( + [ + (name, cls(self, *args, **kwargs)) + for name, cls in self.DEFAULT_BACKENDS.items() + ] + ) + + super().__init__(self.gitpython.root_dir) @property - def root_dir(self) -> str: - return self.repo.working_tree_dir + def dir(self): + return self.gitpython.repo.git_dir + + @property + def gitpython(self): + return self.backends["gitpython"] + + @property + def dulwich(self): + return self.backends["dulwich"] @cached_property def stash(self): return Stash(self) - @staticmethod - def clone(url, to_path, rev=None, shallow_branch=None): - import git - - ld_key = "LD_LIBRARY_PATH" - - env = fix_env(None) - if is_binary() and ld_key not in env.keys(): - # In fix_env, we delete LD_LIBRARY_PATH key if it was empty before - # PyInstaller modified it. GitPython, in git.Repo.clone_from, uses - # env to update its own internal state. When there is no key in - # env, this value is not updated and GitPython re-uses - # LD_LIBRARY_PATH that has been set by PyInstaller. - # See [1] for more info. - # [1] https://github.com/gitpython-developers/GitPython/issues/924 - env[ld_key] = "" - - try: - if shallow_branch is not None and os.path.exists(url): - # git disables --depth for local clones unless file:// url - # scheme is used - url = f"file://{url}" - with TqdmGit(desc="Cloning", unit="obj") as pbar: - clone_from = partial( - git.Repo.clone_from, - url, - to_path, - env=env, # needed before we can fix it in __init__ - no_single_branch=True, - progress=pbar.update_git, - ) - if shallow_branch is None: - tmp_repo = clone_from() - else: - tmp_repo = clone_from(branch=shallow_branch, depth=1) - tmp_repo.close() - except git.exc.GitCommandError as exc: # pylint: disable=no-member - raise CloneError(url, to_path) from exc - - # NOTE: using our wrapper to make sure that env is fixed in __init__ - repo = Git(to_path) - - if rev: + @classmethod + def clone(cls, url, to_path, **kwargs): + for _, backend in cls.DEFAULT_BACKENDS.items(): try: - repo.checkout(rev) - except git.exc.GitCommandError as exc: # pylint: disable=no-member - raise RevError( - "failed to access revision '{}' for repo '{}'".format( - rev, url - ) - ) from exc + backend.clone(url, to_path, **kwargs) + return Git(to_path) + except NotImplementedError: + pass + raise NoGitBackendError("clone") - return repo - - @staticmethod - def is_sha(rev): - import git - - return rev and git.Repo.re_hexsha_shortened.search(rev) + @classmethod + def is_sha(cls, rev): + for _, backend in cls.DEFAULT_BACKENDS.items(): + try: + return backend.is_sha(rev) + except NotImplementedError: + pass + raise NoGitBackendError("is_sha") @staticmethod def _get_git_dir(root_dir): return os.path.join(root_dir, Git.GIT_DIR) - @property - def dir(self): - return self.repo.git_dir - @property def ignore_file(self): return self.GITIGNORE @@ -245,70 +157,6 @@ class Git(Base): self.track_file(relpath(gitignore)) - def add(self, paths): - # NOTE: GitPython is not currently able to handle index version >= 3. - # See https://github.com/iterative/dvc/issues/610 for more details. - try: - self.repo.index.add(paths) - except AssertionError: - msg = ( - "failed to add '{}' to git. You can add those files " - "manually using `git add`. See " - "https://github.com/iterative/dvc/issues/610 for more " - "details.".format(str(paths)) - ) - - logger.exception(msg) - - def commit(self, msg): - self.repo.index.commit(msg) - - def checkout(self, branch, create_new=False, **kwargs): - if create_new: - self.repo.git.checkout("HEAD", b=branch, **kwargs) - else: - self.repo.git.checkout(branch, **kwargs) - - def pull(self, **kwargs): - infos = self.repo.remote().pull(**kwargs) - for info in infos: - if info.flags & info.ERROR: - raise SCMError(f"pull failed: {info.note}") - - def push(self): - infos = self.repo.remote().push() - for info in infos: - if info.flags & info.ERROR: - raise SCMError(f"push failed: {info.summary}") - - def branch(self, branch): - self.repo.git.branch(branch) - - def tag(self, tag): - self.repo.git.tag(tag) - - def untracked_files(self): - files = self.repo.untracked_files - return [os.path.join(self.repo.working_dir, fname) for fname in files] - - def is_tracked(self, path): - return bool(self.repo.git.ls_files(path)) - - def is_dirty(self, **kwargs): - return self.repo.is_dirty(**kwargs) - - def active_branch(self): - return self.repo.active_branch.name - - def list_branches(self): - return [h.name for h in self.repo.heads] - - def list_tags(self): - return [t.name for t in self.repo.tags] - - def list_all_commits(self): - return [c.hexsha for c in self.repo.iter_commits("--all")] - def _install_hook(self, name): hook = self._hook_path(name) with open(hook, "w+") as fobj: @@ -317,8 +165,8 @@ class Git(Base): os.chmod(hook, 0o777) def _install_merge_driver(self): - self.repo.git.config("merge.dvc.name", "DVC merge driver") - self.repo.git.config( + self.gitpython.repo.git.config("merge.dvc.name", "DVC merge driver") + self.gitpython.repo.git.config( "merge.dvc.driver", ( "dvc git-hook merge-driver " @@ -404,48 +252,6 @@ class Git(Base): path_parts = os.path.normpath(path).split(os.path.sep) return basename == self.ignore_file or Git.GIT_DIR in path_parts - def get_tree(self, rev, **kwargs): - from dvc.tree.git import GitTree - - return GitTree(self.repo, self.resolve_rev(rev), **kwargs) - - def get_rev(self): - return self.repo.rev_parse("HEAD").hexsha - - def resolve_rev(self, rev): - from contextlib import suppress - - from git.exc import BadName, GitCommandError - - def _resolve_rev(name): - with suppress(BadName, GitCommandError): - try: - # Try python implementation of rev-parse first, it's faster - return self.repo.rev_parse(name).hexsha - except NotImplementedError: - # Fall back to `git rev-parse` for advanced features - return self.repo.git.rev_parse(name) - except ValueError: - raise RevError(f"unknown Git revision '{name}'") - - # Resolve across local names - sha = _resolve_rev(rev) - if sha: - return sha - - # Try all the remotes and if it resolves unambiguously then take it - if not Git.is_sha(rev): - shas = { - _resolve_rev(f"{remote.name}/{rev}") - for remote in self.repo.remotes - } - {None} - if len(shas) > 1: - raise RevError(f"ambiguous Git revision '{rev}'") - if len(shas) == 1: - return shas.pop() - - raise RevError(f"unknown Git revision '{rev}'") - def has_rev(self, rev): try: self.resolve_rev(rev) @@ -454,9 +260,8 @@ class Git(Base): return False def close(self): - for backend in self.backends: + for backend in self.backends.values(): backend.close() - self.repo.close() @cached_property def _hooks_home(self): @@ -478,34 +283,8 @@ class Git(Base): def no_commits(self): return not self.list_all_commits() - def branch_revs(self, branch: str, end_rev: Optional[str] = None): - """Iterate over revisions in a given branch (from newest to oldest). - - If end_rev is set, iterator will stop when the specified revision is - reached. - """ - commit = self.resolve_commit(branch) - while commit is not None: - yield commit.hexsha - commit = first(commit.parents) - if commit and commit.hexsha == end_rev: - return - - def resolve_commit(self, rev): - """Return Commit object for the specified revision.""" - from git.exc import BadName, GitCommandError - from git.objects.tag import TagObject - - try: - commit = self.repo.rev_parse(rev) - except (BadName, GitCommandError): - commit = None - if isinstance(commit, TagObject): - commit = commit.object - return commit - def _backend_func(self, name, *args, **kwargs): - for backend in self.backends: + for backend in self.backends.values(): try: func = getattr(backend, name) return func(*args, **kwargs) @@ -514,6 +293,26 @@ class Git(Base): raise NoGitBackendError(name) is_ignored = partialmethod(_backend_func, "is_ignored") + add = partialmethod(_backend_func, "add") + commit = partialmethod(_backend_func, "commit") + checkout = partialmethod(_backend_func, "checkout") + pull = partialmethod(_backend_func, "pull") + push = partialmethod(_backend_func, "push") + branch = partialmethod(_backend_func, "branch") + tag = partialmethod(_backend_func, "tag") + untracked_files = partialmethod(_backend_func, "untracked_files") + is_tracked = partialmethod(_backend_func, "is_tracked") + is_dirty = partialmethod(_backend_func, "is_dirty") + active_branch = partialmethod(_backend_func, "active_branch") + list_branches = partialmethod(_backend_func, "list_branches") + list_tags = partialmethod(_backend_func, "list_tags") + list_all_commits = partialmethod(_backend_func, "list_all_commits") + get_tree = partialmethod(_backend_func, "get_tree") + get_rev = partialmethod(_backend_func, "get_rev") + resolve_rev = partialmethod(_backend_func, "resolve_rev") + resolve_commit = partialmethod(_backend_func, "resolve_commit") + branch_revs = partialmethod(_backend_func, "branch_revs") + set_ref = partialmethod(_backend_func, "set_ref") get_ref = partialmethod(_backend_func, "get_ref") remove_ref = partialmethod(_backend_func, "remove_ref") diff --git a/dvc/scm/git/backend/base.py b/dvc/scm/git/backend/base.py index b3f71ef75..dbf179654 100644 --- a/dvc/scm/git/backend/base.py +++ b/dvc/scm/git/backend/base.py @@ -1,11 +1,9 @@ import os from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Callable, Iterable, Optional, Tuple +from typing import Callable, Iterable, Optional, Tuple from dvc.scm.base import SCMError - -if TYPE_CHECKING: - from dvc.scm.git import Git +from dvc.tree.base import BaseTree class NoGitBackendError(SCMError): @@ -16,9 +14,11 @@ class NoGitBackendError(SCMError): class BaseGitBackend(ABC): """Base Git backend class.""" - def __init__(self, scm: "Git", **kwargs): - self.scm = scm - self.root_dir = os.fspath(scm.root_dir) + @abstractmethod + def __init__( + self, scm, root_dir=os.curdir, search_parent_directories=True + ): + pass def close(self): pass @@ -27,6 +27,111 @@ class BaseGitBackend(ABC): def is_ignored(self, path): """Return True if the specified path is gitignored.""" + @property + @abstractmethod + def root_dir(self) -> str: + pass + + @staticmethod + @abstractmethod + def clone( + url: str, + to_path: str, + rev: Optional[str] = None, + shallow_branch: Optional[str] = None, + ): + pass + + @staticmethod + @abstractmethod + def is_sha(rev: str) -> bool: + pass + + @property + @abstractmethod + def dir(self) -> str: + pass + + @abstractmethod + def add(self, paths: Iterable[str]): + pass + + @abstractmethod + def commit(self, msg: str): + pass + + @abstractmethod + def checkout( + self, branch: str, create_new: Optional[bool] = False, **kwargs, + ): + pass + + @abstractmethod + def pull(self, **kwargs): + pass + + @abstractmethod + def push(self): + pass + + @abstractmethod + def branch(self, branch: str): + pass + + @abstractmethod + def tag(self, tag: str): + pass + + @abstractmethod + def untracked_files(self) -> Iterable[str]: + pass + + @abstractmethod + def is_tracked(self, path: str) -> bool: + pass + + @abstractmethod + def is_dirty(self, **kwargs) -> bool: + pass + + @abstractmethod + def active_branch(self) -> str: + pass + + @abstractmethod + def list_branches(self) -> Iterable[str]: + pass + + @abstractmethod + def list_tags(self) -> Iterable[str]: + pass + + @abstractmethod + def list_all_commits(self) -> Iterable[str]: + pass + + @abstractmethod + def get_tree(self, rev: str, **kwargs) -> BaseTree: + pass + + @abstractmethod + def get_rev(self) -> str: + pass + + @abstractmethod + def resolve_rev(self, rev: str) -> str: + pass + + @abstractmethod + def resolve_commit(self, rev: str) -> str: + pass + + @abstractmethod + def branch_revs( + self, branch: str, end_rev: Optional[str] = None + ) -> Iterable[str]: + pass + @abstractmethod def set_ref( self, diff --git a/dvc/scm/git/backend/dulwich.py b/dvc/scm/git/backend/dulwich.py index 66c024221..67ccef4a6 100644 --- a/dvc/scm/git/backend/dulwich.py +++ b/dvc/scm/git/backend/dulwich.py @@ -2,32 +2,121 @@ import fnmatch import logging import os from io import BytesIO -from typing import TYPE_CHECKING, Callable, Iterable, Optional, Tuple +from typing import Callable, Iterable, Optional, Tuple from dvc.scm.base import SCMError +from dvc.tree.base import BaseTree from dvc.utils import relpath from .base import BaseGitBackend -if TYPE_CHECKING: - from dvc.scm.git import Git - logger = logging.getLogger(__name__) class DulwichBackend(BaseGitBackend): # pylint:disable=abstract-method """Dulwich Git backend.""" - def __init__(self, scm: "Git", **kwargs): + def __init__( # pylint:disable=W0231 + self, scm, root_dir=os.curdir, search_parent_directories=True + ): + from dulwich.errors import NotGitRepository from dulwich.repo import Repo - super().__init__(scm, **kwargs) - self.repo = Repo(self.root_dir) + self.scm = scm + + try: + if search_parent_directories: + self.repo = Repo.discover(start=root_dir) + else: + self.repo = Repo(root_dir) + except NotGitRepository as exc: + raise SCMError(f"{root_dir} is not a git repository") from exc + self._stashes: dict = {} def close(self): self.repo.close() + @property + def root_dir(self) -> str: + return self.repo.path + + @staticmethod + def clone( + url: str, + to_path: str, + rev: Optional[str] = None, + shallow_branch: Optional[str] = None, + ): + raise NotImplementedError + + @staticmethod + def is_sha(rev: str) -> bool: + raise NotImplementedError + + @property + def dir(self) -> str: + raise NotImplementedError + + def add(self, paths: Iterable[str]): + raise NotImplementedError + + def commit(self, msg: str): + raise NotImplementedError + + def checkout( + self, branch: str, create_new: Optional[bool] = False, **kwargs, + ): + raise NotImplementedError + + def pull(self, **kwargs): + raise NotImplementedError + + def push(self): + raise NotImplementedError + + def branch(self, branch: str): + raise NotImplementedError + + def tag(self, tag: str): + raise NotImplementedError + + def untracked_files(self) -> Iterable[str]: + raise NotImplementedError + + def is_tracked(self, path: str) -> bool: + raise NotImplementedError + + def is_dirty(self, **kwargs) -> bool: + raise NotImplementedError + + def active_branch(self) -> str: + raise NotImplementedError + + def list_branches(self) -> Iterable[str]: + raise NotImplementedError + + def list_tags(self) -> Iterable[str]: + raise NotImplementedError + + def list_all_commits(self) -> Iterable[str]: + raise NotImplementedError + + def get_tree(self, rev: str, **kwargs) -> BaseTree: + raise NotImplementedError + + def get_rev(self) -> str: + raise NotImplementedError + + def resolve_rev(self, rev: str) -> str: + raise NotImplementedError + + def resolve_commit(self, rev: str) -> str: + raise NotImplementedError + + def branch_revs(self, branch: str, end_rev: Optional[str] = None): + raise NotImplementedError + def _get_stash(self, ref: str): from dulwich.stash import Stash as DulwichStash diff --git a/dvc/scm/git/backend/gitpython.py b/dvc/scm/git/backend/gitpython.py index 726b830b2..2c1afdb09 100644 --- a/dvc/scm/git/backend/gitpython.py +++ b/dvc/scm/git/backend/gitpython.py @@ -1,21 +1,294 @@ import logging +import os +from functools import partial from typing import Callable, Iterable, Optional, Tuple +from funcy import first + +from dvc.progress import Tqdm +from dvc.scm.base import CloneError, RevError, SCMError +from dvc.tree.base import BaseTree +from dvc.utils import fix_env, is_binary + from .base import BaseGitBackend logger = logging.getLogger(__name__) +class TqdmGit(Tqdm): + def update_git(self, op_code, cur_count, max_count=None, message=""): + op_code = self.code2desc(op_code) + if op_code: + message = (op_code + " | " + message) if message else op_code + if message: + self.postfix["info"] = f" {message} | " + self.update_to(cur_count, max_count) + + @staticmethod + def code2desc(op_code): + from git import RootUpdateProgress as OP + + ops = { + OP.COUNTING: "Counting", + OP.COMPRESSING: "Compressing", + OP.WRITING: "Writing", + OP.RECEIVING: "Receiving", + OP.RESOLVING: "Resolving", + OP.FINDING_SOURCES: "Finding sources", + OP.CHECKING_OUT: "Checking out", + OP.CLONE: "Cloning", + OP.FETCH: "Fetching", + OP.UPDWKTREE: "Updating working tree", + OP.REMOVE: "Removing", + OP.PATHCHANGE: "Changing path", + OP.URLCHANGE: "Changing URL", + OP.BRANCHCHANGE: "Changing branch", + } + return ops.get(op_code & OP.OP_MASK, "") + + class GitPythonBackend(BaseGitBackend): # pylint:disable=abstract-method """git-python Git backend.""" + def __init__( # pylint:disable=W0231 + self, scm, root_dir=os.curdir, search_parent_directories=True + ): + import git + from git.exc import InvalidGitRepositoryError + + self.scm = scm + + try: + self.repo = git.Repo( + root_dir, search_parent_directories=search_parent_directories + ) + except InvalidGitRepositoryError: + msg = "{} is not a git repository" + raise SCMError(msg.format(root_dir)) + + # NOTE: fixing LD_LIBRARY_PATH for binary built by PyInstaller. + # http://pyinstaller.readthedocs.io/en/stable/runtime-information.html + env = fix_env(None) + libpath = env.get("LD_LIBRARY_PATH", None) + self.repo.git.update_environment(LD_LIBRARY_PATH=libpath) + + def close(self): + self.repo.close() + @property def git(self): - return self.scm.repo.git + return self.repo.git def is_ignored(self, path): raise NotImplementedError + @property + def root_dir(self) -> str: + return self.repo.working_tree_dir + + @staticmethod + def clone( + url: str, + to_path: str, + rev: Optional[str] = None, + shallow_branch: Optional[str] = None, + ): + import git + + ld_key = "LD_LIBRARY_PATH" + + env = fix_env(None) + if is_binary() and ld_key not in env.keys(): + # In fix_env, we delete LD_LIBRARY_PATH key if it was empty before + # PyInstaller modified it. GitPython, in git.Repo.clone_from, uses + # env to update its own internal state. When there is no key in + # env, this value is not updated and GitPython re-uses + # LD_LIBRARY_PATH that has been set by PyInstaller. + # See [1] for more info. + # [1] https://github.com/gitpython-developers/GitPython/issues/924 + env[ld_key] = "" + + try: + if shallow_branch is not None and os.path.exists(url): + # git disables --depth for local clones unless file:// url + # scheme is used + url = f"file://{url}" + with TqdmGit(desc="Cloning", unit="obj") as pbar: + clone_from = partial( + git.Repo.clone_from, + url, + to_path, + env=env, # needed before we can fix it in __init__ + no_single_branch=True, + progress=pbar.update_git, + ) + if shallow_branch is None: + tmp_repo = clone_from() + else: + tmp_repo = clone_from(branch=shallow_branch, depth=1) + tmp_repo.close() + except git.exc.GitCommandError as exc: # pylint: disable=no-member + raise CloneError(url, to_path) from exc + + # NOTE: using our wrapper to make sure that env is fixed in __init__ + repo = GitPythonBackend(None, to_path) + + if rev: + try: + repo.checkout(rev) + except git.exc.GitCommandError as exc: # pylint: disable=no-member + raise RevError( + "failed to access revision '{}' for repo '{}'".format( + rev, url + ) + ) from exc + + @staticmethod + def is_sha(rev): + import git + + return rev and git.Repo.re_hexsha_shortened.search(rev) + + @property + def dir(self) -> str: + return self.repo.git_dir + + def add(self, paths: Iterable[str]): + # NOTE: GitPython is not currently able to handle index version >= 3. + # See https://github.com/iterative/dvc/issues/610 for more details. + try: + self.repo.index.add(paths) + except AssertionError: + msg = ( + "failed to add '{}' to git. You can add those files " + "manually using `git add`. See " + "https://github.com/iterative/dvc/issues/610 for more " + "details.".format(str(paths)) + ) + + logger.exception(msg) + + def commit(self, msg: str): + self.repo.index.commit(msg) + + def checkout( + self, branch: str, create_new: Optional[bool] = False, **kwargs + ): + if create_new: + self.repo.git.checkout("HEAD", b=branch, **kwargs) + else: + self.repo.git.checkout(branch, **kwargs) + + def pull(self, **kwargs): + infos = self.repo.remote().pull(**kwargs) + for info in infos: + if info.flags & info.ERROR: + raise SCMError(f"pull failed: {info.note}") + + def push(self): + infos = self.repo.remote().push() + for info in infos: + if info.flags & info.ERROR: + raise SCMError(f"push failed: {info.summary}") + + def branch(self, branch): + self.repo.git.branch(branch) + + def tag(self, tag): + self.repo.git.tag(tag) + + def untracked_files(self): + files = self.repo.untracked_files + return [os.path.join(self.repo.working_dir, fname) for fname in files] + + def is_tracked(self, path): + return bool(self.repo.git.ls_files(path)) + + def is_dirty(self, **kwargs): + return self.repo.is_dirty(**kwargs) + + def active_branch(self): + return self.repo.active_branch.name + + def list_branches(self): + return [h.name for h in self.repo.heads] + + def list_tags(self): + return [t.name for t in self.repo.tags] + + def list_all_commits(self): + return [c.hexsha for c in self.repo.iter_commits("--all")] + + def get_tree(self, rev: str, **kwargs) -> BaseTree: + from dvc.tree.git import GitTree + + return GitTree(self.repo, self.resolve_rev(rev), **kwargs) + + def get_rev(self): + return self.repo.rev_parse("HEAD").hexsha + + def resolve_rev(self, rev): + from contextlib import suppress + + from git.exc import BadName, GitCommandError + + def _resolve_rev(name): + with suppress(BadName, GitCommandError): + try: + # Try python implementation of rev-parse first, it's faster + return self.repo.rev_parse(name).hexsha + except NotImplementedError: + # Fall back to `git rev-parse` for advanced features + return self.repo.git.rev_parse(name) + except ValueError: + raise RevError(f"unknown Git revision '{name}'") + + # Resolve across local names + sha = _resolve_rev(rev) + if sha: + return sha + + # Try all the remotes and if it resolves unambiguously then take it + if not self.is_sha(rev): + shas = { + _resolve_rev(f"{remote.name}/{rev}") + for remote in self.repo.remotes + } - {None} + if len(shas) > 1: + raise RevError(f"ambiguous Git revision '{rev}'") + if len(shas) == 1: + return shas.pop() + + raise RevError(f"unknown Git revision '{rev}'") + + def resolve_commit(self, rev): + """Return Commit object for the specified revision.""" + from git.exc import BadName, GitCommandError + from git.objects.tag import TagObject + + try: + commit = self.repo.rev_parse(rev) + except (BadName, GitCommandError): + commit = None + if isinstance(commit, TagObject): + commit = commit.object + return commit + + def branch_revs( + self, branch: str, end_rev: Optional[str] = None + ) -> Iterable[str]: + """Iterate over revisions in a given branch (from newest to oldest). + + If end_rev is set, iterator will stop when the specified revision is + reached. + """ + commit = self.resolve_commit(branch) + while commit is not None: + yield commit.hexsha + commit = first(commit.parents) + if commit and commit.hexsha == end_rev: + return + def set_ref( self, name: str, @@ -35,7 +308,7 @@ class GitPythonBackend(BaseGitBackend): # pylint:disable=abstract-method def iter_refs(self, base: Optional[str] = None): from git import Reference - for ref in Reference.iter_items(self.scm.repo, common_path=base): + for ref in Reference.iter_items(self.repo, common_path=base): if base and base.endswith("/"): base = base[:-1] yield "/".join([base, ref.name]) @@ -86,7 +359,7 @@ class GitPythonBackend(BaseGitBackend): # pylint:disable=abstract-method if include_untracked: args.append("--include-untracked") self.git.stash(*args) - commit = self.scm.resolve_commit("stash@{0}") + commit = self.resolve_commit("stash@{0}") if ref != Stash.DEFAULT_STASH: # `git stash` CLI doesn't support using custom refspecs, # so we push a commit onto refs/stash, make our refspec @@ -94,7 +367,9 @@ class GitPythonBackend(BaseGitBackend): # pylint:disable=abstract-method # `git stash create` is intended to be used for this kind of # behavior but it doesn't support --include-untracked so we need to # use push - self.scm.set_ref(ref, commit.hexsha, message=commit.message) + self.scm.dulwich.set_ref( + ref, commit.hexsha, message=commit.message + ) self.git.stash("drop") return commit.hexsha, False diff --git a/dvc/scm/git/stash.py b/dvc/scm/git/stash.py index 6646f12c0..476d9b7e3 100644 --- a/dvc/scm/git/stash.py +++ b/dvc/scm/git/stash.py @@ -19,10 +19,6 @@ class Stash: self.ref = ref if ref else self.DEFAULT_STASH self.scm = scm - @property - def git(self): - return self.scm.repo.git - def __iter__(self): yield from self.scm._stash_iter(self.ref) @@ -49,7 +45,7 @@ class Stash: self.ref, message=message, include_untracked=include_untracked ) if reset: - self.git.reset(hard=True) + self.scm.gitpython.repo.git.reset(hard=True) return rev def pop(self): diff --git a/tests/func/experiments/test_checkpoints.py b/tests/func/experiments/test_checkpoints.py index 3c0a2594a..c909abe71 100644 --- a/tests/func/experiments/test_checkpoints.py +++ b/tests/func/experiments/test_checkpoints.py @@ -60,8 +60,8 @@ def test_reset_checkpoint(tmp_dir, scm, dvc, checkpoint_stage, caplog): from dvc.repo.experiments.base import CheckpointExistsError dvc.experiments.run(checkpoint_stage.addressing, name="foo") - scm.repo.git.reset(hard=True) - scm.repo.git.clean(force=True) + scm.gitpython.repo.git.reset(hard=True) + scm.gitpython.repo.git.clean(force=True) with pytest.raises(CheckpointExistsError): dvc.experiments.run( @@ -113,6 +113,6 @@ def test_resume_branch(tmp_dir, scm, dvc, checkpoint_stage): with pytest.raises(MultipleBranchError): dvc.experiments.get_branch_containing(branch_rev) - assert branch_rev == dvc.experiments.scm.repo.git.merge_base( + assert branch_rev == dvc.experiments.scm.gitpython.repo.git.merge_base( checkpoint_a, checkpoint_b ) diff --git a/tests/func/experiments/test_experiments.py b/tests/func/experiments/test_experiments.py index 6e6e318d8..69d7c265a 100644 --- a/tests/func/experiments/test_experiments.py +++ b/tests/func/experiments/test_experiments.py @@ -258,7 +258,7 @@ def test_detached_parent(tmp_dir, scm, dvc, exp_stage, mocker): scm.commit("v2") scm.checkout(detached_rev) - assert scm.repo.head.is_detached + assert scm.gitpython.repo.head.is_detached results = dvc.experiments.run(exp_stage.addressing, params=["foo=3"]) exp_rev = first(results) diff --git a/tests/func/experiments/test_show.py b/tests/func/experiments/test_show.py index 70e1fc53b..bd4b7bd56 100644 --- a/tests/func/experiments/test_show.py +++ b/tests/func/experiments/test_show.py @@ -41,7 +41,7 @@ def test_show_experiment(tmp_dir, scm, dvc): scm.commit("baseline") baseline_rev = scm.get_rev() timestamp = datetime.fromtimestamp( - scm.repo.rev_parse(baseline_rev).committed_date + scm.gitpython.repo.rev_parse(baseline_rev).committed_date ) dvc.experiments.run(PIPELINE_FILE, params=["foo=2"]) diff --git a/tests/func/test_checkout.py b/tests/func/test_checkout.py index fd0419cee..a3784cc8d 100644 --- a/tests/func/test_checkout.py +++ b/tests/func/test_checkout.py @@ -580,7 +580,7 @@ def test_stats_on_checkout(tmp_dir, dvc, scm): tmp_dir.gen({"lorem": "lorem", "bar": "new bar", "dir2": {"file": "file"}}) (tmp_dir / "foo").unlink() - scm.repo.git.rm("foo.dvc") + scm.gitpython.repo.git.rm("foo.dvc") tmp_dir.dvc_add(["bar", "lorem", "dir2"], commit="second") scm.checkout("HEAD~") diff --git a/tests/func/test_import.py b/tests/func/test_import.py index 0983e1bb0..b90139dbd 100644 --- a/tests/func/test_import.py +++ b/tests/func/test_import.py @@ -25,7 +25,7 @@ def test_import(tmp_dir, scm, dvc, erepo_dir): assert os.path.isfile("foo_imported") assert (tmp_dir / "foo_imported").read_text() == "foo content" - assert scm.repo.git.check_ignore("foo_imported") + assert scm.is_ignored("foo_imported") assert stage.deps[0].def_repo == { "url": os.fspath(erepo_dir), "rev_lock": erepo_dir.scm.get_rev(), @@ -42,7 +42,7 @@ def test_import_git_file(tmp_dir, scm, dvc, git_dir, src_is_dvc): stage = tmp_dir.dvc.imp(os.fspath(git_dir), "src", "dst") assert (tmp_dir / "dst").read_text() == "hello" - assert tmp_dir.scm.repo.git.check_ignore(os.fspath(tmp_dir / "dst")) + assert tmp_dir.scm.is_ignored(os.fspath(tmp_dir / "dst")) assert stage.deps[0].def_repo == { "url": os.fspath(git_dir), "rev_lock": git_dir.scm.get_rev(), @@ -77,7 +77,7 @@ def test_import_git_dir(tmp_dir, scm, dvc, git_dir, src_is_dvc): stage = dvc.imp(os.fspath(git_dir), "src", "dst") assert (tmp_dir / "dst").read_text() == {"file.txt": "hello"} - assert tmp_dir.scm.repo.git.check_ignore(os.fspath(tmp_dir / "dst")) + assert tmp_dir.scm.is_ignored(os.fspath(tmp_dir / "dst")) assert stage.deps[0].def_repo == { "url": os.fspath(git_dir), "rev_lock": git_dir.scm.get_rev(), @@ -91,7 +91,7 @@ def test_import_dir(tmp_dir, scm, dvc, erepo_dir): stage = dvc.imp(os.fspath(erepo_dir), "dir", "dir_imported") assert (tmp_dir / "dir_imported").read_text() == {"foo": "foo content"} - assert scm.repo.git.check_ignore("dir_imported") + assert scm.is_ignored("dir_imported") assert stage.deps[0].def_repo == { "url": os.fspath(erepo_dir), "rev_lock": erepo_dir.scm.get_rev(), @@ -114,7 +114,7 @@ def test_import_file_from_dir(tmp_dir, scm, dvc, erepo_dir): stage = dvc.imp(os.fspath(erepo_dir), os.path.join("dir", "1")) assert (tmp_dir / "1").read_text() == "1" - assert scm.repo.git.check_ignore("1") + assert scm.is_ignored("1") assert stage.deps[0].def_repo == { "url": os.fspath(erepo_dir), "rev_lock": erepo_dir.scm.get_rev(), @@ -175,7 +175,7 @@ def test_import_non_cached(erepo_dir, tmp_dir, dvc, scm): assert (tmp_dir / dst).is_file() assert filecmp.cmp(erepo_dir / src, tmp_dir / dst, shallow=False) - assert tmp_dir.scm.repo.git.check_ignore(dst) + assert tmp_dir.scm.is_ignored(dst) assert stage.deps[0].def_repo == { "url": os.fspath(erepo_dir), "rev_lock": erepo_dir.scm.get_rev(), @@ -191,7 +191,7 @@ def test_import_rev(tmp_dir, scm, dvc, erepo_dir): stage = dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported", rev="branch") assert (tmp_dir / "foo_imported").read_text() == "foo content" - assert scm.repo.git.check_ignore("foo_imported") + assert scm.is_ignored("foo_imported") assert stage.deps[0].def_repo == { "url": os.fspath(erepo_dir), "rev": "branch", @@ -231,7 +231,7 @@ def test_cache_type_is_properly_overridden(tmp_dir, scm, dvc, erepo_dir): assert not System.is_symlink("foo_imported") assert (tmp_dir / "foo_imported").read_text() == "foo content" - assert scm.repo.git.check_ignore("foo_imported") + assert scm.is_ignored("foo_imported") def test_pull_imported_directory_stage(tmp_dir, dvc, erepo_dir): @@ -340,8 +340,8 @@ def test_import_from_bare_git_repo( erepo_dir.dvc_gen({"foo": "foo"}, commit="initial") erepo_dir.dvc.push() - erepo_dir.scm.repo.create_remote("origin", os.fspath(tmp_dir)) - erepo_dir.scm.repo.remote("origin").push("master") + erepo_dir.scm.gitpython.repo.create_remote("origin", os.fspath(tmp_dir)) + erepo_dir.scm.gitpython.repo.remote("origin").push("master") dvc_repo = make_tmp_dir("dvc-repo", scm=True, dvc=True) with dvc_repo.chdir(): diff --git a/tests/func/test_install.py b/tests/func/test_install.py index 4ecd37487..0b3fb3981 100644 --- a/tests/func/test_install.py +++ b/tests/func/test_install.py @@ -43,7 +43,7 @@ class TestInstall: # scm.commit bypasses hooks with pytest.raises(GitCommandError, match=r"modified:\s*file"): - scm.repo.git.commit(m="file modified") + scm.gitpython.repo.git.commit(m="file modified") def test_post_checkout(self, tmp_dir, scm, dvc): tmp_dir.dvc_gen({"file": "file content"}, commit="add") @@ -69,13 +69,13 @@ class TestInstall: storage_path / file_checksum[:2] / file_checksum[2:] ) - scm.repo.clone(os.fspath(git_remote)) - scm.repo.create_remote("origin", os.fspath(git_remote)) + scm.gitpython.repo.clone(os.fspath(git_remote)) + scm.gitpython.repo.create_remote("origin", os.fspath(git_remote)) scm.install() assert not expected_storage_path.is_file() - scm.repo.git.push("origin", "master") + scm.gitpython.repo.git.push("origin", "master") assert expected_storage_path.is_file() assert expected_storage_path.read_text() == "file_content" @@ -97,7 +97,9 @@ def test_merge_driver_no_ancestor(tmp_dir, scm, dvc): scm.install() (tmp_dir / ".gitattributes").write_text("*.dvc merge=dvc") - scm.repo.git.merge("one", m="merged", no_gpg_sign=True, no_signoff=True) + scm.gitpython.repo.git.merge( + "one", m="merged", no_gpg_sign=True, no_signoff=True + ) # NOTE: dvc shouldn't checkout automatically as it might take a long time assert (tmp_dir / "data").read_text() == {"bar": "bar"} @@ -132,7 +134,9 @@ def test_merge_driver(tmp_dir, scm, dvc): scm.install() (tmp_dir / ".gitattributes").write_text("*.dvc merge=dvc") - scm.repo.git.merge("one", m="merged", no_gpg_sign=True, no_signoff=True) + scm.gitpython.repo.git.merge( + "one", m="merged", no_gpg_sign=True, no_signoff=True + ) # NOTE: dvc shouldn't checkout automatically as it might take a long time assert (tmp_dir / "data").read_text() == {"master": "master", "two": "two"} diff --git a/tests/func/test_scm.py b/tests/func/test_scm.py index 644fda3b2..f0ef1fb1c 100644 --- a/tests/func/test_scm.py +++ b/tests/func/test_scm.py @@ -50,7 +50,7 @@ class TestSCMGit(TestGit): G.commit("add") self.assertTrue(G.is_tracked(foo)) self.assertTrue(G.is_tracked(self.FOO)) - G.repo.index.remove([self.FOO], working_tree=True) + G.gitpython.repo.index.remove([self.FOO], working_tree=True) self.assertFalse(G.is_tracked(foo)) self.assertFalse(G.is_tracked(self.FOO)) self.assertFalse(G.is_tracked("not-existing-file")) @@ -183,9 +183,9 @@ def test_git_stash_workspace(tmp_dir, scm): tmp_dir.gen("file", "1") with scm.stash_workspace(): - assert not scm.repo.is_dirty() + assert not scm.is_dirty() assert "0" == (tmp_dir / "file").read_text() - assert scm.repo.is_dirty() + assert scm.is_dirty() assert "1" == (tmp_dir / "file").read_text() diff --git a/tests/func/test_status.py b/tests/func/test_status.py index bf578104f..c0c1024a9 100644 --- a/tests/func/test_status.py +++ b/tests/func/test_status.py @@ -58,7 +58,7 @@ def test_status_before_and_after_dvc_init(tmp_dir, dvc, git_dir): with git_dir.chdir(): git_dir.init(dvc=True) - git_dir.scm.repo.index.remove(["file"]) + git_dir.scm.gitpython.repo.index.remove(["file"]) os.remove("file") git_dir.dvc_gen("file", "second version", commit="with dvc") new_rev = git_dir.scm.get_rev() diff --git a/tests/func/test_update.py b/tests/func/test_update.py index 26fce3e9d..6f406a7cd 100644 --- a/tests/func/test_update.py +++ b/tests/func/test_update.py @@ -97,7 +97,7 @@ def test_update_import_after_remote_updates_to_dvc(tmp_dir, dvc, erepo_dir): new_rev = None with erepo_dir.branch("branch", new=False), erepo_dir.chdir(): - erepo_dir.scm.repo.index.remove(["version"]) + erepo_dir.scm.gitpython.repo.index.remove(["version"]) erepo_dir.dvc_gen( "version", "updated", commit="upgrade to DVC tracking" ) @@ -134,7 +134,7 @@ def test_update_before_and_after_dvc_init(tmp_dir, dvc, git_dir): with git_dir.chdir(): git_dir.init(dvc=True) - git_dir.scm.repo.index.remove(["file"]) + git_dir.scm.gitpython.repo.index.remove(["file"]) os.remove("file") git_dir.dvc_gen("file", "second version", commit="with dvc") new_rev = git_dir.scm.get_rev() diff --git a/tests/unit/scm/test_git.py b/tests/unit/scm/test_git.py index 43bf94473..6a430156d 100644 --- a/tests/unit/scm/test_git.py +++ b/tests/unit/scm/test_git.py @@ -25,7 +25,9 @@ def test_walk_with_submodules(tmp_dir, scm, git_dir): {"foo": "foo", "bar": "bar", "dir": {"data": "data"}}, commit="add dir and files", ) - scm.repo.create_submodule("submodule", "submodule", url=os.fspath(git_dir)) + scm.gitpython.repo.create_submodule( + "submodule", "submodule", url=os.fspath(git_dir) + ) scm.commit("added submodule") files = []