diff --git a/dvc/repo/scm_context.py b/dvc/repo/scm_context.py index ff086084a..436bbb4a3 100644 --- a/dvc/repo/scm_context.py +++ b/dvc/repo/scm_context.py @@ -4,6 +4,8 @@ from contextlib import contextmanager from functools import wraps from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Set +from dvc.utils import relpath + if TYPE_CHECKING: from dvc.repo import Repo from dvc.scm.base import Base @@ -47,7 +49,7 @@ class SCMContext: raise SCMError(str(exc)) if gitignore_file: - self.track_file(gitignore_file) + self.track_file(relpath(gitignore_file)) return self.ignored_paths.append(path) def ignore_remove(self, path: str) -> None: @@ -61,7 +63,7 @@ class SCMContext: raise SCMError(str(exc)) if gitignore_file: - return self.track_file(gitignore_file) + return self.track_file(relpath(gitignore_file)) @contextmanager def __call__( diff --git a/dvc/scm/git/__init__.py b/dvc/scm/git/__init__.py index 3e10e9f26..797363405 100644 --- a/dvc/scm/git/__init__.py +++ b/dvc/scm/git/__init__.py @@ -17,8 +17,6 @@ from dvc.scm.exceptions import ( GitHookAlreadyExists, RevError, ) -from dvc.utils import relpath -from dvc.utils.fs import path_isin from .backend.base import BaseGitBackend, NoGitBackendError from .backend.dulwich import DulwichBackend @@ -148,14 +146,14 @@ class Git(Base): assert os.path.isabs(path) assert os.path.isabs(ignore_file_dir) - entry = relpath(path, ignore_file_dir).replace(os.sep, "/") + entry = os.path.relpath(path, ignore_file_dir).replace(os.sep, "/") # NOTE: using '/' prefix to make path unambiguous if len(entry) > 0 and entry[0] != "/": entry = "/" + entry gitignore = os.path.join(ignore_file_dir, self.GITIGNORE) - if not path_isin(os.path.realpath(gitignore), self.root_dir): + if not os.path.realpath(gitignore).startswith(self.root_dir + os.sep): raise FileNotInRepoError( f"'{path}' is outside of git repository '{self.root_dir}'" ) @@ -168,7 +166,7 @@ class Git(Base): return None self._add_entry_to_gitignore(entry, gitignore) - return relpath(gitignore) + return gitignore def _add_entry_to_gitignore(self, entry, gitignore): entry = GitWildMatchPattern.escape(entry) @@ -204,7 +202,7 @@ class Git(Base): with open(gitignore, "w", encoding="utf-8") as fobj: fobj.writelines(filtered) - return relpath(gitignore) + return gitignore def verify_hook(self, name): if (self.hooks_dir / name).exists(): diff --git a/dvc/scm/git/backend/dulwich/__init__.py b/dvc/scm/git/backend/dulwich/__init__.py index c41cb42a2..680b70423 100644 --- a/dvc/scm/git/backend/dulwich/__init__.py +++ b/dvc/scm/git/backend/dulwich/__init__.py @@ -19,14 +19,12 @@ from typing import ( from funcy import cached_property from dvc.scm.exceptions import AuthError, InvalidRemote, SCMError -from dvc.utils import relpath from ...objects import GitObject from ..base import BaseGitBackend if TYPE_CHECKING: from dvc.scm.progress import GitProgressEvent - from dvc.types import StrPath from ...objects import GitCommit @@ -145,8 +143,6 @@ class DulwichBackend(BaseGitBackend): # pylint:disable=abstract-method return self.repo.commondir() def add(self, paths: Union[str, Iterable[str]], update=False): - from dvc.utils.fs import walk_files - assert paths or update if isinstance(paths, str): @@ -164,21 +160,26 @@ class DulwichBackend(BaseGitBackend): # pylint:disable=abstract-method # parent git repo root). We append path to root_dir here so # that the result of relpath(path, root_dir) is actually the # path relative to the submodule root. - fs_path = relpath(path, self.root_dir) + fs_path = os.path.relpath(path, self.root_dir) for sm_path in self._submodules.values(): if fs_path.startswith(sm_path): path = os.path.join( self.root_dir, - relpath(fs_path, sm_path), + os.path.relpath(fs_path, sm_path), ) break if os.path.isdir(path): files.extend( - os.fsencode(relpath(fpath, self.root_dir)) - for fpath in walk_files(path) + os.fsencode( + os.path.relpath( + os.path.join(root, fpath), self.root_dir + ) + ) + for root, _, fs in os.walk(path) + for fpath in fs ) else: - files.append(os.fsencode(relpath(path, self.root_dir))) + files.append(os.fsencode(os.path.relpath(path, self.root_dir))) # NOTE: this doesn't check gitignore, same as GitPythonBackend.add if update: @@ -243,10 +244,14 @@ class DulwichBackend(BaseGitBackend): # pylint:disable=abstract-method return untracked def is_tracked(self, path: str) -> bool: - rel = relpath(path, self.root_dir).replace(os.path.sep, "/").encode() + rel = ( + os.path.relpath(path, self.root_dir) + .replace(os.path.sep, "/") + .encode() + ) rel_dir = rel + b"/" - for path in self.repo.open_index(): - if path == rel or path.startswith(rel_dir): + for p in self.repo.open_index(): + if p == rel or p.startswith(rel_dir): return True return False @@ -297,10 +302,10 @@ class DulwichBackend(BaseGitBackend): # pylint:disable=abstract-method return IgnoreFilterManager.from_repo(self.repo) - def is_ignored(self, path: "StrPath") -> bool: + def is_ignored(self, path: "Union[str, os.PathLike[str]]") -> bool: # `is_ignored` returns `false` if excluded in `.gitignore` and # `None` if it's not mentioned at all. `True` if it is ignored. - relative_path = relpath(path, self.root_dir) + relative_path = os.path.relpath(path, self.root_dir) # if checking a directory, a trailing slash must be included if str(path)[-1] == os.sep: relative_path += os.sep diff --git a/dvc/scm/git/backend/gitpython.py b/dvc/scm/git/backend/gitpython.py index 0a6d1e3da..2280fa0c7 100644 --- a/dvc/scm/git/backend/gitpython.py +++ b/dvc/scm/git/backend/gitpython.py @@ -2,10 +2,12 @@ import io import locale import logging import os +import sys from functools import partial from typing import ( TYPE_CHECKING, Callable, + Dict, Iterable, List, Mapping, @@ -22,18 +24,39 @@ from dvc.scm.exceptions import ( RevError, SCMError, ) -from dvc.utils import fix_env, is_binary, relpath from ..objects import GitCommit, GitObject from .base import BaseGitBackend if TYPE_CHECKING: from dvc.scm.progress import GitProgressEvent - from dvc.types import StrPath + logger = logging.getLogger(__name__) +# NOTE: Check if we are in a bundle +# https://pythonhosted.org/PyInstaller/runtime-information.html +def is_binary() -> bool: + return getattr(sys, "frozen", False) + + +def fix_env(env: Dict[str, str] = None) -> Dict[str, str]: + if env is None: + environ = os.environ.copy() + else: + environ = env.copy() + + if is_binary(): + lp_key = "LD_LIBRARY_PATH" + lp_orig = environ.get(lp_key + "_ORIG", None) + if lp_orig is not None: + environ[lp_key] = lp_orig + else: + environ.pop(lp_key, None) + return environ + + class GitPythonObject(GitObject): def __init__(self, obj): self.obj = obj @@ -93,7 +116,7 @@ class GitPythonBackend(BaseGitBackend): # pylint:disable=abstract-method # NOTE: fixing LD_LIBRARY_PATH for binary built by PyInstaller. # http://pyinstaller.readthedocs.io/en/stable/runtime-information.html - env = fix_env(None) + env = fix_env() libpath = env.get("LD_LIBRARY_PATH", None) self.repo.git.update_environment(LD_LIBRARY_PATH=libpath) @@ -104,7 +127,7 @@ class GitPythonBackend(BaseGitBackend): # pylint:disable=abstract-method def git(self): return self.repo.git - def is_ignored(self, path: "StrPath") -> bool: + def is_ignored(self, path: "Union[str, os.PathLike[str]]") -> bool: from git.exc import GitCommandError func = ignore(GitCommandError)(self.repo.git.check_ignore) @@ -127,7 +150,7 @@ class GitPythonBackend(BaseGitBackend): # pylint:disable=abstract-method ld_key = "LD_LIBRARY_PATH" - env = fix_env(None) + env = fix_env() if is_binary() and ld_key not in env.keys(): # In fix_env, we delete LD_LIBRARY_PATH key if it was empty before # PyInstaller modified it. GitPython, in git.Repo.clone_from, uses @@ -534,7 +557,7 @@ class GitPythonBackend(BaseGitBackend): # pylint:disable=abstract-method def reset(self, hard: bool = False, paths: Iterable[str] = None): if paths: paths_list: Optional[List[str]] = [ - relpath(path, self.root_dir) for path in paths + os.path.relpath(path, self.root_dir) for path in paths ] if os.name == "nt": paths_list = [ @@ -567,7 +590,7 @@ class GitPythonBackend(BaseGitBackend): # pylint:disable=abstract-method else: if paths: paths_list: Optional[List[str]] = [ - relpath(path, self.root_dir) for path in paths + os.path.relpath(path, self.root_dir) for path in paths ] if os.name == "nt": paths_list = [ diff --git a/dvc/scm/git/backend/pygit2.py b/dvc/scm/git/backend/pygit2.py index 5ccc04189..c68eabdab 100644 --- a/dvc/scm/git/backend/pygit2.py +++ b/dvc/scm/git/backend/pygit2.py @@ -18,7 +18,6 @@ from typing import ( from funcy import cached_property from dvc.scm.exceptions import MergeConflictError, RevError, SCMError -from dvc.utils import relpath from ..objects import GitCommit, GitObject from .base import BaseGitBackend @@ -28,7 +27,6 @@ logger = logging.getLogger(__name__) if TYPE_CHECKING: from dvc.scm.progress import GitProgressEvent - from dvc.types import StrPath # NOTE: constant from libgit2 git2/checkout.h # This can be removed after next pygit2 release: @@ -283,8 +281,8 @@ class Pygit2Backend(BaseGitBackend): # pylint:disable=abstract-method def _get_stash(self, ref: str): raise NotImplementedError - def is_ignored(self, path: "StrPath") -> bool: - rel = relpath(path, self.root_dir) + def is_ignored(self, path: "Union[str, os.PathLike[str]]") -> bool: + rel = os.path.relpath(path, self.root_dir) if os.name == "nt": rel.replace("\\", "/") return self.repo.path_is_ignored(rel) @@ -473,7 +471,7 @@ class Pygit2Backend(BaseGitBackend): # pylint:disable=abstract-method if paths is not None: tree = self.repo.revparse_single("HEAD").tree for path in paths: - rel = relpath(path, self.root_dir) + rel = os.path.relpath(path, self.root_dir) if os.name == "nt": rel = rel.replace("\\", "/") obj = tree[rel] @@ -512,7 +510,7 @@ class Pygit2Backend(BaseGitBackend): # pylint:disable=abstract-method index = self.repo.index if paths: path_list: Optional[List[str]] = [ - relpath(path, self.root_dir) for path in paths + os.path.relpath(path, self.root_dir) for path in paths ] if os.name == "nt": path_list = [