dvc: bump to 2.0.4

* pull: proper path join for partial pulls

* use os.path.join in the tests for windows

.github/ISSUE_TEMPLATE/bug_report.md

@@ -48,14 +48,14 @@ A clear and concise description of what you expect to happen.
 This is required to ensure that we can reproduce the bug.
 -->
 
-**Output of `dvc version`:**
+**Output of `dvc doctor`:**
 
 ```console
-$ dvc version
+$ dvc doctor
 ```
 
 **Additional Information (if any):**
 
 <!--
 If applicable, please also provide a `--verbose` output of the command, eg: `dvc add --verbose`.
--->
+-->

README.rst

@@ -222,6 +222,9 @@ Citation
 Iterative, *DVC: Data Version Control - Git for Data & Models* (2020)
 `DOI:10.5281/zenodo.012345 <https://doi.org/10.5281/zenodo.3677553>`_.
 
+Barrak, A., Eghan, E.E. and Adams, B. On the Co-evolution of ML Pipelines and Source Code - Empirical Study of DVC Projects, in Proceedings of the 28th IEEE International Conference on Software Analysis, Evolution, and Reengineering, SANER 2021. Hawaii, USA, to appear.
+
+
 .. |Banner| image:: https://dvc.org/img/logo-github-readme.png
    :target: https://dvc.org
    :alt: DVC logo

dvc/fs/gdrive.py

@@ -126,6 +126,9 @@ class GDriveFileSystem(BaseFileSystem):
         self._client_id = config.get("gdrive_client_id")
         self._client_secret = config.get("gdrive_client_secret")
         self._validate_config()
+        self._gdrive_service_credentials_path = tmp_fname(
+            os.path.join(self.repo.tmp_dir, "")
+        )
         self._gdrive_user_credentials_path = (
             tmp_fname(os.path.join(self.repo.tmp_dir, ""))
             if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA)
@@ -208,8 +211,15 @@ class GDriveFileSystem(BaseFileSystem):
         from pydrive2.auth import GoogleAuth
         from pydrive2.drive import GoogleDrive
 
-        if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA):
-            with open(self._gdrive_user_credentials_path, "w") as cred_file:
+        temporary_save_path = self._gdrive_user_credentials_path
+        is_credentials_temp = os.getenv(
+            GDriveFileSystem.GDRIVE_CREDENTIALS_DATA
+        )
+        if self._use_service_account:
+            temporary_save_path = self._gdrive_service_credentials_path
+
+        if is_credentials_temp:
+            with open(temporary_save_path, "w") as cred_file:
                 cred_file.write(
                     os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA)
                 )
@@ -217,6 +227,7 @@ class GDriveFileSystem(BaseFileSystem):
         auth_settings = {
             "client_config_backend": "settings",
             "save_credentials": True,
+            "save_credentials_backend": "file",
             "save_credentials_file": self._gdrive_user_credentials_path,
             "get_refresh_token": True,
             "oauth_scope": [
@@ -226,13 +237,16 @@ class GDriveFileSystem(BaseFileSystem):
         }
 
         if self._use_service_account:
-            auth_settings["save_credentials_backend"] = "json"
             auth_settings["service_config"] = {
                 "client_user_email": self._service_account_user_email,
                 "client_json_file_path": self._service_account_json_file_path,
             }
+            if is_credentials_temp:
+                auth_settings["service_config"][
+                    "client_json_file_path"
+                ] = temporary_save_path
+
         else:
-            auth_settings["save_credentials_backend"] = "file"
             auth_settings["client_config"] = {
                 "client_id": self._client_id or self.DEFAULT_GDRIVE_CLIENT_ID,
                 "client_secret": self._client_secret
@@ -267,8 +281,8 @@ class GDriveFileSystem(BaseFileSystem):
         except Exception as exc:
             raise GDriveAuthError(self.credentials_location) from exc
         finally:
-            if os.getenv(GDriveFileSystem.GDRIVE_CREDENTIALS_DATA):
-                os.remove(self._gdrive_user_credentials_path)
+            if is_credentials_temp:
+                os.remove(temporary_save_path)
 
         return GoogleDrive(gauth)
 

dvc/fs/git.py

@@ -158,12 +158,6 @@ class GitFileSystem(BaseFileSystem):  # pylint:disable=abstract-method
                 errno.ENOENT, os.strerror(errno.ENOENT), path
             )
 
-    @property
-    def hash_jobs(self):  # pylint: disable=invalid-overridden-method
-        # NOTE: gitpython is not threadsafe. See
-        # https://github.com/iterative/dvc/issues/4079
-        return 1
-
     def walk_files(self, top):  # pylint: disable=arguments-differ
         for root, _, files in self.walk(top):
             for file in files:

dvc/fs/ssh/connection.py

@@ -40,7 +40,9 @@ class SSHConnection:
                 host=host, **kwargs
             )
         )
-        self.timeout = kwargs.get("timeout", 1800)
+
+        kwargs.setdefault("timeout", 1800)
+        self.timeout = kwargs["timeout"]
 
         self._ssh = paramiko.SSHClient()
 

dvc/output/base.py

@@ -475,7 +475,7 @@ class BaseOutput:
             return 0
 
         return ilen(
-            filter_info.isin_or_eq(self.path_info / relpath)
+            filter_info.isin_or_eq(self.path_info.joinpath(*relpath))
             for relpath, _ in self.hash_info.dir_info.items()
         )
 

dvc/scm/git/__init__.py

@@ -352,7 +352,7 @@ class Git(Base):
         from .objects import GitTrie
 
         resolved = self.resolve_rev(rev)
-        tree_obj = self._backend_func("get_tree_obj", rev=resolved)
+        tree_obj = self.pygit2.get_tree_obj(rev=resolved)
         trie = GitTrie(tree_obj, resolved)
         return GitFileSystem(self.root_dir, trie, **kwargs)
 
@@ -386,7 +386,7 @@ class Git(Base):
     _stash_iter = partialmethod(_backend_func, "_stash_iter")
     _stash_push = partialmethod(_backend_func, "_stash_push")
     _stash_apply = partialmethod(_backend_func, "_stash_apply")
-    reflog_delete = partialmethod(_backend_func, "reflog_delete")
+    _stash_drop = partialmethod(_backend_func, "_stash_drop")
     describe = partialmethod(_backend_func, "describe")
     diff = partialmethod(_backend_func, "diff")
     reset = partialmethod(_backend_func, "reset")

dvc/scm/git/backend/base.py

@@ -96,7 +96,7 @@ class BaseGitBackend(ABC):
         pass
 
     @abstractmethod
-    def is_dirty(self, **kwargs) -> bool:
+    def is_dirty(self, untracked_files: bool = False) -> bool:
         pass
 
     @abstractmethod
@@ -245,17 +245,21 @@ class BaseGitBackend(ABC):
         message: Optional[str] = None,
         include_untracked: Optional[bool] = False,
     ) -> Tuple[Optional[str], bool]:
-        """Push a commit onto the specified stash."""
+        """Push a commit onto the specified stash.
+
+        Returns a tuple of the form (rev, need_reset) where need_reset
+        indicates whether or not the workspace should be `reset --hard`
+        (some backends will not clean the workspace after creating a stash
+        commit).
+        """
 
     @abstractmethod
     def _stash_apply(self, rev: str):
         """Apply the specified stash revision."""
 
     @abstractmethod
-    def reflog_delete(
-        self, ref: str, updateref: bool = False, rewrite: bool = False
-    ):
-        """Delete the specified reflog entry."""
+    def _stash_drop(self, ref: str, index: int):
+        """Drop the specified stash revision."""
 
     @abstractmethod
     def describe(

dvc/scm/git/backend/dulwich.py

@@ -222,8 +222,9 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
                 return True
         return False
 
-    def is_dirty(self, **kwargs) -> bool:
-        raise NotImplementedError
+    def is_dirty(self, untracked_files: bool = False) -> bool:
+        staged, unstaged, untracked = self.status()
+        return bool(staged or unstaged or (untracked_files and untracked))
 
     def active_branch(self) -> str:
         raise NotImplementedError
@@ -244,7 +245,10 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
         return DulwichObject(self.repo, ".", stat.S_IFDIR, tree.id)
 
     def get_rev(self) -> str:
-        raise NotImplementedError
+        rev = self.get_ref("HEAD")
+        if rev:
+            return rev
+        raise SCMError("Empty git repo")
 
     def resolve_rev(self, rev: str) -> str:
         raise NotImplementedError
@@ -535,10 +539,17 @@ class DulwichBackend(BaseGitBackend):  # pylint:disable=abstract-method
     def _stash_apply(self, rev: str):
         raise NotImplementedError
 
-    def reflog_delete(
-        self, ref: str, updateref: bool = False, rewrite: bool = False
-    ):
-        raise NotImplementedError
+    def _stash_drop(self, ref: str, index: int):
+        from dvc.scm.git import Stash
+
+        if ref == Stash.DEFAULT_STASH:
+            raise NotImplementedError
+
+        stash = self._get_stash(ref)
+        try:
+            stash.drop(index)
+        except ValueError as exc:
+            raise SCMError("Failed to drop stash entry") from exc
 
     def describe(
         self,

dvc/scm/git/backend/gitpython.py

@@ -249,8 +249,8 @@ class GitPythonBackend(BaseGitBackend):  # pylint:disable=abstract-method
     def is_tracked(self, path):
         return bool(self.repo.git.ls_files(path))
 
-    def is_dirty(self, **kwargs):
-        return self.repo.is_dirty(**kwargs)
+    def is_dirty(self, untracked_files: bool = False) -> bool:
+        return self.repo.is_dirty(untracked_files=untracked_files)
 
     def active_branch(self):
         return self.repo.active_branch.name
@@ -495,16 +495,22 @@ class GitPythonBackend(BaseGitBackend):  # pylint:disable=abstract-method
                 ) from exc
             raise SCMError("Could not apply stash") from exc
 
-    def reflog_delete(
-        self, ref: str, updateref: bool = False, rewrite: bool = False
-    ):
-        args = ["delete"]
-        if updateref:
-            args.append("--updateref")
-        if rewrite:
-            args.append("--rewrite")
-        args.append(ref)
-        self.git.reflog(*args)
+    def _stash_drop(self, ref: str, index: int):
+        from git.exc import GitCommandError
+
+        from dvc.scm.git import Stash
+
+        if ref == Stash.DEFAULT_STASH:
+            self.git.stash("drop", index)
+            return
+
+        self.git.reflog(
+            "delete", "--updateref", "--rewrite", f"{ref}@{{{index}}}"
+        )
+        try:
+            self.git.reflog("exists", ref)
+        except GitCommandError:
+            self.remove_ref(ref)
 
     def describe(
         self,

dvc/scm/git/backend/pygit2.py

@@ -1,9 +1,12 @@
 import locale
 import logging
 import os
+import stat
 from io import BytesIO, StringIO
 from typing import Callable, Iterable, List, Mapping, Optional, Tuple, Union
 
+from funcy import cached_property
+
 from dvc.scm.base import MergeConflictError, RevError, SCMError
 from dvc.utils import relpath
 
@@ -31,6 +34,8 @@ class Pygit2Object(GitObject):
 
     @property
     def mode(self):
+        if not self.obj.filemode and self.obj.type_str == "tree":
+            return stat.S_IFDIR
         return self.obj.filemode
 
     def scandir(self) -> Iterable["Pygit2Object"]:
@@ -61,12 +66,20 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
         self._stashes: dict = {}
 
     def close(self):
+        if hasattr(self, "_refdb"):
+            del self._refdb
         self.repo.free()
 
     @property
     def root_dir(self) -> str:
         return self.repo.workdir
 
+    @cached_property
+    def _refdb(self):
+        from pygit2 import RefdbFsBackend
+
+        return RefdbFsBackend(self.repo)
+
     @staticmethod
     def clone(
         url: str,
@@ -133,7 +146,7 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
     def is_tracked(self, path: str) -> bool:
         raise NotImplementedError
 
-    def is_dirty(self, **kwargs) -> bool:
+    def is_dirty(self, untracked_files: bool = False) -> bool:
         raise NotImplementedError
 
     def active_branch(self) -> str:
@@ -210,12 +223,16 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
         if old_ref and old_ref != self.get_ref(name, follow=False):
             raise SCMError(f"Failed to set '{name}'")
 
-        if symbolic:
-            ref = self.repo.create_reference_symbolic(name, new_ref, True)
-        else:
-            ref = self.repo.create_reference_direct(name, new_ref, True)
         if message:
-            ref.set_target(new_ref, message)
+            self._refdb.ensure_log(name)
+        if symbolic:
+            self.repo.create_reference_symbolic(
+                name, new_ref, True, message=message
+            )
+        else:
+            self.repo.create_reference_direct(
+                name, new_ref, True, message=message
+            )
 
     def get_ref(self, name, follow: bool = True) -> Optional[str]:
         from pygit2 import GIT_REF_SYMBOLIC
@@ -302,15 +319,64 @@ class Pygit2Backend(BaseGitBackend):  # pylint:disable=abstract-method
         message: Optional[str] = None,
         include_untracked: Optional[bool] = False,
     ) -> Tuple[Optional[str], bool]:
-        raise NotImplementedError
+        from dvc.scm.git import Stash
+
+        oid = self.repo.stash(
+            self.repo.default_signature,
+            message=message,
+            include_untracked=include_untracked,
+        )
+        commit = self.repo[oid]
+
+        if ref != Stash.DEFAULT_STASH:
+            self.set_ref(ref, commit.id, message=commit.message)
+            self.repo.stash_drop()
+        return str(oid), False
 
     def _stash_apply(self, rev: str):
-        raise NotImplementedError
+        from pygit2 import (
+            GIT_CHECKOUT_RECREATE_MISSING,
+            GIT_CHECKOUT_SAFE,
+            GitError,
+        )
 
-    def reflog_delete(
-        self, ref: str, updateref: bool = False, rewrite: bool = False
-    ):
-        raise NotImplementedError
+        from dvc.scm.git import Stash
+
+        def _apply(index):
+            try:
+                self.repo.index.read(False)
+                self.repo.stash_apply(
+                    index,
+                    strategy=GIT_CHECKOUT_SAFE | GIT_CHECKOUT_RECREATE_MISSING,
+                )
+            except GitError as exc:
+                raise MergeConflictError(
+                    "Stash apply resulted in merge conflicts"
+                ) from exc
+
+        # libgit2 stash apply only accepts refs/stash items by index. If rev is
+        # not in refs/stash, we will push it onto the stash, and then pop it
+        commit, _ref = self.repo.resolve_refish(rev)
+        stash = self.repo.references.get(Stash.DEFAULT_STASH)
+        if stash:
+            for i, entry in enumerate(stash.log()):
+                if entry.oid_new == commit.id:
+                    _apply(i)
+                    return
+
+        self.set_ref(Stash.DEFAULT_STASH, commit.id, message=commit.message)
+        try:
+            _apply(0)
+        finally:
+            self.repo.stash_drop()
+
+    def _stash_drop(self, ref: str, index: int):
+        from dvc.scm.git import Stash
+
+        if ref != Stash.DEFAULT_STASH:
+            raise NotImplementedError
+
+        self.repo.stash_drop(index)
 
     def describe(
         self,

dvc/scm/git/stash.py

@@ -1,11 +1,9 @@
 """git stash convenience wrapper."""
 
 import logging
-import os
 from typing import Optional
 
 from dvc.scm.base import SCMError
-from dvc.utils.fs import remove
 
 logger = logging.getLogger(__name__)
 
@@ -62,19 +60,12 @@ class Stash:
         self.scm._stash_apply(rev)  # pylint: disable=protected-access
 
     def drop(self, index: int = 0):
-        ref = "{0}@{{{1}}}".format(self.ref, index)
         if index < 0 or index >= len(self):
-            raise SCMError(f"Invalid stash ref '{ref}'")
-        logger.debug("Dropping '%s'", ref)
-        self.scm.reflog_delete(ref, updateref=True, rewrite=True)
-
-        # if we removed the last reflog entry, delete the ref and reflog
-        if len(self) == 0:
-            self.scm.remove_ref(self.ref)
-            parts = self.ref.split("/")
-            reflog = os.path.join(self.scm.root_dir, ".git", "logs", *parts)
-            if os.path.exists(reflog):
-                remove(reflog)
+            raise SCMError(f"Invalid stash ref '{self.ref}@{{{index}}}'")
+        logger.debug("Dropping '%s@{%d}'", self.ref, index)
+        self.scm._stash_drop(  # pylint: disable=protected-access
+            self.ref, index
+        )
 
     def clear(self):
         logger.debug("Clear stash '%s'", self.ref)

dvc/state.py

@@ -76,7 +76,10 @@ class State(StateBase):  # pylint: disable=too-many-instance-attributes
         if not tmp_dir:
             return
 
-        config = {"eviction_policy": "least-recently-used"}
+        config = {
+            "eviction_policy": "least-recently-used",
+            "disk_pickle_protocol": 4,
+        }
         self.links = Cache(directory=os.path.join(tmp_dir, "links"), **config)
         self.md5s = Cache(directory=os.path.join(tmp_dir, "md5s"), **config)
 

dvc/utils/fs.py

@@ -180,7 +180,10 @@ def makedirs(path, exist_ok=False, mode=None):
         if not exist_ok or not os.path.isdir(path):
             raise
 
-    os.chmod(path, mode)
+    try:
+        os.chmod(path, mode)
+    except OSError:
+        logger.trace("failed to chmod '%o' '%s'", mode, path, exc_info=True)
 
 
 def copyfile(src, dest, no_progress_bar=False, name=None):

dvc/version.py

@@ -6,7 +6,7 @@
 import os
 import subprocess
 
-_BASE_VERSION = "2.0.2"
+_BASE_VERSION = "2.0.4"
 
 
 def _generate_version(base_version):

setup.py

@@ -51,7 +51,7 @@ install_requires = [
     "configobj>=5.0.6",
     "gitpython>3",
     "dulwich>=0.20.20",
-    "pygit2>=1.4.0",
+    "pygit2>=1.5.0",
     "setuptools>=34.0.0",
     "nanotime>=0.5.2",
     "pyasn1>=0.4.1",
@@ -95,9 +95,9 @@ install_requires = [
 # Extra dependencies for remote integrations
 
 gs = ["gcsfs>=0.7.2"]
-gdrive = ["pydrive2>=1.7.3", "six >= 1.13.0"]
+gdrive = ["pydrive2>=1.8.1", "six >= 1.13.0"]
 s3 = ["boto3>=1.9.201"]
-azure = ["adlfs>=0.6.3", "azure.identity>=1.4.0", "knack"]
+azure = ["adlfs>=0.6.3", "azure-identity>=1.4.0", "knack"]
 # https://github.com/Legrandin/pycryptodome/issues/465
 oss = ["oss2==2.6.1", "pycryptodome<3.9.9"]
 ssh = ["paramiko[invoke]>=2.7.0"]

tests/func/test_data_cloud.py

@@ -537,3 +537,13 @@ def test_push_pull_fetch_pipeline_stages(tmp_dir, dvc, run_copy, local_remote):
 
     dvc.fetch("copy-foo-bar")
     assert len(recurse_list_dir(dvc.odb.local.cache_dir)) == 1
+
+
+def test_pull_partial(tmp_dir, dvc, local_remote):
+    tmp_dir.dvc_gen({"foo": {"bar": {"baz": "baz"}, "spam": "spam"}})
+    dvc.push()
+    clean(["foo"], dvc)
+
+    stats = dvc.pull(os.path.join("foo", "bar"))
+    assert stats["fetched"] == 1
+    assert (tmp_dir / "foo").read_text() == {"bar": {"baz": "baz"}}

tests/func/test_scm.py

@@ -1,6 +1,7 @@
 import os
 
 import pytest
+from flaky.flaky_decorator import flaky
 from git import Repo
 
 from dvc.scm import SCM, Git, NoSCM
@@ -242,6 +243,9 @@ def test_git_stash_drop(tmp_dir, scm, ref):
     assert len(stash) == 1
 
 
+# libgit2 stash_save() is flaky on linux when run inside pytest, see:
+# https://github.com/iterative/dvc/pull/5286#issuecomment-792574294
+@flaky(max_runs=5, min_passes=1)
 @pytest.mark.parametrize("ref", [None, "refs/foo/stash"])
 def test_git_stash_pop(tmp_dir, scm, ref):
     from dvc.scm.git import Stash
@@ -279,6 +283,7 @@ def test_git_stash_clear(tmp_dir, scm, ref):
 
     parts = list(stash.ref.split("/"))
     assert not os.path.exists(os.path.join(os.fspath(tmp_dir), ".git", *parts))
-    assert not os.path.exists(
-        os.path.join(os.fspath(tmp_dir), ".git", "logs", *parts)
-    )
+    # NOTE: some backends will completely remove reflog file on clear, some
+    # will only truncate it, either case means an empty stash
+    log_path = os.path.join(os.fspath(tmp_dir), ".git", "logs", *parts)
+    assert not os.path.exists(log_path) or not open(log_path).read()

tests/unit/scm/test_git.py

@@ -165,6 +165,25 @@ def test_set_ref(tmp_dir, git):
     )
 
 
+def test_set_ref_with_message(tmp_dir, git):
+    tmp_dir.scm_gen({"file": "0"}, commit="init")
+    init_rev = tmp_dir.scm.get_rev()
+    tmp_dir.scm_gen({"file": "1"}, commit="commit")
+    commit_rev = tmp_dir.scm.get_rev()
+
+    git.set_ref("refs/foo/bar", init_rev, message="init message")
+    assert (
+        "init message"
+        in (tmp_dir / ".git" / "logs" / "refs" / "foo" / "bar").read_text()
+    )
+
+    git.set_ref("refs/foo/bar", commit_rev, message="modify message")
+    assert (
+        "modify message"
+        in (tmp_dir / ".git" / "logs" / "refs" / "foo" / "bar").read_text()
+    )
+
+
 def test_get_ref(tmp_dir, git):
     tmp_dir.scm_gen({"file": "0"}, commit="init")
     init_rev = tmp_dir.scm.get_rev()