git: add support for working with refspecs and stashes (#4952)

* git: add support for dulwich Stash

* git: start adding support for custom refs

* git: add Stash and helper context managers for preserving
workspace state

* scm: cleanup refs/stash and add push_refspec

* tests: add tests for git refs and stash

* fix push_refspec

* tests: add tests for git push_refspec

* scm: add git fetch-refspecs

* add test for fetch refspecs

* scm: add test for git refs containing

* remove extraneous log msgs

* fix typing issues

* stash: only try to push if workspace is dirty

* update dulwich to 0.20.14

* scm: move dulwich into scm.backend

* move stash internals to backends

* scm: move git code into dvc/scm/git/... module

* git: use ABC for BaseGitBackend

* fix rebase issue

* git: fix refspec tests on windows

- git file:// URLs need posix style paths

* git: move get_refs_containing to backend

* scm: add callback for handling diverged refs during fetch

tests/unit/scm/test_git.py

@@ -2,6 +2,7 @@ import os
 
 import pytest
 
+from dvc.scm.base import SCMError
 from tests.basic_env import TestDvcGit
 
 
@@ -123,3 +124,125 @@ def test_branch_revs(tmp_dir, scm):
     for rev in scm.branch_revs("master", init_rev):
         assert rev == expected.pop()
     assert len(expected) == 0
+
+
+def test_set_ref(tmp_dir, scm):
+    tmp_dir.scm_gen({"file": "0"}, commit="init")
+    init_rev = scm.get_rev()
+    tmp_dir.scm_gen({"file": "1"}, commit="commit")
+    commit_rev = scm.get_rev()
+
+    scm.set_ref("refs/foo/bar", init_rev)
+    assert (
+        init_rev
+        == (tmp_dir / ".git" / "refs" / "foo" / "bar").read_text().strip()
+    )
+
+    with pytest.raises(SCMError):
+        scm.set_ref("refs/foo/bar", commit_rev, old_ref=commit_rev)
+    scm.set_ref("refs/foo/bar", commit_rev, old_ref=init_rev)
+    assert (
+        commit_rev
+        == (tmp_dir / ".git" / "refs" / "foo" / "bar").read_text().strip()
+    )
+
+    scm.set_ref("refs/foo/baz", "refs/heads/master", symbolic=True)
+    assert (
+        "ref: refs/heads/master"
+        == (tmp_dir / ".git" / "refs" / "foo" / "baz").read_text().strip()
+    )
+
+
+def test_get_ref(tmp_dir, scm):
+    tmp_dir.scm_gen({"file": "0"}, commit="init")
+    init_rev = scm.get_rev()
+    tmp_dir.gen(
+        {
+            os.path.join(".git", "refs", "foo", "bar"): init_rev,
+            os.path.join(
+                ".git", "refs", "foo", "baz"
+            ): "ref: refs/heads/master",
+        }
+    )
+
+    assert init_rev == scm.get_ref("refs/foo/bar")
+    assert init_rev == scm.get_ref("refs/foo/baz")
+    assert "refs/heads/master" == scm.get_ref("refs/foo/baz", follow=False)
+    assert scm.get_ref("refs/foo/qux") is None
+
+
+def test_remove_ref(tmp_dir, scm):
+    tmp_dir.scm_gen({"file": "0"}, commit="init")
+    init_rev = scm.get_rev()
+    tmp_dir.gen(os.path.join(".git", "refs", "foo", "bar"), init_rev)
+    tmp_dir.scm_gen({"file": "1"}, commit="commit")
+    commit_rev = scm.get_rev()
+
+    with pytest.raises(SCMError):
+        scm.remove_ref("refs/foo/bar", old_ref=commit_rev)
+    scm.remove_ref("refs/foo/bar", old_ref=init_rev)
+    assert not (tmp_dir / ".git" / "refs" / "foo" / "bar").exists()
+
+
+def test_refs_containing(tmp_dir, scm):
+    tmp_dir.scm_gen({"file": "0"}, commit="init")
+    init_rev = scm.get_rev()
+    tmp_dir.gen(
+        {
+            os.path.join(".git", "refs", "foo", "bar"): init_rev,
+            os.path.join(".git", "refs", "foo", "baz"): init_rev,
+        }
+    )
+
+    expected = {"refs/foo/bar", "refs/foo/baz", "refs/heads/master"}
+    assert expected == set(scm.get_refs_containing(init_rev))
+
+
+def test_push_refspec(tmp_dir, scm, make_tmp_dir):
+    tmp_dir.scm_gen({"file": "0"}, commit="init")
+    init_rev = scm.get_rev()
+    tmp_dir.gen(
+        {
+            os.path.join(".git", "refs", "foo", "bar"): init_rev,
+            os.path.join(".git", "refs", "foo", "baz"): init_rev,
+        }
+    )
+    remote_dir = make_tmp_dir("git-remote", scm=True)
+    url = "file://{}".format(remote_dir.resolve().as_posix())
+
+    scm.push_refspec(url, "refs/foo/bar", "refs/foo/bar")
+    assert init_rev == remote_dir.scm.get_ref("refs/foo/bar")
+
+    remote_dir.scm.checkout("refs/foo/bar")
+    assert init_rev == remote_dir.scm.get_rev()
+    assert "0" == (remote_dir / "file").read_text()
+
+    scm.push_refspec(url, "refs/foo/", "refs/foo/")
+    assert init_rev == remote_dir.scm.get_ref("refs/foo/baz")
+
+    scm.push_refspec(url, None, "refs/foo/baz")
+    assert remote_dir.scm.get_ref("refs/foo/baz") is None
+
+
+def test_fetch_refspecs(tmp_dir, scm, make_tmp_dir):
+    remote_dir = make_tmp_dir("git-remote", scm=True)
+    url = "file://{}".format(remote_dir.resolve().as_posix())
+
+    remote_dir.scm_gen({"file": "0"}, commit="init")
+    init_rev = remote_dir.scm.get_rev()
+    remote_dir.gen(
+        {
+            os.path.join(".git", "refs", "foo", "bar"): init_rev,
+            os.path.join(".git", "refs", "foo", "baz"): init_rev,
+        }
+    )
+
+    scm.fetch_refspecs(
+        url, ["refs/foo/bar:refs/foo/bar", "refs/foo/baz:refs/foo/baz"]
+    )
+    assert init_rev == scm.get_ref("refs/foo/bar")
+    assert init_rev == scm.get_ref("refs/foo/baz")
+
+    remote_dir.scm.checkout("refs/foo/bar")
+    assert init_rev == remote_dir.scm.get_rev()
+    assert "0" == (remote_dir / "file").read_text()