dvc/tests/unit/scm/test_git.py

import os

import pytest

from dvc.scm.base import SCMError


# Behaves the same as SCM but will test against all supported Git backends.
# tmp_dir.scm will still contain a default SCM instance.
@pytest.fixture(params=["gitpython", "dulwich", "pygit2"])
def git(tmp_dir, scm, request):
    from dvc.scm.git import Git

    git_ = Git(os.fspath(tmp_dir), backends=[request.param])
    git_.test_backend = request.param
    yield git_
    git_.close()


@pytest.mark.parametrize(
    "path, expected",
    [
        (os.path.join("path", "to", ".gitignore"), True),
        (os.path.join("path", "to", ".git", "internal", "file"), True),
        (os.path.join("some", "non-.git", "file"), False),
    ],
    ids=["gitignore_file", "git_internal_file", "non_git_file"],
)
def test_belongs_to_scm(scm, path, expected):
    assert scm.belongs_to_scm(path) == expected


def test_walk_with_submodules(tmp_dir, scm, git_dir):
    git_dir.scm_gen(
        {"foo": "foo", "bar": "bar", "dir": {"data": "data"}},
        commit="add dir and files",
    )
    scm.gitpython.repo.create_submodule(
        "submodule", "submodule", url=os.fspath(git_dir)
    )
    scm.commit("added submodule")

    files = []
    dirs = []
    fs = scm.get_fs("HEAD")
    for _, dnames, fnames in fs.walk("."):
        dirs.extend(dnames)
        files.extend(fnames)

    # currently we don't walk through submodules
    assert not dirs
    assert set(files) == {".gitmodules", "submodule"}


def test_walk_onerror(tmp_dir, scm):
    def onerror(exc):
        raise exc

    tmp_dir.scm_gen({"foo": "foo"}, commit="init")
    fs = scm.get_fs("HEAD")

    # path does not exist
    for _ in fs.walk("dir"):
        pass
    with pytest.raises(OSError):
        for _ in fs.walk("dir", onerror=onerror):
            pass

    # path is not a directory
    for _ in fs.walk("foo"):
        pass
    with pytest.raises(OSError):
        for _ in fs.walk("foo", onerror=onerror):
            pass


def test_is_tracked(tmp_dir, scm):
    tmp_dir.scm_gen(
        {
            "tracked": "tracked",
            "dir": {"data": "data", "subdir": {"subdata": "subdata"}},
        },
        commit="add dirs and files",
    )
    tmp_dir.gen({"untracked": "untracked", "dir": {"untracked": "untracked"}})

    # sanity check
    assert (tmp_dir / "untracked").exists()
    assert (tmp_dir / "tracked").exists()
    assert (tmp_dir / "dir" / "untracked").exists()
    assert (tmp_dir / "dir" / "data").exists()
    assert (tmp_dir / "dir" / "subdir" / "subdata").exists()

    assert not scm.is_tracked("untracked")
    assert not scm.is_tracked(os.path.join("dir", "untracked"))

    assert scm.is_tracked("tracked")
    assert scm.is_tracked("dir")
    assert scm.is_tracked(os.path.join("dir", "data"))
    assert scm.is_tracked(os.path.join("dir", "subdir"))
    assert scm.is_tracked(os.path.join("dir", "subdir", "subdata"))


def test_is_tracked_unicode(tmp_dir, scm):
    tmp_dir.scm_gen("ṭṝḁḉḵḗḋ", "tracked", commit="add unicode")
    tmp_dir.gen("ṳṋṭṝḁḉḵḗḋ", "untracked")
    assert scm.is_tracked("ṭṝḁḉḵḗḋ")
    assert not scm.is_tracked("ṳṋṭṝḁḉḵḗḋ")


def test_no_commits(tmp_dir):
    from dvc.scm.git import Git
    from tests.dir_helpers import git_init

    git_init(".")
    assert Git().no_commits

    tmp_dir.gen("foo", "foo")
    Git().add(["foo"])
    Git().commit("foo")

    assert not Git().no_commits


def test_branch_revs(tmp_dir, scm):
    tmp_dir.scm_gen({"file": "0"}, commit="init")
    init_rev = scm.get_rev()

    expected = []
    for i in range(1, 5):
        tmp_dir.scm_gen({"file": f"{i}"}, commit=f"{i}")
        expected.append(scm.get_rev())

    for rev in scm.branch_revs("master", init_rev):
        assert rev == expected.pop()
    assert len(expected) == 0


def test_set_ref(tmp_dir, git):
    tmp_dir.scm_gen({"file": "0"}, commit="init")
    init_rev = tmp_dir.scm.get_rev()
    tmp_dir.scm_gen({"file": "1"}, commit="commit")
    commit_rev = tmp_dir.scm.get_rev()

    git.set_ref("refs/foo/bar", init_rev)
    assert (
        init_rev
        == (tmp_dir / ".git" / "refs" / "foo" / "bar").read_text().strip()
    )

    with pytest.raises(SCMError):
        git.set_ref("refs/foo/bar", commit_rev, old_ref=commit_rev)
    git.set_ref("refs/foo/bar", commit_rev, old_ref=init_rev)
    assert (
        commit_rev
        == (tmp_dir / ".git" / "refs" / "foo" / "bar").read_text().strip()
    )

    git.set_ref("refs/foo/baz", "refs/heads/master", symbolic=True)
    assert (
        "ref: refs/heads/master"
        == (tmp_dir / ".git" / "refs" / "foo" / "baz").read_text().strip()
    )


def test_set_ref_with_message(tmp_dir, git):
    tmp_dir.scm_gen({"file": "0"}, commit="init")
    init_rev = tmp_dir.scm.get_rev()
    tmp_dir.scm_gen({"file": "1"}, commit="commit")
    commit_rev = tmp_dir.scm.get_rev()

    git.set_ref("refs/foo/bar", init_rev, message="init message")
    assert (
        "init message"
        in (tmp_dir / ".git" / "logs" / "refs" / "foo" / "bar").read_text()
    )

    git.set_ref("refs/foo/bar", commit_rev, message="modify message")
    assert (
        "modify message"
        in (tmp_dir / ".git" / "logs" / "refs" / "foo" / "bar").read_text()
    )


def test_get_ref(tmp_dir, git):
    tmp_dir.scm_gen({"file": "0"}, commit="init")
    init_rev = tmp_dir.scm.get_rev()
    tmp_dir.gen(
        {
            os.path.join(".git", "refs", "foo", "bar"): init_rev,
            os.path.join(
                ".git", "refs", "foo", "baz"
            ): "ref: refs/heads/master",
        }
    )

    assert init_rev == git.get_ref("refs/foo/bar")
    assert init_rev == git.get_ref("refs/foo/baz")
    assert "refs/heads/master" == git.get_ref("refs/foo/baz", follow=False)
    assert git.get_ref("refs/foo/qux") is None


def test_remove_ref(tmp_dir, git):
    tmp_dir.scm_gen({"file": "0"}, commit="init")
    init_rev = tmp_dir.scm.get_rev()
    tmp_dir.gen(os.path.join(".git", "refs", "foo", "bar"), init_rev)
    tmp_dir.scm_gen({"file": "1"}, commit="commit")
    commit_rev = tmp_dir.scm.get_rev()

    with pytest.raises(SCMError):
        git.remove_ref("refs/foo/bar", old_ref=commit_rev)
    git.remove_ref("refs/foo/bar", old_ref=init_rev)
    assert not (tmp_dir / ".git" / "refs" / "foo" / "bar").exists()


def test_refs_containing(tmp_dir, scm):
    tmp_dir.scm_gen({"file": "0"}, commit="init")
    init_rev = scm.get_rev()
    tmp_dir.gen(
        {
            os.path.join(".git", "refs", "foo", "bar"): init_rev,
            os.path.join(".git", "refs", "foo", "baz"): init_rev,
        }
    )

    expected = {"refs/foo/bar", "refs/foo/baz", "refs/heads/master"}
    assert expected == set(scm.get_refs_containing(init_rev))


@pytest.mark.parametrize("use_url", [True, False])
def test_push_refspec(tmp_dir, scm, make_tmp_dir, use_url):
    tmp_dir.scm_gen({"file": "0"}, commit="init")
    init_rev = scm.get_rev()
    tmp_dir.gen(
        {
            os.path.join(".git", "refs", "foo", "bar"): init_rev,
            os.path.join(".git", "refs", "foo", "baz"): init_rev,
        }
    )
    remote_dir = make_tmp_dir("git-remote", scm=True)
    url = f"file://{remote_dir.resolve().as_posix()}"
    scm.gitpython.repo.create_remote("origin", url)

    with pytest.raises(SCMError):
        scm.push_refspec("bad-remote", "refs/foo/bar", "refs/foo/bar")

    remote = url if use_url else "origin"
    scm.push_refspec(remote, "refs/foo/bar", "refs/foo/bar")
    assert init_rev == remote_dir.scm.get_ref("refs/foo/bar")

    remote_dir.scm.checkout("refs/foo/bar")
    assert init_rev == remote_dir.scm.get_rev()
    assert "0" == (remote_dir / "file").read_text()

    scm.push_refspec(remote, "refs/foo/", "refs/foo/")
    assert init_rev == remote_dir.scm.get_ref("refs/foo/baz")

    scm.push_refspec(remote, None, "refs/foo/baz")
    assert remote_dir.scm.get_ref("refs/foo/baz") is None


def test_fetch_refspecs(tmp_dir, scm, make_tmp_dir):
    remote_dir = make_tmp_dir("git-remote", scm=True)
    url = f"file://{remote_dir.resolve().as_posix()}"

    remote_dir.scm_gen({"file": "0"}, commit="init")
    init_rev = remote_dir.scm.get_rev()
    remote_dir.gen(
        {
            os.path.join(".git", "refs", "foo", "bar"): init_rev,
            os.path.join(".git", "refs", "foo", "baz"): init_rev,
        }
    )

    scm.fetch_refspecs(
        url, ["refs/foo/bar:refs/foo/bar", "refs/foo/baz:refs/foo/baz"]
    )
    assert init_rev == scm.get_ref("refs/foo/bar")
    assert init_rev == scm.get_ref("refs/foo/baz")

    remote_dir.scm.checkout("refs/foo/bar")
    assert init_rev == remote_dir.scm.get_rev()
    assert "0" == (remote_dir / "file").read_text()


def test_list_all_commits(tmp_dir, scm):
    tmp_dir.scm_gen("1", "1", commit="1")
    rev_a = scm.get_rev()
    tmp_dir.scm_gen("2", "2", commit="2")
    rev_b = scm.get_rev()
    scm.tag("tag")
    tmp_dir.scm_gen("3", "3", commit="3")
    rev_c = scm.get_rev()
    scm.gitpython.git.reset(rev_a, hard=True)
    scm.set_ref("refs/foo/bar", rev_c)

    assert {rev_a, rev_b} == set(scm.list_all_commits())


def test_ignore_remove_empty(tmp_dir, scm, git):
    if git.test_backend == "pygit2":
        pytest.skip()

    test_entries = [
        {"entry": "/foo1", "path": f"{tmp_dir}/foo1"},
        {"entry": "/foo2", "path": f"{tmp_dir}/foo2"},
    ]

    path_to_gitignore = tmp_dir / ".gitignore"

    with open(path_to_gitignore, "a", encoding="utf-8") as f:
        for entry in test_entries:
            f.write(entry["entry"] + "\n")

    assert path_to_gitignore.exists()

    git.ignore_remove(test_entries[0]["path"])
    assert path_to_gitignore.exists()

    git.ignore_remove(test_entries[1]["path"])
    assert not path_to_gitignore.exists()


@pytest.mark.skipif(
    os.name == "nt", reason="Git hooks not supported on Windows"
)
@pytest.mark.parametrize("hook", ["pre-commit", "commit-msg"])
def test_commit_no_verify(tmp_dir, scm, git, hook):
    import stat

    if git.test_backend == "pygit2":
        pytest.skip()

    hook_file = os.path.join(".git", "hooks", hook)
    tmp_dir.gen(hook_file, "#!/usr/bin/env python\nimport sys\nsys.exit(1)")
    os.chmod(hook_file, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)

    tmp_dir.gen("foo", "foo")
    git.add(["foo"])
    with pytest.raises(SCMError):
        git.commit("commit foo")
    git.commit("commit foo", no_verify=True)


@pytest.mark.parametrize("squash", [True, False])
def test_merge(tmp_dir, scm, git, squash):
    from dvc.scm.base import MergeConflictError

    if git.test_backend == "dulwich":
        pytest.skip()

    tmp_dir.scm_gen("foo", "foo", commit="init")
    init_rev = scm.get_rev()

    scm.checkout("branch", create_new=True)
    tmp_dir.scm_gen("foo", "bar", commit="bar")
    branch = scm.resolve_rev("branch")

    scm.checkout("master")

    with pytest.raises(MergeConflictError):
        tmp_dir.scm_gen("foo", "baz", commit="baz")
        git.merge(branch, commit=not squash, squash=squash, msg="merge")

    scm.gitpython.git.reset(init_rev, hard=True)
    merge_rev = git.merge(
        branch, commit=not squash, squash=squash, msg="merge"
    )
    assert (tmp_dir / "foo").read_text() == "bar"
    if squash:
        assert merge_rev is None
        assert scm.get_rev() == init_rev
    else:
        assert scm.get_rev() == merge_rev


def test_checkout_index(tmp_dir, scm, git):
    if git.test_backend == "dulwich":
        pytest.skip()

    tmp_dir.scm_gen(
        {"foo": "foo", "bar": "bar", "dir": {"baz": "baz"}}, commit="init"
    )
    tmp_dir.gen({"foo": "baz", "dir": {"baz": "foo"}})

    with (tmp_dir / "dir").chdir():
        git.checkout_index([os.path.join("..", "foo"), "baz"], force=True)
    assert (tmp_dir / "foo").read_text() == "foo"
    assert (tmp_dir / "dir" / "baz").read_text() == "baz"

    tmp_dir.gen({"foo": "baz", "bar": "baz", "dir": {"baz": "foo"}})
    git.checkout_index(force=True)
    assert (tmp_dir / "foo").read_text() == "foo"
    assert (tmp_dir / "bar").read_text() == "bar"
    assert (tmp_dir / "dir" / "baz").read_text() == "baz"


@pytest.mark.parametrize(
    "strategy, expected", [("ours", "baz"), ("theirs", "bar")]
)
def test_checkout_index_conflicts(tmp_dir, scm, git, strategy, expected):
    from dvc.scm.base import MergeConflictError

    if git.test_backend == "dulwich":
        pytest.skip()

    tmp_dir.scm_gen({"file": "foo"}, commit="init")
    scm.checkout("branch", create_new=True)
    tmp_dir.scm_gen({"file": "bar"}, commit="bar")
    bar = scm.get_rev()
    scm.checkout("master")
    tmp_dir.scm_gen({"file": "baz"}, commit="baz")

    try:
        git.merge(bar, commit=False, squash=True)
    except MergeConflictError:
        if strategy == "ours":
            git.checkout_index(ours=True)
        else:
            git.checkout_index(theirs=True)
    assert (tmp_dir / "file").read_text() == expected


def test_resolve_rev(tmp_dir, scm, make_tmp_dir, git):
    from dvc.scm.base import RevError

    if git.test_backend == "dulwich":
        pytest.skip()

    remote_dir = make_tmp_dir("git-remote", scm=True)
    url = f"file://{remote_dir.resolve().as_posix()}"
    scm.gitpython.repo.create_remote("origin", url)
    scm.gitpython.repo.create_remote("upstream", url)

    tmp_dir.scm_gen({"file": "0"}, commit="init")
    init_rev = scm.get_rev()
    tmp_dir.scm_gen({"file": "1"}, commit="1")
    rev = scm.get_rev()
    scm.checkout("branch", create_new=True)
    tmp_dir.gen(
        {
            os.path.join(".git", "refs", "foo"): rev,
            os.path.join(".git", "refs", "remotes", "origin", "bar"): rev,
            os.path.join(".git", "refs", "remotes", "origin", "baz"): rev,
            os.path.join(
                ".git", "refs", "remotes", "upstream", "baz"
            ): init_rev,
        }
    )

    assert git.resolve_rev(rev) == rev
    assert git.resolve_rev(rev[:7]) == rev
    assert git.resolve_rev("HEAD") == rev
    assert git.resolve_rev("branch") == rev
    assert git.resolve_rev("refs/foo") == rev
    assert git.resolve_rev("bar") == rev
    assert git.resolve_rev("origin/baz") == rev

    with pytest.raises(RevError):
        git.resolve_rev("qux")

    with pytest.raises(RevError):
        git.resolve_rev("baz")


def test_checkout(tmp_dir, scm, git):
    if git.test_backend == "dulwich":
        pytest.skip()

    tmp_dir.scm_gen({"foo": "foo"}, commit="foo")
    foo_rev = scm.get_rev()
    tmp_dir.scm_gen("foo", "bar", commit="bar")
    bar_rev = scm.get_rev()

    git.checkout("branch", create_new=True)
    assert git.get_ref("HEAD", follow=False) == "refs/heads/branch"
    assert (tmp_dir / "foo").read_text() == "bar"

    git.checkout("master", detach=True)
    assert git.get_ref("HEAD", follow=False) == bar_rev

    git.checkout("master")
    assert git.get_ref("HEAD", follow=False) == "refs/heads/master"

    git.checkout(foo_rev[:7])
    assert git.get_ref("HEAD", follow=False) == foo_rev
    assert (tmp_dir / "foo").read_text() == "foo"


def test_reset(tmp_dir, scm, git):
    if git.test_backend == "dulwich":
        pytest.skip()

    tmp_dir.scm_gen({"foo": "foo", "dir": {"baz": "baz"}}, commit="init")

    tmp_dir.gen({"foo": "bar", "dir": {"baz": "bar"}})
    scm.add(["foo", os.path.join("dir", "baz")])
    git.reset()
    assert (tmp_dir / "foo").read_text() == "bar"
    assert (tmp_dir / "dir" / "baz").read_text() == "bar"
    staged, unstaged, _ = scm.status()
    assert len(staged) == 0
    assert set(unstaged) == {"foo", "dir/baz"}

    scm.add(["foo", os.path.join("dir", "baz")])
    git.reset(hard=True)
    assert (tmp_dir / "foo").read_text() == "foo"
    assert (tmp_dir / "dir" / "baz").read_text() == "baz"
    staged, unstaged, _ = scm.status()
    assert len(staged) == 0
    assert len(unstaged) == 0

    tmp_dir.gen({"foo": "bar", "bar": "bar", "dir": {"baz": "bar"}})
    scm.add(["foo", "bar", os.path.join("dir", "baz")])
    with (tmp_dir / "dir").chdir():
        git.reset(paths=[os.path.join("..", "foo"), os.path.join("baz")])
    assert (tmp_dir / "foo").read_text() == "bar"
    assert (tmp_dir / "bar").read_text() == "bar"
    assert (tmp_dir / "dir" / "baz").read_text() == "bar"
    staged, unstaged, _ = scm.status()
    assert len(staged) == 1
    assert len(unstaged) == 2


def test_add(tmp_dir, scm, git):
    if git.test_backend == "pygit2":
        pytest.skip()

    tmp_dir.gen({"foo": "foo", "bar": "bar", "dir": {"baz": "baz"}})
    git.add(["foo", "dir"])
    staged, unstaged, untracked = scm.status()
    assert set(staged["add"]) == {"foo", "dir/baz"}
    assert len(unstaged) == 0
    assert len(untracked) == 1

    scm.commit("commit")
    tmp_dir.gen({"foo": "bar", "dir": {"baz": "bar"}})
    git.add([], update=True)
    staged, unstaged, _ = scm.status()
    assert set(staged["modify"]) == {"foo", "dir/baz"}
    assert len(unstaged) == 0
    assert len(untracked) == 1

    scm.reset()
    git.add(["dir"], update=True)
    staged, unstaged, _ = scm.status()
    assert set(staged["modify"]) == {"dir/baz"}
    assert len(unstaged) == 1
    assert len(untracked) == 1


@pytest.mark.parametrize("use_sha", [True, False])
def test_pygit_resolve_refish(tmp_dir, scm, git, use_sha):
    import pygit2

    if git.test_backend != "pygit2":
        pytest.skip()

    tmp_dir.scm_gen("foo", "foo", commit="foo")
    head = scm.get_rev()
    tag = "my_tag"
    scm.gitpython.git.tag("-a", tag, "-m", "create annotated tag")

    if use_sha:
        # refish will be annotated tag SHA (not commit SHA)
        ref = git.pygit2.repo.references.get(f"refs/tags/{tag}")
        refish = str(ref.target)
    else:
        refish = tag

    assert refish != head
    commit, ref = git.pygit2._resolve_refish(refish)
    assert isinstance(commit, pygit2.Commit)
    assert str(commit.id) == head
    if not use_sha:
        assert ref.name == f"refs/tags/{tag}"


@pytest.mark.skipif(os.name != "nt", reason="Windows only")
def test_pygit_checkout_subdir(tmp_dir, scm, git):
    if git.test_backend != "pygit2":
        pytest.skip()

    tmp_dir.scm_gen("foo", "foo", commit="init")
    rev = scm.get_rev()
    tmp_dir.scm_gen({"dir": {"bar": "bar"}}, commit="dir")

    with (tmp_dir / "dir").chdir():
        git.checkout(rev)
        assert not (tmp_dir / "dir" / "bar").exists()


@pytest.mark.parametrize(
    "algorithm", [b"ssh-rsa", b"rsa-sha2-256", b"rsa-sha2-512"]
)
def test_dulwich_github_compat(mocker, algorithm):
    from asyncssh.misc import ProtocolError

    from dvc.scm.git.backend.dulwich.asyncssh_vendor import (
        _process_public_key_ok_gh,
    )

    key_data = b"foo"
    auth = mocker.Mock(
        _keypair=mocker.Mock(algorithm=algorithm, public_data=key_data),
    )
    packet = mocker.Mock()

    with pytest.raises(ProtocolError):
        strings = iter((b"ed21556", key_data))
        packet.get_string = lambda: next(strings)
        _process_public_key_ok_gh(auth, None, None, packet)

    strings = iter((b"ssh-rsa", key_data))
    packet.get_string = lambda: next(strings)
    _process_public_key_ok_gh(auth, None, None, packet)