mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-22 07:18:28 +01:00
git: use asyncssh as dulwich SSH vendor (#6797)
* git: support client kwargs for auth in remote methods * tests: use docker for git/ssh server * move dulwich backend into subdir * add base sync<->async support * dulwich: add initial asyncssh SSH vendor * dulwich: use asyncssh as the default SSH vendor * add test for git over ssh * test read (fetch) over ssh * ci: re-enable ssh tests temporary workaround for #6788, #6789 * move async stubs into dvc.scm.asyn * tests: use sshfs and not dvc.fs.ssh * allow explicit 0-length read in asyncssh vendor
This commit is contained in:
@@ -37,3 +37,14 @@ services:
|
||||
- 2222
|
||||
volumes:
|
||||
- ./remotes/user.key.pub:/tmp/key
|
||||
|
||||
git-server:
|
||||
image: ghcr.io/linuxserver/openssh-server
|
||||
environment:
|
||||
- USER_NAME=user
|
||||
- PUBLIC_KEY_FILE=/tmp/key
|
||||
ports:
|
||||
- 2222
|
||||
volumes:
|
||||
- ./remotes/user.key.pub:/tmp/key
|
||||
- ./remotes/git-init:/config/custom-cont-init.d
|
||||
|
||||
@@ -331,3 +331,52 @@ def test_git_stash_clear(tmp_dir, scm, ref):
|
||||
not os.path.exists(log_path)
|
||||
or not open(log_path, encoding="utf-8").read()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.needs_internet
|
||||
@pytest.mark.parametrize("server", [pytest.lazy_fixture("git_ssh")])
|
||||
def test_git_ssh(tmp_dir, scm, server):
|
||||
from dulwich.repo import Repo as DulwichRepo
|
||||
from sshfs import SSHFileSystem
|
||||
|
||||
from dvc.utils.fs import remove
|
||||
from tests.remotes.ssh import TEST_SSH_KEY_PATH, TEST_SSH_USER
|
||||
|
||||
fs = SSHFileSystem(
|
||||
host=server.host,
|
||||
port=server.port,
|
||||
username=TEST_SSH_USER,
|
||||
client_keys=[TEST_SSH_KEY_PATH],
|
||||
)
|
||||
server._ssh.execute("git init --bare test-repo.git")
|
||||
url = f"ssh://{TEST_SSH_USER}@{server.host}:{server.port}/~/test-repo.git"
|
||||
|
||||
tmp_dir.scm_gen("foo", "foo", commit="init")
|
||||
rev = scm.get_rev()
|
||||
|
||||
scm.push_refspec(
|
||||
url,
|
||||
"refs/heads/master",
|
||||
"refs/heads/master",
|
||||
force=True,
|
||||
key_filename=TEST_SSH_KEY_PATH,
|
||||
)
|
||||
|
||||
assert (
|
||||
rev.encode("ascii")
|
||||
== fs.open("test-repo.git/refs/heads/master").read().strip()
|
||||
)
|
||||
|
||||
remove(tmp_dir / ".git")
|
||||
remove(tmp_dir / "foo")
|
||||
DulwichRepo.init(str(tmp_dir))
|
||||
|
||||
scm.fetch_refspecs(
|
||||
url,
|
||||
["refs/heads/master"],
|
||||
force=True,
|
||||
key_filename=TEST_SSH_KEY_PATH,
|
||||
)
|
||||
assert rev == scm.get_ref("refs/heads/master")
|
||||
scm.checkout("master")
|
||||
assert "foo" == (tmp_dir / "foo").read_text()
|
||||
|
||||
@@ -8,6 +8,7 @@ from .gdrive import ( # noqa: F401; noqa: F401
|
||||
GDrive,
|
||||
gdrive,
|
||||
)
|
||||
from .git_server import git_server, git_ssh # noqa: F401
|
||||
from .gs import ( # noqa: F401; noqa: F401
|
||||
GCP,
|
||||
TEST_GCP_CREDS_FILE,
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
#!/bin/bash
|
||||
apk add --no-cache git
|
||||
@@ -0,0 +1,60 @@
|
||||
import pytest
|
||||
|
||||
from .ssh import SSH, TEST_SSH_KEY_PATH, TEST_SSH_USER
|
||||
|
||||
|
||||
class GitSSH(SSH):
|
||||
@staticmethod
|
||||
def get_url(host, port): # pylint: disable=arguments-differ
|
||||
return f"ssh://{host}:{port}/tmp/data/git"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def git_server(test_config, docker_compose, docker_services):
|
||||
import asyncssh
|
||||
from sshfs import SSHFileSystem
|
||||
|
||||
test_config.requires("ssh")
|
||||
conn_info = {
|
||||
"host": "127.0.0.1",
|
||||
"port": docker_services.port_for("git-server", 2222),
|
||||
}
|
||||
|
||||
def get_fs():
|
||||
return SSHFileSystem(
|
||||
**conn_info,
|
||||
username=TEST_SSH_USER,
|
||||
client_keys=[TEST_SSH_KEY_PATH],
|
||||
)
|
||||
|
||||
def _check():
|
||||
try:
|
||||
fs = get_fs()
|
||||
fs.exists("/")
|
||||
fs.execute("git --version")
|
||||
except asyncssh.Error:
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
docker_services.wait_until_responsive(timeout=30.0, pause=1, check=_check)
|
||||
return conn_info
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def git_ssh_connection(git_server):
|
||||
from sshfs import SSHFileSystem
|
||||
|
||||
yield SSHFileSystem(
|
||||
host=git_server["host"],
|
||||
port=git_server["port"],
|
||||
username=TEST_SSH_USER,
|
||||
client_keys=[TEST_SSH_KEY_PATH],
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def git_ssh(git_server, monkeypatch):
|
||||
url = GitSSH(GitSSH.get_url(**git_server))
|
||||
url.mkdir(exist_ok=True, parents=True)
|
||||
return url
|
||||
Reference in New Issue
Block a user