mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-06 15:28:28 +01:00
Peter Rowlands (변기호)
73f2fa8c93
* git: support client kwargs for auth in remote methods * tests: use docker for git/ssh server * move dulwich backend into subdir * add base sync<->async support * dulwich: add initial asyncssh SSH vendor * dulwich: use asyncssh as the default SSH vendor * add test for git over ssh * test read (fetch) over ssh * ci: re-enable ssh tests temporary workaround for #6788, #6789 * move async stubs into dvc.scm.asyn * tests: use sshfs and not dvc.fs.ssh * allow explicit 0-length read in asyncssh vendor
146 lines
3.8 KiB
Python
146 lines
3.8 KiB
Python
import subprocess
|
|
|
|
import pytest
|
|
|
|
from .azure import Azure, azure, azure_server # noqa: F401
|
|
from .gdrive import ( # noqa: F401; noqa: F401
|
|
TEST_GDRIVE_REPO_BUCKET,
|
|
GDrive,
|
|
gdrive,
|
|
)
|
|
from .git_server import git_server, git_ssh # noqa: F401
|
|
from .gs import ( # noqa: F401; noqa: F401
|
|
GCP,
|
|
TEST_GCP_CREDS_FILE,
|
|
TEST_GCP_REPO_BUCKET,
|
|
gs,
|
|
)
|
|
from .hdfs import HDFS, hadoop, hdfs, hdfs_server, real_hdfs # noqa: F401
|
|
from .http import HTTP, http, http_server # noqa: F401
|
|
from .local import Local, local_cloud, local_remote, make_local # noqa: F401
|
|
from .oss import ( # noqa: F401
|
|
OSS,
|
|
TEST_OSS_REPO_BUCKET,
|
|
oss,
|
|
oss_server,
|
|
real_oss,
|
|
)
|
|
from .s3 import ( # noqa: F401
|
|
S3,
|
|
TEST_AWS_REPO_BUCKET,
|
|
real_s3,
|
|
s3,
|
|
s3_fake_creds_file,
|
|
s3_server,
|
|
)
|
|
from .ssh import SSH, ssh, ssh_connection, ssh_server # noqa: F401; noqa: F401
|
|
from .webdav import Webdav, webdav, webdav_server # noqa: F401
|
|
from .webhdfs import WebHDFS, webhdfs # noqa: F401
|
|
|
|
TEST_REMOTE = "upstream"
|
|
TEST_CONFIG = {
|
|
"cache": {},
|
|
"core": {"remote": TEST_REMOTE},
|
|
"remote": {TEST_REMOTE: {"url": ""}},
|
|
}
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def docker():
|
|
import os
|
|
|
|
# See https://travis-ci.community/t/docker-linux-containers-on-windows/301
|
|
if os.environ.get("CI") and os.name == "nt":
|
|
pytest.skip("disabled for Windows on Github Actions")
|
|
|
|
try:
|
|
subprocess.check_output("docker ps", shell=True)
|
|
except (subprocess.CalledProcessError, OSError):
|
|
pytest.skip("no docker installed")
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def docker_compose(docker):
|
|
try:
|
|
subprocess.check_output("docker-compose version", shell=True)
|
|
except (subprocess.CalledProcessError, OSError):
|
|
pytest.skip("no docker-compose installed")
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def docker_compose_project_name():
|
|
return "pytest-dvc-test"
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def docker_services(
|
|
docker_compose_file, docker_compose_project_name, tmp_path_factory
|
|
):
|
|
# overriding `docker_services` fixture from `pytest_docker` plugin to
|
|
# only launch docker images once.
|
|
|
|
from filelock import FileLock
|
|
from pytest_docker.plugin import DockerComposeExecutor, Services
|
|
|
|
executor = DockerComposeExecutor(
|
|
docker_compose_file, docker_compose_project_name
|
|
)
|
|
|
|
# making sure we don't accidentally launch docker-compose in parallel,
|
|
# as it might result in network conflicts. Inspired by:
|
|
# https://github.com/pytest-dev/pytest-xdist#making-session-scoped-fixtures-execute-only-once
|
|
lockfile = tmp_path_factory.getbasetemp().parent / "docker-compose.lock"
|
|
with FileLock(str(lockfile)): # pylint:disable=abstract-class-instantiated
|
|
executor.execute("up --build -d")
|
|
|
|
return Services(executor)
|
|
|
|
|
|
@pytest.fixture
|
|
def make_cloud(request):
|
|
def _make_cloud(typ):
|
|
return request.getfixturevalue(f"make_{typ}")()
|
|
|
|
return _make_cloud
|
|
|
|
|
|
@pytest.fixture
|
|
def make_remote(tmp_dir, dvc, make_cloud):
|
|
def _make_remote(name, typ="local", **kwargs):
|
|
cloud = make_cloud(typ)
|
|
tmp_dir.add_remote(name=name, config=cloud.config, **kwargs)
|
|
return cloud
|
|
|
|
return _make_remote
|
|
|
|
|
|
@pytest.fixture
|
|
def remote(tmp_dir, dvc, request):
|
|
cloud = request.param
|
|
assert cloud
|
|
tmp_dir.add_remote(config=cloud.config)
|
|
yield cloud
|
|
|
|
|
|
@pytest.fixture
|
|
def workspace(tmp_dir, dvc, request):
|
|
from dvc.objects.db import ODBManager
|
|
|
|
cloud = request.param
|
|
|
|
assert cloud
|
|
|
|
tmp_dir.add_remote(name="workspace", config=cloud.config, default=False)
|
|
tmp_dir.add_remote(
|
|
name="cache", url="remote://workspace/cache", default=False
|
|
)
|
|
|
|
scheme = getattr(cloud, "scheme", "local")
|
|
if scheme != "http":
|
|
with dvc.config.edit() as conf:
|
|
conf["cache"][scheme] = "cache"
|
|
|
|
dvc.odb = ODBManager(dvc)
|
|
|
|
return cloud
|