mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-02 07:18:25 +01:00
Batuhan Taskaya
d4a3df73df
* s3: migrate to s3fs * s3: strip etag * s3: disable instance cache * tests: use the proper remote:// form in test_checkout * tests: move/drop some tests * setup: depend on s3fs * tests: normalize s3 tests with some cleanups * Use _download/_upload with old implementation * fsspec: cache .exists() results * setup: pin the boto version * setup: install s3fs[boto3] * s3: proper exception management * s3: config improvements * setup: reference without quotes * setup: install boto separately * setup: install moto server * s3: mocking adjustments for moto_server * s3: mark some fixtures with scope=session * s3: s3_fake_creds_file scopes to func * fsspec: add some docs regarding exists() optimization * fsspec: exists() fallback * fsspec: revert exists() optimization * tests: ssl_verify => verify * setup: temporarily fetch from master * s3: add boto as a requirement * tests: do an extra check regarding race conditions * tests: use motoserver docker image * tests: wrap config deletion with try/finally * tests: disable s3 by default * setup: upgrade fsspec
132 lines
3.4 KiB
Python
132 lines
3.4 KiB
Python
import subprocess
|
|
|
|
import pytest
|
|
|
|
from .azure import Azure, azure, azure_server # noqa: F401
|
|
from .gdrive import ( # noqa: F401; noqa: F401
|
|
TEST_GDRIVE_REPO_BUCKET,
|
|
GDrive,
|
|
gdrive,
|
|
)
|
|
from .gs import ( # noqa: F401; noqa: F401
|
|
GCP,
|
|
TEST_GCP_CREDS_FILE,
|
|
TEST_GCP_REPO_BUCKET,
|
|
gs,
|
|
)
|
|
from .hdfs import HDFS, hadoop, hdfs, hdfs_server, real_hdfs # noqa: F401
|
|
from .http import HTTP, http, http_server # noqa: F401
|
|
from .local import Local, local_cloud, local_remote # noqa: F401
|
|
from .oss import ( # noqa: F401
|
|
OSS,
|
|
TEST_OSS_REPO_BUCKET,
|
|
oss,
|
|
oss_server,
|
|
real_oss,
|
|
)
|
|
from .s3 import ( # noqa: F401
|
|
S3,
|
|
TEST_AWS_REPO_BUCKET,
|
|
real_s3,
|
|
s3,
|
|
s3_fake_creds_file,
|
|
s3_server,
|
|
)
|
|
from .ssh import ( # noqa: F401; noqa: F401
|
|
SSHMocked,
|
|
ssh,
|
|
ssh_connection,
|
|
ssh_server,
|
|
)
|
|
from .webdav import Webdav, webdav, webdav_server # noqa: F401
|
|
from .webhdfs import WebHDFS, real_webhdfs, webhdfs # noqa: F401
|
|
|
|
TEST_REMOTE = "upstream"
|
|
TEST_CONFIG = {
|
|
"cache": {},
|
|
"core": {"remote": TEST_REMOTE},
|
|
"remote": {TEST_REMOTE: {"url": ""}},
|
|
}
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def docker():
|
|
import os
|
|
|
|
# See https://travis-ci.community/t/docker-linux-containers-on-windows/301
|
|
if os.environ.get("CI") and os.name == "nt":
|
|
pytest.skip("disabled for Windows on Github Actions")
|
|
|
|
try:
|
|
subprocess.check_output("docker ps", shell=True)
|
|
except (subprocess.CalledProcessError, OSError):
|
|
pytest.skip("no docker installed")
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def docker_compose(docker):
|
|
try:
|
|
subprocess.check_output("docker-compose version", shell=True)
|
|
except (subprocess.CalledProcessError, OSError):
|
|
pytest.skip("no docker-compose installed")
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def docker_compose_project_name():
|
|
return "pytest-dvc-test"
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def docker_services(
|
|
docker_compose_file, docker_compose_project_name, tmp_path_factory
|
|
):
|
|
# overriding `docker_services` fixture from `pytest_docker` plugin to
|
|
# only launch docker images once.
|
|
|
|
from filelock import FileLock
|
|
from pytest_docker.plugin import DockerComposeExecutor, Services
|
|
|
|
executor = DockerComposeExecutor(
|
|
docker_compose_file, docker_compose_project_name,
|
|
)
|
|
|
|
# making sure we don't accidentally launch docker-compose in parallel,
|
|
# as it might result in network conflicts. Inspired by:
|
|
# https://github.com/pytest-dev/pytest-xdist#making-session-scoped-fixtures-execute-only-once
|
|
lockfile = tmp_path_factory.getbasetemp().parent / "docker-compose.lock"
|
|
with FileLock(str(lockfile)):
|
|
executor.execute("up --build -d")
|
|
|
|
return Services(executor)
|
|
|
|
|
|
@pytest.fixture
|
|
def remote(tmp_dir, dvc, request):
|
|
cloud = request.param
|
|
assert cloud
|
|
tmp_dir.add_remote(config=cloud.config)
|
|
yield cloud
|
|
|
|
|
|
@pytest.fixture
|
|
def workspace(tmp_dir, dvc, request):
|
|
from dvc.objects.db import ODBManager
|
|
|
|
cloud = request.param
|
|
|
|
assert cloud
|
|
|
|
tmp_dir.add_remote(name="workspace", config=cloud.config, default=False)
|
|
tmp_dir.add_remote(
|
|
name="cache", url="remote://workspace/cache", default=False
|
|
)
|
|
|
|
scheme = getattr(cloud, "scheme", "local")
|
|
if scheme != "http":
|
|
with dvc.config.edit() as conf:
|
|
conf["cache"][scheme] = "cache"
|
|
|
|
dvc.odb = ODBManager(dvc)
|
|
|
|
return cloud
|