Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-02 07:18:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

tests: launch oss/azure containers from fixtures (#4178)

Browse Source 
* tests: launch oss/azure containers from fixtures

Related to #4054

* ci: hdfs: remove unneeded fix

We are using pyarrow 0.17.0 already.
This commit is contained in:
Ruslan Kuprieiev
2020-07-08 19:54:17 +03:00
committed by GitHub
parent 253a0eae13
commit 0899b277c0
10 changed files with 106 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/ci/before_install.sh
-2
View File
@@ -32,8 +32,6 @@ if [[ "$TRAVIS_BUILD_STAGE_NAME" == "test" ]]; then
fi
if [ "$TRAVIS_OS_NAME" == "linux" ]; then
bash "$scriptdir/install_azurite.sh"
bash "$scriptdir/install_oss.sh"
bash "$scriptdir/install_hadoop.sh"
fi
scripts/ci/install_azurite.sh
-14
View File
@@ -1,14 +0,0 @@
#!/usr/bin/env bash
set -e
set -x
# run azurite
sudo docker run -d --restart always -e executable=blob -p 10000:10000 \
--tmpfs /opt/azurite/folder \
mcr.microsoft.com/azure-storage/azurite:3.3.0-preview \
azurite -l /data --blobHost 0.0.0.0
# save secrets
echo "export AZURE_STORAGE_CONTAINER_NAME='travis-tests'" >>env.sh
echo "export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;'" >>env.sh
scripts/ci/install_hadoop.sh
-7
View File
@@ -24,13 +24,6 @@ echo "export HADOOP_HDFS_HOME=/usr/local/hadoop" >>env.sh
echo "export YARN_HOME=/usr/local/hadoop" >>env.sh
echo "export HADOOP_COMMON_LIB_NATIVE_DIR=/usr/local/hadoop/lib/native" >>env.sh
# PyArrow==0.16.0 regression https://issues.apache.org/jira/browse/ARROW-7841
# retrieves native library from $HADOOP_HOME directory instead of
# `$HADOOP_HOME/lib/native`.
# Fix: force search for `libhdfs.so` inside `$HADOOP_HOME/lib/native`.
# Note: not needed for PyArrow==0.17.0.
echo "export ARROW_LIBHDFS_DIR=/usr/local/hadoop/lib/native" >> env.sh
echo "export JAVA_HOME=/usr/" >>env.sh
echo "export PATH=\$PATH:/usr/local/hadoop/sbin:/usr/local/hadoop/bin:$JAVA_PATH/bin" >>env.sh
scripts/ci/install_oss.sh
-10
View File
@@ -1,10 +0,0 @@
#!/bin/bash
set -euo pipefail
git clone https://github.com/iterative/oss-emulator.git
sudo docker image build -t oss:1.0 oss-emulator
sudo docker run --detach --restart always -p 8880:8880 --name oss-emulator oss:1.0
echo "export OSS_ENDPOINT='localhost:8880'" >>env.sh
echo "export OSS_ACCESS_KEY_ID='AccessKeyID'" >>env.sh
echo "export OSS_ACCESS_KEY_SECRET='AccessKeySecret'" >>env.sh
setup.py
+1
View File
@@ -103,6 +103,7 @@ tests_requirements = [
"wheel>=0.31.1",
# Test requirements:
"pytest>=4.6.0",
"pytest-docker>=0.7.2",
"pytest-timeout>=1.3.3",
"pytest-cov>=2.6.1",
"pytest-xdist>=1.26.1",
tests/docker-compose.yml
+11
View File
@@ -0,0 +1,11 @@
---
version: '3.2'
services:
azurite:
image: mcr.microsoft.com/azure-storage/azurite:3.3.0-preview
ports:
- "10000"
oss:
image: rkuprieiev/oss-emulator
ports:
- "8880"
tests/remotes/__init__.py
+12 -2
View File
@@ -1,10 +1,12 @@
import subprocess
import pytest
from .azure import Azure, azure # noqa: F401
from .azure import Azure, azure, azure_server # noqa: F401
from .hdfs import HDFS, hdfs # noqa: F401
from .http import HTTP, http, http_server # noqa: F401
from .local import Local, local_cloud, local_remote # noqa: F401
from .oss import OSS, TEST_OSS_REPO_BUCKET, oss # noqa: F401
from .oss import OSS, TEST_OSS_REPO_BUCKET, oss, oss_server # noqa: F401
from .s3 import S3, TEST_AWS_REPO_BUCKET, real_s3, s3 # noqa: F401
TEST_REMOTE = "upstream"
@@ -35,6 +37,14 @@ from .ssh import ( # noqa: F401; noqa: F401
)
@pytest.fixture(scope="session")
def docker_compose():
try:
subprocess.check_output("docker-compose version", shell=True)
except (subprocess.CalledProcessError, OSError):
pytest.skip("no docker-compose installed")
@pytest.fixture
def remote(tmp_dir, dvc, request):
cloud = request.param
tests/remotes/azure.py
+44 -19
View File
@@ -1,36 +1,61 @@
# pylint:disable=abstract-method
import os
import uuid
import pytest
from dvc.path_info import CloudURLInfo
from dvc.utils import env2bool
from .base import Base
TEST_AZURE_CONTAINER = "tests"
TEST_AZURE_CONNECTION_STRING = (
"DefaultEndpointsProtocol=http;"
"AccountName=devstoreaccount1;"
"AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSR"
"Z6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;"
"BlobEndpoint=http://127.0.0.1:{port}/devstoreaccount1;"
)
class Azure(Base, CloudURLInfo):
@staticmethod
def should_test():
do_test = env2bool("DVC_TEST_AZURE", undefined=None)
if do_test is not None:
return do_test
pass
return os.getenv("AZURE_STORAGE_CONTAINER_NAME") and os.getenv(
"AZURE_STORAGE_CONNECTION_STRING"
)
@staticmethod
def get_url():
container_name = os.getenv("AZURE_STORAGE_CONTAINER_NAME")
assert container_name is not None
return "azure://{}/{}".format(container_name, str(uuid.uuid4()))
@pytest.fixture(scope="session")
def azure_server(docker_compose, docker_services):
from azure.storage.blob import ( # pylint: disable=no-name-in-module
BlockBlobService,
)
from azure.common import ( # pylint: disable=no-name-in-module
AzureException,
)
port = docker_services.port_for("azurite", 10000)
connection_string = TEST_AZURE_CONNECTION_STRING.format(port=port)
def _check():
try:
BlockBlobService(
connection_string=connection_string,
).list_containers()
return True
except AzureException:
return False
docker_services.wait_until_responsive(
timeout=60.0, pause=0.1, check=_check
)
return connection_string
@pytest.fixture
def azure():
if not Azure.should_test():
pytest.skip("no azure running")
yield Azure(Azure.get_url())
def azure(azure_server):
url = f"azure://{TEST_AZURE_CONTAINER}/{uuid.uuid4()}"
ret = Azure(url)
ret.config = {
"url": url,
"connection_string": azure_server,
}
return ret
tests/remotes/oss.py
+35 -22
View File
@@ -1,41 +1,54 @@
# pylint:disable=abstract-method
import os
import uuid
import pytest
from dvc.path_info import CloudURLInfo
from dvc.utils import env2bool
from .base import Base
TEST_OSS_REPO_BUCKET = "dvc-test"
TEST_OSS_ENDPOINT = "127.0.0.1:{port}"
TEST_OSS_ACCESS_KEY_ID = "AccessKeyID"
TEST_OSS_ACCESS_KEY_SECRET = "AccessKeySecret"
class OSS(Base, CloudURLInfo):
@staticmethod
def should_test():
do_test = env2bool("DVC_TEST_OSS", undefined=None)
if do_test is not None:
return do_test
pass
return (
os.getenv("OSS_ENDPOINT")
and os.getenv("OSS_ACCESS_KEY_ID")
and os.getenv("OSS_ACCESS_KEY_SECRET")
)
@staticmethod
def _get_storagepath():
return f"{TEST_OSS_REPO_BUCKET}/{uuid.uuid4()}"
@pytest.fixture(scope="session")
def oss_server(docker_compose, docker_services):
import oss2
@staticmethod
def get_url():
return f"oss://{OSS._get_storagepath()}"
port = docker_services.port_for("oss", 8880)
endpoint = TEST_OSS_ENDPOINT.format(port=port)
def _check():
try:
auth = oss2.Auth(
TEST_OSS_ACCESS_KEY_ID, TEST_OSS_ACCESS_KEY_SECRET
)
oss2.Bucket(auth, endpoint, "mybucket").get_bucket_info()
return True
except oss2.exceptions.NoSuchBucket:
return True
except oss2.exceptions.OssError:
return False
docker_services.wait_until_responsive(timeout=30.0, pause=5, check=_check)
return endpoint
@pytest.fixture
def oss():
if not OSS.should_test():
pytest.skip("no oss running")
yield OSS(OSS.get_url())
def oss(oss_server):
url = f"oss://{TEST_OSS_REPO_BUCKET}/{uuid.uuid4()}"
ret = OSS(url)
ret.config = {
"url": url,
"oss_key_id": TEST_OSS_ACCESS_KEY_ID,
"oss_key_secret": TEST_OSS_ACCESS_KEY_SECRET,
"oss_endpoint": oss_server,
}
return ret
tests/unit/remote/test_azure.py
+3 -9
View File
@@ -1,8 +1,5 @@
import pytest
from dvc.path_info import PathInfo
from dvc.remote.azure import AzureRemoteTree
from tests.remotes import Azure
container_name = "container-name"
connection_string = (
@@ -32,14 +29,11 @@ def test_init(dvc):
assert tree._conn_kwargs["connection_string"] == connection_string
def test_get_file_hash(tmp_dir):
if not Azure.should_test():
pytest.skip("no azurite running")
def test_get_file_hash(tmp_dir, azure):
tmp_dir.gen("foo", "foo")
tree = AzureRemoteTree(None, {})
to_info = tree.PATH_CLS(Azure.get_url())
tree = AzureRemoteTree(None, azure.config)
to_info = azure
tree.upload(PathInfo("foo"), to_info)
assert tree.exists(to_info)
hash_ = tree.get_file_hash(to_info)