diff --git a/.gitignore b/.gitignore index 8f4a190e2..e4688cdf1 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,9 @@ innosetup/config.ini pip-wheel-metadata/ .vscode/ + +azurite +env.sh +tests/remotes_env +scripts/ci/gcp-creds.json +.gcp-creds.json diff --git a/scripts/ci/install_hadoop.sh b/scripts/ci/install_hadoop.sh old mode 100644 new mode 100755 index f59f8beb9..4ef1f2ee4 --- a/scripts/ci/install_hadoop.sh +++ b/scripts/ci/install_hadoop.sh @@ -6,7 +6,7 @@ set -x set -e sudo apt-get update -y -sudo apt-get install default-jdk +sudo apt-get install openjdk-8-jdk java -version pushd /usr/local @@ -23,7 +23,7 @@ echo "export HADOOP_HDFS_HOME=/usr/local/hadoop" >> env.sh echo "export YARN_HOME=/usr/local/hadoop" >> env.sh echo "export HADOOP_COMMON_LIB_NATIVE_DIR=/usr/local/hadoop/lib/native" >> env.sh echo "export JAVA_HOME=/usr/" >> env.sh -echo "export PATH=$PATH:/usr/local/hadoop/sbin:/usr/local/hadoop/bin:$JAVA_PATH/bin" >> env.sh +echo "export PATH=\$PATH:/usr/local/hadoop/sbin:/usr/local/hadoop/bin:$JAVA_PATH/bin" >> env.sh source env.sh diff --git a/scripts/ci/remove_hadoop.sh b/scripts/ci/remove_hadoop.sh new file mode 100755 index 000000000..c8b56d596 --- /dev/null +++ b/scripts/ci/remove_hadoop.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +stop-dfs.sh +sudo rm -rf /usr/local/hadoop +sudo rm -rf /usr/local/hadoop-2.6.5 +sudo rm -rf /usr/local/hadoop-2.6.5.tar.gz +rm env.sh diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index 45067b0c1..4bd90ab75 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -43,8 +43,8 @@ TEST_CONFIG = { TEST_SECTION: {Config.SECTION_REMOTE_URL: ""}, } -TEST_AWS_REPO_BUCKET = "dvc-test" -TEST_GCP_REPO_BUCKET = "dvc-test" +TEST_AWS_REPO_BUCKET = os.environ.get("DVC_TEST_AWS_REPO_BUCKET", "dvc-test") +TEST_GCP_REPO_BUCKET = os.environ.get("DVC_TEST_GCP_REPO_BUCKET", "dvc-test") TEST_OSS_REPO_BUCKET = "dvc-test" @@ -68,6 +68,8 @@ def _should_test_gcp(): if not os.path.exists(TestDvc.GCP_CREDS_FILE): return False + # NOTE: we copy because GOOGLE_APPLICATION_CREDENTIALS uses rel path. + # This is only my guess and I don't know why won't we use absolute. creds = os.getenv("GOOGLE_APPLICATION_CREDENTIALS") if creds and os.getenv("GCP_CREDS"): if os.path.exists(creds): diff --git a/tests/remotes_env.sample b/tests/remotes_env.sample new file mode 100644 index 000000000..13e671052 --- /dev/null +++ b/tests/remotes_env.sample @@ -0,0 +1,13 @@ +# Uncomment and fill in to test against S3 +# export AWS_ACCESS_KEY_ID="...SET-ME..." +# export AWS_SECRET_ACCESS_KEY="...SET-ME..." +# export DVC_TEST_AWS_REPO_BUCKET="...SET-ME..." + +# Uncomment and set a bucket name to test against Google Storage +# export GOOGLE_APPLICATION_CREDENTIALS=".gcp-creds.json" +# export GCP_CREDS="yes" +# export DVC_TEST_GCP_REPO_BUCKET="...SET-ME..." + +# Uncomment to test against Microsoft Azure via Azurite +# export AZURE_STORAGE_CONTAINER_NAME="dvc-test" +# export AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;"