test: update/add helpers to run tests against remotes

.gitignore

@@ -28,3 +28,9 @@ innosetup/config.ini
 
 pip-wheel-metadata/
 .vscode/
+
+azurite
+env.sh
+tests/remotes_env
+scripts/ci/gcp-creds.json
+.gcp-creds.json

scripts/ci/install_hadoop.sh

@@ -6,7 +6,7 @@ set -x
 set -e
 
 sudo apt-get update -y
-sudo apt-get install default-jdk
+sudo apt-get install openjdk-8-jdk
 java -version
 
 pushd /usr/local
@@ -23,7 +23,7 @@ echo "export HADOOP_HDFS_HOME=/usr/local/hadoop" >> env.sh
 echo "export YARN_HOME=/usr/local/hadoop" >> env.sh
 echo "export HADOOP_COMMON_LIB_NATIVE_DIR=/usr/local/hadoop/lib/native" >> env.sh
 echo "export JAVA_HOME=/usr/" >> env.sh
-echo "export PATH=$PATH:/usr/local/hadoop/sbin:/usr/local/hadoop/bin:$JAVA_PATH/bin" >> env.sh
+echo "export PATH=\$PATH:/usr/local/hadoop/sbin:/usr/local/hadoop/bin:$JAVA_PATH/bin" >> env.sh
 
 source env.sh
 

scripts/ci/remove_hadoop.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+stop-dfs.sh
+sudo rm -rf /usr/local/hadoop
+sudo rm -rf /usr/local/hadoop-2.6.5
+sudo rm -rf /usr/local/hadoop-2.6.5.tar.gz
+rm env.sh

tests/func/test_data_cloud.py

@@ -43,8 +43,8 @@ TEST_CONFIG = {
     TEST_SECTION: {Config.SECTION_REMOTE_URL: ""},
 }
 
-TEST_AWS_REPO_BUCKET = "dvc-test"
-TEST_GCP_REPO_BUCKET = "dvc-test"
+TEST_AWS_REPO_BUCKET = os.environ.get("DVC_TEST_AWS_REPO_BUCKET", "dvc-test")
+TEST_GCP_REPO_BUCKET = os.environ.get("DVC_TEST_GCP_REPO_BUCKET", "dvc-test")
 TEST_OSS_REPO_BUCKET = "dvc-test"
 
 
@@ -68,6 +68,8 @@ def _should_test_gcp():
     if not os.path.exists(TestDvc.GCP_CREDS_FILE):
         return False
 
+    # NOTE: we copy because GOOGLE_APPLICATION_CREDENTIALS uses rel path.
+    #       This is only my guess and I don't know why won't we use absolute.
     creds = os.getenv("GOOGLE_APPLICATION_CREDENTIALS")
     if creds and os.getenv("GCP_CREDS"):
         if os.path.exists(creds):

tests/remotes_env.sample

@@ -0,0 +1,13 @@
+# Uncomment and fill in to test against S3
+# export AWS_ACCESS_KEY_ID="...SET-ME..."
+# export AWS_SECRET_ACCESS_KEY="...SET-ME..."
+# export DVC_TEST_AWS_REPO_BUCKET="...SET-ME..."
+
+# Uncomment and set a bucket name to test against Google Storage
+# export GOOGLE_APPLICATION_CREDENTIALS=".gcp-creds.json"
+# export GCP_CREDS="yes"
+# export DVC_TEST_GCP_REPO_BUCKET="...SET-ME..."
+
+# Uncomment to test against Microsoft Azure via Azurite
+# export AZURE_STORAGE_CONTAINER_NAME="dvc-test"
+# export AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;"