From 9fdb922ab7b717cf69a81fc5351b86ecf31953b9 Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Thu, 31 Jan 2019 05:55:54 +0200 Subject: [PATCH] travis: test azure and hadoop only during cron jobs Signed-off-by: Ruslan Kuprieiev --- .travis.yml | 33 +-------------------------------- scripts/ci/before_install.sh | 26 ++++++++++++++++++++++++-- scripts/ci/install_azurite.sh | 4 ++-- scripts/ci/install_hadoop.sh | 33 ++++++++++++++++++++++----------- 4 files changed, 49 insertions(+), 47 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0d23b3f69..210582b07 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,44 +59,13 @@ matrix: env: - PYTHON_VER=3.5.0 before_install: - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - brew update; - brew upgrade python; - python --version; - fi - - pip install -r scripts/ci/requirements.txt - # stop the build if there are any readme formatting errors - - python setup.py checkdocs - # stop the build if there are any black errors - - black dvc --check - - black tests --check - bash ./scripts/ci/before_install.sh - if [[ -n "$PYTHON_VER" ]]; then eval "$(pyenv init -)"; pyenv install --skip-existing --keep --verbose "$PYTHON_VER"; pyenv shell "$PYTHON_VER"; fi - - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - brew install openssl; - brew cask install google-cloud-sdk; - fi - - python --version - - if [[ "$TRAVIS_OS_NAME" != "osx" ]]; then - bash ./scripts/ci/install_hadoop.sh; - source ~/.bashrc; - hadoop fs -help; - pushd ~; - sudo mkdir -p /home/hadoop/hadoopinfra/hdfs/namenode/; - sudo chmod 777 /home/hadoop/hadoopinfra/hdfs/namenode/; - sudo chmod 777 -R /usr/local/hadoop; - hdfs namenode -format; - start-dfs.sh; - jps; - hadoop fs -ls /; - hadoop fs -ls hdfs://127.0.0.1/; - hadoop fs -mkdir hdfs://127.0.0.1/ololo; - popd; - fi + - source env.sh install: - bash ./scripts/ci/install.sh script: diff --git a/scripts/ci/before_install.sh b/scripts/ci/before_install.sh index 8be55f10b..37a862d69 100644 --- a/scripts/ci/before_install.sh +++ b/scripts/ci/before_install.sh @@ -3,6 +3,21 @@ set -x set -e +if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then + brew update + brew upgrade python + python --version +fi + +pip install -r scripts/ci/requirements.txt + +# stop the build if there are any readme formatting errors +python setup.py checkdocs + +# stop the build if there are any black errors +black dvc --check +black tests --check + #NOTE: ssh keys for ssh test to be able to ssh to the localhost ls -la ~/.ssh/ @@ -20,7 +35,14 @@ ssh localhost ls &> /dev/null scriptdir="$(dirname $0)" -if [ -n "$TRAVIS_OS_NAME" ] && [ "$TRAVIS_OS_NAME" != "osx" ]; then +echo > env.sh +if [ -n "$TRAVIS_OS_NAME" ] && [ "$TRAVIS_OS_NAME" != "osx" ] \ + && [ "$TRAVIS_EVENT_TYPE" == "cron" ]; then bash "$scriptdir/install_azurite.sh" - source ~/.bashrc + bash "$scriptdir/install_hadoop.sh" +fi + +if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then + brew install openssl + brew cask install google-cloud-sdk fi diff --git a/scripts/ci/install_azurite.sh b/scripts/ci/install_azurite.sh index c753d4933..094864db4 100755 --- a/scripts/ci/install_azurite.sh +++ b/scripts/ci/install_azurite.sh @@ -13,5 +13,5 @@ sudo apt-get install -y docker-ce sudo docker run --restart always -e executable=blob -p 10000:10000 --tmpfs /opt/azurite/folder -d arafato/azurite:2.6.5 # save secrets -echo "export AZURE_STORAGE_CONTAINER_NAME='travis-tests'" >> ~/.bashrc -echo "export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;'" >> ~/.bashrc +echo "export AZURE_STORAGE_CONTAINER_NAME='travis-tests'" >> env.sh +echo "export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;'" >> env.sh diff --git a/scripts/ci/install_hadoop.sh b/scripts/ci/install_hadoop.sh index 6088fcd14..f59f8beb9 100644 --- a/scripts/ci/install_hadoop.sh +++ b/scripts/ci/install_hadoop.sh @@ -16,20 +16,31 @@ sudo mkdir hadoop sudo mv hadoop-2.6.5/* hadoop/ popd -echo "export HADOOP_HOME=/usr/local/hadoop" >> ~/.bashrc -echo "export HADOOP_MAPRED_HOME=/usr/local/hadoop" >> ~/.bashrc -echo "export HADOOP_COMMON_HOME=/usr/local/hadoop" >> ~/.bashrc -echo "export HADOOP_HDFS_HOME=/usr/local/hadoop" >> ~/.bashrc -echo "export YARN_HOME=/usr/local/hadoop" >> ~/.bashrc -echo "export HADOOP_COMMON_LIB_NATIVE_DIR=/usr/local/hadoop/lib/native" >> ~/.bashrc -echo "export JAVA_HOME=/usr/" >> ~/.bashrc -echo "export PATH=$PATH:/usr/local/hadoop/sbin:/usr/local/hadoop/bin:$JAVA_PATH/bin" >> ~/.bashrc +echo "export HADOOP_HOME=/usr/local/hadoop" >> env.sh +echo "export HADOOP_MAPRED_HOME=/usr/local/hadoop" >> env.sh +echo "export HADOOP_COMMON_HOME=/usr/local/hadoop" >> env.sh +echo "export HADOOP_HDFS_HOME=/usr/local/hadoop" >> env.sh +echo "export YARN_HOME=/usr/local/hadoop" >> env.sh +echo "export HADOOP_COMMON_LIB_NATIVE_DIR=/usr/local/hadoop/lib/native" >> env.sh +echo "export JAVA_HOME=/usr/" >> env.sh +echo "export PATH=$PATH:/usr/local/hadoop/sbin:/usr/local/hadoop/bin:$JAVA_PATH/bin" >> env.sh -cat ~/.bashrc - -source ~/.bashrc +source env.sh sudo bash -c 'echo "export JAVA_HOME=/usr/" >> /usr/local/hadoop/etc/hadoop/hadoop-env.sh' sudo cp scripts/ci/core-site.xml /usr/local/hadoop/etc/hadoop sudo cp scripts/ci/hdfs-site.xml /usr/local/hadoop/etc/hadoop + +hadoop fs -help +pushd ~ +sudo mkdir -p /home/hadoop/hadoopinfra/hdfs/namenode/ +sudo chmod 777 /home/hadoop/hadoopinfra/hdfs/namenode/ +sudo chmod 777 -R /usr/local/hadoop +hdfs namenode -format +start-dfs.sh +jps +hadoop fs -ls / +hadoop fs -ls hdfs://127.0.0.1/ +hadoop fs -mkdir hdfs://127.0.0.1/ololo +popd