Stone_Red/dvc
1
0
Fork 0
mirror of https://github.com/Stone-Red-Code/dvc.git synced 2026-11-11 15:28:27 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1559 from efiop/master

Browse Source 
travis: test azure and hadoop only during cron jobs
This commit is contained in:
Ruslan Kuprieiev
2019-01-31 06:33:07 +02:00
committed by GitHub
parent 4335336e0a 9fdb922ab7
commit 2a284e3fd4
4 changed files with 49 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.travis.yml
+1 -32
View File
@@ -59,44 +59,13 @@ matrix:
env:
- PYTHON_VER=3.5.0
before_install:
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
brew update;
brew upgrade python;
python --version;
fi
- pip install -r scripts/ci/requirements.txt
# stop the build if there are any readme formatting errors
- python setup.py checkdocs
# stop the build if there are any black errors
- black dvc --check
- black tests --check
- bash ./scripts/ci/before_install.sh
- if [[ -n "$PYTHON_VER" ]]; then
eval "$(pyenv init -)";
pyenv install --skip-existing --keep --verbose "$PYTHON_VER";
pyenv shell "$PYTHON_VER";
fi
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
brew install openssl;
brew cask install google-cloud-sdk;
fi
- python --version
- if [[ "$TRAVIS_OS_NAME" != "osx" ]]; then
bash ./scripts/ci/install_hadoop.sh;
source ~/.bashrc;
hadoop fs -help;
pushd ~;
sudo mkdir -p /home/hadoop/hadoopinfra/hdfs/namenode/;
sudo chmod 777 /home/hadoop/hadoopinfra/hdfs/namenode/;
sudo chmod 777 -R /usr/local/hadoop;
hdfs namenode -format;
start-dfs.sh;
jps;
hadoop fs -ls /;
hadoop fs -ls hdfs://127.0.0.1/;
hadoop fs -mkdir hdfs://127.0.0.1/ololo;
popd;
fi
- source env.sh
install:
- bash ./scripts/ci/install.sh
script:
scripts/ci/before_install.sh
+24 -2
View File
@@ -3,6 +3,21 @@
set -x
set -e
if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
brew update
brew upgrade python
python --version
fi
pip install -r scripts/ci/requirements.txt
# stop the build if there are any readme formatting errors
python setup.py checkdocs
# stop the build if there are any black errors
black dvc --check
black tests --check
#NOTE: ssh keys for ssh test to be able to ssh to the localhost
ls -la ~/.ssh/
@@ -20,7 +35,14 @@ ssh localhost ls &> /dev/null
scriptdir="$(dirname $0)"
if [ -n "$TRAVIS_OS_NAME" ] && [ "$TRAVIS_OS_NAME" != "osx" ]; then
echo > env.sh
if [ -n "$TRAVIS_OS_NAME" ] && [ "$TRAVIS_OS_NAME" != "osx" ] \
&& [ "$TRAVIS_EVENT_TYPE" == "cron" ]; then
bash "$scriptdir/install_azurite.sh"
source ~/.bashrc
bash "$scriptdir/install_hadoop.sh"
fi
if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
brew install openssl
brew cask install google-cloud-sdk
fi
scripts/ci/install_azurite.sh
+2 -2
View File
@@ -13,5 +13,5 @@ sudo apt-get install -y docker-ce
sudo docker run --restart always -e executable=blob -p 10000:10000 --tmpfs /opt/azurite/folder -d arafato/azurite:2.6.5
# save secrets
echo "export AZURE_STORAGE_CONTAINER_NAME='travis-tests'" >> ~/.bashrc
echo "export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;'" >> ~/.bashrc
echo "export AZURE_STORAGE_CONTAINER_NAME='travis-tests'" >> env.sh
echo "export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;'" >> env.sh
scripts/ci/install_hadoop.sh
+22 -11
View File
@@ -16,20 +16,31 @@ sudo mkdir hadoop
sudo mv hadoop-2.6.5/* hadoop/
popd
echo "export HADOOP_HOME=/usr/local/hadoop" >> ~/.bashrc
echo "export HADOOP_MAPRED_HOME=/usr/local/hadoop" >> ~/.bashrc
echo "export HADOOP_COMMON_HOME=/usr/local/hadoop" >> ~/.bashrc
echo "export HADOOP_HDFS_HOME=/usr/local/hadoop" >> ~/.bashrc
echo "export YARN_HOME=/usr/local/hadoop" >> ~/.bashrc
echo "export HADOOP_COMMON_LIB_NATIVE_DIR=/usr/local/hadoop/lib/native" >> ~/.bashrc
echo "export JAVA_HOME=/usr/" >> ~/.bashrc
echo "export PATH=$PATH:/usr/local/hadoop/sbin:/usr/local/hadoop/bin:$JAVA_PATH/bin" >> ~/.bashrc
echo "export HADOOP_HOME=/usr/local/hadoop" >> env.sh
echo "export HADOOP_MAPRED_HOME=/usr/local/hadoop" >> env.sh
echo "export HADOOP_COMMON_HOME=/usr/local/hadoop" >> env.sh
echo "export HADOOP_HDFS_HOME=/usr/local/hadoop" >> env.sh
echo "export YARN_HOME=/usr/local/hadoop" >> env.sh
echo "export HADOOP_COMMON_LIB_NATIVE_DIR=/usr/local/hadoop/lib/native" >> env.sh
echo "export JAVA_HOME=/usr/" >> env.sh
echo "export PATH=$PATH:/usr/local/hadoop/sbin:/usr/local/hadoop/bin:$JAVA_PATH/bin" >> env.sh
cat ~/.bashrc
source ~/.bashrc
source env.sh
sudo bash -c 'echo "export JAVA_HOME=/usr/" >> /usr/local/hadoop/etc/hadoop/hadoop-env.sh'
sudo cp scripts/ci/core-site.xml /usr/local/hadoop/etc/hadoop
sudo cp scripts/ci/hdfs-site.xml /usr/local/hadoop/etc/hadoop
hadoop fs -help
pushd ~
sudo mkdir -p /home/hadoop/hadoopinfra/hdfs/namenode/
sudo chmod 777 /home/hadoop/hadoopinfra/hdfs/namenode/
sudo chmod 777 -R /usr/local/hadoop
hdfs namenode -format
start-dfs.sh
jps
hadoop fs -ls /
hadoop fs -ls hdfs://127.0.0.1/
hadoop fs -mkdir hdfs://127.0.0.1/ololo
popd