travis: enable hdfs test

Signed-off-by: Ruslan Kuprieiev <[email protected]>

scripts/ci/before_install.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -x
+set -e
+
+#NOTE: ssh keys for ssh test to be able to ssh to the localhost
+ls -la ~/.ssh/
+
+ssh-keygen -t rsa -N "" -f mykey
+mkdir -p ~/.ssh
+cp mykey ~/.ssh/id_rsa
+cp mykey.pub ~/.ssh/id_rsa.pub
+cat mykey.pub >> ~/.ssh/authorized_keys
+ssh-keyscan localhost >> ~/.ssh/known_hosts
+ssh-keyscan 127.0.0.1 >> ~/.ssh/known_hosts
+ssh-keyscan 0.0.0.0 >> ~/.ssh/known_hosts
+ssh 0.0.0.0 ls &> /dev/null
+ssh 127.0.0.1 ls &> /dev/null
+ssh localhost ls &> /dev/null

scripts/ci/core-site.xml

@@ -0,0 +1,6 @@
+<configuration>
+<property>
+<name>fs.defaultFS</name>
+<value>hdfs://localhost:8020</value>
+</property>
+</configuration>

scripts/ci/hdfs-site.xml

@@ -0,0 +1,6 @@
+<configuration>
+<property>
+<name>dfs.replication</name>
+<value>1</value>
+</property>
+</configuration>

scripts/ci/install_hadoop.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# NOTE: based on http://sharafjaffri.blogspot.com/2015/04/installing-single-node-hadoop-26-using.html
+
+set -x
+set -e
+
+# This command is used to tell shell, turn installation mode to non interactive
+# and set auto selection of agreement for Sun Java
+export DEBIAN_FRONTEND=noninteractive
+echo debconf shared/accepted-oracle-license-v1-1 select true | sudo debconf-set-selections
+echo debconf shared/accepted-oracle-license-v1-1 seen true | sudo debconf-set-selections
+
+sudo apt-get update -y
+sudo apt-get install python-software-properties -y
+sudo add-apt-repository ppa:webupd8team/java -y
+sudo apt-get update -y
+sudo -E apt-get install oracle-java8-installer -y
+
+java -version
+
+pushd /usr/local
+sudo wget https://s3-us-west-2.amazonaws.com/dvc-share/test/hadoop-2.6.5.tar.gz
+sudo tar xzf hadoop-2.6.5.tar.gz
+sudo mkdir hadoop
+sudo mv hadoop-2.6.5/* hadoop/
+popd
+
+echo "export HADOOP_HOME=/usr/local/hadoop" >> ~/.bashrc
+echo "export HADOOP_MAPRED_HOME=/usr/local/hadoop" >> ~/.bashrc
+echo "export HADOOP_COMMON_HOME=/usr/local/hadoop" >> ~/.bashrc
+echo "export HADOOP_HDFS_HOME=/usr/local/hadoop" >> ~/.bashrc
+echo "export YARN_HOME=/usr/local/hadoop" >> ~/.bashrc
+echo "export HADOOP_COMMON_LIB_NATIVE_DIR=/usr/local/hadoop/lib/native" >> ~/.bashrc
+echo "export JAVA_HOME=/usr/" >> ~/.bashrc
+echo "export PATH=$PATH:/usr/local/hadoop/sbin:/usr/local/hadoop/bin:$JAVA_PATH/bin" >> ~/.bashrc
+
+cat ~/.bashrc
+
+source ~/.bashrc
+
+sudo bash -c 'echo "export JAVA_HOME=/usr/" >> /usr/local/hadoop/etc/hadoop/hadoop-env.sh'
+
+sudo cp scripts/ci/core-site.xml /usr/local/hadoop/etc/hadoop
+sudo cp scripts/ci/hdfs-site.xml /usr/local/hadoop/etc/hadoop

scripts/ci/script.sh

@@ -3,17 +3,6 @@
 set -x
 set -e
 
-#NOTE: ssh keys for ssh test to be able to ssh to the localhost
-ls -la ~/.ssh/
-
-ssh-keygen -t rsa -N "" -f mykey
-mkdir -p ~/.ssh
-cp mykey ~/.ssh/id_rsa
-cp mykey.pub ~/.ssh/id_rsa.pub
-cat mykey.pub >> ~/.ssh/authorized_keys
-ssh-keyscan 127.0.0.1 >> ~/.ssh/known_hosts
-ssh 127.0.0.1 ls &> /dev/null
-
 python -mtests
 
 if [[ "$TRAVIS_PULL_REQUEST" == "false" && \