travis: enable hdfs test

Signed-off-by: Ruslan Kuprieiev <[email protected]>

.travis.yml

@@ -1,7 +1,5 @@
 language: python
 sudo: required
-services:
-  - docker
 dist: trusty
 group: deprecated-2017Q4
 python:
@@ -45,6 +43,7 @@ matrix:
     - PYENV_ROOT="$HOME/.pyenv"
     - PATH="$PYENV_ROOT/bin:$PATH"
 before_install:
+  - bash ./scripts/ci/before_install.sh
   - if  [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
       eval "$(pyenv init -)";
       PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing --keep --verbose "$PYTHON_VER";
@@ -53,6 +52,21 @@ before_install:
   - which python
   - which pip
   - python --version
+  - bash ./scripts/ci/install_hadoop.sh
+  - source ~/.bashrc
+  - hadoop fs -help
+  - pushd ~
+  - sudo mkdir -p /home/hadoop/hadoopinfra/hdfs/namenode/
+  - sudo chmod 777 /home/hadoop/hadoopinfra/hdfs/namenode/
+  - sudo chmod 777 -R /usr/local/hadoop
+  - hdfs namenode -format
+  - start-dfs.sh
+  - jps
+  - hadoop fs -ls /
+  - hadoop fs -ls hdfs://127.0.0.1/
+  - hadoop fs -mkdir hdfs://127.0.0.1/ololo
+  - popd
+  - export HADOOP_CONTAINER_IP=127.0.0.1
 install:
   - bash ./scripts/ci/install.sh
 script:

scripts/ci/before_install.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -x
+set -e
+
+#NOTE: ssh keys for ssh test to be able to ssh to the localhost
+ls -la ~/.ssh/
+
+ssh-keygen -t rsa -N "" -f mykey
+mkdir -p ~/.ssh
+cp mykey ~/.ssh/id_rsa
+cp mykey.pub ~/.ssh/id_rsa.pub
+cat mykey.pub >> ~/.ssh/authorized_keys
+ssh-keyscan localhost >> ~/.ssh/known_hosts
+ssh-keyscan 127.0.0.1 >> ~/.ssh/known_hosts
+ssh-keyscan 0.0.0.0 >> ~/.ssh/known_hosts
+ssh 0.0.0.0 ls &> /dev/null
+ssh 127.0.0.1 ls &> /dev/null
+ssh localhost ls &> /dev/null

scripts/ci/core-site.xml

@@ -0,0 +1,6 @@
+<configuration>
+<property>
+<name>fs.defaultFS</name>
+<value>hdfs://localhost:8020</value>
+</property>
+</configuration>

scripts/ci/hdfs-site.xml

@@ -0,0 +1,6 @@
+<configuration>
+<property>
+<name>dfs.replication</name>
+<value>1</value>
+</property>
+</configuration>

scripts/ci/install_hadoop.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# NOTE: based on http://sharafjaffri.blogspot.com/2015/04/installing-single-node-hadoop-26-using.html
+
+set -x
+set -e
+
+# This command is used to tell shell, turn installation mode to non interactive
+# and set auto selection of agreement for Sun Java
+export DEBIAN_FRONTEND=noninteractive
+echo debconf shared/accepted-oracle-license-v1-1 select true | sudo debconf-set-selections
+echo debconf shared/accepted-oracle-license-v1-1 seen true | sudo debconf-set-selections
+
+sudo apt-get update -y
+sudo apt-get install python-software-properties -y
+sudo add-apt-repository ppa:webupd8team/java -y
+sudo apt-get update -y
+sudo -E apt-get install oracle-java8-installer -y
+
+java -version
+
+pushd /usr/local
+sudo wget https://s3-us-west-2.amazonaws.com/dvc-share/test/hadoop-2.6.5.tar.gz
+sudo tar xzf hadoop-2.6.5.tar.gz
+sudo mkdir hadoop
+sudo mv hadoop-2.6.5/* hadoop/
+popd
+
+echo "export HADOOP_HOME=/usr/local/hadoop" >> ~/.bashrc
+echo "export HADOOP_MAPRED_HOME=/usr/local/hadoop" >> ~/.bashrc
+echo "export HADOOP_COMMON_HOME=/usr/local/hadoop" >> ~/.bashrc
+echo "export HADOOP_HDFS_HOME=/usr/local/hadoop" >> ~/.bashrc
+echo "export YARN_HOME=/usr/local/hadoop" >> ~/.bashrc
+echo "export HADOOP_COMMON_LIB_NATIVE_DIR=/usr/local/hadoop/lib/native" >> ~/.bashrc
+echo "export JAVA_HOME=/usr/" >> ~/.bashrc
+echo "export PATH=$PATH:/usr/local/hadoop/sbin:/usr/local/hadoop/bin:$JAVA_PATH/bin" >> ~/.bashrc
+
+cat ~/.bashrc
+
+source ~/.bashrc
+
+sudo bash -c 'echo "export JAVA_HOME=/usr/" >> /usr/local/hadoop/etc/hadoop/hadoop-env.sh'
+
+sudo cp scripts/ci/core-site.xml /usr/local/hadoop/etc/hadoop
+sudo cp scripts/ci/hdfs-site.xml /usr/local/hadoop/etc/hadoop

scripts/ci/script.sh

@@ -3,17 +3,6 @@
 set -x
 set -e
 
-#NOTE: ssh keys for ssh test to be able to ssh to the localhost
-ls -la ~/.ssh/
-
-ssh-keygen -t rsa -N "" -f mykey
-mkdir -p ~/.ssh
-cp mykey ~/.ssh/id_rsa
-cp mykey.pub ~/.ssh/id_rsa.pub
-cat mykey.pub >> ~/.ssh/authorized_keys
-ssh-keyscan 127.0.0.1 >> ~/.ssh/known_hosts
-ssh 127.0.0.1 ls &> /dev/null
-
 python -mtests
 
 if [[ "$TRAVIS_PULL_REQUEST" == "false" && \

tests/test_data_cloud.py

@@ -86,7 +86,7 @@ def get_ssh_url():
 
 
 def get_hdfs_url():
-    return 'hdfs://root@{}{}'.format(os.getenv('HADOOP_CONTAINER_IP'), get_local_storagepath())
+    return 'hdfs://{}@{}{}'.format(getpass.getuser(), os.getenv('HADOOP_CONTAINER_IP'), get_local_storagepath())
 
 
 def get_aws_storagepath():
@@ -295,7 +295,7 @@ class TestDataCloudCLIBase(TestDvc):
         self.assertEqual(ret, 0)
 
     def _test_cloud(self, remote=None):
-        args = ['-j', '2']
+        args = ['-v', '-j', '2']
         if remote:
             args += ['-r', remote]
         else:

tests/test_repro.py

@@ -398,14 +398,14 @@ class TestReproExternalHDFS(TestReproExternalBase):
 
     @property
     def bucket(self):
-        return 'root@' + os.getenv('HADOOP_CONTAINER_IP')
+        return getpass.getuser() + '@' + os.getenv('HADOOP_CONTAINER_IP')
 
     def cmd(self, i, o):
-        return 'HADOOP_USER_NAME=root hadoop fs -cp {} {}'.format(i, o)
+        return 'hadoop fs -cp {} {}'.format(i, o)
 
     def write(self, bucket, key, body):
         url = self.scheme + '://' + bucket + '/' + key
-        p = Popen('HADOOP_USER_NAME=root hadoop fs -rm -f {}'.format(url),
+        p = Popen('hadoop fs -rm -f {}'.format(url),
                   shell=True,
                   executable=os.getenv('SHELL'),
                   stdin=PIPE,
@@ -413,7 +413,7 @@ class TestReproExternalHDFS(TestReproExternalBase):
                   stderr=PIPE)
         p.communicate()
 
-        p = Popen('HADOOP_USER_NAME=root hadoop fs -mkdir -p {}'.format(posixpath.dirname(url)),
+        p = Popen('hadoop fs -mkdir -p {}'.format(posixpath.dirname(url)),
                   shell=True,
                   executable=os.getenv('SHELL'),
                   stdin=PIPE,
@@ -428,7 +428,7 @@ class TestReproExternalHDFS(TestReproExternalBase):
         with open('tmp', 'w+') as fd:
             fd.write(body)
 
-        p = Popen('HADOOP_USER_NAME=root hadoop fs -copyFromLocal {} {}'.format('tmp', url),
+        p = Popen('hadoop fs -copyFromLocal {} {}'.format('tmp', url),
                   shell=True,
                   executable=os.getenv('SHELL'),
                   stdin=PIPE,