diff --git a/.travis.yml b/.travis.yml index 41163d7b5..9ae46ef4c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,5 @@ language: python sudo: required -services: - - docker dist: trusty group: deprecated-2017Q4 python: @@ -45,6 +43,7 @@ matrix: - PYENV_ROOT="$HOME/.pyenv" - PATH="$PYENV_ROOT/bin:$PATH" before_install: + - bash ./scripts/ci/before_install.sh - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then eval "$(pyenv init -)"; PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing --keep --verbose "$PYTHON_VER"; @@ -53,6 +52,21 @@ before_install: - which python - which pip - python --version + - bash ./scripts/ci/install_hadoop.sh + - source ~/.bashrc + - hadoop fs -help + - pushd ~ + - sudo mkdir -p /home/hadoop/hadoopinfra/hdfs/namenode/ + - sudo chmod 777 /home/hadoop/hadoopinfra/hdfs/namenode/ + - sudo chmod 777 -R /usr/local/hadoop + - hdfs namenode -format + - start-dfs.sh + - jps + - hadoop fs -ls / + - hadoop fs -ls hdfs://127.0.0.1/ + - hadoop fs -mkdir hdfs://127.0.0.1/ololo + - popd + - export HADOOP_CONTAINER_IP=127.0.0.1 install: - bash ./scripts/ci/install.sh script: diff --git a/scripts/ci/before_install.sh b/scripts/ci/before_install.sh new file mode 100644 index 000000000..a77c9197a --- /dev/null +++ b/scripts/ci/before_install.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -x +set -e + +#NOTE: ssh keys for ssh test to be able to ssh to the localhost +ls -la ~/.ssh/ + +ssh-keygen -t rsa -N "" -f mykey +mkdir -p ~/.ssh +cp mykey ~/.ssh/id_rsa +cp mykey.pub ~/.ssh/id_rsa.pub +cat mykey.pub >> ~/.ssh/authorized_keys +ssh-keyscan localhost >> ~/.ssh/known_hosts +ssh-keyscan 127.0.0.1 >> ~/.ssh/known_hosts +ssh-keyscan 0.0.0.0 >> ~/.ssh/known_hosts +ssh 0.0.0.0 ls &> /dev/null +ssh 127.0.0.1 ls &> /dev/null +ssh localhost ls &> /dev/null diff --git a/scripts/ci/core-site.xml b/scripts/ci/core-site.xml new file mode 100644 index 000000000..3879811c4 --- /dev/null +++ b/scripts/ci/core-site.xml @@ -0,0 +1,6 @@ + + +fs.defaultFS +hdfs://localhost:8020 + + diff --git a/scripts/ci/hdfs-site.xml b/scripts/ci/hdfs-site.xml new file mode 100644 index 000000000..e80ac2021 --- /dev/null +++ b/scripts/ci/hdfs-site.xml @@ -0,0 +1,6 @@ + + +dfs.replication +1 + + diff --git a/scripts/ci/install_hadoop.sh b/scripts/ci/install_hadoop.sh new file mode 100644 index 000000000..25a748989 --- /dev/null +++ b/scripts/ci/install_hadoop.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# NOTE: based on http://sharafjaffri.blogspot.com/2015/04/installing-single-node-hadoop-26-using.html + +set -x +set -e + +# This command is used to tell shell, turn installation mode to non interactive +# and set auto selection of agreement for Sun Java +export DEBIAN_FRONTEND=noninteractive +echo debconf shared/accepted-oracle-license-v1-1 select true | sudo debconf-set-selections +echo debconf shared/accepted-oracle-license-v1-1 seen true | sudo debconf-set-selections + +sudo apt-get update -y +sudo apt-get install python-software-properties -y +sudo add-apt-repository ppa:webupd8team/java -y +sudo apt-get update -y +sudo -E apt-get install oracle-java8-installer -y + +java -version + +pushd /usr/local +sudo wget https://s3-us-west-2.amazonaws.com/dvc-share/test/hadoop-2.6.5.tar.gz +sudo tar xzf hadoop-2.6.5.tar.gz +sudo mkdir hadoop +sudo mv hadoop-2.6.5/* hadoop/ +popd + +echo "export HADOOP_HOME=/usr/local/hadoop" >> ~/.bashrc +echo "export HADOOP_MAPRED_HOME=/usr/local/hadoop" >> ~/.bashrc +echo "export HADOOP_COMMON_HOME=/usr/local/hadoop" >> ~/.bashrc +echo "export HADOOP_HDFS_HOME=/usr/local/hadoop" >> ~/.bashrc +echo "export YARN_HOME=/usr/local/hadoop" >> ~/.bashrc +echo "export HADOOP_COMMON_LIB_NATIVE_DIR=/usr/local/hadoop/lib/native" >> ~/.bashrc +echo "export JAVA_HOME=/usr/" >> ~/.bashrc +echo "export PATH=$PATH:/usr/local/hadoop/sbin:/usr/local/hadoop/bin:$JAVA_PATH/bin" >> ~/.bashrc + +cat ~/.bashrc + +source ~/.bashrc + +sudo bash -c 'echo "export JAVA_HOME=/usr/" >> /usr/local/hadoop/etc/hadoop/hadoop-env.sh' + +sudo cp scripts/ci/core-site.xml /usr/local/hadoop/etc/hadoop +sudo cp scripts/ci/hdfs-site.xml /usr/local/hadoop/etc/hadoop diff --git a/scripts/ci/script.sh b/scripts/ci/script.sh index 3cf2b0807..39af14f81 100644 --- a/scripts/ci/script.sh +++ b/scripts/ci/script.sh @@ -3,17 +3,6 @@ set -x set -e -#NOTE: ssh keys for ssh test to be able to ssh to the localhost -ls -la ~/.ssh/ - -ssh-keygen -t rsa -N "" -f mykey -mkdir -p ~/.ssh -cp mykey ~/.ssh/id_rsa -cp mykey.pub ~/.ssh/id_rsa.pub -cat mykey.pub >> ~/.ssh/authorized_keys -ssh-keyscan 127.0.0.1 >> ~/.ssh/known_hosts -ssh 127.0.0.1 ls &> /dev/null - python -mtests if [[ "$TRAVIS_PULL_REQUEST" == "false" && \ diff --git a/tests/test_data_cloud.py b/tests/test_data_cloud.py index ba8a16ab4..012143b75 100644 --- a/tests/test_data_cloud.py +++ b/tests/test_data_cloud.py @@ -86,7 +86,7 @@ def get_ssh_url(): def get_hdfs_url(): - return 'hdfs://root@{}{}'.format(os.getenv('HADOOP_CONTAINER_IP'), get_local_storagepath()) + return 'hdfs://{}@{}{}'.format(getpass.getuser(), os.getenv('HADOOP_CONTAINER_IP'), get_local_storagepath()) def get_aws_storagepath(): @@ -295,7 +295,7 @@ class TestDataCloudCLIBase(TestDvc): self.assertEqual(ret, 0) def _test_cloud(self, remote=None): - args = ['-j', '2'] + args = ['-v', '-j', '2'] if remote: args += ['-r', remote] else: diff --git a/tests/test_repro.py b/tests/test_repro.py index 620b9ab2e..00e3694b3 100644 --- a/tests/test_repro.py +++ b/tests/test_repro.py @@ -398,14 +398,14 @@ class TestReproExternalHDFS(TestReproExternalBase): @property def bucket(self): - return 'root@' + os.getenv('HADOOP_CONTAINER_IP') + return getpass.getuser() + '@' + os.getenv('HADOOP_CONTAINER_IP') def cmd(self, i, o): - return 'HADOOP_USER_NAME=root hadoop fs -cp {} {}'.format(i, o) + return 'hadoop fs -cp {} {}'.format(i, o) def write(self, bucket, key, body): url = self.scheme + '://' + bucket + '/' + key - p = Popen('HADOOP_USER_NAME=root hadoop fs -rm -f {}'.format(url), + p = Popen('hadoop fs -rm -f {}'.format(url), shell=True, executable=os.getenv('SHELL'), stdin=PIPE, @@ -413,7 +413,7 @@ class TestReproExternalHDFS(TestReproExternalBase): stderr=PIPE) p.communicate() - p = Popen('HADOOP_USER_NAME=root hadoop fs -mkdir -p {}'.format(posixpath.dirname(url)), + p = Popen('hadoop fs -mkdir -p {}'.format(posixpath.dirname(url)), shell=True, executable=os.getenv('SHELL'), stdin=PIPE, @@ -428,7 +428,7 @@ class TestReproExternalHDFS(TestReproExternalBase): with open('tmp', 'w+') as fd: fd.write(body) - p = Popen('HADOOP_USER_NAME=root hadoop fs -copyFromLocal {} {}'.format('tmp', url), + p = Popen('hadoop fs -copyFromLocal {} {}'.format('tmp', url), shell=True, executable=os.getenv('SHELL'), stdin=PIPE,