mirror of
https://github.com/Stone-Red-Code/dvc.git
synced 2026-11-09 23:10:24 +01:00
Saugat PachhaiandCasper da Costa-Luis
109ded4068
* py38: make wheels - ci: set ARROW_LIBHDFS_DIR to load native library - deps: pin pyarrow to 0.15.1 for py<=3.7 - pin deps for python<3.8 - pin deps for python==3.8 as well - add windows Py38 on travis - do not use speedcopy for Python3.8 and Windows * use realpath to resolve symbolic links * do not build pkgs for Windows and Linux on Python3.8 * reorder * pyarrow: document regression fix Co-authored-by: Casper da Costa-Luis <[email protected]>
56 lines
1.8 KiB
Bash
Executable File
56 lines
1.8 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# NOTE: based on http://sharafjaffri.blogspot.com/2015/04/installing-single-node-hadoop-26-using.html
|
|
|
|
set -x
|
|
set -e
|
|
|
|
sudo add-apt-repository -y ppa:openjdk-r/ppa
|
|
sudo apt-get update -y
|
|
sudo apt-get install openjdk-7-jdk
|
|
java -version
|
|
|
|
pushd /usr/local
|
|
sudo wget https://s3-us-east-2.amazonaws.com/dvc-public/travis-test/hadoop-2.6.5.tar.gz
|
|
sudo tar xzf hadoop-2.6.5.tar.gz
|
|
sudo mkdir hadoop
|
|
sudo mv hadoop-2.6.5/* hadoop/
|
|
popd
|
|
|
|
echo "export HADOOP_HOME=/usr/local/hadoop" >>env.sh
|
|
echo "export HADOOP_MAPRED_HOME=/usr/local/hadoop" >>env.sh
|
|
echo "export HADOOP_COMMON_HOME=/usr/local/hadoop" >>env.sh
|
|
echo "export HADOOP_HDFS_HOME=/usr/local/hadoop" >>env.sh
|
|
echo "export YARN_HOME=/usr/local/hadoop" >>env.sh
|
|
echo "export HADOOP_COMMON_LIB_NATIVE_DIR=/usr/local/hadoop/lib/native" >>env.sh
|
|
|
|
# PyArrow==0.16.0 regression https://issues.apache.org/jira/browse/ARROW-7841
|
|
# retrieves native library from $HADOOP_HOME directory instead of
|
|
# `$HADOOP_HOME/lib/native`.
|
|
# Fix: force search for `libhdfs.so` inside `$HADOOP_HOME/lib/native`.
|
|
# Note: not needed for PyArrow==0.17.0.
|
|
echo "export ARROW_LIBHDFS_DIR=/usr/local/hadoop/lib/native" >> env.sh
|
|
|
|
echo "export JAVA_HOME=/usr/" >>env.sh
|
|
echo "export PATH=\$PATH:/usr/local/hadoop/sbin:/usr/local/hadoop/bin:$JAVA_PATH/bin" >>env.sh
|
|
|
|
source env.sh
|
|
|
|
sudo bash -c 'echo "export JAVA_HOME=/usr/" >> /usr/local/hadoop/etc/hadoop/hadoop-env.sh'
|
|
|
|
sudo cp scripts/ci/core-site.xml /usr/local/hadoop/etc/hadoop
|
|
sudo cp scripts/ci/hdfs-site.xml /usr/local/hadoop/etc/hadoop
|
|
|
|
hadoop fs -help
|
|
pushd ~
|
|
sudo mkdir -p /home/hadoop/hadoopinfra/hdfs/namenode/
|
|
sudo chmod 777 /home/hadoop/hadoopinfra/hdfs/namenode/
|
|
sudo chmod 777 -R /usr/local/hadoop
|
|
hdfs namenode -format
|
|
start-dfs.sh
|
|
jps
|
|
hadoop fs -ls /
|
|
hadoop fs -ls hdfs://127.0.0.1/
|
|
hadoop fs -mkdir hdfs://127.0.0.1/ololo
|
|
popd
|