source /usr/lib/hustler/bin/qubole-bash-lib.sh
is_master=`nodeinfo is_master`
if [[ "$is_master" == "1" ]]; then
pwd
#wget http://archive.cloudera.com/beta/livy/livy-server-0.3.0.zip
wget http://mirrors.wuchna.com/apachemirror/incubator/livy/0.5.0-incubating/livy-0.5.0-incubating-bin.zip
#unzip livy-server-0.3.0.zip
unzip livy-0.5.0-incubating-bin.zip
export SPARK_HOME=/usr/lib/spark
export HADOOP_CONF_DIR=/etc/hadoop/conf
#cd livy-server-0.3.0
cd livy-0.5.0-incubating-bin
cp conf/log4j.properties.template conf/log4j.properties
mkdir logs
echo livy.spark.master=yarn >> conf/livy.conf
echo livy.spark.deployMode=client >> conf/livy.conf
echo livy.repl.enableHiveContext=true>> conf/livy.conf
nohup ./bin/livy-server > logs/livy.out 2> logs/livy.err < /dev/null&
fi
wget http://central.maven.org/maven2/com/amazonaws/aws-java-sdk-core/1.11.288/aws-java-sdk-core-1.11.288.jar -P /usr/lib/spark/lib/initial
access=`source /usr/lib/hustler/bin/qubole-bash-lib.sh && nodeinfo s3_access_key_id`
secret=`source /usr/lib/hustler/bin/qubole-bash-lib.sh && nodeinfo s3_secret_access_key`
mkdir /home/yarn/.aws
echo "[default]" >> /home/yarn/.aws/credentials
echo "aws_access_key_id=$access" >> /home/yarn/.aws/credentials
echo "aws_secret_access_key=$secret" >> /home/yarn/.aws/credentials
pip install sagemaker_pyspark