magpie-setup-storm

#!/bin/bash
#############################################################################
#  Copyright (C) 2013 Lawrence Livermore National Security, LLC.
#  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
#  Written by Albert Chu <chu11@llnl.gov>
#  LLNL-CODE-644248
#  
#  This file is part of Magpie, scripts for running Hadoop on
#  traditional HPC systems.  For details, see <URL>.
#  
#  Magpie is free software; you can redistribute it and/or modify it
#  under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  (at your option) any later version.
#  
#  Magpie is distributed in the hope that it will be useful, but
#  WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#  General Public License for more details.
#  
#  You should have received a copy of the GNU General Public License
#  along with Magpie.  If not, see <http://www.gnu.org/licenses/>.
#############################################################################

# This script sets up configuration files for jobs.  For the most
# part, it shouldn't be editted.  See job submission files for
# configuration details.

if [ "${STORM_SETUP}" != "yes" ]
then
    exit 0
fi

source ${MAGPIE_SCRIPTS_HOME}/magpie-submission-convert
source ${MAGPIE_SCRIPTS_HOME}/magpie-common-exports
source ${MAGPIE_SCRIPTS_HOME}/magpie-common-functions

# stormnoderank set if succeed
if ! Magpie_am_I_a_storm_node
then
    exit 0
fi

stormlocaldir="${STORM_LOCAL_DIR}/local"

if [ ! -d "${stormlocaldir}" ]
then
    mkdir -p ${stormlocaldir}
    if [ $? -ne 0 ] ; then
	echo "mkdir failed making ${stormlocaldir}"
	exit 1
    fi
fi

# Sets threadstouse and proccount
Magpie_calculate_threadstouse

if [ "${STORM_SUPERVISOR_SLOTS}X" != "X" ]
then
    stormslotcount=${STORM_SUPERVISOR_SLOTS}
else
    stormslotcount=`expr ${proccount} \/ 2`

    if [ "${stormslotcount}" == "0" ]
    then
	stormslotcount="1"
    fi
fi

if [ "${STORM_DAEMON_HEAP_MAX}X" != "X" ]
then
    stormdaemonheapmax="${STORM_DAEMON_HEAP_MAX}"
else
    stormdaemonheapmax="1024"
fi 

if [ "${STORM_WORKER_HEAP_MAX}X" != "X" ]
then
    stormworkerheapmax="${STORM_WORKER_HEAP_MAX}"
else
    stormworkerheapmax="1024"
fi 

#
# Get config files for setup
#

if [ "${STORM_CONF_FILES}X" == "X" ]
then
    stormconffiledir=${MAGPIE_SCRIPTS_HOME}/conf
else
    stormconffiledir=${STORM_CONF_FILES}
fi

pre_stormyaml=${stormconffiledir}/storm.yaml
pre_stormenvini=${stormconffiledir}/storm_env.ini
pre_stormenvsh=${stormconffiledir}/magpie-launch-storm-env.sh

post_stormyaml=${STORM_CONF_DIR}/storm.yaml
post_stormenvini=${STORM_CONF_DIR}/storm_env.ini
post_stormenvsh=${STORM_CONF_DIR}/magpie-launch-storm-env.sh

#
# Setup Storm configuration files and environment files
#

stormlocaldirsubst=`echo "${stormlocaldir}" | sed "s/\\//\\\\\\\\\//g"`
stormlogdirsubst=`echo "${STORM_LOG_DIR}" | sed "s/\\//\\\\\\\\\//g"`

cp ${pre_stormyaml} ${post_stormyaml}

sed -i \
    -e "s/NIMBUSHOST/${STORM_MASTER_NODE}/g" \
    -e "s/STORMLOCALDIR/${stormlocaldirsubst}/g" \
    -e "s/STORMLOGDIR/${stormlogdirsubst}/g" \
    -e "s/STORMZOOKEEPERPORT/${ZOOKEEPER_CLIENT_PORT}/g" \
    -e "s/STORMUIPORT/${STORM_UI_PORT}/g" \
    -e "s/STORMLOGVIEWERPORT/${STORM_LOGVIEWER_PORT}/g" \
    -e "s/STORMDRPCPORT/${STORM_DRPC_PORT}/g" \
    -e "s/STORMDRPCINVOCATIONSPORT/${STORM_DRPC_INVOCATIONS_PORT}/g" \
    -e "s/STORMDRPCHTTPPORT/${STORM_DRPC_HTTP_PORT}/g" \
    -e "s/NIMBUSTHRIFTPORT/${STORM_THRIFT_PORT}/g" \
    -e "s/STORM_DAEMON_HEAP_MAX/${stormdaemonheapmax}/g" \
    -e "s/STORM_WORKER_HEAP_MAX/${stormworkerheapmax}/g" \
    ${post_stormyaml}

# Add Zookeeper servers to conf

echo "storm.zookeeper.servers:" >> ${post_stormyaml}

zookeepernodes=`cat ${ZOOKEEPER_CONF_DIR}/zookeeper_slaves`
servercount=1
for zookeepernode in ${zookeepernodes}
do
    echo "     - \"${zookeepernode}\"" >> ${post_stormyaml}
    servercount=$((servercount+1))
done

echo "" >> ${post_stormyaml}

# Add slots to conf

echo "supervisor.slots.ports:" >> ${post_stormyaml}

slotport=${STORM_SUPERVISOR_SLOTS_STARTING_PORT}
for i in `seq 1 ${stormslotcount}`
do
    echo "     - ${slotport}" >> ${post_stormyaml}
    slotport=$((slotport+1))
done

stormjavahomesubst=`echo "${JAVA_HOME}" | sed "s/\\//\\\\\\\\\//g"`
stormlogdirsubst=`echo "${STORM_LOG_DIR}" | sed "s/\\//\\\\\\\\\//g"`
stormconfdirsubst=`echo "${STORM_CONF_DIR}" | sed "s/\\//\\\\\\\\\//g"`
stormhomesubst=`echo "${STORM_HOME}" | sed "s/\\//\\\\\\\\\//g"`
stormlocaldirsubst=`echo "${STORM_LOCAL_DIR}" | sed "s/\\//\\\\\\\\\//g"`
magpiescriptshome=`echo "${MAGPIE_SCRIPTS_HOME}" | sed "s/\\//\\\\\\\\\//g"`

cp ${pre_stormenvini} ${post_stormenvini}

sed -i \
    -e "s/MAGPIEJAVAHOME/${stormjavahomesubst}/g" \
    ${post_stormenvini}

cp ${pre_stormenvsh} ${post_stormenvsh}

sed -i \
    -e "s/MAGPIEJAVAHOME/${stormjavahomesubst}/g" \
    -e "s/STORMLOGDIR/${stormlogdirsubst}/g" \
    -e "s/STORMCONFDIR/${stormconfdirsubst}/g" \
    -e "s/STORMHOME/${stormhomesubst}/g" \
    -e "s/STORMLOCALDIR/${stormlocaldirsubst}/g" \
    -e "s/MAGPIESCRIPTSHOME/${magpiescriptshome}/g" \
    ${post_stormenvsh}

if [ "${MAGPIE_REMOTE_CMD:-ssh}" != "ssh" ]
then
    echo "export STORM_SSH_CMD=\"${MAGPIE_REMOTE_CMD}\"" >> ${post_stormenvsh}
fi
if [ "${MAGPIE_REMOTE_CMD_OPTS}X" != "X" ]
then
    echo "export STORM_SSH_OPTS=\"${MAGPIE_REMOTE_CMD_OPTS}\"" >> ${post_stormenvsh}
fi

exit 0