#!/bin/bash
#    ________ .___ ___________
#   /  _____/ |   |\__    ___/
#  /   \  ___ |   |  |    |   
#  \    \_\  \|   |  |    |   
#   \______  /|___|  |____|   
#          \/                 
# git clone git:whoisserver
# src/utils/ripe.whois
# ****************************************

##############################################################################################
# determine service name before anything else
export SERVICE=`basename $0 | sed 's/^ripe.//'`
# sanity check
if [ $SERVICE = whois ]; then
	echo "Do not use $0. It is the generic starter script for all whois services."
	echo "Use $0-* instead!"
	exit 1
fi

##############################################################################################
# basedir of whois service
export BASEDIR=/home/dbase/services/$SERVICE
# E-mail address where crash notifications are sent
#export NOTIFY=dbase24-7@ripe.net
export NOTIFY=dbint@ripe.net
# PID file
export PIDFILE=$BASEDIR/var/tmp/whois-server.pid
# ERR file (startup log and stderr of whois-server is redirected here)
export ERRLOG=$BASEDIR/var/whois-server.out
# number of consecutive crashes before giving up
export MAX_CRASHES=5
# time between crashes to consider them consecutive (in seconds)
# after this amount of seconds, the crash counter is reset to the default,
# and dynamic mode is enabled on restart.
# if whois-server crashes within this timeframe, though, crash counter will be
# decreased until it reaches 0, and the dynamic mode will not be enabled, either
export CRASH_IGNORE_TIMEOUT=86400

##############################################################################################
# event handlers - change them according to current failover setup
whois_exited() {
	true
#	my_log "Stopping quagga."
#	/etc/init.d/quagga stop
#	sleep 10
}

whois_started() {
	true
#	my_log "Starting quagga."
#	/etc/init.d/quagga start
#	sleep 10
}

##############################################################################################
# helper functions
my_log() {
	echo "$@"
	echo "`date +'%y%m%d %H:%M:%S'` $@" >>$ERRLOG
}

my_panic() {
	SUBJECT=$1 ; shift
	my_log "$@"
	echo -e "$@" | mail -a "Auto-Submitted: auto-generated" -s "$SUBJECT" "$NOTIFY"
	my_log "Email sent to $NOTIFY"
}

function send_command() {
	echo -e "$@" | nc -q 1 localhost $CONFIG_PORT
}

function send_command_get_ret() {
	return `echo -e "$@" | nc -q 1 localhost $CONFIG_PORT | sed 's/.*=\([0-9]*\)=.*/\1/g' | tail -1`
}

##############################################################################################
# after the server is started, this function checks if it has come up, and screams if not.
# it also handles the switch to dynamic mode.
check_server() {
	# give server a chance
	my_log 'Waiting for the server to come up... (If you know what you are doing, hit ESC to cancel)'
	NUM=0
	key=""; escape=$'\x1b';
	
	while ! send_command help >/dev/null 2>&1; do
		read -t1 -s -n1 key
		if [[ "$key" == "$escape" ]]; then
			my_log "User cancelled wait at $NUM seconds"
			break;
		fi
		NUM=$[NUM+1]
		if [ $NUM -gt 90 ]; then	# admin tables are huge, and mysql can stall quite a bit before coughing up the first rows
			# bail out
			my_log "Couldn't connect to config port after 30 seconds - something's gone wrong!"
			return
		fi
	done

	# wait for rx_load thread to finish
	my_log 'Waiting for rx_load thread to finish... (If you know what you are doing, hit ESC to cancel)'
	RX_LOAD="Init"
	NUM=0
	key=""; escape=$'\x1b';
	
	while [ $NUM -lt 60 ]; do	# radix tables are huge, and mysql can stall quite a bit before coughing up the first rows
		RX_LOAD=`send_command 'show threads' | grep '^rx load' | awk '{printf("%s %s %s", $10, $11, $12);}'`
		echo -en "\r${RX_LOAD}                              "
		if [ "$RX_LOAD" ]; then
			NUM=0
		else 
			NUM=$[NUM+1]
		fi
		
		read -t1 -s -n1 key
		if [[ "$key" == "$escape" ]]; then
			my_log "User cancelled wait at $NUM seconds"
			break;
		fi
	done
	echo -en "\r"

	# first test - query the server (also takes care of any first-time initialization problems we might have)
	if [ `echo '193.0/16' | nc -q 10 localhost $QUERY_PORT | wc -l` -gt 10 ]; then
		my_log "$SERVICE first startup check succeeded."
	else
		my_panic "$SERVICE first startup check failed on `hostname`" "$SERVICE came up, but did not respond to the first test query - SOMETHING IS WRONG, check logs!"
		return
	fi
	
	# only enable updates if the last startup was more than 24 hours ago
	CURRENT_DATE=`date +%s`
	if [ $[CURRENT_DATE - LAST_START_DATE] -gt $CRASH_IGNORE_TIMEOUT ]; then
		my_log 'Executing "set dynamic start"...'
		send_command 'set dynamic start' &>/dev/null
	fi

	# second test - query the server after (also takes care of any first-time initialization problems we might have)
	# we'll sleep a bit (kinda like bears in the winter), so that dynamic mode has some time to start catching up
	sleep 3
	if [ `echo '193.0/16' | nc -q 10 localhost $QUERY_PORT | wc -l` -gt 10 ]; then
		my_log "$SERVICE second startup check succeeded."
	else
		my_panic "$SERVICE second startup check failed on `hostname`" "$SERVICE came up, but did not respond to the second test query - SOMETHING IS WRONG, check logs!"
		return
	fi

	# execute event handler
	whois_started

	my_log "$SERVICE start finished successfully."
}
	

##############################################################################################
# Starts the whois-server
start_server () {
	while true; do
		# only start the server if the remaining crashes number is big enough
		if [ $CRASHES -gt 0 ]; then
			# only start check_server thread if this is our second start
			# (because first start went to the stdout to give feedback to user)
			if [ $FIRST_RUN -eq 1 ]; then
				export FIRST_RUN=0
				CHECK_THREAD_PID=
			else
				my_panic "$SERVICE crashed & restarted on $HOSTNAME" "$HOSTNAME whois-server crashed & restarted"
				check_server </dev/null >/dev/null 2>&1 &
				CHECK_THREAD_PID=$!
			fi

			PREV_LAST_START_DATE=$LAST_START_DATE
			export LAST_START_DATE=`date +%s`
			$BASEDIR/bin/whois-server -p $PIDFILE -c $BASEDIR/conf/rip.config </dev/null >>$ERRLOG 2>&1

			# kill existing whois-server-start check thread (so that it won't accidentally start event handler)
			if [ $CHECK_THREAD_PID ]; then
				if ps uxaw |  egrep '^root *'$CHECK_THREAD_PID" .*`basename $0`" &>/dev/null; then
					kill -9 $CHECK_THREAD_PID
				fi
			fi

			# execute event handler
			whois_exited

			# if pid file exists - that was a crash - restart the server in safe mode
			if [ ! -f $PIDFILE ]; then
				return
			fi
    
			rm -f $PIDFILE

			# check if it is a consecutive crash or not
			if [ $[LAST_START_DATE - PREV_LAST_START_DATE] -gt $CRASH_IGNORE_TIMEOUT ]; then
				export CRASHES=$MAX_CRASHES
			else
				export CRASHES=$[CRASHES-1]
			fi
  		else
			my_panic "ERROR:URGENT: $HOSTNAME $SERVICE is stopped and not restarted" "Number of consecutive whois-server crashes exceeded $MAX_CRASHES"
			return
		fi
	done
}

##############################################################################################
# Stops the whois-server
# it's a bit complicated as we have to handle all kinds of misbehavior on whois-server's side
stop_server() {
	# Stop server using SIGTERM signal
	if [ -f $PIDFILE ]; then
		PID=`cat $PIDFILE`
		if ps uxaw | awk '{if ($2 == '$PID') printf("%s\n", $13);}' | grep whois-server &>/dev/null; then
			my_log "Stopping $SERVICE ..."
			# first execute the event handler
			whois_exited
			# then, signal whois-server that it should stop
			kill $PID
			# then let running threads finish their workloads
			sleep 10
			# signal all remaining threads afterwards
			ps uxaw -L | awk '{printf("%s %s\n", $2, $3);}' | grep "^$PID " | awk '{print $2}' | while read i; do
				my_log "Thread $i didn't exit - sending TERM"
				kill $i
			done

			# whois-server should remove the PIDFILE when it exits, so wait for it.
			WAIT=15
			while [ -s $PIDFILE ]; do
				WAIT=$[WAIT-1]
				if [ $WAIT -lt 0 ]; then
					my_log "$SERVICE failed to stop; killing"
					rm -f $PIDFILE
					kill -9 $PID
					break
				fi
				sleep 1
			done
			
			# in case of a leftover/daemon whois process, kill it
			if ps uxaw | egrep "^dbase *$PID .*whois-server" &>/dev/null; then
				my_log "whois-server ($PID) failed to stop graciously; sending KILL signal"
				kill -9 $PID
			fi
		fi
	else
		my_log "$PIDFILE does not exist."
		my_log "This suggests that you are trying to stop $SERVICE, but it is not running."
		my_log "If this is not the case, manual intervention is required."
	fi
}

##############################################################################################
# MAIN

# Safeguard (relative paths, ...)
cd $BASEDIR

# number of remaining crashes
export CRASHES=$MAX_CRASHES

# hostname, used only as a shortcut for `hostname`
export HOSTNAME=`hostname`

# first time run marker
export FIRST_RUN=1

# set last start date
export LAST_START_DATE=0

# get the config port from the config file
export CONFIG_PORT=`grep ^SVCONFIG_PORT $BASEDIR/conf/rip.config | awk '{print $2}'`
if [ -z $CONFIG_PORT ]; then
	echo 'SVCONFIG_PORT is invalid in config file!'
	exit 1
fi

# get the query port from the config file
export QUERY_PORT=`grep ^SVWHOIS_PORT $BASEDIR/conf/rip.config | awk '{print $2}'`
if [ -z $QUERY_PORT ]; then
	echo 'SVWHOIS_PORT is invalid in config file!'
	exit 1
fi

case "$1" in
	'start')
		# If an old pid file exists, check if the daemon is already running
		if [ -f $PIDFILE ]; then
			PID=`cat $PIDFILE`
			if ps uxaw | awk '{if ($2 == '$PID') printf("%s\n", $13);}' | grep whois-server &>/dev/null; then
				my_log "PID file $PIDFILE exists and points to a running whois-server."
				my_log "This suggests that you are trying to start an already running whois service."
				my_log "You should either remove the pid file if you are sure this whois service is not running yet, or perform some deeper investigation to solve this situation."
				exit 1
			else
				my_log "Stale pid file $PIDFILE removed. This suggests an unclean shutdown."
				rm -f $PIDFILE
			fi
		fi
		
		my_log "Starting $SERVICE: "
		start_server </dev/null >/dev/null 2>&1 &
		check_server
	;;

	'stop')
		stop_server
	;;

	*)
		echo "usage: $0 start|stop"
		exit 1
	;;
esac
