Curl 工具的妙用：shell中用curl实现业务监控和故障自动处理

不时会遇到JVM内存溢出或服务进程僵死导致的应用不可访问的情况，这个时候一般需要重启下服务进程。虽然在zabbix中对web的监控能监控到且可以执行远程命令，但是基于安全考虑，一般不会再zabbix 的agent上开启允许执行远程命令的权限。可以在本机安装curl工具来实现对web的监控，辅助些脚本就可以实现故障和恢复的告警发送和进程重启了。

#!/usr/bin/bash
##############################################
#NAME: monweb.sh
#DESC:mon the web status,if wrong restart container.
#Note: Curl must be installed，curl工具必须提前安装好。
#History:
#   v1.0 2016-1-20 thinker yang
##############################################
#变量说明，调试级别，数字越高输出信息越多
DEBUG=2
#
#define config
#
##local conf##
#需要载入运行中间件用户下的profile文件，因为启停脚本的可能部分变量定义在此
. /.profile
#定义时间格式
TIME=`date`
#定义要监控页面的curl
URL=wwww.baidu.com/123.html
#定义短信告警头
AlertHead=ERROR-TOMCAT
##server conf##
##中间件相关配置##
#定义启动中间件的命令脚本
STARTSERVERCMD=/startapp.sh
#定义正常停止中间件的命令脚本
STOPSERVERCMD=/stopserver.sh
#用以下关键字grep出中间件进程的pid，如果用一个关键字就可以过滤出，则后面两个和第一个一致即可
PSKEY1=java
PSKEY2=java
PSKEY3=java
#sleep时间/秒：完全停止中间件后到再次启动中间件等待的秒数
WAITRESTARTTIME=10
#
#sleep时间/秒：中间件重启完成后，用curl再次检测web页面状态的间隔时间
WAITCHECKHEALTHTIME=60
##java core file conf##
#内存溢出后产生java dump 文件的路径
MEMOUTFILEPATH=/tmp
#用grep关键字查找因内存溢出而产生文件名的关键字（一般不需要改）
MEMOUTFILEKEY1=heapdump
MEMOUTFILEKEY2=phd
#是（Y）否（N）删除这些因内存溢出产生的dump core文件，默认为删除
RMJAVACOREFILEFLAG=Y
##log conf##
#脚本输出日志路径和日志文件名配置
LOGDIR=/tmp/monlog
LOG=$LOGDIR/monweb.sh.log
##alert sms config##

#########################
#
#function:sendsms,此处定义告警信息来实现邮件或者短信告警，使用方式为echosm "告警内容"，需要自己编写。
#   如果不定义，直接定义为echo $1 就可以
#para:@message
#
#########################
echosm()
{
echo $1
}
#########################
#
#function:getHttpStatus
#
#########################
getHttpStatus()
{
HTTPCODE=`curl -o /dev/null -s -w %{http_code} $URL`
return $HTTPCODE
}
#########################
#
#function:getHttpStatus
#return: container's process number.
#
#########################
getServerProcess()
{
test $DEBUG -ge 1 && echo "DEBUG1> IN function getServerProcess echo PSKEY1: $PSKEY1"
test $DEBUG -ge 1 && echo "DEBUG1> IN function getServerProcess echo PSKEY2: $PSKEY2"
test $DEBUG -ge 1 && echo "DEBUG1> IN function getServerProcess echo PSKEY3:$PSKEY3"
serverProcessNum=`ps -ef | grep -v grep | grep $PSKEY1 | grep $PSKEY2 | grep $PSKEY3 | wc -l`
return $serverProcessNum
test $DEBUG -ge 1 && echo "DEBUG1> function getServerProcess return serverProcessNum $serverProcessNum"
}
#########################
#
#function:killServer,Frist use normal cmd stop server,if fail then kill.
#
#########################
killServer()
{
test $DEBUG -ge 1 && echo "DEBUG1> Func killServer Begin!"
$STOPSERVERCMD
sleep $WAITRESTARTTIME
getServerProcess
#If staill has apache process，then killed
if [ $serverProcessNum -gt 0 ];then
ps -ef | grep -v grep | grep $PSKEY1 | grep $PSKEY2 | grep $PSKEY3 | awk '{print $2}'|xargs kill -9
fi
}
#########################
#
#function:startServer,normal start container server.
#
#########################
startServer()
{
$STARTSERVERCMD
}
#########################
#
#function:restartServer
#
#########################
restartServer()
{
killServer
sleep $WAITRESTARTTIME
startServer
echo "$AlertHead has been restarted! at $TIME on `hostname`" | tee -a  $LOG
echosm "$AlertHead has been restarted! at $TIME on `hostname`" | tee -a  $LOG
}
#########################
#
#function:getOutMemFileNum,when Java memory out ,it will be dump some core files.
#return: dump core files number
#
#########################
getOutMemFileNum()
{
if [ ! -d $MEMOUTFILEPATH ];then
echo "$MEMOUTFILEPATH not exist,please check again!"
else
if [ ${RMJAVACOREFILEFLAG} = "Y" ];then
heapdumpNum=`ls $MEMOUTFILEPATH | grep $MEMOUTFILEKEY1 | grep $MEMOUTFILEKEY2 | wc -l`
return $heapdumpNum
else
heapdumpNum=0
return $heapdumpNum
fi
fi
}
#########################
#
#function:delMemOutFile,Those Java memory dump core files very too larger,should be delete.
#
#########################
delMemOutFile()
{
cd $MEMOUTFILEPATH
ls $MEMOUTFILEPATH | grep $MEMOUTFILEKEY1 | grep $MEMOUTFILEKEY2 | xargs /usr/bin/rm -f
}
#########################
#
#function:main
#
#########################
main()
{
#test ! -d $LOGDIR && mkdir -p $LOGDIR
if [ ! -d $LOGDIR ];then
mkdir -p $LOGDIR
fi
getOutMemFileNum
getHttpStatus
if [ "$HTTPCODE"x != "200"x ] || [ $heapdumpNum -ge 1 ];then
echo "$AlertHead httpcode is $HTTPCODE ! at $TIME on `hostname` " | tee -a  $LOG
#If exist apache process but httpcode is not 200 it will be restart,if there is no apache process then no work on it.
getServerProcess
test $DEBUG -ge 1 && echo "Frist run getServerProcess in func main return serverProcessNum: $serverProcessNum "
if [ $serverProcessNum -ge 1 ];then
echosm "$AlertHead httpcode is $HTTPCODE and will be restarted ! at $TIME on `hostname` " | tee -a  $LOG
fi
test $DEBUG -ge 2 && test $serverProcessNum -ge 1 && echo "$AlertHead httpcode is $HTTPCODE and will be restarted ! at $TIME on `hostname` " | tee -a  $LOG
test $DEBUG -ge 1 && echo "Second run getServerProcess in func main return serverProcessNum: ${serverProcessNum} ,before restart!"
getServerProcess
if [ $serverProcessNum -ge 1 ];then
restartServer
fi
test $DEBUG -ge 2 && echo "DEBUG2> After restartServer the serverProcessNum : ${serverProcessNum} "
sleep $WAITCHECKHEALTHTIME
getHttpStatus
echo "$AlertHead status is $HTTPCODE, at $TIME on `hostname` ." | tee -a  $LOG
if [ "$HTTPCODE" = "200" ];then
echosm "$AlertHead has OK now!at $TIME on `hostname` " | tee -a  $LOG
else
echosm "$AlertHead restart failed or after restart the web staill wrong,please check !at $TIME on `hostname` " | tee -a  $LOG
fi
else
echo "$AlertHead status is $HTTPCODE, at $TIME on `hostname` ." | tee -a $LOG
fi
if [ ${RMJAVACOREFILEFLAG} = "Y" ];then
delMemOutFile
if [ $? -eq 0 ];then
echo "JAVA Core Files in $MEMOUTFILEPATH has been rm, at $TIME ON `hostname` . " | tee -a  $LOG
fi
else
echo "JAVA Core Files in $MEMOUTFILEPATH staill exist,Please check you free space on filesystem at $TIME ON `hostname` . " | tee -a  $LOG
fi
}
#
#MAIN EXE
#
main