话单比对统一解决方案(基于Shell)

在外实习，除了工作，自己还可以做一些感兴趣的事情，这段时间一致在做话单比对的工作，最近又在数据中心的比对，哎呀除了核对还是核对，真是单调！

为什么了不做一个全自动的核对机了，就像全自动的洗衣机你把洗衣粉放好了，水龙头打开，设定一个模式，他自己就会洗、漂、脱一体服务。

嘿嘿，我这两天断断续续做的这个解决方案就是基于这个想法哦，是不是很帅哦....

##################### Uniform Solution for Bill Comparison ################################

##################### Date: 2009-3-6 #####################################################

##################### Author: landry yi ##################################################

##################### Email: yijiyong100@163.com #########################################

##################### Copyright: Harbin Normal University@copyright2009 ###################

##################### Version: 1.0.0#######################################################

##################### Histroy: if modified ,please add here and release new veresion#######

############## Config Part ################################################################

SystemBak="/transfer/yijy/cmp_shell/bak"

SystemTmp="/transfer/yijy/cmp_shell/tmp"

SystemLog="/transfer/yijy/cmp_shell/log"

SystemErr="/transfer/yijy/cmp_shell/err"

NewDataSourceDir="/transfer/yijy/cmp_shell/bin/vtof/bak"

NewDataSourceExp="*20090101*"

OldDataSourceDir="/transfer/yijy/cmp_shell/bin/vtof/bak"

OldDataSourceExp="*20090101*"

NewConvertFlag="on"

NewConvertBinDir="/transfer/yijy/cmp_shell/bin/vtof/new/vtof1"

NewConvertBinCfg="/transfer/yijy/cmp_shell/bin/vtof/cfg/new/vtof.cfg"

NewConvertBinIn="/transfer/yijy/cmp_shell/bin/vtof/new/in"

NewConvertBinOut="/transfer/yijy/cmp_shell/bin/vtof/new/out"

NewConvertSplit="|"

OldConvertFlag="on"

OldConvertBinDir="/transfer/yijy/cmp_shell/bin/vtof/old/vtof1"

OldConvertBinCfg="/transfer/yijy/cmp_shell/bin/vtof/cfg/old/vtof.cfg"

OldConvertBinIn="/transfer/yijy/cmp_shell/bin/vtof/old/in"

OldConvertBinOut="/transfer/yijy/cmp_shell/bin/vtof/old/out"

OldConvertSplit="|"

NewCmpExtractFlag="on"

NewCmpExtractKey="1,3,6,8,10"

NewCmpExtractSem="msisdn:2,chat_type:11,fee_type:16,fee1:14,old_fee1:2"

OldCmpExtractFlag="on"

OldCmpExtractKey="1,3,6,8,10"

OldCmpExtractSem="msisdn:2,chat_type:11,fee_type:16,fee1:14,old_fee1:2"

NewCmpFilePrex="new."

OldCmpFilePrex="old."

CmpFileNewIn="/transfer/yijy/cmp_shell/cmp/new/new_in"

CmpFileOldIn="/transfer/yijy/cmp_shell/cmp/new/old_in"

CmpBinDir="/transfer/yijy/cmp_shell/cmp/bin/o2n_cmp"

CmpBinCfg="/transfer/yijy/cmp_shell/cmp/cfg/wa.cfg"

CheckFile()

{

if [ ! -d ${SystemBak} -o ! -d ${SystemTmp} -o ! -d ${CmpFileNewIn} -o ! -d ${CmpFileOldIn} ]

then

echo 'APP:ERROR:Config error -- DIR'

exit

fi

if [ ! -d ${SystemLog} -o ! -d ${SystemErr} ]

then

echo 'APP:ERROR:Config error -- System Dir'

exit

fi

if [ ! -d ${NewConvertBinIn} -o ! -d ${NewConvertBinOut} -o ! -d ${OldConvertBinIn} -o ! -d ${OldConvertBinOut} ]

then

echo 'APP:ERROR:Convert error -- DIR'

exit

fi

if [ ! -f $CmpBinDir -o ! -f $CmpBinCfg ]

then

echo 'APP:ERROR:Config error -- FILE'

exit

fi

if [ ! -d $NewDataSourceDir -o ! -d $OldDataSourceDir ]

then

echo 'APP:ERROR:Config error -- Datasource'

exit

fi

if [ ! ${NewConvertSplit} -o ! ${OldConvertSplit} ]

then

echo 'APP:ERROR:Config error -- ConvertSplitchar can not be NULL'

exit

fi

if [ ! $NewDataSourceExp -o ! $OldDataSourceExp ]

then

echo 'APP:ERROR:Config error -- DataSourceExp can not be NULL'

exit

fi

if [ $NewConvertFlag = "on" ]

then

if [ ! -d ${NewConvertBinIn} -o ! -d ${NewConvertBinOut} -o ! -f ${NewConvertBinDir} -o ! -f ${NewConvertBinCfg} ]

then

echo 'APP:ERROR:Config error -- NewConvert'

exit

fi

fi

if [ $OldConvertFlag = "on" ]

then

if [ ! -d ${OldConvertBinIn} -o ! -d ${OldConvertBinOut} -o ! -f ${OldConvertBinDir} -o ! -f ${OldConvertBinCfg} ]

then

echo 'APP:ERROR:Config error -- OldConvert'

exit

fi

fi

if [ ! $NewCmpFilePrex -o ! $OldCmpFilePrex ]

then

echo 'APP:ERROR:Config error -- prex'

exit

fi

if [ -n ${NewCmpExtractKey} -a -n ${NewCmpExtractSem} -a -n ${OldCmpExtractKey} -a -n ${OldCmpExtractSem} ]

then

echo ${NewCmpExtractKey} | awk -F ',' '{

printf("APP:MSG:NewExkey number is [%d] -- and the New key are [%s]\n",NF,$0);

}'

echo ${NewCmpExtractSem} | awk -F ',' '{

NewExsemNum=NF

printf("APP:MSG:NewExsem number is [%d] -- and the New sem are [%s]\n",NF,$0);

}'

echo ${OldCmpExtractKey} | awk -F ',' '{

OldExkeyNum=NF

printf("APP:MSG:OldExkey number is [%d] -- and the Old key are [%s]\n",NF,$0);

}'

echo ${OldCmpExtractSem} | awk -F ',' '{

OldExsemNum=NF

printf("APP:MSG:OldExsem number is [%d] -- and the Old sem are [%s]\n",NF,$0);

}'

newExtractKeyNumber=`echo ${NewCmpExtractKey} | awk -F ',' '{printf("%d",NF)}'`

newExtractSemNumber=`echo ${NewCmpExtractSem} | awk -F ',' '{printf("%d",NF)}'`

if [ ${newExtractKeyNumber} -ne ${newExtractSemNumber} ]

then

echo 'APP:ERROR:The new extract key number not equal to sem number'

exit

fi

oldExtractKeyNumber=`echo ${OldCmpExtractKey} | awk -F ',' '{printf("%d",NF)}'`

oldExtractSemNumber=`echo ${OldCmpExtractSem} | awk -F ',' '{printf("%d",NF)}'`

if [ ${oldExtractKeyNumber} -ne ${oldExtractSemNumber} ]

then

echo 'APP:ERROR:The old extract key number not equal to sem number'

exit

fi

else

echo 'APP:ERROR:Config error -- Extract'

exit

fi

echo 'APP:MSG:Config part check successful!'

}

############## Clean Enviroment ###########################################################

CleanEnviroment()

{

echo 'APP:MSG:Start clean the system enviroment...'

########### <--add your own script here to clean the enviroment if you like--> ##########

########### <--add your own script here to clean the enviroment if you like--> ##########

echo 'APP:MSG:Clean the system enviroment over...'

}

############## Data Prepare ###############################################################

totalNewConvertIn=0

totalOldConvertIn=0

totalNewExtractIn=0

totalOldExtractIn=0

DataPrepare()

{

if [ ! -d ${NewDataSourceDir} -o ! -d ${OldDataSourceDir} ]

then

echo 'APP:ERROR:Data source error'

exit

fi

echo 'APP:MSG:Data start prepare...'

if [ ${NewConvertFlag} = "on" ]

then

cp -r ${NewDataSourceDir}/${NewDataSourceExp} ${NewConvertBinIn}

let totalNewConvertIn=`ls -1 ${NewConvertBinIn} | wc -l`

echo 'APP:MSG:The total number of new file need convert in ['${NewConvertBinIn}'] is ['${totalNewConvertIn}']'

else

cp -r ${NewDataSourceDir}/${NewDataSourceExp} ${NewConvertBinOut}

let totalNewExtractIn=`ls -1 ${NewConvertBinOut} | wc -l`

echo 'APP:MSG:The total number of new file need extract in ['${NewConvertBinOut}'] is ['${totalNewExtractIn}']'

fi

if [ ${OldConvertFlag} = "on" ]

then

cp -r ${OldDataSourceDir}/${OldDataSourceExp} ${OldConvertBinIn}

let totalOldConvertIn=`ls -1 ${OldConvertBinIn} | wc -l`

echo 'APP:MSG:The total number of new file need convert in ['${OldConvertBinIn}'] is ['${totalOldConvertIn}']'

else

cp -r ${OldDataSourceDir}/${OldDataSourceExp} ${OldConvertBinOut}

let totalOldExtractIn=`ls -1 ${OldConvertBinOut} | wc -l`

echo 'APP:MSG:The total number of new file need extract in ['${OldConvertBinOut}'] is ['${totalOldExtractIn}']'

fi

echo 'APP:MSG:Data prepare over...'

}

############## Convert Part ###############################################################

ConvertFile()

{

if [ ${NewConvertFlag} = "on" ]

then

if [ 0 -eq `ls -1 ${NewConvertBinIn} | wc -l` ]

then

echo 'APP:MSG:No file exist in '${NewConvertBinIn}

exit

else

let totalNewConvertIn=`ls -1 ${NewConvertBinIn} | wc -l`

newCurPercent=0

newTmpPercent=0

echo 'APP:MSG:Start convert new file:'

echo 'APP:MSG:The total file number in the ['${NewConvertBinIn}'] is ['${totalNewConvertIn}']'

echo 'APP:MSG:\c'

nohup ${NewConvertBinDir} ${NewConvertBinCfg} &

sleep 1

echo 'APP:MSG:Process Bar:____________________________________________50%_________________________________________________100%'

echo 'APP:MSG:Convert Pro:\c'

while [ 0 -lt `ls -1 ${NewConvertBinIn} | wc -l` ]

do

newExtCurFile=`ls -1 ${NewConvertBinIn} | wc -l`

let NewDValue=totalNewConvertIn-newExtCurFile

let newCurPercent=${NewDValue}\*100/${totalNewConvertIn}

while [ ${newTmpPercent} -lt ${newCurPercent} ]

do

echo '#\c'

let newTmpPercent=newTmpPercent+1

done

done

while [ $newTmpPercent -lt 100 ]

do

echo '#\c'

let newTmpPercent=newTmpPercent+1

done

newConvertId=`ps -ef | grep ${NewConvertBinDir} | grep ${NewConvertBinCfg} | awk '{printf("%s",$2);}'`

echo '\nAPP:MSG:New convert program id is ['${newConvertId}']'

kill ${newConvertId}

echo 'APP:MSG:New convert program was killed!'

echo 'APP:MSG:New file Convert over!'

fi

fi

if [ ${OldConvertFlag} = "on" ]

then

if [ 0 -eq `ls -1 ${OldConvertBinIn} | wc -l` ]

then

echo 'APP:MSG:No file exist in '${OldConvertBinIn}

exit

else

let totalOldConvertIn=`ls -1 ${OldConvertBinIn} | wc -l`

oldCurPercent=0

oldTmpPercent=0

echo 'APP:MSG:Start convert old file:'

echo 'APP:MSG:The total file number in the ['${OldConvertBinIn}'] is ['${totalOldConvertIn}']'

echo 'APP:MSG:\c'

nohup ${OldConvertBinDir} ${OldConvertBinCfg} &

sleep 1

echo 'APP:MSG:Process Bar:____________________________________________50%_________________________________________________100%'

echo 'APP:MSG:Convert Pro:\c'

while [ 0 -lt `ls -1 ${OldConvertBinIn} | wc -l` ]

do

oldExtCurFile=`ls -1 ${OldConvertBinIn} | wc -l`

let OldDValue=totalOldConvertIn-oldExtCurFile

let oldCurPercent=${OldDValue}\*100/${totalOldConvertIn}

while [ ${oldTmpPercent} -lt ${oldCurPercent} ]

do

echo '#\c'

let oldTmpPercent=oldTmpPercent+1

done

done

while [ $oldTmpPercent -lt 100 ]

do

echo '#\c'

let oldTmpPercent=oldTmpPercent+1

done

oldConvertId=`ps -ef | grep ${OldConvertBinDir} | grep ${OldConvertBinCfg} | awk '{printf("%s",$2);}'`

echo '\nAPP:MSG:Old convert program id is ['${oldConvertId}']'

kill ${oldConvertId}

echo 'APP:MSG:Old convert program was killed!'

echo 'APP:MSG:Old file Convert over!'

fi

fi

}

############## Extract Part ###############################################################

ExtractFile()

{

if [ ${NewCmpExtractFlag} = "on" ]

then

if [ 0 -eq `find ${NewConvertBinOut} -type f | wc -l` ]

then

echo 'APP:MSG:No file exist in '${NewConvertBinOut}

else

echo 'APP:MSG:Start Extract new file,and copy the extract file to compare program in dir,then move it to SystemBak dir'

let totalNewExtractIn=`ls -1 ${NewConvertBinOut} | wc -l`

echo 'APP:MSG:The total file in ['${NewConvertBinOut}'] is ['${totalNewExtractIn}']'

BakDate=`date +%Y%m%d`

if [ ! -d ${SystemBak}/${BakDate} ]

then

mkdir ${SystemBak}/${BakDate}

fi

if [ 0 -lt `find ${NewConvertBinOut} -type f | wc -l` ]

then

echo 'APP:MSG:Process Bar:____________________________________________50%_________________________________________________100%'

echo 'APP:MSG:Extract Pro:\c'

newExtractTmpPercent=0

newExtractCurPercent=0

newExtractTotal=0

find ${NewConvertBinOut} -type f | while read newConvertFile

do

newConvertTmp=`basename ${newConvertFile}`

newExtractTmpFile=`echo ${NewCmpFilePrex}${newConvertTmp}`

cat ${newConvertFile} | awk -F ${NewConvertSplit} '{

newExStr=""

newExKeyStr="'$NewCmpExtractKey'"

newExSemStr="'$NewCmpExtractSem'"

while(length(newExKeyStr) || length(newExSemStr))

{

newKeyTmpCur=index(newExKeyStr,",")

newSemTmpCur=index(newExSemStr,",")

if (newKeyTmpCur == 0)

{

newKeyTmpCur=length(newExKeyStr)+1;

}

if (newSemTmpCur == 0)

{

newSemTmpCur=length(newExSemStr)+1;

}

newSemTmp=substr(newExSemStr,1,newSemTmpCur-1)

newSemTmpIndex=index(newSemTmp,":")

newSemTmpLen=substr(newSemTmp,newSemTmpIndex+1)

newKeyTmp=substr(newExKeyStr,1,newKeyTmpCur-1)

newExStrTmp=$newKeyTmp

newExStrFormat=formatStr(newExStrTmp,newSemTmpLen)

if (newExStr == "")

{

newExStr=newExStrFormat;

}

else

{

newExStr=sprintf("%s %s",newExStr,newExStrFormat)

}

if(index(newExKeyStr,",") == 0)

{

newExKeyStr=""

}

newExKeyStr=substr(newExKeyStr,newKeyTmpCur+1)

if(index(newExSemStr,",") == 0)

{

newExSemStr=""

}

newExSemStr=substr(newExSemStr,newSemTmpCur+1)

}

printf("%s\n",newExStr);

}

function formatStr (str,len)

{

formatstr=""

strLen=length(str)

sprintf("len %d,strlen%d",len,strLen)

if(strLen>len)

{

formatstr=substr(str,1,len);

}

else

{

formatstr=str

dValue=len-strLen

while(dValue)

{

formatstr=sprintf("%s ",formatstr)

dValue--

}

}

return formatstr

}

'>${SystemTmp}/${newExtractTmpFile}

let newExtractTotal=newExtractTotal+1

let newExtractCurPercent=${newExtractTotal}\*100/${totalNewExtractIn}

while [ ${newExtractTmpPercent} -lt ${newExtractCurPercent} ]

do

echo '#\c'

let newExtractTmpPercent=newExtractTmpPercent+1

done

cp ${SystemTmp}/${newExtractTmpFile} ${CmpFileNewIn}

mv ${SystemTmp}/${newExtractTmpFile} ${SystemBak}/${BakDate}

done

fi

echo '\nAPP:MSG:Extract new file over,copy and move the file sucessfully!'

fi

fi

if [ ${OldCmpExtractFlag} = "on" ]

then

if [ 0 -eq `find ${OldConvertBinOut} -type f | wc -l` ]

then

echo 'APP:MSG:No file exist in '${OldConvertBinOut}

else

echo 'APP:MSG:Start Extract old file,and copy the extract file to compare program in dir,then move it to SystemBak dir'

let totalOldExtractIn=`ls -1 ${OldConvertBinOut} | wc -l`

echo 'APP:MSG:The total file in ['${OldConvertBinOut}'] is ['${totalOldExtractIn}']'

BakDate=`date +%Y%m%d`

if [ ! -d ${SystemBak}/${BakDate} ]

then

mkdir ${SystemBak}/${BakDate}

fi

if [ 0 -lt `find ${OldConvertBinOut} -type f | wc -l` ]

then

echo 'APP:MSG:Process Bar:____________________________________________50%_________________________________________________100%'

echo 'APP:MSG:Extract Pro:\c'

oldExtractTmpPercent=0

oldExtractCurPercent=0

oldExtractTotal=0

find ${OldConvertBinOut} -type f | while read oldConvertFile

do

oldConvertTmp=`basename ${oldConvertFile}`

oldExtractTmpFile=`echo ${OldCmpFilePrex}${oldConvertTmp}`

cat ${oldConvertFile} | awk -F ${OldConvertSplit} '{

oldExStr=""

oldExKeyStr="'$OldCmpExtractKey'"

oldExSemStr="'$OldCmpExtractSem'"

while(length(oldExKeyStr) || length(oldExSemStr))

{

oldKeyTmpCur=index(oldExKeyStr,",")

oldSemTmpCur=index(oldExSemStr,",")

if (oldKeyTmpCur == 0)

{

oldKeyTmpCur=length(oldExKeyStr)+1;

}

if (oldSemTmpCur == 0)

{

oldSemTmpCur=length(oldExSemStr)+1;

}

oldSemTmp=substr(oldExSemStr,1,oldSemTmpCur-1)

oldSemTmpIndex=index(oldSemTmp,":")

oldSemTmpLen=substr(oldSemTmp,oldSemTmpIndex+1)

oldKeyTmp=substr(oldExKeyStr,1,oldKeyTmpCur-1)

oldExStrTmp=$oldKeyTmp

oldExStrFormat=formatStr(oldExStrTmp,oldSemTmpLen)

if (oldExStr == "")

{

oldExStr=oldExStrFormat;

}

else

{

oldExStr=sprintf("%s %s",oldExStr,oldExStrFormat)

}

if(index(oldExKeyStr,",") == 0)

{

oldExKeyStr=""

}

oldExKeyStr=substr(oldExKeyStr,oldKeyTmpCur+1)

if(index(oldExSemStr,",") == 0)

{

oldExSemStr=""

}

oldExSemStr=substr(oldExSemStr,oldSemTmpCur+1)

}

printf("%s\n",oldExStr);

}

function formatStr (str,len)

{

formatstr=""

strLen=length(str)

sprintf("len %d,strlen%d",len,strLen)

if(strLen>len)

{

formatstr=substr(str,1,len);

}

else

{

formatstr=str

dValue=len-strLen

while(dValue)

{

formatstr=sprintf("%s ",formatstr)

dValue--

}

}

return formatstr

}

'>${SystemTmp}/${oldExtractTmpFile}

let oldExtractTotal=oldExtractTotal+1

let oldExtractCurPercent=${oldExtractTotal}\*100/${totalOldExtractIn}

while [ ${oldExtractTmpPercent} -lt ${oldExtractCurPercent} ]

do

echo '#\c'

let oldExtractTmpPercent=oldExtractTmpPercent+1

done

cp ${SystemTmp}/${oldExtractTmpFile} ${CmpFileOldIn}

mv ${SystemTmp}/${oldExtractTmpFile} ${SystemBak}/${BakDate}

done

fi

echo '\nAPP:MSG:Extract old file over,copy and move the file sucessfully!'

fi

fi

}

############## Compare Part ###############################################################

CompareFile()

{

if [ 0 -eq `find ${CmpFileNewIn} -type f | wc -l` ]

then

echo 'APP:MSG:There is no file exist in ['${CmpFileNewIn}']'

exit

fi

if [ 0 -eq `find ${CmpFileOldIn} -type f | wc -l` ]

then

echo 'APP:MSG:There is no file exist in ['${CmpFileOldIn}']'

exit

fi

echo 'APP:MSG:Start comprae file...'

totalCmpNewFileIn=`ls -1 ${CmpFileNewIn} | wc -l`

totalCmpOldFileIn=`ls -1 ${CmpFileOldIn} | wc -l`

echo 'APP:MSG:The total number of file in ['${CmpFileNewIn}'] is ['${totalCmpNewFileIn}']'

echo 'APP:MSG:The total number of file in ['${CmpFileOldIn}'] is ['${totalCmpOldFileIn}']'

cmpCurPercent=0

cmpTmpPercent=0

echo 'APP:MSG:\c'

nohup ${CmpBinDir} ${CmpBinCfg} &

sleep 1

echo 'APP:MSG:Process Bar:____________________________________________50%_________________________________________________100%'

echo 'APP:MSG:Compare Pro:\c'

while [ 0 -lt `ls -1 ${CmpFileNewIn} | wc -l` ]

do

totalCurCmpFileIn=`ls -1 ${CmpFileNewIn} | wc -l`

let fileDvalue=totalCmpNewFileIn-totalCurCmpFileIn

let cmpCurPercent=${fileDvalue}\*100/${totalCmpNewFileIn}

while [ ${cmpTmpPercent} -lt ${cmpCurPercent} ]

do

echo '#\c'

let cmpTmpPercent=cmpTmpPercent+1

done

done

while [ ${cmpTmpPercent} -lt 100 ]

do

echo '#\c'

let cmpTmpPercent=cmpTmpPercent+1

done

sleep 5

theCompareId=`ps -ef | grep ${CmpBinDir} | grep ${CmpBinCfg} | awk '{printf("%s",$2);}'`

if [ -n "$theCompareId" ]

then

kill $theCompareId

fi

echo '\nAPP:MSG:Compare Result:'

tail nohup.out

echo 'APP:MSG:Compare file over...'

}

############## Main Program ###############################################################

CheckFile

#CleanEnviroment

DataPrepare

ConvertFile

ExtractFile

CompareFile