shell sed 积累

xshell

a, 远程登录ssh 等； ssh 可以证书或者 用户名密码登录

b， 文件上传下载：

配置；

rz 上传文件到unix 服务器

sz 下载

find and 统计

find ./ -type d -name "*v1*"|wc -l

find ./ -type d -name "*wan*"|wc -l

list：

for i in `find ./ -type d -name "*tonn*"`;do echo $i;done

查询 拷贝

for i in `find ./ -type d -name "*tonn*"`;do rsync -avP $i /data/aa/;done

1， 生成全目录的list：

[root@10-10-159-137 ip_all_my_used]# pwd

/data/actlog/ip_all_my_used

[root@10-10-159-137 ip_all_my_used]# find /data/actlog/ip_all_my_used/ -name "*.tar.gz" > service.list

/data/actlog/ip_all_my_used/wefasdierfxoede1/act.log.2015-10-06.tar.gz

/data/actlog/ip_all_my_used/wefasdierfxoede1/act.log.2015-08-31.tar.gz

2， 全list文件修改内容：

#!/bin/bash

i=0

echo "current time is: "$(date)

for tarfile in `cat ./service.list`

do

echo $tarfile

tar -zxvf $tarfile

rm -fr $tarfile

logfile=`ls act.log.*`

echo "here is my: "$logfile

echo `ll $logfile`

sed -i 's/S771/S257/g' $logfile

tar czf $tarfile $logfile

echo "current time is: "$(date)

rm ./act.log.*

done

service.list 内容如下：

/data/actlog/ip_all_my_used/wefasdierfxoede1/act.log.2015-11-15.tar.gz

3， 几个 用过的sed 替换 或者行删除

锚定行的结束 如：/sed$/匹配所有以sed结尾的行

删除 reason=65 结束的行；

sed -i 's/reason=65$//g' act.log.2015-11-15

sed -i 's/reason=74$//g' act.log.2015-11-15

删除所有 && 之间， 含有 banlance的 列； &balance=14044& ---》》》 &&

sed -i 's/&[^&]*balance[^&]*//g' act.log.2015-11-15

删除所有 && 之间， 含有 banlance的 列（包括前面的&）； &balance=14044& ---》》》 &

sed -i 's/&balance[^&]*//g' act.log.2015-11-15

4， shell 脚本例子

[root@10-10-159-137 ip_all_my_used]# find /data/actlog/ip_all_my_used/ -name "*.tar.gz" > service.list

循环遍历 目录中的tar.gz 文件， 解压， 搜索文件， 然后满足一定条件的， 输出到指定文件（目录拼接而成）中。

#!/bin/bash

i=0

echo "current time is: "$(date)

for tarfile in `cat ./service.list`

do

a=${tarfile%".tar.gz"}

path1=${a#"/data/actlog/"}

path=${path1//"/"/"_"}

tar -zxvf $tarfile

for line in `cat ./act.log.*`

do

#echo $line

if [[ $line == *interface=yb_expend* ]]

then

echo $line >> ./yb_expend/$path"yb_expend".txt

elif [[ $line == *interface=shop_buy* ]]

then

echo $line >> ./shop_buy/$path"shop_buy".txt

elif [[ $line == *interface=recharge* ]]

then

echo $line >> ./recharge/$path"recharge".txt

elif [[ $line == *interface=yb_income* ]]

then

echo $line >> ./yb_income/$path"yb_income".txt

elif [[ $line == *interface=logout* ]]

then

echo $line >> ./logout/$path"logout".txt

fi

done

i=$(($i+1))

echo "current time is: "$(date)

echo "is first parseing log file of: "$i" and name is: "$tarfile

rm ./act.log.*

done

5， 批量更改当前目录：

#!/bin/bash

echo "current time is: "$(date)

i=0

for tarfile in `ls .`

do

mv $tarfile "bf4c52cbd9cd_"$i

i=$(($i+1))

done

6， 批量运行 一系列脚本：

# cat run_parse.sh

echo "here make dir" >> timeinfo.txt

date >> timeinfo.txt

./mymkdir.sh

echo "here firstparse" >> timeinfo.txt

date >> timeinfo.txt

./firstparse.sh

echo "here make dir" >> timeinfo.txt

date >> timeinfo.txt

./recharge.sh

echo "here shop_buy" >> timeinfo.txt

date >> timeinfo.txt

./shop_buy.sh

echo "here parse yb_expend" >> timeinfo.txt

date >> timeinfo.txt

./yb_expend.sh

echo "here parse yb_income" >> timeinfo.txt

date >> timeinfo.txt

./yb_income.sh

echo "here parse logout" >> timeinfo.txt

date >> timeinfo.txt

./logout.sh

7， 去除 某些 表头的 列名；

#!/bin/bash

rm -fr tt.txt

ls yb_expend > tt.txt

#find ./yb_expend/ -name "*.txt" > tt.txt

let i=0

for tarfile in `cat tt.txt`

do

echo ./"yb_expend"/$tarfile

sed -i 's/interface=/\ /g' ./"yb_expend"/$tarfile

sed -i 's/&gname=/, /g' ./"yb_expend"/$tarfile

sed -i 's/&gid=/, /g' ./"yb_expend"/$tarfile

sed -i 's/&sid=/, /g' ./"yb_expend"/$tarfile

sed -i 's/&oldsid=/, /g' ./"yb_expend"/$tarfile

sed -i 's/&user=/, /g' ./"yb_expend"/$tarfile

sed -i 's/&roleid=/, /g' ./"yb_expend"/$tarfile

sed -i 's/&dept=/, /g' ./"yb_expend"/$tarfile

sed -i 's/&time=/, /g' ./"yb_expend"/$tarfile

sed -i 's/&amount=/, /g' ./"yb_expend"/$tarfile

sed -i 's/&balance=/, /g' ./"yb_expend"/$tarfile

sed -i 's/&level=/, /g' ./"yb_expend"/$tarfile

sed -i 's/&reason=/, /g' ./"yb_expend"/$tarfile

cat ./"yb_expend"/$tarfile >> yb_expend_after.txt

i=$(($i+1))

done

echo "yb_expend file number is"$i

echo "done yb_expend , and yb_expend file number is "$i >> result_log.txt

rm -fr tt.txt

#!/bin/bash

rm -fr tt.txt

ls yb_expend > tt.txt

#find ./yb_expend/ -name "*.txt" > tt.txt

let i=0

for tarfile in `cat tt.txt`

do

echo ./"yb_expend"/$tarfile

sed -i 's/reason=65$//g' ./"yb_expend"/$tarfile

sed -i 's/reason=74$//g' ./"yb_expend"/$tarfile #删除reason=74的行列

sed -i 's/&balance[^&]*//g' ./"yb_expend"/$tarfile #删除balance 列

sed -i 's/interface=/\ /g' ./"yb_expend"/$tarfile

sed -i 's/&gname=/, /g' ./"yb_expend"/$tarfile

sed -i 's/&gid=/, /g' ./"yb_expend"/$tarfile

sed -i 's/&sid=/, /g' ./"yb_expend"/$tarfile

sed -i 's/&oldsid=/, /g' ./"yb_expend"/$tarfile

sed -i 's/&user=/, /g' ./"yb_expend"/$tarfile

sed -i 's/&roleid=/, /g' ./"yb_expend"/$tarfile

sed -i 's/&dept=/, /g' ./"yb_expend"/$tarfile

sed -i 's/&time=/, /g' ./"yb_expend"/$tarfile

sed -i 's/&amount=/, /g' ./"yb_expend"/$tarfile

sed -i 's/&balance=/, /g' ./"yb_expend"/$tarfile

sed -i 's/&level=/, /g' ./"yb_expend"/$tarfile

sed -i 's/&reason=/, /g' ./"yb_expend"/$tarfile

cat ./"yb_expend"/$tarfile >> yb_expend_after.txt

i=$(($i+1))

done

echo "yb_expend file number is"$i

echo "done yb_expend , and yb_expend file number is "$i >> result_log.txt

rm -fr tt.txt

#!/bin/bash

rm -fr tt.txt

ls shop_buy > tt.txt

#find ./shop_buy/ -name "*.txt" > tt.txt

let i=0

for tarfile in `cat tt.txt`

do

echo ./"shop_buy"/$tarfile

sed -i 's/interface=/\ /g' ./"shop_buy"/$tarfile

sed -i 's/&gname=/, /g' ./"shop_buy"/$tarfile

sed -i 's/&gid=/, /g' ./"shop_buy"/$tarfile

sed -i 's/&sid=/, /g' ./"shop_buy"/$tarfile

sed -i 's/&oldsid=/, /g' ./"shop_buy"/$tarfile

sed -i 's/&user=/, /g' ./"shop_buy"/$tarfile

sed -i 's/&roleid=/, /g' ./"shop_buy"/$tarfile

sed -i 's/&dept=/, /g' ./"shop_buy"/$tarfile

sed -i 's/&time=/, /g' ./"shop_buy"/$tarfile

sed -i 's/&balance=/, /g' ./"shop_buy"/$tarfile

sed -i 's/&itemid=/, /g' ./"shop_buy"/$tarfile

sed -i 's/&amount=/, /g' ./"shop_buy"/$tarfile

sed -i 's/&money=/, /g' ./"shop_buy"/$tarfile

sed -i 's/&level=/, /g' ./"shop_buy"/$tarfile

sed -i 's/&remain=/, /g' ./"shop_buy"/$tarfile

sed -i 's/&map_id=/, /g' ./"shop_buy"/$tarfile

cat ./"shop_buy"/$tarfile >> shop_buy_after.txt

i=$(($i+1))

done

echo "shop_buy file number is"$i

echo "done shop_buy , and shop_buy file number is "$i >> result_log.txt

rm -fr tt.txt

#!/bin/bash

rm -fr tt.txt

ls yb_income > tt.txt

#find ./yb_income/ -name "*.txt" > tt.txt

let i=0

for tarfile in `cat tt.txt`

do

echo ./"yb_income"/$tarfile

sed -i 's/interface=/\ /g' ./"yb_income"/$tarfile

sed -i 's/&gname=/, /g' ./"yb_income"/$tarfile

sed -i 's/&gid=/, /g' ./"yb_income"/$tarfile

sed -i 's/&sid=/, /g' ./"yb_income"/$tarfile

sed -i 's/&oldsid=/, /g' ./"yb_income"/$tarfile

sed -i 's/&user=/, /g' ./"yb_income"/$tarfile

sed -i 's/&roleid=/, /g' ./"yb_income"/$tarfile

sed -i 's/&dept=/, /g' ./"yb_income"/$tarfile

sed -i 's/&time=/, /g' ./"yb_income"/$tarfile

sed -i 's/&amount=/, /g' ./"yb_income"/$tarfile

sed -i 's/&balance=/, /g' ./"yb_income"/$tarfile

sed -i 's/&level=/, /g' ./"yb_income"/$tarfile

sed -i 's/&reason=/, /g' ./"yb_income"/$tarfile

cat ./"yb_income"/$tarfile >> yb_income_after.txt

i=$(($i+1))

done

echo "yb_income file number is"$i

echo "done yb_income, and yb_income file number is "$i >> result_log.txt

rm -fr tt.txt

#!/bin/bash

rm -fr tt.txt

ls logout > tt.txt

#find ./logout/ -name "*.txt" > tt.txt

let i=0

for tarfile in `cat tt.txt`

do

echo ./"logout"/$tarfile

sed -i 's/interface=/\ /g' ./"logout"/$tarfile

sed -i 's/&gname=/, /g' ./"logout"/$tarfile

sed -i 's/&gid=/, /g' ./"logout"/$tarfile

sed -i 's/&sid=/, /g' ./"logout"/$tarfile

sed -i 's/&oldsid=/, /g' ./"logout"/$tarfile

sed -i 's/&user=/, /g' ./"logout"/$tarfile

sed -i 's/&roleid=/, /g' ./"logout"/$tarfile

sed -i 's/&dept=/, /g' ./"logout"/$tarfile

sed -i 's/&time=/, /g' ./"logout"/$tarfile

sed -i 's/&level=/, /g' ./"logout"/$tarfile

sed -i 's/&map_id=/, /g' ./"logout"/$tarfile

sed -i 's/&ip=/, /g' ./"logout"/$tarfile

sed -i 's/&onlinetime=/, /g' ./"logout"/$tarfile

sed -i 's/&group=/, /g' ./"logout"/$tarfile

cat ./"logout"/$tarfile >> logout_after.txt

i=$(($i+1))

done

echo "logout file number is"$i

echo "done logout, and logout file number is "$i >> result_log.txt

rm -fr tt.txt

#!/bin/bash

#rm -fr tt.txt

ls recharge > tt.txt

#find ./recharge/ -name "*.txt" > tt.txt

i=0

for tarfile in `cat tt.txt`

do

echo ./"recharge"/$tarfile

sed -i 's/interface=/\ /g' ./"recharge"/$tarfile

sed -i 's/&gname=/, /g' ./"recharge"/$tarfile

sed -i 's/&gid=/, /g' ./"recharge"/$tarfile

sed -i 's/&sid=/, /g' ./"recharge"/$tarfile

sed -i 's/&oldsid=/, /g' ./"recharge"/$tarfile

sed -i 's/&user=/, /g' ./"recharge"/$tarfile

sed -i 's/&roleid=/, /g' ./"recharge"/$tarfile

sed -i 's/&dept=/, /g' ./"recharge"/$tarfile

sed -i 's/&time=/, /g' ./"recharge"/$tarfile

sed -i 's/&amount=/, /g' ./"recharge"/$tarfile

sed -i 's/&money=/, /g' ./"recharge"/$tarfile

sed -i 's/&balance=/, /g' ./"recharge"/$tarfile

sed -i 's/&level=/, /g' ./"recharge"/$tarfile

sed -i 's/&order=/, /g' ./"recharge"/$tarfile

sed -i 's/&channel=/, /g' ./"recharge"/$tarfile

cat ./"recharge"/$tarfile >> recharge_after.txt

i=$(($i+1))

done

echo "recharge file number is"$i

echo "done recharge , and recharge file number is "$i >> result_log.txt

rm -fr tt.txt

~

8,

#cat list_make_base_para.sh

遍历101.× 的目录， 然后查找所有 *.tar.gz 文件 到 service.list 中。

[root@10-10-159-137 ip_101.198]# vim list_make_base_para.sh

#!/bin/bash

#echo $1

find . -name "101.*" > ip_101.198.txt

sed -i 's/.\// /g' ip_101.198.txt

for ip_addr in `cat ./ip_101.198.txt`

do

find "/data/actlog/"$ip_addr -name "*.gz" > ip_101.198_list.txt

done

9， shell 分隔符

#!/bin/bash

oldIFS=$IFS

echo $IFS

IFS=$':'

for i in `cat /etc/passwd`;

do

echo $i

done;

IFS=$oldIFS

10， 追加文件到 文件末尾

#!/bin/bash

i=0

echo "current time is: "$(date)

for fullfile in `ls yb_expend`

do

fill=${fullfile:0:18}

tarfile="bf4c52cbd9cd/"$fill".tar.gz"

echo $tarfile

echo $fullfile

tar -zxvf $tarfile

rm -fr $tarfile

logfile=`ls act.log.*`

echo "here is my: "$logfile

sed -i 's/gameinfo interface=yb_expend*//g' $logfile

linenumbers=`sed -n '$=' $logfile`

centerline=`expr $linenumbers / 2`

echo ./yb_expend/$fullfile

cat ./yb_expend/$fullfile >> $logfile

#sed -i '200r ./yb_expend/$fullfile ' $logfile

tar czf $tarfile $logfile

echo "current time is: "$(date)

rm ./act.log.*

done

11： 修改 修改时间

for fullfile in `ls .`

do

fill=${fullfile:8:4}${fullfile:13:2}${fullfile:16:2}

fill=$(($fill+1))

touch -d "$fill 01:03:00" $fullfile

echo "$fill 01:03:00"

done