(OK) 刘姐实验中的大数据分析—awk—paste—system

[root@localhost 给同光的数据]# pwd

/opt/cBPM-设计文档/张同光—设计文档/给同光的数据

[root@localhost 给同光的数据]# head -5 task-usage.csv

点击(此处)折叠或打开

2070226000000,2070237000000,2916101306,0,38675718,2.193e-05,0,0,0,0,0,,0,0.0002365,,8.215,0.04253,0,0,0

2070226000000,2070300000000,4665712499,305,460145180,0.04279,0.06628,0.07739,0.001984,0.002632,0.06641,,0.0003786,0.4438,,5.141,0.01595,0,0,0.01657

2070226000000,2070300000000,4665712499,1605,32064642,0.01996,0.06,0.07727,0.007782,0.008438,0.0603,,0.0003805,0.07349,,2.237,0.006193,0,0,0.03137

2070226000000,2070300000000,4665896876,310,905033,0.01558,0.06702,0.07727,0.0047,0.005348,0.06714,,0.0003691,0.03802,,5.537,0.01843,0,0,0.01036

2070226000000,2070300000000,6371483842,190,2854214373,0.0008802,0.000865,0.001848,0.0001869,0.0002813,0.001354,,0,0.02026,,6.742,0.02581,0,0,0.000124

[root@localhost 给同光的数据]# head -5 task-event.csv

点击(此处)折叠或打开

2070226325733,,6397857072,512,,0,sd97khHSDKGM3BF42qtSNY39C4ZR1IRQWe3h9vSy4mU=,0,1,0.01562,0.03979,0.000309,0

2070226331535,,3996125377,36,2097345596,5,/ADqb6ab3/Bxrz3dyZaDPRX7DAfUdiR+JTSVDrtJ+qU=,3,9,0.1257,0.01271,0.0001183,0

2070226331538,,3996125377,36,,0,/ADqb6ab3/Bxrz3dyZaDPRX7DAfUdiR+JTSVDrtJ+qU=,3,9,0.1257,0.01271,0.0001183,0

2070227032231,,515042969,8,,5,/fk1fVcVxZ6iM6gHZzqbIyq56m5zrmHfpdcZ/zzkq4c=,2,0,0.01562,0.01553,0.0002155,0

2070227032234,,515042969,8,,0,/fk1fVcVxZ6iM6gHZzqbIyq56m5zrmHfpdcZ/zzkq4c=,2,0,0.01562,0.01553,0.0002155,0

+++++++++++++++++++++++++

-----------------------需求1.

+++++++++++++++++++++++++

合并task-usage.csv和task-event.csv文件，生成新文件task.csv169, 新文件由以下字段组成：

-----------------------文件task-usage.csv中的 $3$4,$1,$6

job ID（第3列,$3)

task index（第4列,$4)

1. start time of the measurement period（第1列,$1）、

3. mean CPU usage rate（第6列,$6)

-----------执行下面命令，生成task-usage.csv3416

awk -F, '{print $3,$4}' OFS='' task-usage.csv > task-usage.csv34

awk -F, '{print $1,$6}' OFS=',' task-usage.csv > task-usage.csv16

paste -d"," task-usage.csv34 task-usage.csv16 > task-usage.csv3416

-----------------------和 文件task-event.csv中的 $3$4,$9

4.job ID(第3列，$3)、

5.task index(第4列,$4)、

6.priority(第9列,$9)，

-----------执行下面命令，生成task-event.csv349

awk -F, '{print $3,$4}' OFS='' task-event.csv > task-event.csv34

awk -F, '{print $9}' OFS='' task-event.csv > task-event.csv9

paste -d"," task-event.csv34 task-event.csv9 > task-event.csv349

-----------------------各字段仍以空格隔开。合并时需判断：两个文件的job ID和task index一样的（即task-event.csv的$3、$4分别等于task-usage.csv的$3、$4），其余字段才在一行；

-----------执行下面命令，合并task-usage.csv3416，task-event.csv349，生成task.csv169

awk -F, 'ARGIND==1 {w[$1]=$2}

ARGIND==2 {

flag=0;

for(a in w)

if($1==a) {

flag=1;

print $2,$3,w[a];

break;

}

}' task-event.csv349 task-usage.csv3416 > task.csv169

点击(此处)折叠或打开

awk -F, 'ARGIND==1 {w[$1]=$2}

ARGIND==2 {

flag=0;

for(a in w)

if($1==a) {

flag=1;

print $2,$3,w[a];

break;

}

}' task-event.csv349 task-usage.csv3416 > task.csv169

**********************************************************************

由于task-event.csv349 和 task-usage.csv3416 两个文件很大，有300M+，实际数据更大，上面算法很低效，计算时间长达几十个小时，因此，使用如下算法生成 task.csv169 文件

**********************************************************************

[root@localhost 给同光的数据]# gedit binarysearch.awk

点击(此处)折叠或打开

#!/bin/awk -f

# date :2010-10-02

# binary search by awk ,just for fun

# awk -f binarysearch.awk a b

# 参考 http://bbs.chinaunix.net/forum.php?mod=viewthread&tid=1794824
# sort task-usage.csv3416 > task-usage.csv3416s

# sort task-event.csv349 > task-event.csv349s

# cp task-event.csv349s a; cp task-usage.csv3416s b

# mv task-event.csv349s a; mv task-usage.csv3416s b

# awk -f binarysearch.awk a b > task.csv169temp

# sort task.csv169temp > task.csv169

NR==FNR { a[k++] = $0 }

NR>FNR {

start= 0;end = k-1

while(start<= end) {

mid =int(start+ ((end - start)/2))

#start,end都很大时，比如元素达到 2^30 时，平常做法mid =int((start+ end)/2)将超过整数的最大值 2^31 -1，此时讲溢 #出，值为负了。所以要用这个办法，O(∩_∩)O~

#第1个文件(task-event.csv349s)，各字段在b[x]

split(a[mid], b, ",")

#第2个文件(task-usage.csv3416s b)，各字段在c[x]

split($1, c, ",")

#if($1==b[1]) {print "ok "$1 " was found";break}

#下面比较字符串

#if(b[1]""==c[1]"") {print c[2],c[3],b[2],b[1],c[1]; break}

if(b[1]""==c[1]"") {print c[2],c[3],b[2]; break}

else if (c[1]"" > b[1]"") start = mid+1

#else if (c[1]"" > b[1]"") {print b[1],c[1]; start = mid+1}

#else if (sprintf("%s",c[1]) > sprintf("%s",b[1])) {print b[1],c[1]; start = mid+1}

else end= mid-1

}

}

**********************************************************************

+++++++++++++++++++++++++

-----------------------需求2.

+++++++++++++++++++++++++

对于合并后的文件task.csv，进行条件统计：

条件1：以5分钟（task.csv的start time of the measurement period字段（$1）间隔差=300000000）为时间间隔，

条件2：按priority字段($4)的取值，分别统计mean_CPU_usage字段值：

priority>=9的任务，统计该时间间隔内mean CPU usage rate和的中位数，生成新文件task-usage_normal.csv；

priority<9的任务，统计该时间间隔内mean CPU usage rate和的中位数，生成新文件task-usage_lower.csv；

生成的新文件，包含字段

1.autoNo、 5. mean CPU usage rate。

-----------执行下面命令，生成文件task-usage_normal.csv，task-usage_lower.csv

//执行之前，先删除相关文件

[root@localhost 给同光的数据]# rm 20* task-usage_normal.csv temp -f

[root@localhost 给同光的数据]# rm 20* task-usage_lower.csv temp -f

//查看开始时间，下面命令输出的 第一行 第一列 即是开始时间，赋值给starttime，根据需要调整steplen

[root@localhost 给同光的数据]# less task.csv169

//执行下面命令，生成文件task-usage_normal.csv

awk 'BEGIN{

starttime=2070226000000;

steplen=1000000;

endtime=starttime+steplen;

}

{

if($1 < endtime)<endtime)

{

/*优先级判断*/

if($3>=1){

/*printf("time:%d, endtime:%u\n", $1, endtime);*/

/*printf("priority:%d\n", $3);*/

print $2 >> endtime;

}

}else{

/*判断endtime文件是否存在*/

/*print endtime;*/

cmd="test -f "endtime"";

ret=system(cmd);

if(!ret){ /*endtime文件存在， 则执行下面*/

/*endtime先排序，放入temp文件，然后取中位数*/

/*print endtime;*/

cmd="sort "endtime" >temp";

system(cmd);

/*system("sleep 5");*/

print endtime-steplen "~" endtime >> "task-usage_normal.csv";

cmd="wc temp >tmp";

system(cmd);

getline var < "tmp"; split(var,a," "); midline=int(a[1]/2+1);

cmd="awk NR=="midline" temp >> task-usage_normal.csv"; /* task-usage_lower.csv */

system(cmd);

close("tmp");

close("temp");

close(endtime);

}

endtime+=steplen;

}

}' task.csv169

上面代码见下面附件



aa.txt

[root@localhost 给同光的数据]# ls

date.tmp task-event.csv9 task-usage.csv34 需求.txt

task.csv169 task-event.csv.org task-usage.csv3416 需求.txt~

task-event.csv task-usage.csv task-usage.csv.org

task-event.csv34 task-usage.csv16 temp.txt

task-event.csv349 task_usage.csv34

[root@localhost 给同光的数据]#

+++++++++++++++++++++++++

-----------------------需求3.

+++++++++++++++++++++++++

cpu_by_time_prio.txt 内容如下，包含数千行，要求 根据 第一列 大小 排序：

点击(此处)折叠或打开

6.220000000000000000e+03 0.000000000000000000e+00 1.582216674804687500e+03 3.727216339111328100e+01 1.667797279357910200e+01 0.000000000000000000e+00 3.194387207031250000e+02 0.000000000000000000e+00 3.024159669876098600e+00 2.463942146301269500e+01 2.162471771240234400e+01 1.739675781250000000e+03 1.963522148132324200e+01 1.394829864501953100e+02

2.980000000000000000e+03 0.000000000000000000e+00 3.961403503417968700e+02 4.579420471191406200e+01 4.849912643432617200e+01 0.000000000000000000e+00 3.451186523437500000e+02 0.000000000000000000e+00 2.817885160446167000e+00 2.533868217468261700e+01 4.820736694335937500e+01 1.439588256835937500e+03 2.016125297546386700e+01 1.407745971679687500e+02

方法：

点击(此处)折叠或打开

cut -d' ' -f1 cpu_by_time_prio.txt > cpu_by_time_prio.txt1tmp

awk '{print sprintf("%04d", $0);}' cpu_by_time_prio.txt1tmp > cpu_by_time_prio.txt1

cut -d' ' -f2- cpu_by_time_prio.txt > cpu_by_time_prio.txt2-14

paste -d" " cpu_by_time_prio.txt1 cpu_by_time_prio.txt2-14 | sort > cpu_by_time_prio.txt-ok

++++++++++++++++++++++++++下面杂项，不用看

//取中位数

awk 'BEGIN{

endtime=2070227000000;

temp=2070228000000;

cmd="wc "temp" >tmp";

system(cmd);

getline var < "tmp"; split(var,a," "); midline=int(a[1]/2+1);

cmd="test -f "endtime"";

ret=system(cmd);

if(!ret){ print endtime}

cmd="sort "endtime" >temp";

system(cmd);

"wc -l temp" | getline var; split(var,a," "); midline=int(a[1]/2+1);

print midline;

cmd="awk NR=="midline" temp > temp.txt";

system(cmd);

close("temp");

}'

cat temp.txt

cat task.csv169 | grep 2070249000000

cat task.csv169 | grep 2070249000000|cut -d' ' -f2|sort

cmd="awk NR=="midline" temp > temp.txt";

print midline;

"sed -n 5p temp" | getline var; print var;

FILENAME==ARGV[1] {

array[array_size] = strtonum($0);

array_size++;

}

awk 'BEGIN { "sort 2070235000000 >temp"; "wc -l temp" | getline var; split(var,a," "); midline=int(a[1]/2+1); print midline}'

awk 'BEGIN { "sort 2070235000000 >temp"; "wc -l temp" | getline var; split(var,a," "); midline=int(a[1]/2+1); print midline; NR==midline{print} < temp}'<temp}'

awk '{i=1;while((getline<"testdata")>0)print $0;}'

awk 判断文件是否存在

awk 'BEGIN{a=system("test -f /etc/passwdd");if(a) {print "file is exist."}}'

<temp}'

</temp}'

</temp}'

</endtime)