压力测试监控数据通过python matplotlib进行可视化

执行压力测试过程中，监控服务器性能变化，通过常用的命令可以观察当前时刻性能情况，但是无法判断整个测试过程中性能的变化趋势。

随后，将压力测试过程中服务器性能指标按照10s一次进行采集并写入文件中。完成测试后，通过脚本解析采集的性能数据，使用python的三方模块matplotlib对指标进行绘图。

不足之处是，目前这个脚本只能在压力测试结束后进行曲线图绘制，而不能在压力测试过程中实施绘制

指标文件数据格式如下：

Network_Flow
In_link Out_link
Linking_Number Net_Traffic(MB)
1.87925 0.119033
4.5772 0.141904
2.62263 0.126162
2.50861 0.12293
4.92332 0.139697
3.24791 0.126943

解析指标文件的代码如下：

# coding:utf-8
__author__ = 'Libiao'
import os,sys
curr_dir = os.getcwd()

def is_multi_lines(linelabel):
if ' ' in linelabel.strip():
return True
else:
return False

def read_datas(args=()):
dir_name,files_dict = args
result = {}
for groupname,files_list in files_dict.items():
res = []
res_dir = curr_dir + "\\" + dir_name + "\\"
if not os.listdir(res_dir):
continue
for files in files_list:
try:
with file(res_dir + files,'r') as f:
datas = [data for data in f]
title,line_labels,x_y_labels = datas.pop(0),datas.pop(0),datas.pop(0)
x_label,y_label = x_y_labels.strip("\n").split(' ')
if is_multi_lines(line_labels):
line_label_first,line_label_second = line_labels.strip("\n").split(" ")
line_data_first = [d.strip("\n").split(' ')[0] for d in datas]
line_data_second = [d.strip("\n").split(' ')[1] for d in datas if d]
res.append((title,line_label_first,line_label_second,x_label,y_label,line_data_first,line_data_second))
else:
line_data = [d.strip() for d in datas]
res.append((title,line_labels,x_label,y_label,line_data))
except IOError as ioerr:
print str(ioerr)
result[groupname] = res
return (dir_name,result)

解析配置文件的代码如下：

# coding:utf-8
__author__ = 'Libiao'

import os
from xml.etree import ElementTree as ET

def read_xml(xmlpath):
res = []
root = ET.parse(xmlpath)
rootnode_list = root.getiterator("dir")
for node in rootnode_list:
files_map = {}
dir_name = node.attrib["name"]
for second_node in node.getiterator("group"):
group_name = second_node.attrib['name']
tmp = []
for child in second_node.getchildren():
tmp.append(child.text)
files_map[group_name] = tmp
res.append((dir_name,files_map))
return res

进行曲线图绘制的代码：

#coding:utf-8

import matplotlib.pyplot as plt
from pylab import *
import numpy as np
import random
import read_kpi_data,xml_parser
import threading,subprocess,multiprocessing
import os

curr_dir = os.getcwd()

color = ['r','g','b','y','c','k']

#获取测试时间值
def get_test_time(dir_name):
with file(curr_dir + "\\" + dir_name + "\\timeConsum",'r') as f:
times = int(float(f.readline().strip()))
return times

#获取服务连接数
def get_server_num(dir_name):
ser_num = []
try:
with file(curr_dir + "\\" + dir_name + "\\linking_number","r") as f:
for value in f:
if value:
ser_num.append(int(value))
except Exception as e:
print str(e)
return ser_num

#获取图表布局rows和cols
def get_layout(graph_num):
rows,cols = None,None
if graph_num == 0:
return None
elif graph_num == 1:
rows,cols = 1,1
elif graph_num < 4 and graph_num not in (0,1):
rows,cols = graph_num,1
elif graph_num >= 4 and graph_num <= 10:
if graph_num % 2 == 0:
rows = graph_num / 2
else:
rows = graph_num / 2 + 1
cols = 2
else:
if graph_num % 3 == 0:
rows = graph_num / 3
else:
rows = graph_num / 3 + 1
cols = 3
return (rows,cols)

def graph_display(plot_name,all_datas):
plt.figure(plot_name,figsize=(10,8),dpi=80)
#设置子图在画板中的布局，主要是子图高宽（hspace、wspace）、上下边距（bottom、top），左右边距（left、right）
# plt.subplots_adjust(left=0.08,right=0.95,bottom=0.05,top=0.95,wspace=0.25,hspace=0.45)
if len(all_datas) > 10:
plt.subplots_adjust(left=0.08,right=0.95,bottom=0.05,top=0.95,hspace=0.75)
elif len(all_datas)>6 and len(all_datas) <= 10:
plt.subplots_adjust(bottom=0.08,top=0.95,hspace=0.35)
#从time_consum文件读取性能指标采集消耗的时间
#x_axle = get_test_time(plot_name)
#从linking_number文件读取连接数
x_axle = get_server_num(plot_name)
#获取图表布局
rows,cols = get_layout(len(all_datas))
colorLen = len(color)
for index, datas in enumerate(all_datas):
yMax,yMin = 0.0,0.0
subplt = plt.subplot(rows,cols,index+1)
title,first_line_label,second_line_label,x_label,y_label,first_line_datas,second_line_datas = None,None,None,None,None,None,None
try:
title,first_line_label,second_line_label,x_label,y_label,first_line_datas,second_line_datas = datas
except:
title,first_line_label,x_label,y_label,first_line_datas = datas

tmp = None
#设置子图标题
subplt.set_title(title.strip("\n"))
x = np.linspace(0,np.array(get_server_num(plot_name)).max(),len(first_line_datas))
#x = np.array(get_server_num(plot_name))
if second_line_label == None:
tmp = [float(d) for d in first_line_datas]
subplt.plot(x,first_line_datas,color = color[index % colorLen],linewidth=1.5,linestyle='-',label=first_line_label.strip("\n"))
else:
tmp1 = [float(d) for d in first_line_datas if d]
tmp2 = [float(d) for d in second_line_datas if d]
try:
tmp = [max(tmp1),max(tmp2),min(tmp1),min(tmp2)]
except ValueError as e:
print title
num = -1 #num=-1,是为了防止color = color[index % colorLen + num]计算中color下标越界
for line_label,line_data in zip([first_line_label,second_line_label],[first_line_datas,second_line_datas]):
subplt.plot(x,line_data,color = color[index % colorLen + num],linewidth=1.5,linestyle='-',label=line_label.strip("\n"))
subplt.legend()
num += 1
#读取连接数，绘制到每个图表
with open("SYS_KPI/srs_number",'r') as f:
datas = f.readlines()
yMax = max(tmp)
yMin = min(tmp)
dx = (x.max() - x.min()) * 0.1
dy = (yMax - yMin) * 0.1
subplt.set_ylim(yMin - dy,yMax + dy)
subplt.set_xlim(x.min() - dx,x.max() + dx)
#{'position':(1.0,1.0)}，其中1.0相当于x轴最右边，0表示最左边，0.5便是中间
subplt.set_xlabel(x_label,{'color':'b','position':(1.0,1.0)})
subplt.set_ylabel(y_label,{'color':'b'})
#子图显示网格
subplt.grid(True)
subplt.legend()
plt.show()

if __name__ == '__main__':
xml_path = curr_dir + "\config\kpi.xml"
files_lists = xml_parser.read_xml(xml_path)

for files_list in files_lists:
all_datas = []
#print read_kpi_data.read_datas(files_list)
plot_name,plot_datas = read_kpi_data.read_datas(files_list)
datas = plot_datas.values()
all_datas.extend(datas)

for d in all_datas:
multiprocessing.Process(target=graph_display,args=(plot_name,d)).start()

数据可视化效果：