爬虫学习实例 - selenium Webdirver爬取集思录债券数据

#!python3
#coding:utf-8

'''
################################
@ MyBlog: blog.csdn.net/hjxzt1
www.mykurol.com
github: https://github.com/kurolz ################################

爬取集思录网站债券数据
目前可存为txt或xlsx两种格式
超过定义的涨幅或跌幅，可邮件通知
#### 请填写发送的邮箱密码 ####
'''

'''邮件通知'''
def sendMail(id,uplift):
import smtplib
from email.mime.text import MIMEText
from email.header import Header

# 第三方 SMTP 服务
mail_host = "smtp.163.com"  # 设置smtp服务器，例如：smtp.163.com
mail_user = "kurolz@163.com"  # 发送的邮箱用户名
mail_pass = "******"  # 发送的邮箱密码

sender = 'kurolz@163.com'  # 发送的邮箱
receivers = 'kurolz@163.com'  # 接收的邮箱
if uplift < format(0, '.0%'):  # uplift为涨幅或跌幅
text = '债券代码：' + id + ', 跌幅为：' + uplift
else:
text = '债券代码：' + id + ', 涨幅为：' + uplift  # 发送的文本
message = MIMEText(text)
message['From'] = sender
message['To'] = receivers

subject = text
message['Subject'] = Header(subject)

try:
smtpObj = smtplib.SMTP()
smtpObj.connect(mail_host, 25)  # 25 为 SMTP 端口号
smtpObj.login(mail_user, mail_pass)    # login
smtpObj.sendmail(sender, receivers, message.as_string())    # 发送邮件
print("邮件发送成功")
except smtplib.SMTPException as e:
print("Error: 无法发送邮件" + str(e))

'''当债券涨幅超过定义的limit值时，发送邮件通知'''
def limitsendMail(limit,i,data):
if limit < 0:
if data < format(0, '.0%'):
limit = format(limit, '.3%')
if data.split('-')[1] > limit.split('-')[1]:
sendMail(i, data)
else:
limit = format(limit, '.0%')
if data > limit:
sendMail(i, data)

'''定义抓取数据的行'''
def data_row():
jisilu_id = []  # 要爬取的债券代码
p = 0   # 可选，0为取双行，-1为取单行

while True:
try:
p += 2  # 可选，2为取单双行，1位取所有行
p = str(p)  # 转换为str，用于抓取数据
element = driver.find_element_by_xpath('//*[@id="flex3"]/tbody/tr['+ p +']')
data_id = element.get_attribute("id")   # 抓取所有需要爬取的债券代码
p = int(p)  # 转换为int，维持增量循环
except:
break
jisilu_id.append(data_id)
return jisilu_id

'''定义抓取数据的列'''
def data_colum():
c = []  # 存储要抓取的列
for num in range(1,24):
c.append(str(num))
# 删除三个无关数据的列
del c[20]
del c[12]
del c[11]
return c

'''定义爬取的操作'''
try:
from selenium import webdriver
import ssl

html = 'https://www.jisilu.cn/data/cbnew/#tlink_3'  # 定义爬取的网站
ssl._create_default_https_context = ssl._create_unverified_context  # 取消证书认证
try:
driver = webdriver.PhantomJS()
driver.get(html)
driver.implicitly_wait(3)  # 等待3秒
except:
print ('请安装phantomjs')

except ImportError:
print ('No module named selenium. 请安装selenium模块')

'''抓取数据'''
a = {}  # 存储数据，存储格式：a = {债券代码:{title:data,title:data, ...}, ...}
for i in data_row():
b = {}  # 存储格式：b = {title：data, ...}
for lie in data_colum():
title = driver.find_element_by_xpath('//*[@id="flex3"]/thead/tr[2]/th['+lie+']').text   # 抓取title
data = driver.find_element_by_xpath('//*[@id='+i+']/td['+lie+']').text  # 抓取数值
title = title.replace("\n", "") # 去掉title中的换行符
b[title] = data
if lie == "4":
limitsendMail(0.05, i, data)
a[i] = b

'''数据输出保存'''
class print_data(object):
def __init__(self, filename):
self.filename = filename

'''输出到TXT'''
def printTxt(self):
import time
with open(self.filename, 'ab+') as w:
nowtime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
w.write(("\n@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ " + nowtime + " @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@\n").encode())
w.close()
for key in a:

cb20
for key2 in a[key]:
with open(self.filename, 'ab+') as w:
w.write((key2 + ":" + a[key][key2] + "，").encode('utf-8'))
w.close()
with open(self.filename, 'ab+') as w:
w.write(("\n").encode())
w.close()

'''输出到Xlsx'''
def printXlsx(self):
try:
import xlsxwriter
workbook = xlsxwriter.Workbook(self.filename)  # 创建一个Excel文件
worksheet = workbook.add_worksheet()  # 创建一个工作表对象

colum_len = ''
for key in a:
colum_len = key
if colum_len != '':
break

'''写入title'''
colum_num = 0
while (colum_num < len(a[colum_len].keys())):
for i in list(a[colum_len].keys()):
worksheet.write(0, colum_num, i)  # 写入行列表示法的单元格
colum_num += 1

'''写入数值'''
row_num = 1
colum_num_2 = 0
while (colum_num_2 < len(a[colum_len].keys())):
for key in a:
colum_num_2 = 0
for key2 in a[key]:
worksheet.write(row_num, colum_num_2, a[key][key2])  # 写入行列表示法的单元格
colum_num_2 += 1
row_num += 1

except ImportError:
print ('No module named xlsxwriter，输出为xlsx文件需要安装xlsxwriter模块，或重新定义输出为txt文件')

if __name__ == "__main__":
printfilename = '07150240-2.xlsx'
file = print_data(printfilename)
if printfilename.split('.')[1] == 'xlsx':
file.printXlsx()
elif printfilename.split('.')[1] == 'txt':
file.printTxt()
else:
print ('输出文件名定义错误，无法输出，只能为xlsx或txt格式')