[Python爬虫] 之十四：Selenium +phantomjs抓取媒介360数据

# coding=utf-8
import os
import re
from selenium import webdriver
import selenium.webdriver.support.ui as ui
from selenium.webdriver.common.keys import Keys
import time
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.select import Select
import IniFile
from selenium.webdriver.common.keys import Keys
from threading import Thread
import thread
import LogFile
import urllib
import mongoDB
#抓取数据线程类
class ScrapyData_Thread(Thread):
#抓取数据线程类
def __init__(self,webSearchUrl,inputTXTIDLabel,searchlinkIDLabel,htmlLable,htmltime,keyword,invalid_day,db):
'''
构造函数
:param webSearchUrl: 搜索页url
:param inputTXTIDLabel: 搜索输入框的标签
:param searchlinkIDLabel: 搜索链接的标签
:param htmlLable: 要搜索的标签
:param htmltime: 要搜索的时间
:param invalid_day: 要搜索的关键字，多个关键字中间用分号(;)隔开
:param keywords: #资讯发布的有效时间，默认是3天以内
:param db: 保存数据库引擎
'''
Thread.__init__(self)

self.webSearchUrl = webSearchUrl
self.inputTXTIDLabel = inputTXTIDLabel
self.searchlinkIDLabel = searchlinkIDLabel
self.htmlLable = htmlLable
self.htmltime = htmltime
self.keyword = keyword
self.invalid_day = invalid_day
self.db = db

self.driver = webdriver.PhantomJS()
self.wait = ui.WebDriverWait(self.driver, 20)
self.driver.maximize_window()

def Comapre_to_days(self,leftdate, rightdate):
'''
比较连个字符串日期，左边日期大于右边日期多少天
:param leftdate: 格式：2017-04-15
:param rightdate: 格式：2017-04-15
:return: 天数
'''
l_time = time.mktime(time.strptime(leftdate, '%Y-%m-%d'))
r_time = time.mktime(time.strptime(rightdate, '%Y-%m-%d'))
result = int(l_time - r_time) / 86400
return result

def run(self):
print '关键字：%s' % self.keyword

self.driver.get(self.webSearchUrl)
time.sleep(1)

# js = "var obj = document.getElementById('ctl00_ContentPlaceHolder1_txtSearch');obj.value='" + self.keyword + "';"
# self.driver.execute_script(js)
# 点击搜索链接
ss_elements = self.driver.find_element_by_id(self.inputTXTIDLabel)
ss_elements.send_keys(unicode(self.keyword,'utf8'))

search_elements = self.driver.find_element_by_id(self.searchlinkIDLabel)
search_elements.click()
time.sleep(4)

self.wait.until(lambda driver: self.driver.find_elements_by_xpath(self.htmlLable))
Elements = self.driver.find_elements_by_xpath(self.htmlLable)

timeElements = self.driver.find_elements_by_xpath(self.htmltime)
urlList = []
for hrefe in Elements:
urlList.append(hrefe.get_attribute('href').encode('utf8'))

index = 0
strMessage = ' '
strsplit = '\n------------------------------------------------------------------------------------\n'
index = 0
# 每页中有用记录
recordCount = 0
usefulCount = 0
meetingList = []
kword = self.keyword
currentDate = time.strftime('%Y-%m-%d')

for element in Elements:

strDate = timeElements[index].text.encode('utf8')
# 日期在3天内并且搜索的关键字在标题中才认为是复合要求的数据
#因为搜索的记录是安装时间倒序，所以如果当前记录的时间不在3天以内，那么剩下的记录肯定是小于当前日期的，所以就退出
if self.Comapre_to_days(currentDate,strDate) < self.invalid_day:
usefulCount =  1
txt = element.text.encode('utf8')
if txt.find(kword) > -1:
dictM = {'title': txt, 'date': strDate,
'url': urlList[index], 'keyword': kword, 'info': txt}
meetingList.append(dictM)

# print ' '
# print '标题：%s' % txt
# print '日期：%s' % strDate
# print '资讯链接：' + urlList[index]
# print strsplit

# # log.WriteLog(strMessage)
recordCount = recordCount + 1
else:
usefulCount = 0

if usefulCount:
break
index = index + 1

self.db.SaveMeetings(meetingList)
print "共抓取了: %d 个符合条件的资讯记录" % recordCount

self.driver.close()
self.driver.quit()

if __name__ == '__main__':

configfile = os.path.join(os.getcwd(), 'chinaMedia.conf')
cf = IniFile.ConfigFile(configfile)
webSearchUrl = cf.GetValue("section", "webSearchUrl")
inputTXTIDLabel = cf.GetValue("section", "inputTXTIDLabel")
searchlinkIDLabel = cf.GetValue("section", "searchlinkIDLabel")
htmlLable = cf.GetValue("section", "htmlLable")
htmltime = cf.GetValue("section", "htmltime")

invalid_day = int(cf.GetValue("section", "invalid_day"))

keywords= cf.GetValue("section", "keywords")
keywordlist = keywords.split(';')
start = time.clock()
db = mongoDB.mongoDbBase()
for keyword in keywordlist:
if len(keyword) > 0:
t = ScrapyData_Thread(webSearchUrl,inputTXTIDLabel,searchlinkIDLabel,htmlLable,htmltime,keyword,invalid_day,db)
t.setDaemon(True)
t.start()
t.join()

end = time.clock()
print "整个过程用时间: %f 秒" % (end - start)