用python写网络爬虫-使用xpath代替正则表达式

学习使用xpath代替正则表达式，首先得了解xpath语法

xpath语法详解

1.使用xpath遇到的几个问题，一个是不会提取a标签的内容，后来查资料发现需要在后面加/text()

2.提取倒数标签数据使用[last()-xx]的方式

今天还顺便了解了requests模块，发现这个模块蛮好用的

注释写中文出错后再开头加上# -- coding:gbk --即可

附上今天使用xpath改造的项目源码：

test2.py

import sys
import requests
import re

def download(url):
print("downloading:"+url)
headers={'User-agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}
try:
content=requests.get(url,headers=headers)
except urllib.error.URLError as e:
print("download error:"+e.reason)
content=None
return content

def save(file_name, file_content):
print("saving.......")
with open(file_name + ".html", "wb") as f:
f.write(file_content.content)
f.close()

murl="http://blog.csdn.net/Joliph/article/list/1"
html = download(murl)
save(re.split('/',murl)[-1], html)

budejie2.py

# -*- coding:gbk -*-
import sys
import requests
import re
from lxml import etree
from multiprocessing.dummy import Pool as threadpool
import urllib.request
import time

def wp():
print("plz input 1~50")
page=input("which budejie page you want to download?(1~50):")
page=int(page)
while page>50:
print("plz input 1~50")
page=input("which budejie page you want to download?(1~50):")
page=int(page)
return page

pg=wp()
headers={'User-agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}
url="http://www.budejie.com/"+str(pg)

html=requests.get(url,headers=headers).text
selector=etree.HTML(html)

mp4url=[]
start_time=time.time()
mp4list=selector.xpath('//*[@class="j-r-list-tool-l-down f-tar j-down-video j-down-hide ipad-hide"]/a')
for i in mp4list:
mp4url.append(str(i.xpath('@href')[0]))
end_time=time.time()
print("匹配耗时："+str(end_time-start_time)+"Seconds")

for i in mp4url:
print("downloading:"+i)
filename=re.split('/',i)[-1]
urllib.request.urlretrieve(i,filename)

csdn2.py:

# -*- coding:gbk -*-
import sys
import requests
import re
from lxml import etree

def download(url):
headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36"}
html=requests.get(url,headers=headers).text
return html

html2=download("http://blog.csdn.net/Joliph")
selector2=etree.HTML(html2)
pagelist=selector2.xpath('//*[@id="papelist"]/a[last()-2]/text()')[0]
#这里有有个潜在的问题，在我博客写到5页以上时出现...后无法判断页数
pagelist=int(pagelist)
for page in range(1,pagelist+1):
url="http://blog.csdn.net/Joliph/article/list/"+str(page)
html=download(url)
selector=etree.HTML(html)
titlelist=selector.xpath('//*[@class="list_c_t"]/a/text()')
yearlist=selector.xpath('//*[@class="date_t"]/span/text()')
monthlist=selector.xpath('//*[@class="date_t"]/em/text()')
daylist=selector.xpath('//*[@class="date_b"]/text()')
#/text()!!!!!!!!!!!!!!!!!!!!!!!
number=len(titlelist)
for i in range(1,number+1):
print(yearlist[i-1]+"."+monthlist[i-1]+"."+daylist[i-1]+"----"+titlelist[i-1])