python学习记——爬糗事百科

# -*- coding:utf-8 -*-
import urllib
import urllib2
import re

page = 1
url = 'http://www.qiushibaike.com/hot/page/' + str(page)
user_agent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.107 Safari/537.36'
headers = { 'User-Agent' : user_agent }
try:
request = urllib2.Request(url,headers = headers)
response = urllib2.urlopen(request)
content = response.read().decode('utf-8')
pattern = re.compile('<div class="author clearfix">.*?<a.*?>.*?<img.*?>.*?</a>.*?<a.*?>.*?<h2>(.*?)</h2>.*?</a>.*?</div>.*?<div class="content">(.*?)<!--.*?-->.*?</div>(.*?)<div class="stats">',re.S)
items = re.findall(pattern,content)
for item in items:
haveImg = re.search("img",item[2])
if not haveImg:
print item[0],item[1]
except urllib2.URLError, e:
if hasattr(e,"code"):
print e.code
if hasattr(e,"reason"):
print e.reason