爬取糗事百科

# -*- coding:utf-8 -*-
import urllib2
import re
import sys
reload(sys)
sys.setdefaultencoding( "utf-8" )
 

f = open("c.txt", "ab+")
page = 1
for page in range(1, 5):
url = 'https://www.qiushibaike.com/hot/page/%s/' % page  #糗事百科的url
user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
headers = { 'User-Agent' : user_agent }
request = urllib2.Request(url,headers = headers)
response = urllib2.urlopen(request)
content = response.read().decode('utf-8')
pattern1 = re.compile(r'<h2>(.*?)</h2>.+?<span>(.*?)</span>',re.S) #正则匹配源代码
items = re.findall(pattern1,content)
for item in items:
items=re.sub(r'\s','',item[1])
f.write("发布者:"+item[0]+"发布内容:"+items+'\n')
f.close()