python学习之简单正则与爬虫

#-*- coding:utf8 -*-
import requests
import re# 导入正则表达式的模块

#导入下面这几行用中文就不会出错
import sys
reload(sys)
sys.setdefaultencoding("utf-8")

class Spider():
def getGtmlSource(self,url):
"""获得html"""
html = requests.get(url).text
return html

def changepage(self,url,total_page):
"""获得所有url的列表"""
nowPage = int(re.search('pageNum=(\d+)',url).group(1))#re.search只找到第一个符合条件的 .group(1)是只要一个括号里面的
all_link=[]
for i in range(nowPage,total_page+1):
# 替换
link = re.sub('pageNum=\d+','pageNum=%s'%i,url,re.S)
all_link.append(link)
return all_link

def geteveryclass(self,source):
everyclass = re.findall('(deg="0" >.*?</li>)',source,re.S)
return everyclass

def getinfo(self,eachclass):
"""getinfo用来从每个课程块中提取出我们需要的信息"""

info = {}
info['title'] = re.search('class="lessonimg" title="(.*?)>',eachclass,re.S).group(1)
info['content'] = re.search('display: none;">(.*?)</p>',eachclass,re.S).group(1)

return info

if __name__ == "__main__":
s = Spider()
classinfo = []
url = 'http://www.jikexueyuan.com/course/?pageNum=1'
url_list = s.changepage(url,2)
for link in url_list:
print u'正在处理页面：' + link
html = s.getGtmlSource(link)
# print html
everyclass = s.geteveryclass(html)
for each in everyclass:
info = s.getinfo(each)
classinfo.append(info)
print classinfo