Python爬上不得姐 并将段子写入数据库

#Python2.7 可以优化一下 前10页 每页点赞最多的段子  百思不得姐

# -*- coding: utf-8 -*-
import MySQLdb
import urllib,urllib2
import sys,re

reload(sys)
sys.setdefaultencoding('utf-8')

class TestBDJ():
def __init__(self):
pass
def getbdj(self):
lis = []
for h in range(1,10):
cc = []
tt = []
try:
url = 'http://www.budejie.com/text/'+str(h)
res = urllib.urlopen(url).read()
#print res
m = re.compile('<div class="j-r-list-c-desc">(.*?)</div>.*?<i class="icon-up ui-icon-up"></i>  <span>(.*?)</span>',re.S)    #当前版本可以这样匹配  没有匹配作者
tt = re.findall(m,res)
for i in tt:
cc.append(i[1])    #把每个段子点赞的数量放入list
aa = cc
aa = map(eval,aa)             #需要将点赞的值由string转换成int
#print "befor %s",aa
for b in range(len(aa) - 1):           #排序
for i in range(len(aa) - 1):
if aa[i] < aa[i + 1]:
aa[i], aa[i + 1] = aa[i + 1], aa[i]
#print "after %s",aa

for t in range(5):             #取每页前5条段子
duanzi = []
index = cc.index(str(aa[t]))           #将点赞的数量转回string 根据下标 获取对应的段子
#print tt[index][0].replace("<br />","")
#print tt[index][1]
star = tt[index][1]              #点赞人数
word = tt[index][0].replace("<br />","").encode('utf-8')       #文字内容
duanzi.append(star)
duanzi.append(word)
lis.append(duanzi)
except Exception,e:
print e
return lis

def load(self):
key = self.getbdj()
if key:
try:
conn = MySQLdb.connect(host='localhost', user='hehehe', passwd='****', db='tester', port=330,charset="utf8")    # 加上charset="utf8" 指定编码格式  解决写入mysql中文乱码的问题
cur = conn.cursor()
conn.select_db('tester')
cur.execute('create table if not exists budejie(st int,info TEXT)')
for k in key:
cur.execute('insert into budejie values(%s,%s)',k)
cur.close()
conn.commit()
conn.close()
except MySQLdb.Error, e:
print "Mysql Error %d: %s" % (e.args[0], e.args[1])

return 0

if __name__=='__main__':
h = TestBDJ()
h.load()

#coding:utf-8
import urllib.request
import re
import importlib,sys

for h in range(1,10):
cc = []
tt = []
try:
url = 'http://www.budejie.com/text/'+str(h)
res = urllib.request.urlopen(url).read()
res = res.decode("utf-8")
#   print (res)
m = re.compile('<div class="j-r-list-c-desc">(.*?)</div>.*?<i class="icon-up ui-icon-up"></i>  <span>(.*?)</span>',re.S)    #当前版本可以这样匹配  没有匹配作者
tt = re.findall(m,res)
for i in tt:
cc.append(i[1])    #把每个段子点赞的数量放入list
aa = cc
aa = list(map(eval,aa))             #需要将点赞的值由string转换成int
#print "befor %s",aa
for b in range(len(aa)-1):           #排序
for i in range(len(aa) - 1):
if aa[i] < aa[i + 1]:
aa[i], aa[i + 1] = aa[i + 1], aa[i]
#print "after %s",aa
for t in range(5):             #取每页前5条段子
index = cc.index(str(aa[t]))           #将点赞的数量转回string 根据下标 获取对应的段子
print (tt[index][0].replace("<br />",""))
print (tt[index][1])
except Exception as e:
print (e)

# -*- coding: utf-8 -*-
import urllib,urllib2
import sys,re

reload(sys)
sys.setdefaultencoding('utf-8')

for h in range(1,8):
cc = []
tt = []
try:
url = 'http://www.qiushibaike.com/text/page/'+str(h)+'/?s=4891212'
req = urllib2.Request(url)
req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36')
res = urllib2.urlopen(req).read()
#print res
m = re.compile('<div class="content">(.*?)</div>.*?<span class="stats-vote"><i class="number">(.*?)</i> 好笑</span>',re.S)    #当前版本可以这样匹配  没有匹配作者

tt = re.findall(m,res)
for i in tt:
cc.append(i[1])    #把每个段子点赞的数量放入list

　　　　 aa = cc
aa = map(eval,aa)             #需要将点赞的值由string转换成int
#print "befor %s",aa
for b in range(len(aa) - 1):           #排序
for i in range(len(aa) - 1):
if aa[i] < aa[i + 1]:
aa[i], aa[i + 1] = aa[i + 1], aa[i]
#print "after %s",aa
for t in range(5):             #取每页前5条段子
index = cc.index(str(aa[t]))           #将点赞的数量转回string 根据下标 获取对应的段子
　　　　　　　print tt[index][0].replace("<br/>","")
　　　　　　　print tt[index][1]
　　except Exception,e: print e

# -*- coding: utf-8 -*-
import urllib,urllib2
import sys,re
reload(sys)
sys.setdefaultencoding('utf-8')

for h in range(1,3):
cc = []
tt = []
try:
url = 'http://www.qiushibaike.com/imgrank/page/'+str(h)+'/?s=4891221'
req = urllib2.Request(url)
req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36')
res = urllib2.urlopen(req).read()
#print res
m = re.compile('<div class="content">(.*?)</div>.*?target="_blank">.*?<img src=\"(.*?)\" alt=.*?<span class="stats-vote"><i class="number">(.*?)</i> 好笑</span>',re.S)    #当前版本可以这样匹配  没有匹配作者
tt = re.findall(m,res)
for i in tt:
cc.append(i[2])    #把每个段子点赞的数量放入list
#    print i[0]
#    print i[1]
#    print i[2]
aa = cc
aa = map(eval,aa)             #需要将点赞的值由string转换成int
#print "befor %s",aa
for b in range(len(aa) - 1):           #排序
for i in range(len(aa) - 1):
if aa[i] < aa[i + 1]:
aa[i], aa[i + 1] = aa[i + 1], aa[i]
#print "after %s",aa
for t in range(5):             #取每页前5条段子
index = cc.index(str(aa[t]))           #将点赞的数量转回string 根据下标 获取对应的段子
print index
print tt[index][0]
print tt[index][1]
print tt[index][2]

except Exception,e:
print e