python 百度贴吧爬虫（下载图片）

业余时用python写的百度贴吧爬虫程序，算是对学习python程序得一个练习。

本程序可以针对给定的贴吧链接，把帖子楼主的发言或者图片爬取出来，目前主要功能为下载所有楼主发的图片。爬取楼主发言的功能仅支持屏幕输出，没有保存到本地文件，有兴趣的朋友可以进行补充。仅供学习，转载请标明出处。

tieba_spider.py

#coding:utf-8
import urllib2,re,time,threading
import DownQueue

user_agent='Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36' #模拟浏览器访问
url='http://tieba.baidu.com/p/3271638607?see_lz=1&pn=' #贴吧地址，只看楼主
header={'User-Agent' : user_agent}

g_worker=DownQueue.down() #下载器

class Tieba_Spider(threading.Thread):
def __init__(self,url,type):
threading.Thread.__init__(self)
self.url=url
self.type=type
self.num=0

def run(self):
self.start_spider()

def get_info(self):
try:
req=urllib2.Request(self.url,headers=header)
response=urllib2.urlopen(req)
htm=response.read().decode('gbk')
self.num=self.get_page_num(htm)
print 'It has %d page' % self.num
self.title=self.get_title(htm)
print 'It\'s title is %s'%self.title

except urllib2.URLError,e:
if hasattr(e,'code'):
print 'Error code :',e.code
if hasattr(e,'reason'):
print 'Reason :',e.reason

def start_spider(self):
global g_worker
self.get_info()

for i in range(1,self.num+1,1):
print 'start : ',i
try:
req=urllib2.Request(self.url+str(i),headers=header)
response=urllib2.urlopen(req)
htm=response.read().decode('gbk')
if self.type==0:
self.page_deal(htm)
elif self.type==1:
self.down_pic(htm)

except urllib2.URLError,e:
if hasattr(e,'code'):
print 'Error code :',e.code
if hasattr(e,'reason'):
print 'Reason :',e.reason

g_worker.set_flag(True)

def get_page_num(self,htm):

match=re.search(r'<span class="red">(\d*)</span>',htm)
if match:
return int(match.group(1))
else:
return 0

def get_title(self,htm):

match=re.search(r'class="core_title_txt(\s+)"(\s+)title="(.*?)"',htm)
if match:
return match.group(3)

else:
print 'no match title'
return ''

def page_deal(self,htm):
match=re.findall(r'id="post_content_(.*?)">(.*?)</div>',htm)
if match:
for it in match:
print it[1],'\n'
else:
print 'no deal'

def down_pic(self,htm):
global g_worker
match=re.findall(r'<img class="BDE_Image" pic_type=(.*?)src="(.*?)"',htm)
if match:
for it in match:
print 'picture url :',it[1],'\n'
g_worker.push(it[1])
else:
print 'no deal'

if __name__=='__main__':

spider=Tieba_Spider(url,1)#参数1为下载图片。默认为0，功能为抓取楼主的发言在屏幕上显示
spider.start()

g_worker.start()

DownQueue.py

#coding:utf-8
import threading,Queue,re,time
import urllib2

class down(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.queue=Queue.Queue(1000)
self.semaphore=threading.Semaphore(0)
self.flag=False #是否停止

def push(self,obj):
self.queue.put(obj)
self.semaphore.release()

def set_flag(self,f):
self.flag=f

def run(self):
while True:
if self.semaphore.acquire():
obj=self.queue.get()
data=urllib2.urlopen(obj).read()
pic=re.search(r'.*/(.*)',obj)

print 'dowing ',pic.group(1)
fd=open('./spider_pic/%s'%pic.group(1),'wb')
fd.write(data)
fd.close()

if self.queue.empty() and self.flag:    #线程结束条件，队列为空并且退出标志为真
break

Tieba_Spider 类为爬虫类，负责爬出楼主发言中的图片链接，并将其推入down类的队列中。down类的工作为下载图片。两个类均继承自threading.Thread。仅供学习，转载请标明出处。