如何利用python爬虫技术将百度贴吧上面的帖子名称爬取下来

下面的程序可以将百度贴吧上的发帖标题爬取下来，源码如下：

#coding=utf-8

import urllib2

import urllib

import json

from lxml import etree

page=open('tieba.json','w')

request=urllib2.Request('http://tieba.baidu.com/f?ie=utf-8&kw=%E7%BD%91%E7%BB%9C%E7%88%AC%E8%99%AB&fr=search')

response=urllib2.urlopen(request)

rehtml=response.read().decode('utf-8')#这一步是模拟浏览器解码

html=etree.HTML(rehtml)

result=html.xpath('//li//a[@class="j_th_tit "]')#注意空格

for site in result:
item={}
name=site.text
item['name']=name
line = json.dumps(item,ensure_ascii=False) + '\n'
page.write(line.encode('utf-8'))#这一步是将ascii码转化成utf-8

page.close()