python beautifulsoup 爬虫学习

爬取IMDB上的电影关键词keyword

源HTML文档，参看文档源码

# -*- coding: utf-8 -*-
import  urllib2
from bs4 import BeautifulSoup
import unicodedata
page=urllib2.urlopen("http://www.imdb.com/title/tt1619029/keywords?ref_=tt_stry_kw")
soup=BeautifulSoup(page,"lxml")
print soup.find_all(attrs={"class":"sodatext"})#正则获取标签值
print soup.select(' div[class="sodatext"]')#获取标签值
f=open('F:\\keyw.txt','w')#打开文档准备写入
kwinfo=[]#字典，准备获取字段

for keyw in soup.select(' div[class="sodatext"]'):
kw=keyw.get_text()
#print k
kw.strip()
line = unicodedata.normalize('NFKD', kw).encode('ascii', 'ignore')#将Unicode类型转换为str类型
if(line.startswith("\n")):#如果以换行符开头，
line=line.replace("\n","")#去除所有换行符
print type(line)
kwinfo.append(line)#加入到字典中
print line
print kwinfo#打印字典数据
for item in range(len(kwinfo)):
f.write(kwinfo[item]+"|")#写入字典数据到文件，作为一行，以|标记各关键词

f.close()#关闭文档

loosely based on real events

widow

period film

1960s

american politics

jackie kennedy

title spoken by character

character name in title

[‘first lady’, ‘american history’, ‘kubrickian’, ‘34 year old’, ‘forename as title’, ‘one word title’, ‘female lead’, ‘kennedy assassination’, ‘death of husband’, ‘year 1963’, ‘loosely based on real events’, ‘widow’, ‘period film’, ‘1960s’, ‘american politics’, ‘jackie kennedy’, ‘title spoken by character’, ‘character name in title’]

写入文档：first lady|american history|kubrickian|34 year old|forename as title|one word title|female lead|kennedy assassination|death of husband|year 1963|loosely based on real events|widow|period film|1960s|american politics|jackie kennedy|title spoken by character|character name in title|