Xpath爬取豆瓣top250电影

python爬取信息的方式有很多，今天就教大家如何使用xpath爬取豆瓣top250的电影名称、评分、链接、引言，并存储到csv中

python版本：3.7.4

相关模块：lxml模块；requests模块；以及python自带模块csv

环境搭建：安装python并添加到环境变量，pip安装需要的相关模块

代码如下：

[code]import requests
from lxml import etree
import csv

#豆瓣top250网址
doubanUrl='https://movie.douban.com/top250?start={}&filter='

#获取网页源码
def getSource(url):
headers={
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'
}
res=requests.get(url,headers=headers)
res.encoding='utf-8'
return res.text

# 获取电影信息
def getEveryItem(source):
html_element=etree.HTML(source)
movieItemList=html_element.xpath('//div[@class="info"]')

#定义一个空列表
movieList=[]

for eachMovie in movieItemList:

#创建一个字典，向列表中存储数据[{电影一},{电影二}...]
movieDict={}

title=eachMovie.xpath('div[@class="hd"]/a/span[@class="title"]/text()') #电影名
otherTitle=eachMovie.xpath('div[@class="hd"]/a/span[@class="other"]/text()') #其他名称
link=eachMovie.xpath('div[@class="hd"]/a/@href')[0] #链接
star=eachMovie.xpath('div[@class="bd"]/div[@class="star"]/span[@class="rating_num"]/text()')[0] #评分
quote=eachMovie.xpath('div[@class="bd"]/p[@class="quote"]/span/text()') #引言

#条件语句：如果有引言则写引言，若没有则不写
if quote:
quote=quote[0]
else:
quote=''

#保存数据
movieDict['title']=''.join(title+otherTitle)
movieDict['url']=link
movieDict['star']=star
movieDict['quote']=quote

movieList.append(movieDict)
print(movieList)
return  movieList

#保存数据
def writeData(movieList):
with open('douban_top250.csv','w',encoding='utf-8') as f:
writer=csv.DictWriter(f,fieldnames=['title','star','quote','url'])
writer.writeheader() #写入表头
for each in movieList:
writer.writerow(each)

if __name__ == "__main__":
movieList=[]

#一共10页，循环10次
for i in range(10):

#调用函数
pageLink=doubanUrl.format(i*25)
source=getSource(pageLink)
movieList+=getEveryItem(source)

writeData(movieList)