python爬取猫眼top100

用Xpath和requests爬取猫眼上的top100电影，爬取豆瓣top250也可以用相同的方法

import requests
from lxml import etree

def url_open(url):
header={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0'}
response = requests.get(url,headers=header)
return response

def get_data(response):
dom = etree.HTML(response.text)
for i in dom.xpath('//div[@id="app"]//dl//dd'):
#用.来进行二次提取//表示任意，用@来查找需要搜索的属性
print(i.xpath('.//p[@class="name"]//a/@title')[0])
print(i.xpath('.//p[@class="star"]/text()')[0].strip())
print(i.xpath('.//p[@class="releasetime"]/text()')[0])
print("".join(i.xpath('.//p[@class="score"]//i/text()')))

def next_url(url):
x = 0
while x != 100:
next_url = url + '?offset=' +str(x)
get_data(url_open(next_url))
x += 10

if __name__ == '__main__':
url = 'https://maoyan.com/board/4'
next_url(url)