Python爬虫 爬取今日头条街拍部分图片

import requests
import json,os,re
from urllib import request

for i in range(0,60,20):
# 循环3页
url = 'https://www.toutiao.com/search_content/?offset={}&format=json&keyword=%E8%A1%97%E6%8B%8D&autoload=true&count=20&cur_tab=1&from=search_tab'.format(i)
# print(url)

response = requests.get(url)
# json
html = response.json()
# print(html)
html_url = html['data']
# print(html_url)

for html_item_url in html_url:
# print(html_item_url)
# 判断是否有article_url
if 'article_url' in html_item_url:
info_url = html_item_url['article_url']
# print(info_url)
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
}
response = requests.get(info_url,headers=headers)
# print(response)
# 转换为str
html_str = response.text
# print(html_str)
# 用正则定位图片位置
patt = r'gallery: JSON\.parse\((.*)\),'
math_res = re.search(patt, html_str)
# print(math_res)

# 打开文件 没有则创建
if not os.path.exists('Down'):
os.mkdir('Down')

if math_res :
patt_origin = math_res.group(1)
# print(patt_origin)
# 转换为str类型
res_str = json.loads(patt_origin)
# 转换为dict类型
res_dict = json.loads(res_str)
images_list = res_dict['sub_images']
# print(images_list)
for img in images_list:
# print(img['url'])
img_url = img['url']
# 给文件取名 (嘿嘿嘿)
# 'split('/')[-1]' 这个为切割 取最后值
filename = 'Down/' + img_url.split('/')[-1] + '.jpg'
# 下载图片
request.urlretrieve(img_url, filename)
print(img_url + '正在下载此图ding...。')
print('--'*50)
else:
#因为有些url里 正则没有匹配到 所有会有部分失败 纯属正常
print('下载失败')
print('--' * 50)