python多线程爬取-今日头条的街拍数据（附源码加思路注释）

这里用的是json+re+requests+beautifulsoup+多线程

1 import json
2 import re
3 from multiprocessing.pool import Pool
4
5 import requests
6 from bs4 import BeautifulSoup
7 from config import *
8 from requests import RequestException
9
10
11 def get_page_index(offset, keyword):
12     '''得到一个页面的索引'''
13     data = {
14         'offset': offset,
15         'format': 'json',
16         'keyword': keyword,
17         'autoload': 'true',
18         'count': '20',
19         'cur_tab': '1',
20         'from': 'search_tab'
21     }
22     # 请求方式一
23     # url = 'https://www.toutiao.com/search_content/?'+urlencode(data)
24     # response = requests.get(url)
25
26     # 请求方式二
27     url = 'https://www.toutiao.com/search_content/'
28     try:
29         response = requests.get(url, params=data)
30         if response.status_code == 200:
31             return response.text
32         return None
33     except RequestException:
34         return None
35
36
37 def parse_page_index(html):
38     '''解析json数据'''
39     data = json.loads(html)
40     if data and 'data' in data.keys():
41         for item in data.get('data'):
42             yield item.get('article_url')
43
44
45 def get_page_detail(url):
46     '''得到详情页的数据'''
47     # 添加的请求头
48     headers = {
49         'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
50     }
51     try:
52         response = requests.get(url, headers=headers)
53         if response.status_code == 200:
54             return response.text
55         return None
56     except RequestException:
57         return None
58
59
60 def parse_page_detail(html, url):
61     '''解析详情页数据'''
62     soup = BeautifulSoup(html, 'lxml')
63     t = soup.select('title')
64     for i in t:
65         title = i.get_text()
66
67     pattern = re.compile('gallery: JSON.parse\("(.*?)"\),', re.S)
68     result = re.search(pattern, html)
69     if result:
70
71         # print(result.group(1))
72         d = re.sub('\\\\', '', result.group(1))
73         # print(d)
74         data = json.loads(d)
75         if data:
76             images = [item.get('url') for item in data.get('sub_images')]
77             for image in images:
78                 download_image(image, title)
79             return {
80                 'title': title,
81                 'url': url,
82                 'images': images
83             }
84     else:
85         None
86
87
88 def download_image(url, title):
89     '''
90     图片下载
91     :param url: 下载的连接
92     :return:
93     '''
94     print('正在下载', url)
95     try:
96         response = requests.get(url)
97         if response.status_code == 200:
98             content = response.content
99             save_to_image(content, title)
100         return None
101     except RequestException:
102         return None
103
104
105 count = 0
106
107
108 def save_to_image(content, title):
109     global count
110     '''
111     保存图片文件
112     :param content: 图片文件的内容
113     :return:
114     '''
115     name = title + str(count)
116     file_path = './头条/{}.{}'.format(name, 'jpg')
117     with open(file_path, 'wb') as f:
118         count += 1
119         f.write(content)
120
121
122 def main(offset):
123     '''主程序入口'''
124     html = get_page_index(offset, '街拍')
125
126     # print(html)
127     for url in parse_page_index(html):
128
129         if url:
130             # print(url)
131             html = get_page_detail(url)
132             if html:
133                 # print(parse_page_detail(html, url))
134                 result = parse_page_detail(html, url)
135                 if result:
136                     print(result)
137                     # save_to_mongo(result)
138
139
140 GROUP_START = 1
141 GROUP_END = 20
142 if __name__ == '__main__':
143     groups = [i * 20 for i in range(GROUP_START, GROUP_END)]
144     pool = Pool()
145     pool.map(main, groups)