【Python】如何获取知乎最有价值的内容

一 前言 
   相信大部分能看到这篇blog的人都听说过知乎吧？如果你没有听说过，那么链接在这里   知乎  作为一个知乎er，为了更加深入的理解“xxx 是一种什么体验”（的图片），为了践行 “技术改变生活”（实则有些wuliao) ，使用requsets 爬取知乎中最优价值的内容，本文本着探索的精神，写一段获取内容的python程序。

二 践行

1. #!/usr/bin/env python
   
2. #-*- coding:utf-8 -*-
   
3. import re
   
4. import requests
   
5. import os
   
6. from urlparse import urlsplit
   
7. from os.path import basename
8. def getHtml(url):
   
9.     session = requests.Session()
   
10.     # 模拟浏览器访问
    
11.     header = {
    
12.         'User-Agent': "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36",
    
13.         'Accept-Encoding': 'gzip, deflate'}
    
14.     res = session.get(url, headers=header)
    
15.     if res.status_code == 200:
    
16.         content = res.content
    
17.     else:
    
18.         content = ''
    
19.     return content
    
20. 
    
21. def mkdir(path):
    
22.     if not os.path.exists(path):
    
23.         print '新建文件夹:', path
    
24.         os.makedirs(path)
    
25.         return True
    
26.     else:
    
27.         print u"图片存放于:", os.getcwd() + os.sep + path
    
28.         return False
    
29. 
    
30. def download_pic(img_lists, dir_name):
    
31.     print "一共有 {num} 张照片".format(num=len(img_lists))
    
32.     for image_url in img_lists:
    
33.         response = requests.get(image_url, stream=True)
    
34.         if response.status_code == 200:
    
35.             image = response.content
    
36.         else:
    
37.             continue
    
38.         file_name = dir_name + os.sep + basename(urlsplit(image_url)[2])
    
39.         try:
    
40.             with open(file_name, "wb") as picture:
    
41.                 picture.write(image)
    
42.         except IOError:
    
43.             print("IO Error\n")
    
44.             return
    
45.         finally:
    
46.             picture.close
    
47.             print "下载 {pic_name} 完成!".format(pic_name=file_name)
    
48. 
    
49. def getAllImg(html):
    
50.     # 利用正则表达式把源代码中的图片地址过滤出来
    
51.     #reg = r'data-actualsrc="(.*?)">'
    
52.     reg = r'https://pic\d.zhimg.com/[a-fA-F0-9]{5,32}_\w+.jpg'
    
53.     imgre = re.compile(reg, re.S)
    
54.     tmp_list = imgre.findall(html) # 表示在整个网页中过滤出所有图片的地址，放在imglist中
    
55.     # 清理掉头像和去重 获取data-original的内容
    
56.     tmp_list = list(set(tmp_list)) # 去重
    
57.     imglist = []
    
58.     for item in tmp_list:
    
59.         if item.endswith('r.jpg'):
    
60.             img_list.append(item)
    
61.     print 'num : %d' % (len(imglist))
    
62.     return imglist
    
63. 
    
64. 
    
65. if __name__ == '__main__':
    
66.     question_id = 35990613
    
67.     zhihu_url = "https://www.zhihu.com/question/{qid}".format(qid=question_id)
    
68.     html_content = getHtml(zhihu_url)
    
69.     path = 'zhihu_pic'
    
70.     mkdir(path) # 创建本地文件夹
    
71.     img_list = getAllImg(html_content) # 获取图片的地址列表
    
72.     download_pic(img_list, path)       # 保存图片

本代码还存在一些不足的地方，无法完全获取全部的图片，需要在兼容 自动点击 ”更多“ 加载更多答案。
代码第二版解决了第一版代码中不能自动加载的问题。

1. #!/usr/bin/env python
   
2. #-*- coding:utf-8 -*-
   
3. import re
   
4. import requests
   
5. import os
   
6. from urlparse import urlsplit
   
7. from os.path import basename
   
8. 
   
9. headers = {
   
10.     'User-Agent': "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36",
    
11.     'Accept-Encoding': 'gzip, deflate'}
    
12. 
    
13. 
    
14. def mkdir(path):
    
15.     if not os.path.exists(path):
    
16.         print '新建文件夹:', path
    
17.         os.makedirs(path)
    
18.         return True
    
19.     else:
    
20.         print u"图片存放于:", os.getcwd() + os.sep + path
    
21.         return False
    
22. 
    
23. 
    
24. def download_pic(img_lists, dir_name):
    
25.     print "一共有 {num} 张照片".format(num=len(img_lists))
    
26.     for image_url in img_lists:
    
27.         response = requests.get(image_url, stream=True)
    
28.         if response.status_code == 200:
    
29.             image = response.content
    
30.         else:
    
31.             continue
    
32.         file_name = dir_name + os.sep + basename(urlsplit(image_url)[2])
    
33.         try:
    
34.             with open(file_name, "wb") as picture:
    
35.                 picture.write(image)
    
36.         except IOError:
    
37.             print("IO Error\n")
    
38.             continue
    
39.         finally:
    
40.             picture.close
    
41.             print "下载 {pic_name} 完成!".format(pic_name=file_name)
    
42. 
    
43. 
    
44. def get_image_url(qid, headers):
    
45.     # 利用正则表达式把源代码中的图片地址过滤出来
    
46.     #reg = r'data-actualsrc="(.*?)">'
    
47.     tmp_url = "https://www.zhihu.com/node/QuestionAnswerListV2"
    
48.     size = 10
    
49.     image_urls = []
    
50.     session = requests.Session()
    
51.     # 利用循环自动完成需要点击 “更多” 获取所有答案，每个分页作为一个answer集合。
52.     while True:
    
53.         postdata = {'method': 'next', 'params': '{"url_token":' +
    
54.                     str(qid) + ',"pagesize": "10",' + '"offset":' + str(size) + "}"}
    
55.         page = session.post(tmp_url, headers=headers, data=postdata)
    
56.         ret = eval(page.text)
    
57.         answers = ret['msg']
58.         size += 10
    
59.         if not answers:
    
60.             print "图片URL获取完毕, 页数: ", (size - 10) / 10
    
61.             return image_urls
    
62.         #reg = r'https://pic\d.zhimg.com/[a-fA-F0-9]{5,32}_\w+.jpg'
    
63.         imgreg = re.compile('data-original="(.*?)"', re.S)
    
64.         for answer in answers:
    
65.             tmp_list = []
    
66.             url_items = re.findall(imgreg, answer)
    
67.             for item in url_items: # 这里去掉得到的图片URL中的转义字符'\\'
    
68.                 image_url = item.replace("\\", "")
    
69.                 tmp_list.append(image_url)
    
70.             # 清理掉头像和去重 获取data-original的内容
    
71.             tmp_list = list(set(tmp_list)) # 去重
    
72.             for item in tmp_list:
    
73.                 if item.endswith('r.jpg'):
    
74.                     print item
    
75.                     image_urls.append(item)
    
76.         print 'size: %d, num : %d' % (size, len(image_urls))
    
77. 
    
78. 
    
79. if __name__ == '__main__':
    
80.     question_id = 26037846
    
81.     zhihu_url = "https://www.zhihu.com/question/{qid}".format(qid=question_id)
    
82.     path = 'zhihu_pic'
    
83.     mkdir(path) # 创建本地文件夹
    
84.     img_list = get_image_url(question_id, headers) # 获取图片的地址列表
    
85.     download_pic(img_list, path) # 保存图片

来自 “ ITPUB博客 ” ，链接：http://blog.itpub.net/22664653/viewspace-2138599/，如需转载，请注明出处，否则将追究法律责任。

转载于:http://blog.itpub.net/22664653/viewspace-2138599/