python爬取微信公众号文章
2018-08-31 14:54
866 查看
本文实例为大家分享了python爬取微信公众号文章的具体代码，供大家参考，具体内容如下
# -*- coding: utf-8 -*-
import requests
from bs4 import BeautifulSoup
from requests.exceptions import RequestException
import time
import random
import MySQLdb
import threading
import socket
import math
socket.setdefaulttimeout(60)#这里对整个socket层设置超时时间。后续文件中如果再使用到socket，不必再设置
glock = threading.Lock() #定义全局锁
CATEGORY_URL= ['http://www.we123.com/gzh/onclick/'] #获取地区分类链接
all_url = [] #
ALL_URLS = [] #所有详细页面链接
proxy_list = [] #IP池
URL = 'http://www.we123.com'
PAGE_URL = [] #所有分页链接
#获取Ip池
def get_ip():
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36'}
url = 'http://http-webapi.zhimaruanjian.com'#可以使用芝麻代理，好用稳定还不贵
resp = requests.get(url,headers=headers)
obj = resp.json() #获取json ip池对象
for ip in obj:
arr = 'http://' + str(ip['ip']) + ':' + str(ip['port'])
proxy_list.append(arr)
#获取页面源码函数
def get_html(url):
# headers = {}
user_agent_list = [
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.3538.400 QQBrowser/9.6.12501.400',
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0'
]
# user_agent = random.choice(user_agent_list)
headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.3538.400 QQBrowser/9.6.12501.400'
}
# 代理，免费的代理只能维持一会可能就没用了，自行更换
# proxy_list = [
# "http://27.192.185.62:3252",
# ]
# proxy_ip = random.choice(proxy_list)
# proxies = {'http': proxy_ip}
# print(str(url))
try:
resp = requests.get(url,headers=headers)
# print("72行：status_code = " + str(resp.status_code))
# print(type(resp.text))
# print(resp.url) # 请求的url
if resp.status_code == 200:
return resp
elif resp.status_code == 404:
return resp
elif resp.status_code == 500:
return resp
return resp
except RuntimeError:
print("超时")
return "error"
except ConnectionError:
print("连接超时")
return "error"
except RequestException:
print("http请求父类错误")
with open('url_exception.txt','a+', encoding='utf-8') as f:
f.write(str(url))
f.write('\n')
return "error"
#获取区域分类链接
def get_categoty_url():
url = 'http://www.we123.com/gzh/onclick/'
resp = get_html(url)
soup = BeautifulSoup(resp.text,'lxml')
html = soup.select('div.div-subs2 > div.divst-content > div.divst-subs > li > a')
# 获取区域分类链接
for i in html:
city = i['href'].split("/")[-1]
if (city == '海外' or city == '台湾' or city == '澳门'):
continue
url = URL + i['href']
CATEGORY_URL.append(url)
print(CATEGORY_URL)
#获取每个区域下所有分页链接
def get_page_url(url):
city = url.split('/')[-1]
html = get_html(url)
if html == "error":
print("98行：connect url error")
time.sleep(random.randint(10,20))
return "error"
soup = BeautifulSoup(html.text,'lxml')
#获取总条数
all_nums = soup.select("div.page > a > b")
if len(all_nums) == 0:
return "error"
else:
all_nums = soup.select("div.page > a > b")[0].get_text()
#获取总分页数
all_pages = math.ceil((int(all_nums) / 30))
#获取所有分页链接
all_page_url = []
for i in range(0,int(all_pages)):
page_url = 'http://www.we123.com/e/action/ListInfo.php?page=' + str(i) + '&classid=45&line=30&tempid=10&orderby=onclick&myorder=0&totalnum=' + str(all_nums)
all_page_url.append(page_url)
return all_page_url
# 获取所有详细页面链接
def get_page_urls():
global PAGE_URL
c_url = CATEGORY_URL.pop()
print('121 行：请求链接' + c_url)
PAGE_URL = get_page_url(c_url) #获取每个区域下面的所有分页链接
# 获取所有详细页面链接
def get_info_urls():
while True:
global PAGE_URL #设置全局变量
glock.acquire() #加锁
if len(PAGE_URL) == 0:
glock.release() #解锁
print('131 行：CATEGORY_URL 为空')
break
else:
p_url = PAGE_URL.pop()
print('135 行：请求链接' + p_url)
glock.release() #解锁
glock.acquire() #加锁
html = get_html(p_url)
if html == "error":
print("141行：connect url error")
time.sleep(2)
return
soup = BeautifulSoup(html.text,'lxml')
info_urls = soup.select('div.gzhRight > div.gzh_list > ul > li > a')
for x in info_urls:
i_url = URL + x['href']
ALL_URLS.append(i_url)
print("库存链接共:" + str(len(ALL_URLS)))
glock.release() #解锁
#获取每一页需要的数据
def get_data():
while True:
global ALL_URLS #设置全局变量
glock.acquire() #加锁
print("当前库存："+str(len(ALL_URLS)))
if len(ALL_URLS) == 0:
glock.release() #解锁
print('159 行 ：ALL_URLS 为空')
break
else:
url = ALL_URLS.pop()
print("开始抓取数据：" + url)
glock.release() #解锁
time.sleep(1) #睡眠1秒钟
html = get_html(url)
if html == "error":
print("168行：connect url error")
time.sleep(random.randint(2, 4))
return
html.encoding='utf-8' #显式地指定网页编码，一般情况可以不用
soup = BeautifulSoup(html.text,'lxml')
#公众号名称
names = soup.select('div.artcleLeft > div.xcxnry > div.xcxtop > div.xcxtop_left > div.gzhtop_logo > h1')
#微信号id
accounts = []
accounts.append(soup.select('div.artcleLeft > div.xcxnry > div.xcxtop > div.xcxtop_left > div.gzhtop_logo > p')[0])
#微信头像
imgs = soup.select('div.artcleLeft > div.xcxnry > div.xcxtop > div.xcxtop_left > div.gzhtop_logo > img')
#公众号二维码
QR_codes= soup.select('div.artcleLeft > div.xcxnry > div.xcxtop > div.xcxtop_right > img')
#介绍
descs = soup.select('div.artcleLeft > div.xcxnry > div.xcxinfo')
#公众号分类
categorys = []
category = ''
cate = soup.select('div.artcleLeft > div.xcxnry > div.xcxtop > div.xcxtop_left > div.xcx_p > span > a')
if not len(cate) == 0:
category = cate[0].get_text()
else:
category = '综合'
glock.acquire() #加锁
for name,account,img,QR_code,desc in zip(names,accounts,imgs,QR_codes,descs):
data = {
'name':name.get_text(),
'category':category,
'account':account.get_text().split("：")[-1],
'img':img['src'],
'QR_code':QR_code['src'],
'desc':desc.get_text()
}
add_data(data,url)
glock.release() #解锁
#添加数据
def add_data(data,url):
con = MySQLdb.connect('127.0.0.1','root','root','test',charset="utf8",use_unicode=True)
cursor = con.cursor()
# exit()
insert_sql = """
insert ignore into weixin5(w_name,category,account,img,QR_code,introduce)
VALUES (%s,%s,%s,%s,%s,%s)
"""
print('212行 ：' + data['name'] + '_' + data['account'] + '添加成功！-' + url)
try:
cursor.execute(insert_sql,(data['name'],data['category'],data['account'],data['img'],data['QR_code'],str(data['desc'])))
con.commit()
except:
ALL_URLS.insert(0,url)
print("218行：" + URL + '插入失败')
con.rollback()
con.close()
# 将时间字符串转化为时间戳
def time_to(dt):
timeArray = time.strptime(dt, "%Y年%m月%d日")
timestamp = int(time.mktime(timeArray))
return timestamp
#启动多线程爬取
def main():
for x in range(3):
th = threading.Thread(target=get_info_urls)
th.start()
# get_info_urls()
time.sleep(3)
for x in range(5):
th = threading.Thread(target=get_data)
th.start()
if __name__ == '__main__':
# 计时
t1 = time.time()
# 调用函数
get_ip() #获取ip池
get_page_urls()
time.sleep(2)
# get_categoty_url()
main()
print(time.time() - t1)
以上就是本文的全部内容，希望对大家的学习有所帮助，也希望大家多多支持脚本之家。
您可能感兴趣的文章:
内容来自用户分享和网络整理，不保证内容的准确性，如有侵权内容，可联系管理员处理 
相关文章推荐
- Python2.7下载微信公众号文章的图片（二）
- python 下载微信公众号文章，含图片，生成文本
- 关于python爬取搜狗微信公众号文章永久链接
- 用python爬取微信公众号文章
- python 下载微信公众号文章，含图片，并分词，还搜索分词
- python爬虫(17)爬出新高度_抓取微信公众号文章（selenium+phantomjs）（上）
- python 下载微信公众号文章，含图片，并对文本分词， 并 搜索某个词，然后替换为 粗体 红色
- python scrapy爬取微信公众号文章的爬虫
- python 下载微信公众号文章，含图片，分词，搜索所有分词
- Python批量爬取微信公众号文章中的图片
- [Python爬虫] 之十五：Selenium +phantomjs根据微信公众号抓取微信文章
- Python2.7下载微信公众号文章的图片
- python抓取oschina最新文章
- python开发系列文章更新啦
- 如何实现自动采集微信公众号文章
- 这篇文章记录廖雪峰python教程的习题(九)
- 一篇不错的介绍Python中Unicode码的文章
- 一篇文章快速了解Python的GIL
- [python]使用Counter统计文章中出现频率最高的单词
- python利用微信公众号报警代码