通俗易懂的分析如何用Python实现一只小爬虫，爬取拉勾网的职位信息

效果预览

思路

代码

https.py

# -*- coding: utf-8 -*-
from src.setting import IP, UA
import requests, random
import logging

class Http:
'''
http请求相关的操作
'''

def __init__(self):
pass

def get(self, url, headers=None, cookies=None, proxy=None, timeOut=5, timeOutRetry=5):
'''
获取网页源码
url: 网页链接
headers: headers
cookies: cookies
proxy: 代理
timeOut: 请求超时时间
timeOutRetry: 超时重试次数
return: 源码
'''
if not url:
logging.error('GetError url not exit')
return 'None'

# 这里只展示了一部分代码
# 完整代码已上传到Github

main.py

def getInfo(url, para):
"""
获取信息
"""
generalHttp = Http()
htmlCode = generalHttp.post(url, para=para, headers=headers, cookies=cookies)
generalParse = Parse(htmlCode)
pageCount = generalParse.parsePage()
info = []
for i in range(1, 3):
print('第%s页' % i)
para['pn'] = str(i)
htmlCode = generalHttp.post(url, para=para, headers=headers, cookies=cookies)
generalParse = Parse(htmlCode)
info = info + getInfoDetail(generalParse)
time.sleep(2)
return info

def processInfo(info, para):
"""
信息存储
"""
logging.error('Process start')
try:
title = '公司名称\t公司类型\t融资阶段\t标签\t公司规模\t公司所在地\t职位类型\t学历要求\t福利\t薪资\t工作经验\n'
file = codecs.open('%s职位.xls' % para['city'], 'w', 'utf-8')
file.write(title)
for p in info:
line = str(p['companyName']) + '\t' + str(p['companyType']) + '\t' + str(p['companyStage']) + '\t' + \
str(p['companyLabel']) + '\t' + str(p['companySize']) + '\t' + str(p['companyDistrict']) + '\t' + \
str(p['positionType']) + '\t' + str(p['positionEducation']) + '\t' + str(
p['positionAdvantage']) + '\t' + \
str(p['positionSalary']) + '\t' + str(p['positionWorkYear']) + '\n'
file.write(line)
file.close()
return True
except Exception as e:
print(e)
return None

parse.py

class Parse:
'''
解析网页信息
'''

def __init__(self, htmlCode):
self.htmlCode = htmlCode
self.json = demjson.decode(htmlCode)
pass

def parseTool(self, content):
'''
清除html标签
'''
if type(content) != str: return content
sublist = ['<p.*?>', '</p.*?>', '<b.*?>', '</b.*?>', '<div.*?>', '</div.*?>',
'</br>', '<br />', '<ul>', '</
b83c
ul>', '<li>', '</li>', '<strong>',
'</strong>', '<table.*?>', '<tr.*?>', '</tr>', '<td.*?>', '</td>',
'\r', '\n', '&.*?;', '&', '#.*?;', '<em>', '</em>']
try:
for substring in [re.compile(string, re.S) for string in sublist]:
content = re.sub(substring, "", content).strip()
except:
raise Exception('Error ' + str(substring.pattern))
return content

# 这里只展示了一部分代码
# 完整代码已上传到Github

setting.py

# -*- coding: utf-8 -*-

# headers
headers = {
'Host': 'www.lagou.com',
'Connection': 'keep-alive',
'Content-Length': '23',
'Origin': 'https://www.lagou.com',
'X-Anit-Forge-Code': '0',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'Accept': 'application/json, text/javascript, */*; q=0.01',
'X-Requested-With': 'XMLHttpRequest',
'X-Anit-Forge-Token': 'None',
'Referer': 'https://www.lagou.com/jobs/list_java?city=%E5%B9%BF%E5%B7%9E&cl=false&fromSearch=true&labelWords=&suginput=',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7'
}

测试