爬虫实战：爬取前程无忧（51job）python相关职位信息

import requests
from bs4 import BeautifulSoup
import re
import time

requests.adapters.DEFAULT_RETRIES = 3
strat_url = 'https://search.51job.com/list/030000%252C00,000000,0000,00,9,99,python,2,1.html'
ua = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36'}
resp = requests.get(strat_url,headers = ua)
resp.encoding = 'gb2312'
pages = int(re.findall(r'共(\d+)页',resp.text)[0]) #匹配总页数，确定循环次数
info_list = []

def get_info(url):
resp = requests.get(url,headers = ua)
resp.encoding = 'gb2312'
soup = BeautifulSoup(resp.text,'lxml')
resultList = soup.find(id="resultList")
infos = resultList.find_all(class_='el')
for info in infos:
t1 = info.find(class_='t1').text.strip()
t2 = info.find(class_='t2').text.strip()
t3 = info.find(class_='t3').text.strip()
t4 = info.find(class_='t4').text.strip()
t5 = info.find(class_='t5').text.strip()
format_str = '{0}\t{1}\t{2}\t{3}\t{4}'.format(t1,t2,t3,t4,t5)
#        print(format_str)
info_list.append(format_str)

if __name__=='__main__':
for i in range(1,pages+1):
url = 'https://search.51job.com/list/030000%252C00,000000,0000,00,9,99,python,2,'+str(i)+'.html'
print('\r当前进度：{0}/{1}'.format(i,pages),end = '')
get_info(url)
time.sleep(0.2)
with open('F:/前程无忧广东省python.txt','w',encoding='utf8') as f:
f.write('\n'.join(info_list))
print('爬取结束！')