爬虫实战15：selenium爬取拉勾网Python职位表，并保存到MySQL中

import requests
import time
from lxml import etree
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from selenium.common.exceptions import  NoSuchElementException
import pymysql

def get_page():
url = "https://www.lagou.com/"
browser = webdriver.Chrome()
browser.get(url)
quanguo = browser.find_element_by_id("cboxClose")
quanguo.click()
wait = WebDriverWait(browser, 30)
# input = wait.until(EC.presence_of_all_elements_located((By.ID, 'search_input')))
time.sleep(5)
input = browser.find_element_by_id("search_input")
# print(input)
input.send_keys('Python')
time.sleep(1)
# button = wait.until(EC.presence_of_all_elements_located((By.ID, 'earch_button')))
button = browser.find_element_by_id("search_button")
button.click()
source = browser.page_source
html = etree.HTML(source)
link = html.xpath('//a[@class="position_link"]/@href')
url_list = []
for position_url in link:
url_list.append(position_url)
browser.quit()
return url_list

def parse_page(position_url):
db = pymysql.connect(host='localhost', user='root', password='123456', port=3306, db='mysql')
cursor = db.cursor()
driver = webdriver.Chrome()
driver.get(position_url)
text = driver.page_source
html = etree.HTML(text)
job_info = {}
name = html.xpath('//div[@class="position-content "]//span[@class="name"]/text()')[0]
salary = html.xpath('//div[@class="position-content "]//span[@class="salary"]/text()')[0].strip()
# job_request = html.xpath('//div[@class="position-content "]//dd[@class="job_request"]/p/span/text()')
# print(job_request)
drrs = html.xpath('//div[@class="position-content "]//dd[@class="job_request"]/p/span[2]/text()')[0].split('/')[1].strip()
years = html.xpath('//div[@class="position-content "]//dd[@class="job_request"]/p/span[3]/text()')[0].split('/')[0].strip()
jingyan = html.xpath('//div[@class="position-content "]//dd[@class="job_request"]/p/span[4]/text()')[0].split('/')[0].strip()
zhiye = html.xpath('//div[@class="position-content "]//dd[@class="job_request"]/p/span[5]/text()')[0]
company = html.xpath('//div[@class="job_company_content"]//em[@class="fl-cn"]/text()')[0].strip()
infos = html.xpath('//div[@class="job-detail"]/p/text()')[:]
infos = ''.join(infos)
job_info['职位'] = name
job_info['薪水'] = salary
job_info['地址'] = drrs
job_info['工作年限'] = years
job_info['经验'] = jingyan
job_info['是否全职'] = zhiye
job_info['公司名称'] = company
job_info['职位描述'] = infos
sql = '''INSERT INTO func(names, salarys, drrss , yearss, jingyans, zhiyes, companys, infoss) values('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s')'''\
% (name, salary, drrs, years, jingyan, zhiye, company, infos)
sql = sql.encode('utf-8')
cursor.execute(sql)
db.commit()
db.close()
print(job_info)
driver.quit()

def main():
url_list = get_page()
for positioin_url in url_list:
parse_page(positioin_url)

main()