python scrapy爬取腾讯招聘网站保存为csv文件，而且设定其表头顺序(csv默认是无序的)

要点：1.设定UA

          2.创建csv_item_exporter.py   设定csv

           3.settings.py设定好参数

           4.begin启动程序

目标网址：http://hr.tencent.com/position.php?keywords=&tid=0&star

spider.py(主体很简单)

import os,io,sys,re from scrapy.spider import Spider from scrapy.selector import Selector from lianxi.items import LianxiItem from scrapy import Request class LianxiSpider(Spider): name = "lianxi" allowed_domains = ["hr.tencent.com"] start_urls = [] # 起始urls列表 for i in range(0,200,10): #爬取了20页 url='https://hr.tencent.com/position.php?&start='+str(i)+'#a' start_urls.append(url) def parse(self, response): geduan=response.xpath('//table[@class="tablelist"]/tr[not(@class="h"or@class="f")]') item=LianxiItem() for zhiwei in geduan: name=zhiwei.xpath('.//td[@class="l square"]//a//text()').extract() item['name']="".join(name).strip() genre=zhiwei.xpath('./td[2]/text()').extract() item['genre']="".join(genre).strip() number=zhiwei.xpath('./td[3]/text()').extract() item['number'] = "".join(number).strip() place=zhiwei.xpath('./td[4]/text()').extract() item['place'] = "".join(place).strip() time = zhiwei.xpath('./td[5]/text()').extract() item['time'] = "".join(time ).strip() yield item

items.py

import scrapy class LianxiItem(scrapy.Item): name = scrapy.Field() #招聘单位 genre = scrapy.Field() #招聘类型 number = scrapy.Field() #招聘人数 place = scrapy.Field() #招聘地点 time = scrapy.Field()

middlewares.py   设置了随机UA，可以下载fake_useragent模块

from fake_useragent import UserAgent #下载此模块才有 from scrapy import signals import random import logging from scrapy.utils.response import response_status_message logger = logging.getLogger(__name__) class RandomUserAgent(object): def __init__(self,): self.agent=UserAgent() ##headers的模块 def process_request(self,request,spider): request.headers.setdefault('User-agent',self.agent.random)##随机选择headers @classmethod #文件默认 def from_crawler(cls, crawler): # This method is used by Scrapy to create your spiders. s = cls() crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) return s def spider_opened(self, spider): spider.logger.info('Spider opened: %s' % spider.name)
Pipelines.py   不写也没问题

class LianxiPipeline(object): def process_item(self, item, spider): print (item) return item

重点来了！！！csv_item_exporter.py   ，设置保存csv文件的顺序，跟spider.py同一个文件夹

from scrapy.conf import settings from scrapy.contrib.exporter import CsvItemExporter import io class MyProjectCsvItemExporter(CsvItemExporter): def __init__(self, *args, **kwargs): delimiter = settings.get('CSV_DELIMITER', ',') kwargs['delimiter'] = delimiter fields_to_export = settings.get('FIELDS_TO_EXPORT', []) if fields_to_export : kwargs['fields_to_export'] = fields_to_export super(MyProjectCsvItemExporter, self).__init__(*args, **kwargs)同时，settings.py 也要设置对应的csv参数

BOT_NAME = 'lianxi' SPIDER_MODULES = ['lianxi.spiders'] NEWSPIDER_MODULE = 'lianxi.spiders' ITEM_PIPELINES = { 'lianxi.pipelines.LianxiPipeline': 100, } DOWNLOADER_MIDDLEWARES = { 'lianxi.middlewares.RandomUserAgent': 20, 'doubanmovie.middlewares.RandomProxy':None, 'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware':None, 'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware':100, } RETRY_ENABLED = False DOWNLOAD_DELAY = 0.5 COOKIES_ENABLED = False ##########==========保存为csv文件设定表头顺序！！ FEED_EXPORTERS = { 'csv': 'lianxi.spiders.csv_item_exporter.MyProjectCsvItemExporter', } FIELDS_TO_EXPORT = [ 'name', 'genre', 'number', 'place', 'time', ]

OK，最后设定begin.py，启动程序

from scrapy import cmdline
cmdline.execute("scrapy crawl lianxi -o info.csv -t csv".split())