使用scrapy 抓取顶点小说

# -- coding: utf-8 --

import scrapy

from urllib import parse

from scrapy import Request

from ding.items import DingItem

scrapy spider爬取部分



#将要获取的网址

class DingdianSpider(scrapy.Spider):

name = 'dingdian'   #scrapy框架name
start_urls = ['http://www.x23us.com/'] #顶点小说网址

#解析
def parse(self, response):
title = response.xpath('/html/body/div[2]/ul/li/a/@href').extract()[1:12]#全部类型
for i in title:#便利获取值，传入下一个解析函数
yield Request(url=parse.urljoin(response.url,i), callback=self.parse_get)

#获取最大页数

def parse_get(self, response):

name = response.xpath(‘//*[@id=”pagelink”]/a[14]/text()’).extract()[0]

for i in range(1, int(name)):

if response.url != ‘http://www.x23us.com/quanben/1‘:

left_url = response.url[:-6]

right_url = response.url[-5:]

yield Request(left_url + str(i) + right_url, callback=self.get_parse)#每页的链接

else:

yield Request(parse.urljoin(response.url, str(i)), callback=self.get_parse)

#全本小说的类型与其他类型的url不一致，返回上一级函数重新匹配，在传入下一级函数中



#每章内容url和作者

def get_parse(self, response):

try:

article_url = response.xpath(‘//*[@id=”content”]/dd/table/tr/td/a/@href’).extract()[1]

author = response.xpath(‘//*[@id=”content”]/dd/table/tr/td/text()’).extract()[0]

yield Request(article_url, callback=self.page_list, meta={‘author’:author})#传入下一级函数

except:

pass



#每章标题和标题的url

def page_list(self,response):

title = response.xpath(‘//*[@id=”a_main”]/div/dl/dd/h1/text()’).extract()[0]

content_url = response.xpath(‘//*[@id=”at”]/tr/td/a/@href’).extract()[0]

yield Request(parse.urljoin(response.url, content_url), callback=self.content_html, meta={‘title’:title, ‘author’:response.meta[‘author’]})

#传入下一级函数



#解析每章的内容，从网页中抽取项目，清晰，验证你要抓取的数据进入items

def content_html(self, response):

item = DingItem()

title1 = response.xpath(‘//*[@id=”amain”]/dl/dd[1]/h1/text()’).extract()[0]

item[‘book’] = response.meta[‘title’]

item[‘article_title’] = title1

item[‘author’] = response.meta[‘author’]

# content_all =”

content_con = response.xpath(‘//*[@id=”contents”]/text()’).extract()

for i in content_con:

content_all = i.replace(‘\xa0’,”)

item[‘content’] = content_all

yield item

scrapy item 部分

#定义抓取的字段

import scrapy

class DingItem(scrapy.Item):

author = scrapy.Field()

article_title = scrapy.Field()

content = scrapy.Field()

book = scrapy.Field()

pass

scrapy pipelines部分

#存入PYmongo

import pymongo

class DingPipeline(object):

def init(self):

client = pymongo.MongoClient()

self.db = client[‘Dingdian’][‘xiaoshuo’]

def process_item(self, item, spider):

self.db.insert(dict(item))

return item

运行代码

from scrapy.cmdline import execute

execute([‘scrapy’, ‘crawl’, ‘dingdian’])