爬虫 scrapy 抓取小说实例

#coding=utf-8
import scrapy,os
curpath = os.getcwd()
noveldir = ''
class novelSpider(scrapy.Spider):
name = 'xiaoshuo'
start_urls = ['http://www.biquge.com/2_2970/']
def __init__(self):
self.noveldir = ''

def parse(self,response):
title = response.css('div#info h1::text').extract_first()    #小说名
self.noveldir = os.path.join(curpath,title)
self.log(self.noveldir )
if not os.path.exists(self.noveldir ):
os.makedirs(self.noveldir )                          #创建小说目录
self.log('开始下载%s' % title.encode('utf8'))
for href in  response.css('dd a').css('a::attr(href)'):      #小说章节链接
yield response.follow(href,self.parse_page)
def parse_page(self,response):
filename = response.css('div.bookname h1::text').extract_first().strip() #章节名，同时作为文件名
self.log('开始下载 %s' % filename.encode('utf8'))
with open(os.path.join(self.noveldir ,filename),'w+') as f:
for item in response.css('div#content::text').extract():         #小说章节内容写到文件
f.write(item.encode("utf8")+"\n")