Python+Scrapy爬取起点小说网数据存入Mysql与MongoDB数据库

1.编写item

class BookItem(scrapy.Item):
# 小说名
book_name = scrapy.Field()
# 作者
author = scrapy.Field()
# 小说类型
book_type = scrapy.Field()
# 小说状态
book_state = scrapy.Field()
# 小说更新
book_update = scrapy.Field()

book_time = scrapy.Field()
# 最新一章地址
new_href = scrapy.Field()
# 小说简介
book_intro = scrapy.Field()
pass

2.编写Spider

from scrapy.spiders import Spider
from scrapy.http import Request
from scrapy.selector import Selector

from Mycrawl.items import BookItem
import requests

class BookSpider(Spider):
# 爬虫名字，重要
name = 'book'
# 反爬措施
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.119 Safari/537.36'}
#url = "https://www.qidian.com/rank/yuepiao?style=1"
#start_urls = ['qidian.com']

def start_requests(self):
url = "https://www.qidian.com/rank/yuepiao?style=1"
yield Request(url, headers=self.headers, callback=self.parse)

def parse(self, response):
item = BookItem()
selector = Selector(response)
books = selector.xpath('//div[@class="book-mid-info"]')
for book in books:
name = book.xpath('h4/a/text()').extract()
author = book.xpath('p[@class="author"]/a[@class="name"]/text()').extract()
type = book.xpath('p[@class="author"]/a[@data-eid="qd_C42"]/text()').extract()
state = book.xpath('p[@class="author"]/span/text()').extract()
intro = book.xpath('p[@class="intro"]/text()').extract()
update = book.xpath('p[@class="update"]/a[@target="_blank"]/text()').extract()
href = book.xpath('p[@class="update"]/a/@href').extract()
time = book.xpath('p[@class="update"]/span/text()').extract()

item['book_name'] = name[0]
item['author'] = author[0]
item['book_type'] = type[0]
item['book_state'] = state[0]
item['book_update'] = update[0]
item['book_time'] = time[0]
item['new_href'] = 'https:' + href[0]
item['book_intro'] = ''.join(intro).replace(' ','').replace('\n','')
yield item

3.编写piplines连接数据库

Mysql

import pymysql
class BookPipeline(object):
def __init__(self):
# 连接数据库
self.conn = pymysql.connect(host='127.0.0.1', port=3306, user='root', passwd='1likePython',
db='TESTDB', charset='utf8')
# 建立游标对象
self.cursor = self.conn.cursor()
self.cursor.execute('truncate table Book')
self.conn.commit()

def process_item(self, item, spider):
try:
self.cursor.execute("insert into Book (book_name,author,book_type,book_state,book_update,book_time,new_href,book_intro) \
VALUES (%s,%s,%s,%s,%s,%s,%s,%s)", (item['book_name'], item['author'], item['book_type'],
item['book_state'], item['book_update'], item['book_time'],
item['new_href'], item['book_intro']))
self.conn.commit()
except pymysql.Error:
print("Error%s,%s,%s,%s,%s,%s,%s,%s" % (item['book_name'], item['author'], item['book_type'],
item['book_state'], item['book_update'], item['book_time'],
item['new_href'], item['book_intro']))
return item

MongoDB

import pymongo
class BookPipeline(object):
def __init__(self):
# 连接数据库
self.client = pymongo.MongoClient(host='127.0.0.1', port=27017)
self.test = self.client['TESTDB']
self.post = self.test['movie']
def process_item(self, item, spider):
data = dict(item)
self.post.insert(data)
return item

4.编写setting

BOT_NAME = 'Mycrawl'

SPIDER_MODULES = ['Mycrawl.spiders']
NEWSPIDER_MODULE = 'Mycrawl.spiders'

# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'Mycrawl (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

ITEM_PIPELINES = {
#'Mycrawl.pipelines.MoviePipeline': 100,
'Mycrawl.pipelines.BookPipeline': 300,
}

5.结果

Mysql

MongoDB