scrapy将数据存储到mysql中

一、在配置中设置数据库

#setting.py
ITEM_PIPELINES = {
......
'dushuproject.pipelines.MysqlPipeline': 299,
}
DB_HOST = '127.0.0.1'
DB_PORT = 3306
DB_USER = 'root'
DB_PWD = '123456'
DB_NAME = 'test'
DB_CHARSET = 'utf8'

二、在管道中进行存储

class MysqlPipeline(object):

def __init__(self):
settings = get_project_settings()
self.host = settings['DB_HOST']
self.port = settings['DB_PORT']
self.user = settings['DB_USER']
self.pwd = settings['DB_PWD']
self.name = settings['DB_NAME']
self.charset = settings['DB_CHARSET']

self.connect()

def connect(self):
self.conn = pymysql.connect(host=self.host,
port=self.port,
user=self.user,
password=self.pwd,
db=self.name,
charset=self.charset)
self.cursor = self.conn.cursor()

def close_spider(self, spider):
self.conn.close()
self.cursor.close()

def process_item(self, item, spider):
sql = 'insert into book(image_url, book_name, author, info) values("%s", "%s", "%s", "%s")' % (item['image_url'], item['book_name'], item['author'], item['info'])
# 执行sql语句
self.cursor.execute(sql)
return item