scrapy_redis分布式爬虫 从redis写到mysql数据库中

import redis

import MySQLdb

import json

def process_item():

# 创建redis数据库连接

rediscli = redis.Redis(host = “127.0.0.1”, port = 6379, db = 0)

# 创建mysql数据库连接
mysqlcli = MySQLdb.connect(host = "127.0.0.1", port = 3306, \
user = "", passwd = "", db = "数据库")

offset = 0

while True:
# 将数据从redis里pop出来
source, data = rediscli.blpop("表名:items")
item = json.loads(data)
try:
# 创建mysql 操作游标对象，可以执行mysql语句
cursor = mysqlcli.cursor()

cursor.execute("insert into 表名 (username, age) values (%s, %s)", [item['username'], item['age']])
# 提交事务
mysqlcli.commit()
# 关闭游标
cursor.close()
offset += 1
print offset
except:
pass

if name == “main“:

process_item()