Python爬虫从入门到放弃（十一）之 Scrapy框架整体的一个了解

这里是通过爬取伯乐在线的全部文章为例子，让自己先对scrapy进行一个整理的理解

该例子中的详细代码会放到我的github地址：https://github.com/pythonsite/spider/tree/master/jobboleSpider

注：这个文章并不会对详细的用法进行讲解，是为了让对scrapy各个功能有个了解，建立整体的印象。

在学习Scrapy框架之前，我们先通过一个实际的爬虫例子来理解，后面我们会对每个功能进行详细的理解。
这里的例子是爬取http://blog.jobbole.com/all-posts/ 伯乐在线的全部文章数据

分析要爬去的目标站信息

先看如下图，首先我们要获取下图中所有文章的连接，然后是进入每个文章连接爬取每个文章的详细内容。
每个文章中需要爬取文章标题，发表日期，以及标签，赞赏收藏，评论数，文章内容。

class JobbolespiderPipeline(object):
def process_item(self, item, spider):
return item

class JsonWithEncodingPipeline(object):
'''
返回json数据到文件
'''
def __init__(self):
self.file = codecs.open("article.json",'w',encoding="utf-8")

def process_item(self, item, spider):
lines = json.dumps(dict(item),ensure_ascii=False) + "\n"
self.file.write(lines)
return item

def spider_closed(self,spider):
self.file.close()

class MysqlPipeline(object):
'''
插入mysql数据库
'''
def __init__(self):
self.conn =pymysql.connect(host='192.168.1.19',port=3306,user='root',passwd='123456',db='article_spider',use_unicode=True, charset="utf8")
self.cursor = self.conn.cursor()

def process_item(self,item,spider):
insert_sql = '''
insert into jobbole_article(title,create_date,url,url_object_id,front_image_url,front_image_path,comment_nums,fav_nums,praise_nums,tag,content) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
'''

self.cursor.execute(insert_sql,(item["title"],item["create_date"],item["url"],item["url_object_id"],item["front_image_url"],item["front_image_path"],item["comment_nums"],item["fav_nums"],item["praise_nums"],item["tag"],item["content"]))
self.conn.commit()

class MysqlTwistedPipline(object):
'''
采用异步的方式插入数据
'''
def __init__(self,dbpool):
self.dbpool = dbpool

@classmethod
def from_settings(cls,settings):
dbparms = dict(
host = settings["MYSQL_HOST"],
port = settings["MYSQL_PORT"],
user = settings["MYSQL_USER"],
passwd = settings["MYSQL_PASSWD"],
db = settings["MYSQL_DB"],
use_unicode = True,
charset="utf8",
)
dbpool = adbapi.ConnectionPool("pymysql",**dbparms)
return cls(dbpool)
def process_item(self,item,spider):
'''
使用twisted将mysql插入变成异步
:param item:
:param spider:
:return:
'''
query = self.dbpool.runInteraction(self.do_insert,item)
query.addErrback(self.handle_error)

def handle_error(self,failure):
#处理异步插入的异常
print(failure)

def do_insert(self,cursor,item):
#具体插入数据
insert_sql = '''
insert into jobbole_article(title,create_date,url,url_object_id,front_image_url,front_image_path,comment_nums,fav_nums,praise_nums,tag,content) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
'''
cursor.execute(insert_sql,(item["title"],item["create_date"],item["url"],item["url_object_id"],item["front_image_url"],item["front_image_path"],item["comment_nums"],item["fav_nums"],item["praise_nums"],item["tag"],item["content"]))

class ArticleImagePipeline(ImagesPipeline):
'''
对图片的处理
'''
def item_completed(self, results, item, info):

for ok ,value in results:
if ok:
image_file_path = value["path"]
item['front_image_path'] = image_file_path
else:
item['front_image_path'] = ""

return item

View Code