十.scrapy项目 爬取主页http://cuiqingcai.com/获取所有url与title

rules = (
Rule(LinkExtractor(allow=('\d+\.html$',)), callback='parse_all', follow=True),
# Rule(LinkExtractor(allow=('\d+\.html$',)), callback='parse_pachong', follow=True),
)

#coding:utf-8
from scrapy.spiders import CrawlSpider, Rule, Request
from scrapy.linkextractors import LinkExtractor
from ..items import CuiqingcaiItem

class myspider(CrawlSpider):
name = 'cqc'
allowed_domains = ['cuiqingcai.com']
count_all = 0
url_all = []
start_urls = ['http://cuiqingcai.com']
label_tags = [u'爬虫', 'scrapy', 'selenium']

rules = (
Rule(LinkExtractor(allow=('\d+\.html$',)), callback='parse_all', follow=True),
# Rule(LinkExtractor(allow=('\d+\.html$',)), callback='parse_pachong', follow=True),
)
'''
# 将爬虫相关的数据存入数据库
def parse_pachong(self, response):
print_tag = False
title_name = u""
for tag in self.label_tags:
title_name = response.xpath('//header/h1[1][@class="article-title"]/a/text()').extract()[0]
if tag in title_name.lower().encode("utf-8"):
print_tag = True
if print_tag == True:
self.count_all = self.count_all + 1
self.url_all.append(response.url)
item = CuiqingcaiItem()
item['url'] = response.url
item['title'] = title_name.encode("utf-8")
return item
'''
# 将全站数据存入json文件
def parse_all(self, response):
title_name = None
if response.xpath('//header/h1[1][@class="article-title"]/a/text()').extract()[0]:
title_name = response.xpath('//header/h1[1][@class="article-title"]/a/text()').extract()[0]
item = CuiqingcaiItem()
item['url'] = response.url
item['title'] = title_name
return item

import json
from pymongo import MongoClient
import settings
from items import CuiqingcaiItem

class CuiqingcaiPipeline(object):
def __init__(self):
cn=MongoClient('127.0.0.1',27017)
db=cn[settings.Mongodb_DBNAME]
self.table=db[settings.Mongodb_DBTable]
def process_item(self, item, spider):
if isinstance(item,CuiqingcaiItem):
try:
self.table.insert(dict(item))
except Exception, e:
pass
            return item

import scrapy

class CuiqingcaiItem(scrapy.Item):

title = scrapy.Field()  # 标题
url = scrapy.Field()  # 页面的地址