pyspider采集例子

#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# vim: set et sw=4 ts=4 sts=4 ff=unix fenc=utf8:
# Created on 2014-10-25 14:31:24

import re
import json
from libs.pprint import pprint
from libs.base_handler import *

class Handler(BaseHandler):
'''
this is a sample handler
'''
crawl_config = {
}
proxy = ""

@every(0, 30)
def on_start(self):
self.crawl(self.proxy+'http://www.douban.com/group/haixiuzu/discussion',
force_update=True, callback=self.index_page)

@config(age=10)
def index_page(self, response):
for each in response.doc('tr > .title > a').items():
self.crawl(self.proxy+each.attr.href, callback=self.detail_page)

@config(age=30*24*60*60)
def detail_page(self, response):
assert response.url != "https://www.douban.com/"
return {
"url": response.url,
"title": response.doc("#content h1").text(),
"author": response.doc(".topic-content .from a").text(),
"author_url": response.doc("DIV.topic-doc>H3>SPAN.from>A").attr.href,
"imgs": [x.attr.src for x in response.doc('.topic-doc img').items()]
}

def on_result(self, result):
if not result or not result['imgs']:
return
post_id = re.search("topic/(\d+)", self.response.url).group(1)
self.crawl("https://api.duoshuo.com/posts/import.json#"+post_id, method="POST",
data={
"short_name": "database",
"secret": "8e5a5be8873ad7e9a59147c3cfd10e73",
"posts[0][post_key]": post_id,
"posts[0][thread_key]": "haixiuzu",
"posts[0][message]": json.dumps(result).encode("base64").replace("\n", "")
}, callback=self.post_to_duoshuo)

def post_to_duoshuo(self):
pass