Python 抓取【参考消息网站】的新闻

# -*- coding:utf-8 -*-
'''
Created on 2015-12-8

@author: AndyCoder
'''

import re
import urllib2
import json

class spider(object):
'''
spider
'''

def __init__(self, url="",header=""):
'''
Constructor
'''
self.url = url
self.header = header

def parseUrl(self, urlPatter='"url":"(.*?)",'):
urlList = []
pattern = re.compile(urlPatter, re.DOTALL)
request = urllib2.Request(self.url)
request.add_header('User-Agent', self.header)
response = urllib2.urlopen(request)

html = response.read()
contentHtml =  html.decode('raw_unicode_escape')

items = re.findall(pattern, html)
for item in items:
urls = item.replace('\\','')
urlList.append(urls)
return urlList, contentHtml

def parseContent(self,url,contentPattern='<div class="content">(.*?)</a>(.*?)</strong>(.*?)</div>'):
newsList = []
newsDict = {}
contentPattern = re.compile(contentPattern, re.DOTALL)
titlePattern = re.compile('<title>(.*?)-(.*?)</title>', re.DOTALL)

req = urllib2.Request(url)
resp = urllib2.urlopen(req)
content = resp.read()
utfContent = content.decode('utf8')
title = ''
for item in re.findall(titlePattern, utfContent):
title = item[0]

for item in re.findall(contentPattern, content):
#            news = "{'title':" + "'" + title + "'" + "," + "'url':" + "'" + url + "'"  + "," + "'time':" + "'" + item[1] + "'" + "," + "'content':" + "'" + item[2] + "'}"
newsDict['title'] = title
newsDict['url'] = url
newsDict['time'] = item[1]
newsDict['content'] = item[2]

newsList.append(newsDict)
return newsList,content

# s = spider('http://app.cankaoxiaoxi.com/?app=system&controller=channel&action=wap_index&catid=1&order=publish&num=2weight=60&jsoncallback=?','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36')
# urls, html = s.parseUrl()
# for url in urls:
#     newsList, content = s.parseContent(url)
#     for news in newsList:
#         print news
#
#
s = spider('http://app.cankaoxiaoxi.com/?app=system&controller=channel&action=wap_index&catid=1&order=publish&num=2weight=60&jsoncallback=?','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.86 Safari/537.36')
urls, html = s.parseUrl()
for url in urls:
newsList, content = s.parseContent(url)
news_string = json.dumps(newsList)
decoded = json.loads(news_string)
if len(decoded)>0:
print decoded[0]