提取网址的python练习

import urllib, urllib2, cookielib
from HTMLParser import HTMLParser
import sys

reload(sys)
sys.setdefaultencoding('utf8')

class WebParser(HTMLParser):
def __init__(self, links, path):
HTMLParser.__init__(self)
self.links = links
self.path = path

def handle_starttag(self, tag, attrs):
if tag == 'a':
if len(attrs) == 0:
pass
else:
for (key, val) in attrs:
if key == 'href':
if val.startswith('http'):
self.links.add(val)
elif val.startswith('/'):
self.links.add(self.path + val)

class Crawl:
def __init__(self):
self.path = 'http://www.baidu.com'
self.cookie = cookielib.CookieJar()
handler = urllib2.HTTPCookieProcessor(self.cookie)
self.opener = urllib2.build_opener(handler)

def open(self, path):
self.response = self.opener.open(path)

def showCookie(self):
for item in self.cookie:
print 'Name = ' + item.name
print 'value = ' + item.value

def showResponse(self):
print self.response.read()

def getAllUrl(self, links, path):
try:
self.open(path)
res = self.response.read()
parser = WebParser(links, path)
parser.feed(res)
parser.close()
except Exception, e:
print e

def crawl(self):
src_links = set()
result_links = set()
self.getAllUrl(src_links, self.path)
n = 200
while len(src_links) != 0 and n > 0:
link = src_links.pop()
if link in result_links:
pass
result_links.add(link)
self.getAllUrl(src_links, link)
n -= 1
print n

return result_links | src_links

c = Crawl()
rlt = c.crawl()
for link in rlt:
print link