爬虫实例抓取并download with Beautifulsoap

from bs4 import BeautifulSoup

import urllib2

import urllib, os, re, time, sys

#import socket

def build_request(link):

#             user_agent = 'Mozilla/5.0 (X11; U; Linux i686) Gecko/20071127 Firefox/2.0.0.11'

values = {'name' : 'Michael Foord',

'location' : 'Northampton',

'language' : 'Python' }

headers = { 'User-Agent' : 'Custom User-Agent' }

data=urllib.urlencode(values)

req = urllib2.Request(link, data, headers)

req.add_unredirected_header('User-Agent', 'Custom User-Agent')

return req

def build_urllib2(link):

print time.strftime('%Y-%m-%d_%H:%M:%S',time.localtime(time.time()))

#set timeout

urllib2.socket.setdefaulttimeout(60)

#set proxy

null_proxy_handler = urllib2.ProxyHandler({})
#null_proxy_handler = urllib2.ProxyHandler({})

opener = urllib2.build_opener(null_proxy_handler)

urllib2.install_opener(opener)

print 'after install opener'
print time.strftime('%Y-%m-%d_%H:%M:%S',time.localtime(time.time()))

def get_img(link, path):
print 'start to download '+ link

p = re.compile(r'[/\\:\*\"?|<>]+')

q = re.compile(r'[\.]+')

content=[]

try:

req= build_request(link)

content = urllib2.urlopen(req)

#                             content = opener.open(req)

print 'start to create soup'
soup = BeautifulSoup(content)

my_img = soup.find_all('img')

print 'ccccccccccc'
if my_img ==[]:

print 'no pic there'
sys.exit(0)

print 'ok, start to download'
for img in my_img:

img_link = img.get('src')

#print img_link

filename = img_link.split("/")[-1]

if not q.search(filename):

#add postfix to the filename if it doesn't have

filename = filename+'.jpg'
file_path = os.path.join(path,filename)

if os.path.exists(file_path):

continue
                                                if p.search(filename):

print 'continue'
continue
                                                print 'downloading '+filename

try:

urllib.urlretrieve(img_link,file_path, None)

except:

print 'T_T, Failed to download '+ filename

continue
                except urllib2.HTTPError, e:

print e.code

print e.msg

print e.headers

print e.fp.read()

weblink = "http://club.history.sina.com.cn/thread-5534627-1-1.html"

mypath = "G:\\python\\test\\"
build_urllib2(weblink)

get_img(weblink, mypath)