从豆瓣中抓取图片以及音乐

该脚本可以从豆瓣中抓取图片以及音乐，可以输入自己的正则，正则的合法性我没有去进行验证，这只是一个简单的文件抓取，只是用了一些基本的模块，看到网上好多使用Beautiful Soup
模块的例子，一会研究研究。代码很简单，就不多解释了

#! /usr/bin/python2.6.6

# -- coding:utf-8 --

import urllib, urllib2, gevent, re

import time, os

from gevent import monkey

monkey.patch_all()

def worker(url):

try:

m_arr = []

parent_dir = os.path.join("%s" % time.strftime('%Y-%m-%d',time.localtime(time.time()))) #按日期创建目录

if not os.path.exists(parent_dir):

os.mkdir(parent_dir)

os.chdir(parent_dir)

if not os.path.exists('images'):#创建存储图片的目录

os.mkdir('images')

if not os.path.exists('radios'):#创建存储音乐的目录

os.mkdir('radios')

response=urllib2.urlopen(url)

text=response.read()

groups=re.finditer(reg, text)

for g in groups:

name=g.group(1).strip() + ".mp3"

cover_name=g.group(1).strip() + ".jpg"

cover_path=g.group(2).replace('\\', '')

path=g.group(3).replace('\\', '')

m_arr.append((name, path, cover_name, cover_path))

except Exception, e:

print "<<==get regulare text raised exceptin %s==>>" % e

finally:

return m_arr

def grun(path, name):

try:

urllib.urlretrieve(path, name)

except Exception, e:

print "<<==Fetch material %s in %s raised Exception %s==>>" % (name, path, e)

pass

if __name__ == '__main__':

#匹配音乐url

url = raw_input("input the url to fetch materials ==>>")

re_pat = raw_input("input the regular expression to fetch materials ==>>")

if not url:

url = "http://site.douban.com/dannv/"

if not re_pat:

re_pat = '{"name":"(.+?)",.+?"cover":"(.+?)",.+?"rawUrl":"(.+?)",.+?}'

reg=re.compile(re_pat, re.I)

musicArray = worker(url)

jobs = []

os.chdir('images')

for (name, path, cover_name, cover_path) in musicArray:

jobs.append(gevent.spawn(grun, cover_path, cover_name))

gevent.joinall(jobs, timeout=600)

os.chdir('../radios')

for (name, path, cover_name, cover_path) in musicArray:

jobs.append(gevent.spawn(grun, path, name))

gevent.joinall(jobs, timeout=600)