python抓网页资源小脚本

#!/usr/bin/env python
# coding: utf-8
import urllib

def filter_src(file_name):
resource_list = []
f_obj = open(file_name)
for f_line in f_obj:
if '404' in f_line:
str_goal = f_line.strip().split(' ')[7]
if not str_goal in resource_list:
print str_goal
if '/static' in str_goal:
str_goal = str_goal.replace('/static', '')
resource_list.append(str_goal[:-1])
print resource_list
return resource_list

def down_src(source_list):
base_url = "http://www.ttcrm.com"
down_path = r"src"
for source in source_list:
source_url = base_url + source
source_path = down_path + source
print source_url
source_stram = urllib.urlopen(source_url)
f_obj = open(source_path,'wb')
f_obj.write(source_stram.read())

if __name__=='__main__':
file_name = 'src.txt'
source_list = filter_src(file_name)
down_src(source_list)

f_obj = open(source_path,'wb')
f_obj.write(source_stram.read())