Python彻底解决采集网页乱码问题

实战十几万网页采集后的修正版：

def download_page(indexurl, timeout=10):
"""
下载网站数据并返回
:param indexurl:
:param timeout:
:return:
"""
try:
res = urllib2.urlopen(indexurl, timeout=timeout)
code = res.getcode()
info = res.info()
charset = None

if info:
m = re.findall(r'charset=([a-zA-Z0-9_-]+)', ' '.join(info.headers), re.I)
if m:
charset = str(m[0]).lower()

if code == 200:
html = res.read()
if not charset and html:
charset = pick_charset(html)

# 如果完全采不到 charset,默认使用 gbk 反正都是乱码
if not charset or charset == "gb2312":
charset = 'gbk'

if charset and charset != 'utf-8':
try:
html = html.decode(charset).encode('utf-8')
except:
pass
else:
html = ''
return (code,
res.geturl(),
charset,
html
)
except urllib2.URLError, e:
return str("%r" % e)
except socket.timeout, e:
return str("%r" % e)
except:
return str(sys.exc_info())