python-贴吧图片爬取的一个小脚本

  学了点python，写了个爬取贴吧图片的小脚本，记录一下，其中遇到了一个坑，就是下载下来的html，百度不知道怎么做了特殊处理，加上了注释，结果一开始怎么都提取不到图片地址，最后仔细比较才发现，然后批量把注释取消了才成功获得url。
 真坑！

代码如下：
#!/usr/bin/env python
# -*- coding:utf-8 -*-

import urllib
import urllib2
import ssl
import re
import os
from lxml import etree

totalcount = 0

def mkdir(path):
# 引入模块
import os

# 去除首位空格
path = path.strip()
# 去除尾部 \ 符号
path = path.rstrip("\\")

# 判断路径是否存在
# 存在 True
# 不存在 False
isExists = os.path.exists(path)

# 判断结果
if not isExists:
# 如果不存在则创建目录
# 创建目录操作函数
os.makedirs(path)
os.chdir(path)
print path+ ' 创建成功'
return True
else:
# 如果目录存在则不创建，并提示目录已存在
os.chdir(path)
print path + ' 目录已存在'
return False

def loadPage(url):
"""
作用：根据url发送请求，获取服务器相应文件
:param url: 需要爬取的url地址
:param fileName: 处理的文件名
:return: 读出来的内容
"""

sslNoVerify = ssl._create_unverified_context()
headers = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/604.4.7 (KHTML, like Gecko) Version/11.0.2 Safari/604.4.7"}

request = urllib2.Request(url,headers=headers)
html = urllib2.urlopen(request,context=sslNoVerify ).read()

html = re.sub(r"<!--","<div>",html)
html = re.sub(r"--\>", "</div>", html)

content = etree.HTML(html)
link_list = content.xpath('//div[@class="t_con cleafix"]//a[@class="j_th_tit "]/@href')

for a in link_list:
subfullUrl = "https://tieba.baidu.com/"+a
gotoSubHtml(subfullUrl )
return html

def gotoSubHtml(url):
sslNoVerify = ssl._create_unverified_context()
headers = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/604.4.7 (KHTML, like Gecko) Version/11.0.2 Safari/604.4.7"}

request = urllib2.Request(url, headers=headers)
html = urllib2.urlopen(request, context=sslNoVerify).read()

html = re.sub(r"<!--", "<div>", html)
html = re.sub(r"--\>", "</div>", html)

content = etree.HTML(html)
link_list = content.xpath('//div[@class="d_post_content_main "]//img[@class="BDE_Image"]/@src')
for a in link_list:
response = urllib2.urlopen(a)

pic = response.read()
global totalcount
with open(str(totalcount)+"_"+a[-8:], 'wb') as f:
f.write(pic)

totalcount += 1
print "---下载一张图片成功 ，第"+str(totalcount)+ "张"

def saveImg(url):
headers = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/604.4.7 (KHTML, like Gecko) Version/11.0.2 Safari/604.4.7"}

response = urllib2.Request()

def writePage(html,fileName):
"""
作用：将html内容写入到本地
:param html: 服务器相应文件内容
:param fileName: 保存的文件名
:return:
"""

print "正在保存" + fileName
#文件写入
with open(fileName,"w") as f:
f.write(html)

print "-" * 30

def tiebaSpider(url,beginPage,endPage,folderName):
"""
作用：贴吧爬虫调度器，负责组合吃力每个页面的url
:param url: 贴吧url的前部分
:param beginPage: 起始页
:param endPage: 结束页
:return: nil
"""

for page in range(beginPage,endPage+1):
pn = (page -1) * 50
fileName = "第" + str(page) + "页.html"
fullUrl = url + "&pn=" + str(pn)
# print fullUrl

mkdir(folderName)
html = loadPage(fullUrl)
print "谢谢使用"
print "-"*30

if __name__ == "__main__":
kw = raw_input("请输入需要爬取的贴吧名：")
beginPage =int(raw_input("请输入起始页:"))
endPage= int(raw_input("请输入结束页:"))

url = "http://tieba.baidu.com/f?"

key = urllib.urlencode({"kw":unicode(kw, "utf-8").encode('gb2312')})
print "key = " + key
fullUrl = url + key

tiebaSpider(fullUrl,beginPage,endPage,kw)

爬出来的结果