Python3抓取煎蛋妹子图

#coding=utf-8

import httplib2,re,random,os
urls=[]
imgUrls=[]
h = httplib2.Http(".cache")
reg=r'img src="(http.*?gif)" /></p>'
imgre = re.compile(reg)
filepath='/home/arthur/myapps/imgs'
prefix_url='http://jandan.net/ooxx/page-'

def createJDUrlList(size):
#煎蛋妹子图前900页无内容
for i in range(900,size):
urls.append(prefix_url+str(i))

def getImgUrlsFromPageUrl(pageUrl):
i_headers = {"User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:39.0) Gecko/"+str(random.random())+" Firefox/39.0","Accept": "text/plain"}
response,content = h.request(pageUrl,headers=i_headers)
content=content.decode('utf-8')
pageImgUrls=re.findall(imgre,content)
imgUrls.extend(pageImgUrls)
print('pageImgUrls:'+str(len(pageImgUrls)))

#保存图片到文件，如果图片文件已经存在，则跳过
def getAndSaveImg(imgUrl):
filename=imgUrl.split('/')[-1]
file_path=filepath+'/'+filename
if os.path.exists(file_path):
return
print('save:['+imgUrl+'] to ['+file_path+']')
try:
#经测试煎蛋有防止爬虫功能，所以改变User-Agent，伪装成新请求
i_headers = {"User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:39.0) Gecko/"+str(random.random())+" Firefox/39.0","Accept": "text/plain"}
response,content = h.request(imgUrl,headers=i_headers)
save2File(fcontent=content,mode='wb',path=file_path)
except Exception as e:
print(e)

def save2File(fcontent,path,mode):
wfile=open(path,mode)
wfile.write(content)
wfile.close()

def getJiandanMM():
#最新页面
createJDUrlList(1474)
getImgUrlsFromPageUrl(urls[-1])
for url in urls:
getImgUrlsFromPageUrl(url)
#将图片url暂存到文件，下载图片出现错误可以直接从文件载入图片url
pre=str(imgUrls)
pre=pre.replace("[","")
pre=pre.replace("]","")+"\n"
save2File(fcontent=pre,mode='w',path=filepath+'/allurls_2.txt')

for imgUrl in imgUrls:
print(imgUrl)
getAndSaveImg(imgUrl)

def getJiandanMMFromFile(filepath):
wfile=open(filepath,'r')
line=wfile.readline()
wfile.close()
urls=line.replace('\n','').replace('\'','').replace(' ','').split(',')

for url in urls:
saveImg(url)
if __name__ == "__main__":
getJiandanMM()