爬取图片网站上的美女图片

#coding=utf-8

from bs4 import BeautifulSoup as BS4
import requests
import wget
import sys
import chardet
import os

__author__ = 'Administrator'

HOST = 'http://www.5442.com/meinv/'
START_URL = 'http://www.5442.com/meinv/'
ATTR_HREF = 'href'
ATTR_TITLE = 'title'
ATTR_ALT = 'alt'
ATTR_SRC = 'src'
HOST_CONTENT_ENCODING = 'GB2312'
MAIN_DOWNLOAD_FOLDER = 'G:/meinv/'
NEXT_ALBLUM = '可爱熊吖BOBO'
ALREDY_TO_DOWNLOAD_ALBLUMN = False
MAX_THEME_PAGE_COUNT = 200

def utf82GBK(s):
return s.decode('utf-8').encode('gb2312')

def gbkprint(s):
print(utf82GBK(s))

def uni2utf8(s):
return s.encode('utf-8')

def gbk2utf8(s):
return s.decode('gb2312','ignore').encode('utf-8','ignore')

def myprint(s):
print(s)

_ = myprint

def url2bs4(url):
res = requests.get(url)
if res.status_code != 200:
return

#enc = chardet.detect(res.content)
#print enc
res.encoding = HOST_CONTENT_ENCODING
content = gbk2utf8(res.content)
ret = BS4(content)
return ret

def read_tags(fromURL):
bs = url2bs4(fromURL)
tags_objects = bs.find_all("a",attrs={'class':'yxtag'})
ret = {}
for t in tags_objects:
ret[uni2utf8(t.text)] = uni2utf8(t[ATTR_HREF])

return ret

def dump_tags(tags):
for k in tags:
_('名称:%s 地址：%s' % (k, tags[k]))

def next_download_addr(album_name, folder_name):
full_name = os.path.join(MAIN_DOWNLOAD_FOLDER, album_name)#, folder_name)
folder_full_name = utf82GBK(full_name)
if not os.path.isdir(folder_full_name):
os.makedirs(folder_full_name)

next_num = 1
while True:
file_name = os.path.join(folder_full_name, str(next_num)+ '.jpg')
if not os.path.isfile(file_name):
break
next_num = next_num + 1

return file_name

def download_album_one_page(url, album_name):
bs = url2bs4(url)
cur = bs.find(name='p', attrs={'id': 'contents',})
imgs = cur.find_all(name='img')
for img in imgs:
wget.download(img[ATTR_SRC], next_download_addr(album_name, uni2utf8(img[ATTR_ALT])))

def download_album(url, name):
global ALREDY_TO_DOWNLOAD_ALBLUMN
if not ALREDY_TO_DOWNLOAD_ALBLUMN:
_('已经下载 %s' % (name,))
if NEXT_ALBLUM in name:
_('准备开始继续下载...')
ALREDY_TO_DOWNLOAD_ALBLUMN = True
return

_('正在下载影集:%s 地址：%s' % (name, url))
bs = url2bs4(url)
page = bs.find(name='div', attrs={'class':'page'})
lis = page.find_all(name='li')
sum_page = 0
for li in lis:
page_name = uni2utf8(li.a.text)
if page_name.startswith('共'):
sum_page = int(page_name[3:-5])
break

download_album_one_page(url, name)
cur_page = 2
prefix = url[:url.rfind('.')]
while cur_page <= sum_page:
download_album_one_page('%s_%d.html' % (prefix,cur_page,), name)
cur_page = cur_page + 1

def process_one_tag_page(tag_name, tag_url):
_('当前主题:%s' % (tag_name,))
bs = url2bs4(tag_url)
bricks = bs.find_all(name='div', attrs={'class':'item masonry_brick masonry-brick'})
for i in bricks:
album = i.find(name='a')
if not album:
continue

download_album(uni2utf8(album[ATTR_HREF]), uni2utf8(album[ATTR_TITLE]))

def process_tags(tags):
for k in tags:
#if '全部' in k:
#   continue
#if not '人体艺术' in k and not '丝袜' in k:
#   continue
if not '丝袜' in k:
continue

#process_one_tag_page(k, tags[k])
bs_first_page = url2bs4(tags[k])
page_num = bs_first_page.find(name='li', attrs={'class':'pageinfo',})
if not page_num:
continue
page_num = page_num.text
seprator_index = page_num[page_num.find('/')+1:-1]
page_num = int(seprator_index)

prefix_url = tags[k][0:tags[k].rfind('.')]
url_list = [tags[k],]
start_page_num = 2
while start_page_num < page_num:
next = '%s/%d.html' % (prefix_url,start_page_num)
if len(url_list) >= MAX_THEME_PAGE_COUNT:
break
url_list.append(next)
start_page_num = start_page_num + 1

for n  in url_list:
process_one_tag_page(k, n)

if __name__ == '__main__':
#reload(sys)
#sys.setdefaultencoding('gb2312')

tags = read_tags(START_URL)
#dump_tags(tags)
process_tags(tags)