Python3.6 学习爬虫入门篇

# -*- coding:utf-8 -*-
# 糗事百科爬虫
import urllib.request
import os

class Crawler:

# 获取HTML信息
def open_url(self, url):
req = urllib.request.Request(url)
# 伪装成浏览器访问
req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko)'
' Chrome/49.0.2623.112 Safari/537.36')
response = urllib.request.urlopen(req)
return response.read()

# 查找图片
def find_img(self, url):
# 打开地址并转码
html = self.open_url(url).decode('utf-8')
# 获取html中第一个'img src='的下标
img_start = html.find('img src=')
img_url_arr = []
while img_start != -1:
# 从img_start到img_start + 200查找,获取第一个'.jpg'的下标
img_end = html.find('.jpg', img_start, img_start + 200)
if img_end != -1:
# 根据img_start和img_end截取字符串,获得img地址
img_url = html[img_start+9:img_end+4]
# 加入数组
img_url_arr.append(img_url)
# 获取已'.JPEG'结尾的图片
# img_end_jpeg = html.find('.JPEG', img_start, img_start + 200)
# if img_end_jpeg != -1:
#     img_url = html[img_start+9:img_end_jpeg+5]
#     img_url_arr.append(img_url)
# 继续查找
img_start = html.find('img src=', img_start + 9)
return img_url_arr

def save_img(self, img_url_arr):
# 遍历数组
for img_url in img_url_arr:
print("下载图片地址:%s" % img_url)
# 从倒数第一个'/'到结尾截取字符串作为文件名
filename = img_url.split('/')[-1]
with open(filename, 'wb') as f:
# 打开地址.这里视情况而定,有些网页的图片包含"http:"
img = self.open_url('http:' + img_url)
f.write(img)

def start_download(self, page=10):
# %d处为页码,后面用%替换
url = "http://www.qiushibaike.com/imgrank/page/%d/"
try:
# 创建文件夹
os.mkdir('糗事百科图')
except Exception:
print("")
# 将工作环境移动到创建的文件夹下面
os.chdir('糗事百科图')
for p in range(1, page + 1):
print("开始第%d页" % p)
# %替换页码,获取图片地址
img_url_arr = self.find_img(url % p)
# 保存图片
self.save_img(img_url_arr)

crawler = Crawler()
crawler.start_download(3)