Python爬虫——4.6使用requests和正则表达式、随机代理爬取淘宝网商品信息

# coding:utf-8
'''
使用requests模块，使用代理，进行淘宝网商品信息的爬取
'''
# 引入需要的模块
import requests
import random
import re
from lxml import etree

# 定义免费代理列表
proxy_list = [{"http": "116.8.83.3:8118"}, {"http": "116.8.83.3:8118"},
{"http": "113.89.59.161:8118"}, {"http": "113.67.183.196:8118"},
{"http": "180.155.135.224:31425"}, {"http": "123.161.153.238:22593"}]
# 定义访问路由
url = 'https://uland.taobao.com/sem/tbsearch?refpid=mm_26632360_8858797_29866178&keyword=%E5%A5%B3%E8%A3%85&clk1=51ae9dffa6be64388c59f2c5c20274c7&upsid=51ae9dffa6be64388c59f2c5c20274c7'
# 设置请求头
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
}
# 发送请求
response = requests.get(url, headers=headers, proxies=random.choice(proxy_list), verify=False)
content = response.text

# 图片名称的获取
reg_name = r'<span class="title" title=".*">(.*?)</span>'
pattern_name = re.compile(reg_name)
titles_list = pattern_name.findall(content)
titles = []
for title in titles_list:
titles.append(title)
print "商品名称爬取结束"
# 图片的获取
reg_name = r'<img data-ks-lazyload="(.*?)" s'
pattern_name = re.compile(reg_name)
imgs_list = pattern_name.findall(content)
imgs = []
# 修改连接
for i in range(0, len(imgs_list) - 5):
imgs_list[i] = "https:" + imgs_list[i]
# 获取图片的url,进行第二次爬取

i = 1
for img in imgs_list:
filename = str(i) + '_' + img[-20:]
f = open('tb_img/%s' % filename, 'wb')
url = img
response = requests.get(url, verify=False)
f.write(response.content)
f.close()
i += 1

print "图片爬取结束"