Python初级爬虫（利用多任务协程爬取虎牙MM图片）

Python多任务协程下载虎牙直播MM图片

# coding = utf-8
import re
import gevent
from gevent import monkey, pool
import time, random
import urllib.request

monkey.patch_all()

def downloader_img(img_url, img_name):
"""下载图片"""
req = urllib.request.urlopen(img_url)
img_content = req.read()
# 下载图片保存
with open(r"/home/python/Desktop/01_Python基础/day03/正则表达式/photo/mm%s" % img_name, "wb") as f:
f.write(img_content)
time.sleep(random.random())

def main():
p1 = pool.Pool(10)  # 加入协程
# 读取文件
locals_addr = r"/home/python/Desktop/01_Python基础/day03/正则表达式/1.txt"
url_txt = open(locals_addr, "r", encoding=" utf-8")
url_content = url_txt.read()
jpg_lists = re.findall(r"https://.*?\.jpg", url_content)  # 正则匹配网页的有效地址
num = 0
list1 = []  # 定义一个列表用来存网址
# 遍历图片列表
for j in jpg_lists:
list1.append(gevent.spawn(downloader_img, j, "%d.jpg" % num))
num += 1
gevent.joinall(list1)

if __name__ == '__main__':
main()

方法
主函数
效果