Python-爬取校花网视频(单线程和多线程版本)

一、参考文章

    python爬虫爬取校花网视频，单线程爬取

    爬虫----爬取校花网视频，包含多线程版本

    上述两篇文章都是对校花网视频的爬取，由于时间相隔很久了，校花网上的一些视频已经不存在了，因此上述文章中的代码在运行时会出现一些异常，本篇文章主要是对上述文章中的代码进行了优化和异常处理，在次做笔记记录方便以后查阅，修改如下：

1、添加的异常处理如下红色部分代码

二、单线程版本

#-*- coding=utf-8 -*-
import re
import requests
import hashlib
import time
import os

header = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 UBrowser/6.1.2107.204 Safari/537.36',
'Referer':'http://www.xiaohuar.com'
}

def get_index(url):
respose = requests.get(url, headers = header)
if respose.status_code == 200:
return respose.text

def parse_index(res):
urls = re.findall(r'class="items".*?href="(.*?)"', res, re.S)  # re.S 把文本信息转换成1行匹配
return urls

def get_detail(urls):
for url in urls:
if not url.startswith('http'):
url='http://www.xiaohuar.com%s' %url
result = requests.get(url, headers = header)
if result.status_code == 200 :
mp4_url_list = re.findall(r'id="media".*?src="(.*?)"', result.text, re.S)
if mp4_url_list:
mp4_url = mp4_url_list[0]
save(mp4_url)

path = os.getcwd() + '/video/'

def save(url):
try:#下载视频加异常处理
video = requests.get(url, headers = header)
except requests.exceptions.RequestException as e :
print(repr(e))
return

if video.status_code == 200:
m = hashlib.md5()
m.update(url.encode('utf-8'))
m.update(str(time.time()).encode('utf-8'))
filename = r'%s.mp4' % m.hexdigest()
filepath = path + filename
print(filepath)
with open(filepath, 'wb') as f:
f.write(video.content)
else:
print(f'视频不存在了：{url}')

def main():
for i in range(5):
res1 = get_index('http://www.xiaohuar.com/list-3-%s.html' % i )#拿第一页数据
res2 = parse_index(res1)#提取第一页上的所有url
get_detail(res2)#下载url集合上的视频

if __name__ == '__main__':
main()

三、多线程版本

#-*- coding=utf-8 -*-
# 异步，多线程优化下载速度

import requests
import re
import os
import hashlib,time
from concurrent.futures import ThreadPoolExecutor

p = ThreadPoolExecutor(30)

def get_index(url):
response = requests.get(url)
if response.status_code == 200:
return response.text

def parse_index(res):
res = res.result()
urls = re.findall(r'class="items".*?href="(.*?)"', res, re.S)

p.submit(get_detail, urls)

def get_detail(urls):
for url in urls:
if not url.startswith('http'):
url='http://www.xiaohuar.com%s' %url
r1=requests.get(url)
if r1.status_code == 200:
url_list=re.findall(r'id="media".*?src="(.*?)"', r1.text, re.S)
if url_list:
mp4_url = url_list[0]
save(mp4_url)

path = os.getcwd() + '/video_mutil/'
if not os.path.exists(path):
os.makedirs(path)

def save(url):
try:#下载视频做异常处理，视频可能不存在了
r2 = requests.get(url)
except requests.exceptions.RequestException as e :
print(repr(e))
return

if r2.status_code == 200:
m=hashlib.md5()
m.update(url.encode('utf-8'))
m.update(str(time.time()).encode('utf-8'))
filename = '%s.mp4' %m.hexdigest()
file_path = path + filename
with open(file_path,'wb') as f:
f.write(r2.content)
print('视频下载完成：%s' % file_path)
else:
print(f'视频不存在了：{url}')

def main():
for i in range(5):
p.submit(get_index, 'http://www.xiaohuar.com/list-3-%s.html' % i).add_done_callback(parse_index)

if __name__ == '__main__':
main()

四、资源下载

    资源下载地址：Python爬取校花网视频-单线程和多线程版本

转载声明：本站文章无特别说明，皆为原创，版权所有，转载请注明：朝十晚八