使用多进程，多线程和协程实现爬虫，使用queue完成数据通信

一直在看python的多进程，多线程和协程，动手试了试写个爬虫练习一下。

多线程版本 

[code]import ssl
import threading
import time
import urllib.request
from queue import Queue
from lxml import etree

class myThread(threading.Thread):
def __init__(self,url,q):
super(myThread, self).__init__()
self.q = q
self.headers = {

'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36'

}
self.url = url

def run(self):
self.parse_html()

def send_request(self, url):
request = urllib.request.Request(url=url, headers=self.headers)
#我这里需要加上ssl认证
ssl._create_default_https_context = ssl._create_unverified_context
response = urllib.request.urlopen(request)

html_response = response.read().decode('utf-8')

return html_response

def parse_html(self):
resp = self.send_request(self.url)
html = etree.HTML(resp)
div_list = html.xpath("//div[@class='index_only']//div[@id='container']/div")
for div in div_list:
name = div.xpath(".//img/@alt")[0]
src = div.xpath(".//img/@src2")[0]
# print(src)
# print(name)
self.q.put(name + '\t' + src)

def main():
q = Queue()
base_url = 'http://sc.chinaz.com/tupian/renwutupian_'

url_list = [base_url + str(num)+'.html' for num in range(2, 100)]
print(url_list)
Thread_list = []
# 创建并启动线程
for url in url_list:
p = myThread(url, q)
p.start()
Thread_list.append(p)
# 让主线程等待子线程执行完成
for i in Thread_list:
i.join()
#打印队列里存入的信息
while not q.empty():
print(q.get())

if __name__ == "__main__":
start = time.time()
main()
print('总共耗时：%s' % (time.time() - start))

多进程版本

[code]import multiprocessing
import ssl
import time
import urllib.request
from queue import Queue

from lxml import etree

class ZhanzProcess(multiprocessing.Process):

def __init__(self, url, q):
super(ZhanzProcess, self).__init__()

self.q = q
self.url = url
self.headers = {

'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36'

}

def run(self):
self.parse_html()

def send_request(self, url):
request = urllib.request.Request(url=url, headers=self.headers)
ssl._create_default_https_context = ssl._create_unverified_context
response = urllib.request.urlopen(request)
html_response = response.read().decode('utf-8')

return html_response

def parse_html(self):
res = self.send_request(self.url)
html = etree.HTML(res)
div_list = html.xpath("//div[@class='index_only']//div[@id='container']/div")
for div in div_list:
name = div.xpath(".//img/@alt")[0]
src = div.xpath(".//img/@src2")[0]
# print(name + '\t' + src)

self.q.put(name + '\t' + src)

def main():
q = Queue()
base_url = 'http://sc.chinaz.com/tupian/renwutupian_'

url_list = [base_url + str(num) + '.html' for num in range(2, 100)]
print(url_list)
process_list = []
for url in url_list:
# print(url)
p = ZhanzProcess(url, q)
p.start()
process_list.append(p)

for i in process_list:
i.join()

while not q.empty():
print(q.get())

if __name__ == "__main__":
start = time.time()
main()
print('总共耗时：%s' % (time.time() - start))

协程版本

[code]import ssl
import time
import urllib.request
from queue import Queue

import gevent as gevent
from lxml import etree

# 打猴子补丁
from gevent import monkey

monkey.patch_all()

class Zzspider(object):

def __init__(self):
self.q = Queue()
self.headers = {

'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.109 Safari/537.36'

}

def run(self, url):
self.parse_html(url)

def send_request(self, url):
request = urllib.request.Request(url=url, headers=self.headers)
ssl._create_default_https_context = ssl._create_unverified_context
response = urllib.request.urlopen(request)
html_response = response.read().decode('utf-8')

return html_response

def parse_html(self, url):
res = self.send_request(url)
html = etree.HTML(res)
div_list = html.xpath("//div[@class='index_only']//div[@id='container']/div")
for div in div_list:
name = div.xpath(".//img/@alt")[0]
src = div.xpath(".//img/@src2")[0]

self.q.put(name + '\t' + src)

def main(self):
base_url = 'http://sc.chinaz.com/tupian/renwutupian_'

url_list = [base_url + str(num) + '.html' for num in range(2, 100)]
job_list = [gevent.spawn(self.run, url) for url in url_list]
# 让线程等待所有任务完成，再继续执行。
gevent.joinall(job_list)
while not self.q.empty():
print(self.q.get())

if __name__ == '__main__':
start = time.time()
zz = Zzspider()
zz.main()
print('总共耗时：%s' % (time.time() - start))