python3.6抓取代理ip

有时候我们会用到代理ip，下面我们便抓取代理ip网站的ip

代理网站 点击打开链接

import requests
import random
from bs4 import BeautifulSoup

class Spider_Ips():
url = 'http://www.xicidaili.com/wn/'
headers = [
{'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:34.0) Gecko/20100101 Firefox/34.0'},
{'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'},
{'User-Agent': 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.12 Safari/535.11'},
{'User-Agent': 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)'},
{'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:40.0) Gecko/20100101 Firefox/40.0'},
{'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/44.0.2403.89 Chrome/44.0.2403.89 Safari/537.36'}
]
MAX_RETRIES = 20
session = requests.Session()
adapter = requests.adapters.HTTPAdapter(max_retries=MAX_RETRIES)
session.mount('https://', adapter)
session.mount('http://', adapter)

def list_ip(self):
req = self.session.get(self.url, headers=random.choice(self.headers))
response = BeautifulSoup(req.text, 'html.parser')
res_obj = response.findAll('table')
tab = res_obj[0]
for item in tab.findAll('tr'):
tds = item.findAll('td')
if tds:
with open('HTTPS.txt', 'a', encoding='utf-8') as f:
print(tds[1].text+':'+tds[2].text)
f.write(tds[1].text+':'+tds[2].text+'\n')

def __init__(self):
self.list_ip()

s_ip = Spider_Ips()

运行便可 ，抓取到txt文本中，结果如下图所示，我们便可以使用下面的ip了


当然我们也可以对这些ip存入到数据库等其他操作