导入requests模块

cmd–>pip install requests

测试

import requests
r = requests.get("http://www.baidu.com")
print(r.status_code)
r.encoding = 'utf-8'
print(r.text)

requests 的主要方法

requests方法

爬取网页的通用代码框架

def getHTMLText(url):
try:
r = requests.get(url, timeout=30)
r.raise_for_status() #如果状态不是200，引发HTTPError异常
r.encoding = r.apparent_encoding
return r.text
except:
return "产生异常"

爬取b站网页数据

import requests
def getHTMLText(url):
try:
r = requests.get(url)
r.raise_for_status()
r.encoding = r.apparent_encoding
return r.text
except:
return "出现异常"

url = "https://www.bilibili.com/video/BV1Vi4y147tt"
print(getHTMLText(url))

爬取亚马逊页面数据

import requests
def getHTMLText(url):
try:
r = requests.get(url)
r.raise_for_status()
r.encoding = r.apparent_encoding
return r.text
except:
r.status_code
return "出现异常"

url = "https://www.amazon.cn/dp/B0871VRY5G?ref_=Oct_DLandingS_D_1224d3f5_61&smid=A26HDXW89ZT98L"
print(getHTMLText(url))
------------------------------------输出结果
503
出现异常

修改请求头

kv = {'user-agent': 'Mozilla/5.0'}
r = requests.get(url, headers = kv)

搜索引擎

import requests
kv = {'wd':'python'}
r = requests.get("http://www.baidu.com/s", params=kv)
print(r.status_code)
print(r.request.url)
print(len(r.text))

网络图片爬取和存储

import requests
path = 'D:/abc.jpg'
url = 'http://img0.dili360.com/ga/M00/3A/FE/wKgBzFWsfZGATvOZAAprl5aLOOw352.tub.jpg'
r = requests.get(url)
print(r.status_code)
f = open(path,'wb')
print(f.write(r.content))
f.close()

标准格式

import requests
import os
url = 'http://img0.dili360.com/ga/M00/3A/FE/wKgBzFWsfZGATvOZAAprl5aLOOw352.tub.jpg'
root = 'd://jpg//'
path = root + url.split('/')[-1]
try:
if not os.path.exists(root):
os.mkdir(root)
if not os.path.exists(path):
r = requests.get(url)
f = open(path,'wb')
f.write(r.content)
f.close()
print("保存成功")
else:
print('文件已存在')
except:
print('爬取失败')