初识python爬虫

 没有反爬技术下的代码：

[code]from lxml import etree
import requests

url = 'https://movie.douban.com/chart'
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) .......'}#在开发者模式中获取
response = requests.get(url,headers=headers)#get方法请求响应
html = etree.HTML(response.content)#获取代码字节类型
href_list = html.xpath('//a/img/@src')#xpath获取html里的数据
for img in href_list:
img_response = requests.get(img,headers=headers)
with open(img[-15:],"wb") as f:
f.write(img_response.content)

网页隐藏数据后的代码：

[code]import re #re模块 正则匹配
import requests
from lxml import etree

headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ...'}
url = "https://www.pearvideo.com/category_8"

response = requests.get(url,headers=headers)
html = etree.HTML(response.content)
video_url = html.xpath('//div[@class="vervideo-bd"]/a/@href')
print(video_url)
real_video_url = []
main_url = "https://www.pearvideo.com/"
for i in video_url:
real_video_url.append(main_url+i)
print(real_video_url)

video_list = []
for video in real_video_url:
video_response = requests.get(video,headers=headers)
#正则出要查找的字符串
reg = 'ldUrl="",srcUrl="(.*?)"'
video_list.append(re.findall(reg,video_response.content.decode())[0])

for index,i in enumerate(video_list):
video_response = requests.get(i,headers = headers)
with open(str(index)+i[-4:],"wb") as f:
f.write(video_response.content)

print(video_list)