自己爬虫的几个案例

原创，如转载请注明来源https://www.cnblogs.com/sogeisetsu/

爬取中国大学排名

#这个只用到了requests 和 bs4
#爬取大学排名
import requests
from bs4 import BeautifulSoup as bs
def grthtml(url):
demo=requests.get(url)
demo.encoding=demo.apparent_encoding
demo=demo.text
return(demo)
def listhtml(ulist,html):
soup=bs(html,"html.parser")
soup=soup.tbody
for tr in soup("tr"):
tds=tr("td")
ulist.append([tds[0].string,tds[1].string,tds[2].string])
def pmhtml(ulist,num):
print('{0:^10}\t{1:{3}^7}\t{2:^10}'.format("排名","校名","地址",chr(12288)))
for i in ulist[0:num]:
print("{0:^10}\t{1:{3}^10}\t{2:^10}".format(i[0],i[1],i[2],chr(12288)))
if __name__=="__main__":
url="http://www.zuihaodaxue.cn/zuihaodaxuepaiming2016.html"
html=grthtml(url)
uinfo=[]
listhtml(uinfo,html)
num=int(input())
pmhtml(uinfo,num)

爬取今日头条热点

# 这个爬取的是动态页面
# 爬取动态页面，今日头条
# 源文件里是没有内容的只有js
import requests
from bs4 import BeautifulSoup as bs
import json
def gethtml(url):
try:
#通过F12检查出来自下面这个链接
# url="https://www.toutiao.com/api/pc/feed/?category=news_hot&utm_source=toutiao&widen=1&max_behot_time=0&max_behot_time_tmp=0&tadrequire=true&as=A1D5FDAAB5194AE&cp=5DA599948ACE7E1&_signature=TBeQ-wAAEbOkzbKGAd3hQUwXkO"
head={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36','Cookie':#cookie就用自己的吧}
r=requests.get(url,headers=head)
return r.text #返回json文件
except:
print("oneerror")
def getulist(html,list):
try:
soup=json.loads(html)
soupdata=soup['data']
for one in soupdata:
a='https://www.toutiao.com/a'+one['group_id']
list.append([one['title'],one['abstract'],a])
except:
print("twoerror")
def printulist(list):
for i in list:
print("title:\n{}".format(i[0]))
print("简介：\n{}".format(i[1]))
print("链接:{:^30}".format(i[2]))
if __name__=="__main__":
url = "https://www.toutiao.com/api/pc/feed/?category=news_hot&utm_source=toutiao&widen=1&max_behot_time=0&max_behot_time_tmp=0&tadrequire=true&as=A1D5FDAAB5194AE&cp=5DA599948ACE7E1&_signature=TBeQ-wAAEbOkzbKGAd3hQUwXkO"
html=gethtml(url)
relist=[]
getulist(html,relist)
printulist(relist)

爬取知乎热点

#爬取知乎
import requests
from bs4 import BeautifulSoup as bs
def gethtml(url):
headers={'user-agent':'Mozila/5.0','Cookie':#cookie就用自己的吧}
r=requests.get(url,headers=headers)
r.raise_for_status()
r.encoding=r.apparent_encoding
r=r.text
return r
def getlist(ulist,html):
soup=bs(html,"html.parser")
soup = soup("div", "HotList-list")
c=soup[0].contents
for sect in c:
if sect.a.p:
pp=sect.a.p.string
else:
pp="该题目下没有相应简介"
ulist.append([sect.a.h2.string,pp,sect.a.attrs['href']])
def printlist(ulist,num):
top=1
for i in ulist[0:num]:
print("top{}".format(top))
top+=1
print("题目：{}".format(i[0]))
print("优秀回答简介：\n{}".format(i[1]))
print("链接\n{}".format(i[2]))
# print("\n")
if __name__ == "__main__":
url="https://www.zhihu.com/hot"
ulist=[]
print("您想看知乎热榜top几？")
num=int(input())
html=gethtml(url)
getlist(ulist,html)
printlist(ulist,num)