Python爬取扇贝“【无老师】7天搞定TOEFL单词”

#!/usr/bin/env python3
from bs4 import BeautifulSoup
import requests
import csv
import bs4
import codecs

#检查url地址并返回网页contents
def check_link(url):
try:

r = requests.get(url)
r.raise_for_status()
r.encoding = r.apparent_encoding
return r.text
except:
print('无法链接服务器！！！')

# 判断一个unicode是否是英文字母
def is_alphabet(uchar):
if ('\u0041' <= uchar<='\u005a') or ('\u0061' <= uchar<='\u007a'):
return True
else:
return False

#爬取表格数据
def get_contents(urlist):
'''urlist: a list containing all the useful urls'''
result = []
for url in urlist:
content = check_link(url)
soup = BeautifulSoup(content,'lxml')
trs = soup.find_all('tr')
for tr in trs:
ui = []
for td in tr:
ui.append(td.string)
result.append(ui)
return result

#爬取URL链接
def get_urls(url_content,root_url="https://www.shanbay.com"):
'''get all the urls from url_content and save into a list'''
ulist = []
soup = BeautifulSoup(url_content,'lxml')
urls=soup.find_all('a')
for url in urls:
try:
if url.string.startswith('【无老师7天TOEFL】List'):
ulist.append(root_url+url.get('href'))
for j in range(2,11):
extend_url = root_url+url.get('href')+'?page='+str(j)
ulist.append(extend_url)
except:
pass
return ulist

def save_contents(result):
'''result: all the useful result from urls'''
with codecs.open('shanbay.csv', 'w', 'utf_8_sig') as f:
writer = csv.writer(f)
for i in range(len(result)):
try:
if is_alphabet(result[i][1]):
writer.writerow([result[i][1],result[i][3]])
print("write in line:",i)
except:
print("error in line:{}, contents is:{}".format(i,result[i]))

def main():
src_url = "https://www.shanbay.com/wordbook/5440/"
# get the contents in source page
src_content = check_link(src_url)

#get all the useful urls in source page
urls = get_urls(src_content)

# scrapy all the useful contents from all the urls
result = get_contents(urls)
#save all the useful contents into csv
save_contents(result)

main()

效果图：