爬取搜索出来的电影的下载地址并保存到excel

一、背景

利用Requests模块获取页面，BeautifulSoup来获取需要的内容，最后利用xlsxwriter模块讲内容保存至excel，首先通过讲关键字收拾出来的页面获取到子页面的url，然后再次去抓取获取到子页面的信息保存到excel

二、代码

编写了两个模块，geturldytt和getexceldytt，最后在main内调用



git源码地址

geturldyttd代码如下：

#!/bin/env python
# -*- coding:utf-8 -*-
from urllib import parse
import requests
from bs4 import BeautifulSoup
#http://s.dydytt.net/plus/search.php?keyword=%BF%C6%BB%C3&searchtype=titlekeyword&channeltype=0&orderby=&kwtype=0&pagesize=10&typeid=0&TotalResult=279&PageNo=4
class get_urldic:
def __init__(self):
self.first_url = 'http://s.dydytt.net/plus/search.php?'
self.second_url = '&searchtype=titlekeyword&channeltype=0&orderby=&kwtype=0&pagesize=10&typeid=0&TotalResult=279&PageNo='
self.info_url = 'http://s.dydytt.net'
#获取搜索关键字
def get_url(self):
urlList = []
# first_url = 'http://s.dydytt.net/plus/search.php?'
# second_url = '&searchtype=titlekeyword&channeltype=0&orderby=&kwtype=0&pagesize=10&typeid=0&TotalResult=279&PageNo='
try:
search = input("Please input search name:")
dic = {'keyword':search}
keyword_dic = parse.urlencode(dic,encoding='gb2312')
page = int(input("Please input page:"))
except Exception as e:
print('Input error:',e)
exit()
for num in range(1,page+1):
url = self.first_url + str(keyword_dic) + self.second_url + str(num)
urlList.append(url)
print("Please wait....")
print(urlList)
return urlList,search

#获取网页文件
def get_html(self,urlList):
response_list = []
for r_num in urlList:
request = requests.get(r_num)
response = request.content.decode('gbk','ignore').encode('utf-8')
response_list.append(response)
return response_list

#获取blog_name和blog_url
def get_soup(self,html_doc):
result = {}
for g_num in html_doc:
soup = BeautifulSoup(g_num,'html.parser')
context = soup.find_all('td', width="55%")
for i in context:
title=i.get_text()
result[title.strip()]=self.info_url + i.b.a['href']
return result

def get_info(self,info_dic):
info_tmp = []
for k,v in info_dic.items():
print(v)
response = requests.get(v)
new_response = response.content.decode('gbk').encode('utf-8')
soup = BeautifulSoup(new_response, 'html.parser')
info_dic = soup.find_all('div', class_="co_content8")
info_list1= []
for context in info_dic:
result = list(context.get_text().split())
for i in range(0, len(result)):
if '发布' in result[i]:
public = result[i]
info_list1.append(public)
elif "豆瓣" in result[i]:
douban = result[i] + result[i+1]
info_list1.append(douban)
elif "【下载地址】" in result[i]:
download = result[i] + result[i+1]
info_list1.append(download)
else:
pass
info_tmp.append(info_list1)
return info_tmp

if __name__ == '__main__':
blog = get_urldic()
urllist, search = blog.get_url()
html_doc = blog.get_html(urllist)
result = blog.get_soup(html_doc)
for k,v in result.items():
print('search blog_name is:%s,blog_url is:%s' % (k,v))
info_list = blog.get_info(result)
for list in info_list:
print(list)

getexceldytt代码如下：

#!/bin/env python
# -*- coding:utf-8 -*-
# @Author  : kaliarch

import xlsxwriter

class create_excle:
def __init__(self):
self.tag_list = ["movie_name", "movie_url"]
self.info = "information"

def create_workbook(self,search=" "):
excle_name = search + '.xlsx'
#定义excle名称
workbook = xlsxwriter.Workbook(excle_name)
worksheet_M = workbook.add_worksheet(search)
worksheet_info = workbook.add_worksheet(self.info)
print('create %s....' % excle_name)
return workbook,worksheet_M,worksheet_info

def col_row(self,worksheet):
worksheet.set_column('A:A', 12)
worksheet.set_row(0, 17)
worksheet.set_column('A:A',58)
worksheet.set_column('B:B', 58)

def shell_format(self,workbook):
#表头格式
merge_format = workbook.add_format({
'bold': 1,
'border': 1,
'align': 'center',
'valign': 'vcenter',
'fg_color': '#FAEBD7'
})
#标题格式
name_format = workbook.add_format({
'bold': 1,
'border': 1,
'align': 'center',
'valign': 'vcenter',
'fg_color': '#E0FFFF'
})
#正文格式
normal_format = workbook.add_format({
'align': 'center',
})
return merge_format,name_format,normal_format

#写入title和列名
def write_title(self,worksheet,search,merge_format):
title = search + "搜索结果"
worksheet.merge_range('A1:B1', title, merge_format)
print('write title success')

def write_tag(self,worksheet,name_format):
tag_row = 1
tag_col = 0
for num in self.tag_list:
worksheet.write(tag_row,tag_col,num,name_format)
tag_col += 1
print('write tag success')

#写入内容
def write_context(self,worksheet,con_dic,normal_format):
row = 2
for k,v in con_dic.items():
if row > len(con_dic):
break
col = 0
worksheet.write(row,col,k,normal_format)
col+=1
worksheet.write(row,col,v,normal_format)
row+=1
print('write context success')
#写入子页面详细内容
def write_info(self,worksheet_info,info_list,normal_format):
row = 1
for infomsg in info_list:
for num in range(0,len(infomsg)):
worksheet_info.write(row,num,infomsg[num],normal_format)
num += 1
row += 1

print("wirte info success")

#关闭excel
def workbook_close(self,workbook):
workbook.close()

if __name__ == '__main__':
print('This is create excel mode')

main代码如下：

#!/bin/env python
# -*- coding:utf-8 -*-

import geturldytt
import getexceldytt

#获取url字典
def get_dic():
blog = geturldytt.get_urldic()
urllist, search = blog.get_url()
html_doc = blog.get_html(urllist)
result = blog.get_soup(html_doc)
info_list= blog.get_info(result)
return result,search,info_list

#写入excle
def write_excle(urldic,search,info_list):
excle = getexceldytt.create_excle()
workbook, worksheet, worksheet_info = excle.create_workbook(search)
excle.col_row(worksheet)
merge_format, name_format, normal_format = excle.shell_format(workbook)
excle.write_title(worksheet,search,merge_format)
excle.write_tag(worksheet,name_format)
excle.write_context(worksheet,urldic,normal_format)
excle.write_info(worksheet_info,info_list,normal_format)
excle.workbook_close(workbook)

def main():
url_dic ,search_name, info_list = get_dic()
write_excle(url_dic,search_name,info_list)

if __name__ == '__main__':
main()

三、效果展示

运行代码，填写搜索的关键字，及搜索多少页,生成excel的sheet1内容为首先搜索的页面及url，sheet2为sheet1内url对应的电影下载链接及豆瓣评分，有的子页面存在没有豆瓣评分或下载链接。