python 爬虫 豆瓣 评论及评分

借鉴了不少 hang 的博客：https://segmentfault.com/a/1190000010473819

评分：

# -*- coding: utf-8 -*-

"""

Created on Wed Sep 20 16:19:02 2017

@author: su

"""

from urllib import request

import re

from bs4 import BeautifulSoup as bs

"""resp = request.urlopen("https://movie.douban.com/nowplaying/hangzhou/" )

html_data = resp.read().decode("UTF-8")

soup = bs(html_data,"html.parser")

nowplaying_movie = soup.find_all("div",id="nowplaying")

nowplaying_movie_list = nowplaying_movie[0].find_all("li",class_="list-item")

nowplaying_list = []

for item in nowplaying_movie_list:

    nowplaying_dict={}

    nowplaying_dict["id"]= item["data-subject"]

    nowplaying_dict["title"]= item["data-title"]

    for tag_img_item in item.find_all("img"):

        nowplaying_dict["name"]=tag_img_item["alt"]

        nowplaying_list.append(nowplaying_dict)

"""

for start in range(0,60,20):

    requr = "https://movie.douban.com/subject/"+nowplaying_list[0]["id"]+"/comments?"+"start="+str(start)+"&limit=20"

    resp = request.urlopen(requr)

    html_data = resp.read().decode("UTF-8")

    soup = bs(html_data,"html.parser")

    comment_list=[]

    comment_div_list = soup.find_all("div",class_="comment")

    for item in comment_div_list:

        comment_dict={}

       #if t in range(0,50,10): 

        item_score = item.find_all("h3")[0]

        item_score = item_score.find_all("span")[4]

        comment_dict["得分"]=item_score["class"]

        comment_dict["评级"]=item_score["title"]

        comment_list.append(comment_dict)

        

    print(comment_list)

评论：

@author: su

"""

from urllib import request

import re

import jieba    #分词包

import pandas as pd

import numpy 

"""

resp = request.urlopen('https://movie.douban.com/nowplaying/hangzhou/')

html_data = resp.read().decode('utf-8')

from bs4 import BeautifulSoup as bs

soup = bs(html_data, 'html.parser')    

nowplaying_movie = soup.find_all('div', id='nowplaying')

nowplaying_movie_list = nowplaying_movie[0].find_all('li', class_='list-item') 

nowplaying_list = [] 

for item in nowplaying_movie_list:        

        nowplaying_dict = {}        

        nowplaying_dict['id'] = item['data-subject']       

        for tag_img_item in item.find_all('img'):            

            nowplaying_dict['name'] = tag_img_item['alt']            

            nowplaying_list.append(nowplaying_dict)  

"""

requrl = 'https://movie.douban.com/subject/' + nowplaying_list[3]['id'] + '/comments' +'?' +'start=0' + '&limit=20' 

resp = request.urlopen(requrl) 

html_data = resp.read().decode('utf-8') 

soup = bs(html_data, 'html.parser') 

comment_div_lits = soup.find_all('div', class_='comment') 

eachCommentList = []; 

for item in comment_div_lits: 

        if item.find_all('p')[0].string is not None:     

            eachCommentList.append(item.find_all('p')[0].string)

comments = ''

with open("s.txt","w") as f:

  for k in range(len(eachCommentList)):

      comments = comments + (str(eachCommentList[k])).strip()

      pattern = re.compile(r'[\u4e00-\u9fa5]+')

      filterdata = re.findall(pattern, comments)

      cleaned_comments = ''.join(filterdata)

      f.write(cleaned_comments)

f.close()