新浪微博数据挖掘食谱之十五： 爬虫篇 (抓取用户的朋友)

#!/usr/bin/python
# -*- coding: utf-8 -*-

'''
Created on 2015-1-11
@author: beyondzhou
@name: crawl_friendship_graph.py
'''

# Crawl friendship graph
def crawl_friendship_graph():

# import
#import json
from login import weibo_login
from users import crawl_weibo_followers

# Access to sina api
weibo_api = weibo_login()

screen_name = 'beyondzhou8'
crawl_weibo_followers(weibo_api, screen_name, depth=1, limit=10)

if __name__ == '__main__':
crawl_friendship_graph()

# Crawl a friendship graph
def crawl_weibo_followers(weibo_api, screen_name, limit=1000000, depth=2):

from data import save_to_mongo

# Resolve the ID for screen_name and start working with IDs for consistency in storage
seed_id = str(weibo_api.users.show.get(screen_name=screen_name)['id'])
_, next_queue = get_friends_followers_ids(weibo_api, user_id=seed_id, friends_limit=0, followers_limit=limit)

# Store a seed_id => _follower_ids mapping in MongoDB
save_to_mongo({'followers' : [ _id for _id in next_queue ]}, 'followers_crawl', '{0}-follower_ids'.format(seed_id))

d = 1
while d < depth:
d += 1
(queue, next_queue) = (next_queue, [])
for fid in queue:
follower_ids = get_friends_followers_ids(weibo_api, user_id=fid, friends_limit=0, followers_limit=limit)

# Store a fid => follower_ids mapping in MongoDB
save_to_mongo({'followers' : [ _id for _id in next_queue ]}, 'followers_crawl', '{0}-follower_ids'.format(fid))
next_queue += follower_ids