python 取出 Mongdb 中的数据 转化成DataFrame 然后用pandas处理数据

这段时间再玩python ，数据源来源于mongdb ，数据处理方式用的是pandas

刚开始是用的一个比较麻烦的转化，直接上代码：

方法一：

import pandas as pd
from pymongo import MongoClient

client = MongoClient('192.168.1.5',10070)

db = client.dbtest

collection=db.data_table
items = collection.find()

dateId = []
ai_type = []
ai_name = []
quorum = []
priceUSD = []
ai_disageform = []
country = []
continent  = []
company = []
ai_cap_tr = []
n = 0
for i in items:
n= n+1
print("正在输出 %s 条"%n)
keys = i.keys()
if 'ai_disageform' in keys:
ai_disageform.append(i['ai_disageform'])
else:
ai_disageform.append('')
if 'date' in keys:
t = str(i['date'])
dateId.append(t[:10])
else:
dateId.append('')
if 'ai_type' in keys:
ai_type.append(i['ai_type'])
else:
ai_type.append('')
if 'continent' in keys:
continent.append(i['continent'])
else:
continent.append('')
if 'quorum' in keys:
quorum.append(i['quorum'])
else:
quorum.append('')
if 'priceUSD' in keys:
priceUSD.append(i['priceUSD'])
else:
priceUSD.append('')
if 'country' in keys:
country.append(i['country'])
else:
country.append('')
if 'ai_name' in keys:
ai_name.append(i['ai_name'])
else:
ai_name.append('')
if 'company' in keys:
company.append(i['company'])
else:
company.append('')
if 'ai_cap_tr' in keys:
ai_cap_tr.append(i['ai_cap_tr'])
else:
ai_cap_tr.append('')

df = pd.DataFrame({'dateId':dateId,
'ai_type':ai_type,
'ai_name':ai_name,
'quorum':quorum,
'priceUSD':priceUSD,
'ai_disageform':ai_disageform,
'country':country,
'continent':continent,
'ai_cap_tr':ai_cap_tr,
'company':company})

df.to_csv('../ncbdata/b.csv', encoding = "utf-8",index=None)

具体思路：经测验，每条记录是dict类型的，将每个键里的值放到不同的数组中，然后创建dataframe对象。

方法二：

import pandas as pd
import numpy as np
import  pymongo
from pymongo import MongoClient
import json

#连接mongdb
def connectMongdb():

client = MongoClient('192.168.1.5',10070)

db = client.dbtest

collection = db.data_table
items = collection.find()
return items

#转化为df
def tran_df():
items = connectMongdb()
temp = []
for dict in items:
del dict['_id']
dict['date'] = dict['date'].strftime("%Y-%m-%d")
temp.append(dict)
data_employee = pd.read_json(json.dumps(temp))
data_employee_ri = data_employee.reindex(columns=['date', 'ai_type', 'ai_name'])
data_employee_ri.to_csv('data/a.csv')

def main():
tran_df()

if __name__ == "__main__":
main()

具体思路：将每一个字典放到一个数组里，然后通过read_json() 方法转化为df对象。