python内存管理

import gc
del obj_name
gc.collection()

import  pandas as pd
import numpy as np
test = pd.read_csv("test_data.csv",names=['User_id','Location_id','Merchant_id'],header=None
,dtype = {'User_id':np.uint32,'Location_id':np.uint32,'Merchant_id':np.uint32})

import pandas as pd
import numpy as np
import subprocess

columns = ['User_id','Seller_id','Item_id','Category_id','Online_Action_id','Time_Stamp']
type = {'User_id':np.uint32,'Seller_id':np.uint32,'Item_id':np.uint32,'Category_id':np.uint16,'Online_Action_id':np.uint8,'Time_Stamp':np.uint32}

# for Linux
# Get number of lines in the CSV file
# nlines = subprocess.check_output('wc -l %s' %'ijcai2016_taobao', shell=True)
# nlines = int(nlines.split()[0])

#windows need nmake
#find /v/c "" ijcai2016_taobao

#also can get lines by using SQL
nlines = 44528127

chunksize = 1000

with open(name='pre_taobao.csv',mode='w') as out:
for i in range(0, nlines, chunksize):
df = pd.read_csv("ijcai2016_taobao",
header=None,  # no header, define column header manually later
nrows=chunksize, # number of rows to read at each iteration
skiprows=i)   # skip rows that were already read
# columns to read
df.columns = columns
for i in range(len(df)):
if df['Time_Stamp'][i]  < 20151101:
str_tmp = str(df['User_id'][i])+','+str(df['Seller_id'][i])+','+str(df['Item_id'][i])+','+str(df['Category_id'][i])+','+str(df['Online_Action_id'][i])+','+str(df['Time_Stamp'][i])
out.write(str_tmp)
out.write('\n')