python读取大文件处理方式

.read()

.readline()

.readlines()，当我们的文件有10个G甚至更大时，用上面的方法内存就直接爆掉了。

def read_in_chunks(filePath, chunk_size=1024*1024):
"""
Lazy function (generator) to read a file piece by piece.
Default chunk size: 1M
You can set your own chunk size
"""
file_object = open(filePath)
while True:
chunk_data = file_object.read(chunk_size)
if not chunk_data:
break
yield chunk_data
if __name__ == "__main__":
filePath = './path/filename'
for chunk in read_in_chunks(filePath):
process(chunk) # <do something with chunk>

with open()

#If the file is line based
with open(...) as f:
for line in f:
process(line) # <do something with line>

import fileinput
for line in fileinput.input(['sum.log']):
print line