Python文本去重

# -*- coding:utf-8 -*-
#auther_cclarence_2016_4_6
#open file and read out the characters
from optparse import OptionParser
import sys
reload(sys)
sys.setdefaultencoding("utf-8")
def readfile(filename):
try:
f = open(filename)
except Exception, e:
print "No such file"
exit(0)
text = f.readlines()
f.close()
for i in range(0,len(text)-1):
text[i] = text[i][:-1]
return text
#deduplication
def unique(arr):
arr1 = list(set(arr))
arr1.sort(key = arr.index)
return arr1
def main():
parser = OptionParser()
parser.add_option("-f", "--file", dest="filename",help="write report to FILE", metavar="FILE")
(options, args) = parser.parse_args()
filename = options.filename
text = readfile(filename)
text_dealed = unique(text)
for i in range(0,len(text_dealed)-1):
text_dealed[i] = text_dealed[i] +'\n'
f = open("output.txt","w")
f.writelines(text_dealed)
f.close()
deduplication_num = len(text) - len(text_dealed)
print "success"
print "The num of data from the source file        :" + str(len(text))
print "The num of data from the preprocessed file: :" + str(len(text_dealed))
print "The num of data removed                     :" + str(deduplication_num)
if __name__ == '__main__':
main()