python多线程处理数据

python多线程处理数据

从文件读取数据，多线程处理

#! /usr/bin/env python
#encoding=utf-8

import threading
import time
from Queue import Queue

def readFile():
file_object = open('/opt/dev/python/list.dat')
global queue
for line in file_object:
queue.put(line)

class Consumer(threading.Thread):
def run(self):
global queue
while queue.qsize() > 0:
msg = self.name + '消费了 '+queue.get()
print msg
time.sleep(0.01)

queue = Queue()
def main():
readFile()
for i in range(5):
c = Consumer()
c.start()

if __name__ == '__main__':
main()

测试数据

队列1
队列2
队列3
队列4
队列5
队列6
队列7
队列8
队列9
队列10
队列11
队列12
队列13
队列14
队列15
队列16


运行结果：
Thread-2消费了 队列879
Thread-3消费了 队列880
Thread-5消费了 队列881
Thread-4消费了 队列882
Thread-6消费了 队列883
Thread-2消费了 队列884
Thread-2消费了 队列885
Thread-6消费了 队列886
Thread-3消费了 队列88

shell单词计数
grep -io '队列' 1.log|wc -l

附：python的生产者消费者代码

#! /usr/bin/env python
#encoding=utf-8

import threading
import time
from Queue import Queue

class Producer(threading.Thread):
def run(self):
global queue
count = 0
while True:
for i in range(100):
if queue.qsize() > 1000:
pass
else:
count = count +1
msg = '生成产品'+str(count)
queue.put(msg)
print msg
time.sleep(1)

class Consumer(threading.Thread):
def run(self):
global queue
while True:
for i in range(3):
if queue.qsize() < 100:
pass
else:
msg = self.name + '消费了 '+queue.get()
print msg
time.sleep(1)

queue = Queue()

def test():
for i in range(500):
queue.put('初始产品'+str(i))
for i in range(2):
p = Producer()
p.start()
for i in range(5):
c = Consumer()
c.start()
if __name__ == '__main__':
test()