Python学习之分布式进程初探(1)

所谓分布式进程就是让两个进程运行在不同的电脑上面通过网络来进行数据交流,下面是在windows下通过managers模块来进行的分布式进程!

import random,time,queue
from multiprocessing.managers import BaseManager
from multiprocessing import freeze_support

task_queue = queue.Queue()
result_queue = queue.Queue()
def return_task():
return task_queue
def return_result():
return result_queue
class QueueManager(BaseManager):
pass
def test():
QueueManager.register('get_task',callable = return_task)
QueueManager.register('get_result',callable = return_result)
manager = QueueManager(address = ('127.0.0.1',5000),authkey = b'abc')#绑定到端口5000,并且设置验证码'abc'
manager.start()
#获得经过封装之后的task和result
task,result = manager.get_task(),manager.get_result()
#放一些任务
for i in range(5):
print('put %d in task'%i)
task.put(i)
#获取结果
print('try to get result')
for i in range(5):
r = result.get(timeout = 10)
print('get the result %d'%r)
manager.shutdown()
print('master exit')

if __name__ == '__main__':
freeze_support()
test()

在worker模块里面，由于manager从网络获取相应数据,因此我们只需要给这个manager注册上获取数据的方法就可以了.

from multiprocessing.managers import BaseManager
from multiprocessing import freeze_support

class QueueManager(BaseManager):
pass

if __name__ == '__main__':
QueueManager.register('get_task')
QueueManager.register('get_result')
server_adr = '127.0.0.1'
print('connect to the server%s',server_adr)
manager = QueueManager(address = (server_adr,5000),authkey = b'abc')
manager.connect()
print('connect successfuly')

task = manager.get_task()
result = manager.get_result()
for i in range(5):
try:
t = task.get(timeout = 1)
print('now process the task%d'%t)
result.put(t*t)
except Queue.Empty:
print('the task queue is empty, maybe some task lost?')
print('wordker exit')