Python进程池的简单使用

来源：互联网发布：澳洲听歌用什么软件编辑：程序博客网时间：2024/06/06 04:36

Python中有进程池的概念，相比线程池，更加稳定。在此基础上，进行一个简单封装。

首先写一个工具类ProcessUtil.py

#!/user/bin/env python3# -*- coding: utf-8 -*-from multiprocessing.pool import Poolclass TaskAction(object):    def do_in_action(self):        passclass TaskProcess(object):    def __init__(self, core_size=None):        if core_size is None:            self.pool = Pool()        else:            if type(core_size) is not int:                raise ValueError('core_size can\'t handle type  %s ,only int support ' % type(core_size))            self.pool = Pool(core_size)    # 检查task参数有效性    @staticmethod    def _check_task(tasks):        # check tasks type is list ?        if type(tasks) is not list:            raise ValueError('tasks can\'t handle type %s , only list support' % type(tasks))        # check tasks' subset is TaskAction ?        for child_task in tasks:            if not isinstance(child_task, TaskAction):                raise ValueError(                    'tasks\' subset can\'t handle type %s , only TaskAction support' % type(child_task))    # 异步执行进程    def execute_task(self, tasks):        apply_results = []        # check params        self._check_task(tasks)        # 添加任务至进程池        for child_task in tasks:            apply_results.append(self.pool.apply_async(child_task.do_in_action))        # 拒绝添加新的Process        self.pool.close()        # 等待所有子进程执行完毕        self.pool.join()        # 封装请求结果        results = []        for res in apply_results:            results.append(res.get())        return results

然后是一个测试case：

#!/user/bin/env python3# -*- coding: utf-8 -*-import jsonimport osimport random# 进程的试炼import timeimport unittestfrom util import DateUtilfrom util import ProcessUtilclass ProcessTest(unittest.TestCase):    def setUp(self):        print('============================= %s is ready to start...' % self._testMethodName)    def tearDown(self):        print('============================= %s had finished...' % self._testMethodName)    def test_04(self):        list = []        # 构造任务列        for i in range(100):            user_id = i + 1            access_token = DateUtil.DateParser.parse_stamp_second(DateUtil.DateParser.now()) + user_id + random.randint(                10, 99)            list.append(DemoTest(user_id, access_token))        # 执行任务列        result = ProcessUtil.TaskProcess(3).execute_task(list)        print('All sub processes done.')        print(json.dumps(result))class DemoTest(ProcessUtil.TaskAction):    def __init__(self, user_id, access_token):        self.user_id = user_id        self.access_token = access_token    def do_in_action(self):        second = random.random() * 3        time.sleep(second)        print('UserId : %s , AccessToken : %s , (%s) ' % (self.user_id, self.access_token, os.getpid()))        return secondif __name__ == '__main__':    unittest.main()

控制台部分打印信息：

UserId : 90 , AccessToken : 1504516412.0 , (15272) UserId : 88 , AccessToken : 1504516457.0 , (7456) UserId : 91 , AccessToken : 1504516405.0 , (15272) UserId : 92 , AccessToken : 1504516425.0 , (7456) UserId : 89 , AccessToken : 1504516452.0 , (7096) UserId : 93 , AccessToken : 1504516471.0 , (15272) UserId : 96 , AccessToken : 1504516435.0 , (15272) UserId : 97 , AccessToken : 1504516460.0 , (15272) UserId : 98 , AccessToken : 1504516439.0 , (15272) UserId : 94 , AccessToken : 1504516455.0 , (7456) UserId : 95 , AccessToken : 1504516454.0 , (7096) UserId : 100 , AccessToken : 1504516402.0 , (7456) UserId : 99 , AccessToken : 1504516426.0 , (15272) .----------------------------------------------------------------------Ran 1 test in 48.824s

从结果上看进程设置了3个进程分别在 7456 ，7096，15272 三个pid上运行，提高了效率

最后吐槽下，java中实现线程池好累，要关心池的配置，资源释放，类型校验等等

阅读全文

0 0