Python 线程池

 

1. import Queue, threading, sys
2. from threading import Thread
3. import time,urllib
4. # working thread
5. class Worker(Thread):
6.    worker_count = 0
7.    def __init__( self, workQueue, resultQueue, timeout = 0, **kwds):
8.        Thread.__init__( self, **kwds )
9.        self.id = Worker.worker_count
10.        Worker.worker_count += 1
11.        self.setDaemon( True )
12.        self.workQueue = workQueue
13.        self.resultQueue = resultQueue
14.        self.timeout = timeout
15.        self.start( )
16.    def run( self ):
17.        ''' the get-some-work, do-some-work main loop of worker threads '''
18.        while True:
19.            try:
20.                callable, args, kwds = self.workQueue.get(timeout=self.timeout)
21.                res = callable(*args, **kwds)
22.                print "worker[%2d]: %s" % (self.id, str(res) )
23.                self.resultQueue.put( res )
24.            except Queue.Empty:
25.                break
26.            except :
27.                print 'worker[%2d]' % self.id, sys.exc_info()[:2]
28.                
29. class WorkerManager:
30.    def __init__( self, num_of_workers=10, timeout = 1):
31.        self.workQueue = Queue.Queue()
32.        self.resultQueue = Queue.Queue()
33.        self.workers = []
34.        self.timeout = timeout
35.        self._recruitThreads( num_of_workers )
36.    def _recruitThreads( self, num_of_workers ):
37.        for i in range( num_of_workers ):
38.            worker = Worker( self.workQueue, self.resultQueue, self.timeout )
39.            self.workers.append(worker)
40.    def wait_for_complete( self):
41.        # ...then, wait for each of them to terminate:
42.        while len(self.workers):
43.            worker = self.workers.pop()
44.            worker.join( )
45.            if worker.isAlive() and not self.workQueue.empty():
46.                self.workers.append( worker )
47.        print "All jobs are are completed."
48.    def add_job( self, callable, *args, **kwds ):
49.        self.workQueue.put( (callable, args, kwds) )
50.    def get_result( self, *args, **kwds ):
51.        return self.resultQueue.get( *args, **kwds )

Worker类是一个工作线程，不断地从workQueue队列中获取需要执行的任务，执行之，并将结果写入到resultQueue中，这里的workQueue和resultQueue都是现成安全的，其内部对各个线程的操作做了互斥。当从workQueue中获取任务超时，则线程结束。

WorkerManager负责初始化Worker线程，提供将任务加入队列和获取结果的接口，并能等待所有任务完成。

一个典型的测试例子如下，它用10个线程去下载一个固定页面的内容，实际应用时应该是执行不同的任务。

复制代码

1. def test_job(id, sleep = 0.001 ):
2.    try:
3.        urllib.urlopen('[url]https://www.gmail.com/[/url]').read()
4.    except:
5.        print '[%4d]' % id, sys.exc_info()[:2]
6.    return id
7. def test():
8.    import socket
9.    socket.setdefaulttimeout(10)
10.    print 'start testing'
11.    wm = WorkerManager(10)
12.    for i in range(500):
13.        wm.add_job( test_job, i, i*0.001 )
14.    wm.wait_for_complete()
15.    print 'end testing'