Python 多进程实例

来源：互联网发布：陕西广电网络单位怎样编辑：程序博客网时间：2024/05/03 11:33

1.使用多个进程读取同一个文件，每次取出文件中的一行，并共同写到某个文件中去

import multiprocessingimport timeFILE_SOURCE = 'f:\source.txt'FILE_TARGET = 'f:\\target.txt'ID_LIST = range(1, 999)def create_source():    f = open(FILE_SOURCE, 'w')    s = ''    for i in ID_LIST:        s += str(i) + ' ' + str(10 + float(i)/1000) + '\n'    f.write(s)    f.close()def get_info_from_file(file_path, id_in):    f = open(file_path, 'r')    all_info = {}    lines = f.readlines()    f.close()    for line in lines:        item = line.split(' ')        all_info[item[0]] = line    index = str(id_in)    if index in all_info:        return all_info[index]    return Nonedef write_info_to_target(file_path, info):    f = open(file_path, 'a')    f.write(info)    f.close()class MyTask(object):    def __init__(self, id_in):        self.id = id_indef worker(task_queue, name):    while True:        next_task = task_queue.get()        if next_task is None:            task_queue.task_done()            break        id_in = next_task.id        print name, id_in        write_info_to_target(FILE_TARGET, get_info_from_file(FILE_SOURCE, id_in))        #模拟进程代码运行过程        time.sleep(0.05)        task_queue.task_done()    print "worker exit"    returndef multiprocessing_file_operation():    task_queue = multiprocessing.JoinableQueue()    print "start", time.clock()    #create task    job_num = multiprocessing.cpu_count()*2    jobs = [multiprocessing.Process(target=worker, args=(task_queue, 'Process %d' % i)) for i in xrange(job_num)]    for j in jobs:        j.start()    for i in ID_LIST:        task_queue.put(MyTask(i))    #下面的操作是必须的，否则进程无法从queue中获取None对象，进程就一直死循环，挂死    for j in xrange(job_num):        task_queue.put(None)    task_queue.join()    print "end", time.clock()if __name__=="__main__":    multiprocessing_file_operation()

运行时间上：

CPU是个双核

如果是单进程：为53s

如果起4个进程：为14s

8个进程：为7s

16个进程：为4s

0 0