python 模拟分割大文件后,多线程处理

来源:互联网 发布:时时彩后二软件 编辑:程序博客网 时间:2024/05/20 05:30
#!/usr/bin/env python#--*-- coding:utf-8 --*--from random import randintfrom time import ctimefrom time import sleepimport queueimport threadingclass MyTask(object):    """具体的任务类"""    def __init__(self, name):        self.name = name        self._work_time = randint(1, 5)    def work(self):        print("Task %s is start : %s, sleep time= %d" % (self.name, ctime(), self._work_time))        sleep(self._work_time)        print("Task %s is end : %s" % (self.name, ctime()))class MyThread(threading.Thread):    """多线程的类"""    def __init__(self, my_queue):        self.my_queue = my_queue        super(MyThread, self).__init__()    def run(self):        while True:            if self.my_queue.qsize() > 0:                self.my_queue.get().work()            else:                breakdef print_split_line(num=30):    print("*" * num)if __name__ == "__main__":    print_split_line()    import my_read_file    # 分割文件    sf = my_read_file.SplitFiles(r"F:\multiple_thread_read_file.txt", line_count=300)    file_num = sf.split_file()    queue_length = file_num    my_queue = queue.LifoQueue(queue_length)    threads = []    for i in range(queue_length):        file_name = sf.get_part_file_name(i)        mt = MyTask(file_name)        my_queue.put_nowait(mt)    for i in range(queue_length):        mtd = MyThread(my_queue)        threads.append(mtd)    for i in range(queue_length):        threads[i].start()    for i in range(queue_length):        threads[i].join()    print_split_line()