Python多线程下载文件实例代码

Python实现多线程下载文件的方法与实例代码。

实现简单的多线程下载，需要关注几点：
1.文件的大小：可以从reponse header中提取，如“Content-Length:911”表示大小是911字节
2.任务拆分：指定各个线程下载的文件的哪一块，可以通过request header中添加“Range: bytes=300-400”（表示下载300~400byte的内容），注意可以请求的文件的range是[0, size-1]字节的。
3.下载文件的聚合：各个线程将自己下载的文件块保存为临时文件，所有线程都完成后，再将这些临时文件按顺序聚合写入到最终的一个文件中。
实现代码：

#!/usr/bin/python# -*- coding: utf-8 -*-# filename: paxel.py# edit: www.jbxue.com# Jay modified it a little and save for further potential usage.'''It is a multi-thread downloading tool    It was developed following axel.        Author: volans        E-mail: volansw [at] gmail.com'''import sysimport osimport timeimport urllibfrom threading import Thread# in case you want to use http_proxylocal_proxies = {'http': 'http://131.139.58.200:8080'}class AxelPython(Thread, urllib.FancyURLopener):    '''Multi-thread downloading class.        run() is a vitural method of Thread.    '''    def __init__(self, threadname, url, filename, ranges=0, proxies={}):        Thread.__init__(self, name=threadname)        urllib.FancyURLopener.__init__(self, proxies)        self.name = threadname        self.url = url        self.filename = filename        self.ranges = ranges        self.downloaded = 0    def run(self):        '''vertual function in Thread'''        try:            self.downloaded = os.path.getsize(self.filename)        except OSError:            #print 'never downloaded'            self.downloaded = 0        # rebuild start poind        self.startpoint = self.ranges[0] + self.downloaded        # This part is completed        if self.startpoint >= self.ranges[1]:            print 'Part %s has been downloaded over.' % self.filename            return        self.oneTimeSize = 16384  # 16kByte/time        print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])        self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))        self.urlhandle = self.open(self.url)        data = self.urlhandle.read(self.oneTimeSize)        while data:            filehandle = open(self.filename, 'ab+')            filehandle.write(data)            filehandle.close()            self.downloaded += len(data)            #print "%s" % (self.name)            #progress = u'\r...'            data = self.urlhandle.read(self.oneTimeSize) def GetUrlFileSize(url, proxies={}):    urlHandler = urllib.urlopen(url, proxies=proxies)    headers = urlHandler.info().headers    length = 0    for header in headers:        if header.find('Length') != -1:            length = header.split(':')[-1].strip()            length = int(length)    return length def SpliteBlocks(totalsize, blocknumber):    blocksize = totalsize / blocknumber    ranges = []    for i in range(0, blocknumber - 1):        ranges.append((i * blocksize, i * blocksize + blocksize - 1))    ranges.append((blocksize * (blocknumber - 1), totalsize - 1))    return ranges def islive(tasks):    for task in tasks:        if task.isAlive():            return True    return False def paxel(url, output, blocks=6, proxies=local_proxies):    ''' paxel    '''    size = GetUrlFileSize(url, proxies)    ranges = SpliteBlocks(size, blocks)    threadname = ["thread_%d" % i for i in range(0, blocks)]    filename = ["tmpfile_%d" % i for i in range(0, blocks)]    tasks = []    for i in range(0, blocks):        task = AxelPython(threadname[i], url, filename[i], ranges[i])        task.setDaemon(True)        task.start()        tasks.append(task)    time.sleep(2)    while islive(tasks):        downloaded = sum([task.downloaded for task in tasks])        process = downloaded / float(size) * 100        show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)        sys.stdout.write(show)        sys.stdout.flush()        time.sleep(0.5)    filehandle = open(output, 'wb+')    for i in filename:        f = open(i, 'rb')        filehandle.write(f.read())        f.close()        try:            os.remove(i)            pass        except:            pass    filehandle.close()if __name__ == '__main__':    url = 'http://dldir1.qq.com/qqfile/QQforMac/QQ_V3.1.1.dmg'    output = 'download.file'    paxel(url, output, blocks=4, proxies={})

您可能感兴趣的文章：

• python多线程实例学习
• Python多线程Socket编程的例子
• Python模块教程之thread多线程处理
• python多线程批量修改主机名
• Python多线程探测SSH主机(示例)
• python多线程下载网页的实现代码
• python多线程下载的实现代码
• python多线程Ping网段的实例代码
• python 多线程编程的例子
• python多线程异步同时执行多个函数的方法
• python多线程简明实例