Python实现多线程下载文件的代码实例

1107次阅读  |  发布于5年以前

实现简单的多线程下载,需要关注如下几点:
1.文件的大小:可以从reponse header中提取,如"Content-Length:911"表示大小是911字节
2.任务拆分:指定各个线程下载的文件的哪一块,可以通过request header中添加"Range: bytes=300-400"(表示下载300~400byte的内容),注意可以请求的文件的range是[0, size-1]字节的。
3.下载文件的聚合:各个线程将自己下载的文件块保存为临时文件,所有线程都完成后,再将这些临时文件按顺序聚合写入到最终的一个文件中。

实现代码:

复制代码 代码如下:

!/usr/bin/python

-- coding: utf-8 --

filename: paxel.py

FROM: http://jb51.net/code/view/58/full/

Jay modified it a little and save for further potential usage.

'''It is a multi-thread downloading tool

It was developed following axel.  
    Author: volans  
    E-mail: volansw [at] gmail.com  

'''

import sys
import os
import time
import urllib
from threading import Thread

in case you want to use http_proxy

local_proxies = {'http': 'http://131.139.58.200:8080'}

class AxelPython(Thread, urllib.FancyURLopener):
'''Multi-thread downloading class.

    run() is a vitural method of Thread.  
'''  
def __init__(self, threadname, url, filename, ranges=0, proxies={}):  
    Thread.__init__(self, name=threadname)  
    urllib.FancyURLopener.__init__(self, proxies)  
    self.name = threadname  
    self.url = url  
    self.filename = filename  
    self.ranges = ranges  
    self.downloaded = 0  

def run(self):  
    '''vertual function in Thread'''  
    try:  
        self.downloaded = os.path.getsize(self.filename)  
    except OSError:  
        #print 'never downloaded'  
        self.downloaded = 0  

    # rebuild start poind  
    self.startpoint = self.ranges[0] + self.downloaded  

    # This part is completed  
    if self.startpoint >= self.ranges[1]:  
        print 'Part %s has been downloaded over.' % self.filename  
        return  

    self.oneTimeSize = 16384  # 16kByte/time  
    print 'task %s will download from %d to %d' % (self.name, self.startpoint, self.ranges[1])  

    self.addheader("Range", "bytes=%d-%d" % (self.startpoint, self.ranges[1]))  
    self.urlhandle = self.open(self.url)  

    data = self.urlhandle.read(self.oneTimeSize)  
    while data:  
        filehandle = open(self.filename, 'ab+')  
        filehandle.write(data)  
        filehandle.close()  

        self.downloaded += len(data)  
        #print "%s" % (self.name)  
        #progress = u'\r...'  

        data = self.urlhandle.read(self.oneTimeSize)  

def GetUrlFileSize(url, proxies={}):
urlHandler = urllib.urlopen(url, proxies=proxies)
headers = urlHandler.info().headers
length = 0
for header in headers:
if header.find('Length') != -1:
length = header.split(':')[-1].strip()
length = int(length)
return length

def SpliteBlocks(totalsize, blocknumber):
blocksize = totalsize / blocknumber
ranges = []
for i in range(0, blocknumber - 1):
ranges.append((i blocksize, i blocksize + blocksize - 1))
ranges.append((blocksize * (blocknumber - 1), totalsize - 1))

return ranges  

def islive(tasks):
for task in tasks:
if task.isAlive():
return True
return False

def paxel(url, output, blocks=6, proxies=local_proxies):
''' paxel
'''
size = GetUrlFileSize(url, proxies)
ranges = SpliteBlocks(size, blocks)

threadname = ["thread_%d" % i for i in range(0, blocks)]  
filename = ["tmpfile_%d" % i for i in range(0, blocks)]  

tasks = []  
for i in range(0, blocks):  
    task = AxelPython(threadname[i], url, filename[i], ranges[i])  
    task.setDaemon(True)  
    task.start()  
    tasks.append(task)  

time.sleep(2)  
while islive(tasks):  
    downloaded = sum([task.downloaded for task in tasks])  
    process = downloaded / float(size) * 100  
    show = u'\rFilesize:%d Downloaded:%d Completed:%.2f%%' % (size, downloaded, process)  
    sys.stdout.write(show)  
    sys.stdout.flush()  
    time.sleep(0.5)  

filehandle = open(output, 'wb+')  
for i in filename:  
    f = open(i, 'rb')  
    filehandle.write(f.read())  
    f.close()  
    try:  
        os.remove(i)  
        pass  
    except:  
        pass  

filehandle.close()  

if name == 'main':
url = 'http://dldir1.qq.com/qqfile/QQforMac/QQ_V3.1.1.dmg'
output = 'download.file'
paxel(url, output, blocks=4, proxies={})

Copyright© 2013-2020

All Rights Reserved 京ICP备2023019179号-8