python支持断点续传的多线程下载示例


Posted in Python onJanuary 16, 2014
#! /usr/bin/env python
#coding=utf-8
from __future__ import unicode_literals
from multiprocessing.dummy import Pool as ThreadPool
import threading
import os
import sys
import cPickle
from collections import namedtuple
import urllib2
from urlparse import urlsplit
import time

# global lock
lock = threading.Lock()

# default parameters
defaults = dict(thread_count=10,
    buffer_size=10*1024,
    block_size=1000*1024)

def progress(percent, width=50):
    print "%s %d%%\r" % (('%%-%ds' % width) % (width * percent / 100 * '='), percent),
    if percent >= 100:
        print
        sys.stdout.flush()

def write_data(filepath, data):
    with open(filepath, 'wb') as output:
        cPickle.dump(data, output)

def read_data(filepath):
    with open(filepath, 'rb') as output:
        return cPickle.load(output)

FileInfo = namedtuple('FileInfo', 'url name size lastmodified')

def get_file_info(url):
    class HeadRequest(urllib2.Request):
        def get_method(self):
            return "HEAD"
    res = urllib2.urlopen(HeadRequest(url))
    res.read()
    headers = dict(res.headers)
    size = int(headers.get('content-length', 0))
    lastmodified = headers.get('last-modified', '')
    name = None
    if headers.has_key('content-disposition'):
        name = headers['content-disposition'].split('filename=')[1]
        if name[0] == '"' or name[0] == "'":
            name = name[1:-1]
    else:
        name = os.path.basename(urlsplit(url)[2])
    return FileInfo(url, name, size, lastmodified)

def download(url, output,
        thread_count = defaults['thread_count'],
        buffer_size = defaults['buffer_size'],
        block_size = defaults['block_size']):
    # get latest file info
    file_info = get_file_info(url)
    # init path
    if output is None:
        output = file_info.name
    workpath = '%s.ing' % output
    infopath = '%s.inf' % output
    # split file to blocks. every block is a array [start, offset, end],
    # then each greenlet download filepart according to a block, and
    # update the block' offset.
    blocks = []
    if os.path.exists(infopath):
        # load blocks
        _x, blocks = read_data(infopath)
        if (_x.url != url or
                _x.name != file_info.name or
                _x.lastmodified != file_info.lastmodified):
            blocks = []
    if len(blocks) == 0:
        # set blocks
        if block_size > file_info.size:
            blocks = [[0, 0, file_info.size]]
        else:
            block_count, remain = divmod(file_info.size, block_size)
            blocks = [[i*block_size, i*block_size, (i+1)*block_size-1] for i in range(block_count)]
            blocks[-1][-1] += remain
        # create new blank workpath
        with open(workpath, 'wb') as fobj:
            fobj.write('')
    print 'Downloading %s' % url
    # start monitor
    threading.Thread(target=_monitor, args=(infopath, file_info, blocks)).start()
    # start downloading
    with open(workpath, 'rb+') as fobj:
        args = [(url, blocks[i], fobj, buffer_size) for i in range(len(blocks)) if blocks[i][1] < blocks[i][2]]
        if thread_count > len(args):
            thread_count = len(args)
        pool = ThreadPool(thread_count)
        pool.map(_worker, args)
        pool.close()
        pool.join()

    # rename workpath to output
    if os.path.exists(output):
        os.remove(output)
    os.rename(workpath, output)
    # delete infopath
    if os.path.exists(infopath):
        os.remove(infopath)
    assert all([block[1]>=block[2] for block in blocks]) is True

def _worker((url, block, fobj, buffer_size)):
    req = urllib2.Request(url)
    req.headers['Range'] = 'bytes=%s-%s' % (block[1], block[2])
    res = urllib2.urlopen(req)
    while 1:
        chunk = res.read(buffer_size)
        if not chunk:
            break
        with lock:
            fobj.seek(block[1])
            fobj.write(chunk)
            block[1] += len(chunk)

def _monitor(infopath, file_info, blocks):
    while 1:
        with lock:
            percent = sum([block[1] - block[0] for block in blocks]) * 100 / file_info.size
            progress(percent)
            if percent >= 100:
                break
            write_data(infopath, (file_info, blocks))
        time.sleep(2)

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser(description='Download file by multi-threads.')
    parser.add_argument('url', type=str, help='url of the download file')
    parser.add_argument('-o', type=str, default=None, dest="output", help='output file')
    parser.add_argument('-t', type=int, default=defaults['thread_count'], dest="thread_count", help='thread counts to downloading')
    parser.add_argument('-b', type=int, default=defaults['buffer_size'], dest="buffer_size", help='buffer size')
    parser.add_argument('-s', type=int, default=defaults['block_size'], dest="block_size", help='block size')
    argv = sys.argv[1:]
    if len(argv) == 0:
        argv = ['https://eyes.nasa.gov/eyesproduct/EYES/os/win']
    args = parser.parse_args(argv)
    start_time = time.time()
    download(args.url, args.output, args.thread_count, args.buffer_size, args.block_size)
    print 'times: %ds' % int(time.time()-start_time)
Python 相关文章推荐
python通过pil模块获得图片exif信息的方法
Mar 16 Python
Python类的定义、继承及类对象使用方法简明教程
May 08 Python
在Python的Django框架中调用方法和处理无效变量
Jul 15 Python
深入理解Python对Json的解析
Feb 14 Python
Python+Selenium自动化实现分页(pagination)处理
Mar 31 Python
详解Python用三种方式统计词频的方法
Jul 29 Python
python装饰器练习题及答案
Nov 01 Python
python 实现保存最新的三份文件,其余的都删掉
Dec 22 Python
python logging 日志的级别调整方式
Feb 21 Python
Kears 使用:通过回调函数保存最佳准确率下的模型操作
Jun 17 Python
简单了解如何封装自己的Python包
Jul 08 Python
python实现将中文日期转换为数字日期
Jul 14 Python
python获得图片base64编码示例
Jan 16 #Python
python练习程序批量修改文件名
Jan 16 #Python
python使用urllib模块开发的多线程豆瓣小站mp3下载器
Jan 16 #Python
python使用urllib模块和pyquery实现阿里巴巴排名查询
Jan 16 #Python
python3.3教程之模拟百度登陆代码分享
Jan 16 #Python
python解析发往本机的数据包示例 (解析数据包)
Jan 16 #Python
python多线程扫描端口示例
Jan 16 #Python
You might like
PHP模板引擎SMARTY
2006/10/09 PHP
PHP中for与foreach的区别分析
2011/03/09 PHP
有关php运算符的知识大全
2011/11/03 PHP
一个简单的网页密码登陆php代码
2012/07/17 PHP
Codeigniter通过SimpleXML将xml转换成对象的方法
2015/03/19 PHP
Nginx下ThinkPHP5的配置方法详解
2017/08/01 PHP
php图片裁剪函数
2018/10/31 PHP
基于jquery的一个OutlookBar类,动态创建导航条
2010/11/19 Javascript
JavaScript 更严格的相等 [译]
2012/09/20 Javascript
关于jQuery中.attr()和.prop()的问题探讨
2013/09/06 Javascript
js中的scroll和offset 使用比较的实例与分析
2013/09/29 Javascript
jQuery支持动态参数将函数绑定到事件上的方法
2015/03/17 Javascript
Javascript基于对象三大特性(封装性、继承性、多态性)
2016/01/04 Javascript
AngularJS入门教程之AngularJS模型
2016/04/18 Javascript
Vue.js列表渲染绑定jQuery插件的正确姿势
2017/06/29 jQuery
微信小程序 转发功能的实现
2017/08/04 Javascript
详解weex默认webpack.config.js改造
2018/01/08 Javascript
详解react关于事件绑定this的四种方式
2018/03/09 Javascript
生成无限制的微信小程序码的示例代码
2019/09/20 Javascript
[00:19]CN DOTA NEVER DIE!VG夺冠rOtK接受采访
2019/12/23 DOTA
python实现文件批量编码转换及注意事项
2019/10/14 Python
Python基础之字典常见操作经典实例详解
2020/02/26 Python
Python之字典对象的几种创建方法
2020/09/30 Python
python反编译教程之2048小游戏实例
2021/03/03 Python
html5使用canvas画三角形
2014/12/15 HTML / CSS
英国假发网站:Hothair
2018/02/23 全球购物
十八大闭幕感言
2014/01/22 职场文书
政府采购方案
2014/06/12 职场文书
村党支部书记四风问题个人对照检查材料思想汇报
2014/10/06 职场文书
岗位聘任报告
2015/03/02 职场文书
绿色环保倡议书
2015/04/28 职场文书
无婚姻登记记录证明
2015/06/18 职场文书
运动会运动员赞词
2015/07/22 职场文书
八年级语文教学反思
2016/03/03 职场文书
导游词之江苏溱潼古镇
2019/11/27 职场文书
如何在CocosCreator里画个炫酷的雷达图
2021/04/16 Javascript