#!/usr/bin/env python3

# -*- coding:utf-8 -*-

#  @Time: 2020/12/16 10:42

#  @Author:zhangmingda

#  @File: urllib_multi_download.py

#  @Software: PyCharm

#  Description: 使用urllib 模块 实现多线程下载某个文件测试

from concurrent.futures import ThreadPoolExecutor, as_completed

from urllib.request import urlopen

from urllib.request import Request

from urllib.request import quote

import json

import math

import os

class DownLoader(object):

    def __init__(self):

        self.part_size = 1024 * 1024 * 10  # 分块下载大小

        self.part_thread_num = 10

        self.BUFFER_SIZE = 64 * 1024

    def download_part(self, encode_url, part_filename, offset, end_bytes):

        """

        :param encode_url:经过URL编码的网络地址

        :param part_filename: 文件块儿名字

        :param offset: 下载字节起始点(包含)

        :param end_bytes: 下载字节结束点(包含)

        :return: (下载结果)

        """

        # 构造请求头

        range_header = {

            'Range': 'bytes=%s-%s' % (offset, end_bytes)

        }

        print(range_header)

        cur_task_ret = False

        expected_file_size = end_bytes - offset + 1

        part_req = Request(encode_url,headers=range_header)

        with open(part_filename, 'wb') as local_part_fd:

            with urlopen(part_req) as req_fd:

                while True:

                    # 一直从网络读数据

                    data = req_fd.read(self.BUFFER_SIZE)

                    if not data:

                        break

                    local_part_fd.write(data)

        if expected_file_size == os.stat(part_filename).st_size:

            print('%s 与预期块儿文件大小相符' % part_filename)

            cur_task_ret = True

            # break

        else:

            print('%s 与预期块儿文件大小 不符,预期%s字节,实际得到%s 字节' % (

            part_filename, expected_file_size, os.stat(part_filename).st_size))

        return {part_filename: cur_task_ret}

    def download(self, url):

        finally_filename = os.path.basename(url)

        # 将URL编码成%字符串格式

        encode_url = quote(url, safe=";/?:@&=+$,")

        print(encode_url)

        # 构造请求

        req = Request(encode_url)

        # 发起请求并且获取内容长度

        with urlopen(req) as fp:

            # print(json.dumps(dir(fp),indent=1))

            print(fp.getheaders())

            # length = fp.getheader('content-Range')

            length = fp.getheader('Content-Length')

            length = int(length)

            print(type(length))

            print('length:', length)

        # 分块任务列表

        thread_list = []

        # 每个块儿下载的结果

        multi_chunk_download_result = {}

        chunk_size = self.part_size

        # 计算需要下载的块儿个数

        chunk_count = int(math.ceil(length / float(chunk_size)))

        pool_args_list = []

        # 计算每个块儿请求的字节范围

        for i in range(chunk_count):

            offset = chunk_size * i

            end_bytes = min(chunk_size * (i + 1), length) - 1

            # 将一个文件划分的所有块儿任务，添加到任务列表

            part_num = i + 1

            part_filename = finally_filename + '.' + str(part_num)

            # 每个块儿请求的范围，块儿名字，加到线程参数列表

            pool_args_list.append((encode_url, part_filename, offset, end_bytes))

        # ********开始多线程下载数据，并获取下载结果**************

        # 构建线程池实例

        tp = ThreadPoolExecutor(max_workers=self.part_thread_num)

        # 全部添加到任务队列开始处理

        [thread_list.append(tp.submit(self.download_part, *args)) for args in pool_args_list]

        # 等待所有线程结束，获取全部线程的执行结果

        [multi_chunk_download_result.update(part_thread.result()) for part_thread in as_completed(thread_list)]

        # 下载总结

        print('下载总结')

        # 如果任务数和块儿数对不上，报一下出入

        if len(multi_chunk_download_result) != chunk_count:

            raise RuntimeError(

                "%s part miss,expect=%d,actual=%d" % (finally_filename, chunk_count, len(multi_chunk_download_result)))

        # 如果任务都完毕，检查是否有失败的块儿

        for item in multi_chunk_download_result.keys():

            if not multi_chunk_download_result[item]:

                raise RuntimeError("%s part upload has fail" % item)

        # 都OK 整合文件

        with open(finally_filename, 'wb') as local_fd:

            for i in range(chunk_count):

                part_filename = finally_filename + '.' + str(i + 1)

                with open(part_filename, 'rb') as part_fd:

                    while True:

                        bytes_data = part_fd.read(self.BUFFER_SIZE)

                        if not bytes_data:

                            break

                        local_fd.write(bytes_data)

        if length == os.stat(finally_filename).st_size:

            print('%s  下载完成，文件大小相符' % finally_filename)

            for part_filename in multi_chunk_download_result.keys():

                os.remove(part_filename)

        else:

            print('%s  下载完成，但大小不符,content_length:%s  下载后大小 %s' % (finally_filename, length,os.stat(finally_filename).st_size ))

if __name__ == '__main__':

    downloader = DownLoader()

    url = 'https://ks3-cn-beijing.ksyun.com/zhangmingda/111-3333333.Python安装与命令行操作.mp4'

    print(url)

    downloader.download(url)