python urllib2 实现大文件下载

使用urllib2下载并分块copy：

# from urllib2 import urlopen # Python 2

from urllib.request import urlopen # Python 3

response = urlopen(url)

CHUNK = 16 * 1024

with open(file, 'wb') as f:

    while True:

        chunk = response.read(CHUNK)

        if not chunk:

            break

        f.write(chunk)

另一种大文件copy方式， shutil：

import shutil

try:

    from urllib.request import urlopen # Python 3

except ImportError:

    from urllib2 import urlopen # Python 2

def get_large_file(url, file, length=16*1024):

    req = urlopen(url)

    with open(file, 'wb') as fp:

        shutil.copyfileobj(req, fp, length)

关于shutil的一些介绍：https://www.cnblogs.com/zhangboblogs/p/7821702.html

使用urlib2并显示下载进度，以视频为例：

#coding:utf-8

import urllib

import urllib2

import requests

import random

import uuid

import time

import sys

from threading import Thread

#img_url = "https://p.ssl.qhimg.com/dm/48_48_100/t017aee03b28107657b.jpg"

img="http://vip.zuiku8.com/1810/妖精的尾巴最终季-01.mp4"

my_headers={

    "User-Agent":"Mozilla/5.0 (Windows NT 6.3; Win64; x64) \

    AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36",

   'Referer':'https://www.bilibili.com/bangumi/play/ep250436',

}

def chunk_report(bytes_so_far, chunk_size, total_size):

   percent = float(bytes_so_far) / total_size

   percent = round(percent*100, 2)

   if percent %1==0:

      sys.stdout.write("Downloaded %d of %d bytes (%0.2f%%)\n" %

           (bytes_so_far, total_size, percent))

   if bytes_so_far >= total_size:

      sys.stdout.write('\n')

def chunk_read(response, url,chunk_size=8192, report_hook=None):

   total_size = response.info().getheader('Content-Length').strip()

   total_size = int(total_size)

   bytes_so_far = 0

   path_name=url.split("/")[-1]

   path_name=path_name.replace("\n","")

   path_name=path_name.decode("utf-8")

   print path_name

   with open("%s" % path_name, "wb") as f:

      while 1:

         chunk = response.read(chunk_size)

         f.write(chunk)

         f.flush()

         bytes_so_far += len(chunk)

         if not chunk:

            break

         if report_hook:

            report_hook(bytes_so_far, chunk_size, total_size)

   return bytes_so_far

def down_load(img):

        print img

        request =  urllib2.Request(url=img, headers=my_headers)

        response = urllib2.urlopen(request);

        chunk_read(response,img, report_hook=chunk_report)

        print "downloading with urllib  --->"

if __name__ == '__main__':

     down_load(img)

结果：

如果想在一行显示，打印时加\r，end为空：

print ('\r downloading...{:.1f}'.format(percent), end="")

\r 是移至本行行首
\b 是退一个字符

此外，提一下urllib，之后没有用它，是因为不支持https：

#!/usr/bin/python

#encoding:utf-8

import urllib

import os

img="http://vip.zuiku8.com/1810/妖精的尾巴最终季-01.mp4"

def Schedule(a,b,c):

   '''

   a:已经下载的数据块

   b:数据块的大小

   c:远程文件的大小

   '''

   per = 100.0*a*b/c

   if per > 100:

      per = 100

   print '%.2f%%' % per

def main():

   path=img.split(".")[-1]

   urllib.urlretrieve(img,path,Schedule)

if __name__ == '__main__':

   main()

使用urllib2时，发现https下载不成功，添加了如下代码：

import ssl

ssl._create_default_https_context = ssl._create_unverified_context

巴特西

python urllib2 实现大文件下载

最新文章

热门文章