#coding=utf-8

import requests

import re

import string

import urllib2 as url

import urllib

import Queue

import os

import sys

from contextlib import closing 

class ProgressBar(object):

    def __init__(self, title, count=0.0, run_status=None, fin_status=None, total=100.0,    unit='', sep='/', chunk_size=1.0):

        super(ProgressBar, self).__init__()

        self.info = '[%s] %s %.2f %s %s %.2f %s'

        self.title = title

        self.total = total

        self.count = count

        self.chunk_size = chunk_size

        self.status = run_status or ''

        self.fin_status = fin_status or ' ' * len(self.statue)

        self.unit = unit

        self.seq = sep  

    def __get_info(self):

        # [名称]状态 进度 单位 分割线 总数 单位

        _info = self.info % (self.title, self.status, self.count/self.chunk_size, self.unit, self.seq, self.total/self.chunk_size, self.unit)

        return _info  

    def refresh(self, count=1, status=None):

        self.count += count

        # if status is not None:

        self.status = status or self.status

        end_str = '\r'

        if self.count >= self.total:

            end_str = '\n'

            self.status = status or self.fin_status  

        sys.stdout.flush()

        sys.stdout.write(self.__get_info()+end_str)

        sys.stdout.flush()

def get_regex_outcome_list(text, regex):

    """

    获取文本的正则表达式截取

    """

    lst = []

    reobj = re.compile(regex)

    for match in reobj.finditer(text):

        #match.start()

        #match.end()

        lst.append(match.group())

    return lst

def wget_file(url, file_name):

    """

    下载文件

    """

    #r = requests.get(url)

    #with open(file_name, 'wb') as code:

    #    code.write(r.content)

    with closing(requests.get(url, stream=True)) as response:

        chunk_size = 1024

        content_size = float(response.headers['content-length'])

        '''

        需要根据 response.status_code 的不同添加不同的异常处理

        '''

        print('content_size', content_size,response.status_code ,  )

        progress = ProgressBar('processing:'

                    , total=content_size

                    , unit='KB'

                    , chunk_size=chunk_size

                    , run_status='download...'

                    , fin_status='download ok!')

            # chunk_size = chunk_size < content_size and chunk_size or content_size

        with open(file_name, 'wb') as file:

                for data in response.iter_content(chunk_size=chunk_size):

                    file.write(data)

                    progress.refresh(count=len(data))  

        print('download ok!', ) 

def page_to_url(url, html):

    """

    解析页面上的html，提取href中的url

    """

    lst = get_regex_outcome_list(html, 'href=".*"')

    context = ''

    path_lst = []

    for item in lst:

        item = item.replace('"','')

        item = item.replace('href=', '')

        path_lst.append((url+item, item))

    return path_lst

def parser_path_list(init_url, path_lst, deep_queue):

    """

    解析路径列表，对路径做对应处理

    """

    for item in path_lst:

        url = item[0]

        file_name = item[1]

        if file_name[-1] == '/' and file_name != '../':

            deep_queue.put(url)

            dir_name = url[len(init_url):-1]

            try:

                os.mkdir(dir_name)

            except Exception, e:

                print e

        else:

            #pass

            file_path = url[len(init_url):-1]

            print file_path

            try:

                wget_file(url, file_path)

            except Exception, e:

                print e                

def visit_all_path(init_url):

    """

    遍历所有路径

    """

    r = requests.get(init_url)

    path_lst = page_to_url(init_url, r.text)

    deep_queue = Queue.Queue()

    parser_path_list(init_url, path_lst, deep_queue)

    while not deep_queue.empty():

        print deep_queue.qsize()

        url = deep_queue.get()

        r = requests.get(url)

        path_lst = page_to_url(url, r.text)

        parser_path_list(init_url, path_lst, deep_queue)

    return path_lst

if __name__ == '__main__':

    init_url = 'http://mirrors.aliyun.com/centos/7.2.1511/cloud/x86_64/openstack-liberty/'

    visit_all_path(init_url)

[loacl_network_openstack_liberty]

name=ftpyum

baseurl=ftp://192.168.29.94/pub/cloud/openstack-liberty

enabled=1

gpgcheck=1

gpgkey=ftp://192.168.29.94/pub/centos7/RPM-GPG-KEY-CentOS-7