import requests

import re

import hashlib,time

def get_index(url):

    response=requests.get(url)

    if response.status_code == 200:

        return response.text

def parse_index(res):

    urls=re.findall(r'class="items".*?href="(.*?)"',res,re.S)

    return urls

def get_detail(urls):

    for url in urls:

        if not url.startswith('http'):

            url='http://www.xiaohuar.com%s' %url

        r1=requests.get(url)

        if r1.status_code == 200:

            url_list=re.findall(r'id="media".*?src="(.*?)"',r1.text,re.S)

            if url_list:

                mp4_url=url_list[0]

                # print(mp4_url)

                save(mp4_url)

def save(url):

    print('Download:%s' %url)

    r2=requests.get(url)

    if r2.status_code == 200:

        m=hashlib.md5()

        m.update(url.encode('utf-8'))

        m.update(str(time.time()).encode('utf-8'))

        filename='%s.mp4' %m.hexdigest()

        file_path=r'D:\\爬虫视频\%s' % filename

        with open(file_path,'wb') as f:

            f.write(r2.content)

def main():

    for i in range(5):

        res1=get_index('http://www.xiaohuar.com/list-3-%s.html' %i)

        res2=parse_index(res1)

        get_detail(res2)

if __name__ == '__main__':

    main()

# 异步，多线程优化下载速度

import requests

import re

import hashlib,time

from concurrent.futures import ThreadPoolExecutor

p=ThreadPoolExecutor(30)

def get_index(url):

    response=requests.get(url)

    if response.status_code == 200:

        return response.text

def parse_index(res):

    res=res.result()

    urls=re.findall(r'class="items".*?href="(.*?)"',res,re.S)

    # return urls

    for url in urls:

        p.submit(get_detail,url)

def get_detail(urls):

    for url in urls:

        if not url.startswith('http'):

            url='http://www.xiaohuar.com%s' %url

        r1=requests.get(url)

        if r1.status_code == 200:

            url_list=re.findall(r'id="media".*?src="(.*?)"',r1.text,re.S)

            if url_list:

                mp4_url=url_list[0]

                # print(mp4_url)

                save(mp4_url)

def save(url):

    print('Download:%s' %url)

    r2=requests.get(url)

    if r2.status_code == 200:

        m=hashlib.md5()

        m.update(url.encode('utf-8'))

        m.update(str(time.time()).encode('utf-8'))

        filename='%s.mp4' %m.hexdigest()

        file_path=r'D:\\爬虫视频\%s' % filename

        with open(file_path,'wb') as f:

            f.write(r2.content)

def main():

    for i in range(5):

        p.submit(get_index,'http://www.xiaohuar.com/list-3-%s.html' %i).add_done_callback(parse_index)

        # res1=get_index('http://www.xiaohuar.com/list-3-%s.html' %i)

        # res2=parse_index(res1)

        # get_detail(res2)

if __name__ == '__main__':

    main()