import requests # 导入requests请求库

# #

# 往百度主页发送请求，获取响应对象

response = requests.get(url = 'https:www.baidu.com')

# #

# 设置字符编码为 utf-8

response.encoding = 'utf-8'

# #

# 打印响应文本

print(response.text)

# #

# 把响应文本写入本地

with open('baidu.html','w',encoding='utf-8')as f:

    f.write(response.text)

import requests

video_url = 'https://www.pearvideo.com/video_1570475'

response = requests.get(url=video_url)

print(response.text)

# 往视频源地址发送请求

response = requests.get('https://video.pearvideo.com/mp4/adshort/20190626/cont-1570475-14058832_adpkg-ad_hd.mp4')

# 打印二进制流，比如图片、视频等数据

print(response.content)

#

# 保存视频到本地

with open('video.mp4','wb')as f:

    f.write(response.content)

import requests

import re # 正则，用于解析文本数据

# 1.先往梨视频主页发送请求

response = requests.get('https://www.pearvideo.com/')

print(response.text)

# re正则匹配获取所有视频id

# 参数1：正确匹配规则

# 参数2：解析文本

# 参数3：匹配模式

res_list = re.findall('<a href="video_(.*?)"', response.text, re.S)

 print(res_list)

for v_id in res_list:

    detail_url = 'https://www.pearvideo.com/video_'+ v_id

     print(detail_url)

    # 对每一个视频详情页发送请求获取视频源url

    response = requests.get(url=detail_url)

    # print(response.text)

    # 解析并提取详情页视频url

    # 视频url

    video_url = re.findall('srcUrl="(.*?)"', response.text, re.S)[0]

    print(video_url)

    # 视频名称

    video_name = re.findall('<h1 class="video-tt">(.*?)</h1>', response.text, re.S)[0]

    print(video_name)

    # 往视频url发送请求获取视频二进制流

    v_response = requests.get(video_url)

    with open('%s.mp4'% video_name,'wb')as f:

        f.write(v_response.content)

        print(video_name, '视频爬取完成')

import requests

import re

# 爬虫三部曲

# 1.发送请求

def get_page(base_url):

    response = requests.get(base_url)

    return response

# 2.解析文本

def parse_index(text):

    res = re.findall(

        '<div class="item">.*?<em class="">(.*?)</em>.*?<a href="(.*?)">.*?<span class="title">(.*?)</span>.*?导演:(.*?)</p>.*?<span class="rating_num".*?>(.*?)</span>.*?<span>(.*?)人评价</span>.*?<span class="inq">(.*?)</span>',

        text, re.S)

    return res

# 3.保存数据

def save_data(data):

    with open('douban.txt','a',encoding='utf-8')as f:

        f.write(data)

# main + 回车键

if __name__ == '__main__':

# num = 10

# base_url = 'https://movie.douban.com/top250?start{}&filter='.format(num)

    num = 0

    for line in range(10):

        base_url = f'https://movie.douban.com/top250?start={num}&filter='

        num += 25

        print(base_url)

        # 1.发送请求，调用函数

        response = get_page(base_url)

        # 2.解析文本

        movie_list = parse_index(response.text)

        # 3.保存数据

        # 数据的格式化

        for movie in movie_list:

            # print(movie)

            # 解压赋值

            # 电影排名、电影url、电影名称、导演-主演-类型、电影评分、评价人数、电影简介

            v_top,v_url,v_name,v_daoyan,v_point,v_num,v_desc = movie

            # v_top = movie[0]

            # v_url = movie[1]

            movie_concent = f'''

            电影排名：{v_top}

            电影url：{v_url}

            电影名称：{v_name}

            导演主演：{v_daoyan}

            电影评分：{v_point}

            评价人数：{v_num}

            电影简介：{v_desc}

            \n

            '''

            print(movie_concent)

            # 保存数据

            save_data(movie_concent)