import re

from concurrent.futures import ThreadPoolExecutor

import requests

#获取豆瓣电影top250电影名字、导演、评分、评价人数

def getDoubanRource(url):

    header = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:97.0) Gecko/20100101 Firefox/97.0"}

    res=requests.get(url,headers=header)

    #获取页面源代码

    pagesource = res.text

    #预加热正则表达式对象

    obj=re.compile(r'<span class="title">(?P<filmname>.*?)</span>.*?<p class="">\s*(?P<director>.*?)'

                   r' .*?<span class="rating_num" property="v:average">(?P<score>.*?)</span>.*?<span>(?P<person>.*?)评价</span>',re.S)

    reptile_res=obj.finditer(pagesource)

    with open("D:\dir_ytj\\dome1.csv",mode="a") as f:

        for item in reptile_res :

            filmname=item.group("filmname")

            director = item.group("director")

            score = item.group("score")

            person = item.group("person")

            f.write(f"{filmname},{director},{score},{person}\n")

        print(url,"收取完毕")

if __name__ == '__main__':

    with ThreadPoolExecutor(10) as t:

        for i in range(10):

            t.submit(getDoubanRource,f"https://movie.douban.com/top250?start={25*i}&filter=")

    print("完成全部信息收录")

import asyncio

import sys,io

import re

from concurrent.futures import ThreadPoolExecutor

import requests

sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='gb18030')

#获取豆瓣电影top250电影名字、导演、评分、评价人数

async def writeCsv(filmname,director,score,person):

    with open("D:\dir_ytj\\dome2.csv", mode="a") as f:

        f.write(f"{filmname},{director},{score},{person}\n")

async def getDoubanRource(url):

    header = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:97.0) Gecko/20100101 Firefox/97.0"}

    res=requests.get(url,headers=header)

    #获取页面源代码

    pagesource = res.text

    #预加热正则表达式对象

    obj=re.compile(r'<span class="title">(?P<filmname>.*?)</span>.*?<p class="">\s*(?P<director>.*?)'

                   r' .*?<span class="rating_num" property="v:average">(?P<score>.*?)</span>.*?<span>(?P<person>.*?)评价</span>',re.S)

    reptile_res=obj.finditer(pagesource)

    tasks=[]

    for item in reptile_res :

        filmname=item.group("filmname")

        director = item.group("director")

        score = item.group("score")

        person = item.group("person")

        tasks.append(writeCsv(filmname,director,score,person))

    await asyncio.wait(tasks)

if __name__ == '__main__':

    print("正在收集网页信息......")

    for i in range(10):

        asyncio.run(getDoubanRource(f"https://movie.douban.com/top250?start={25*i}&filter="))

    print("收集完成")