1 import requests

 2 import json

 3 import csv

 4

 5

 6 class DoubantvSpider:

 7     def __init__(self):

 8         # self.proxies = {"http":"http://125.123.152.81:3000"}

 9         self.url = "https://movie.douban.com/j/search_subjects?type=tv&tag=%E5%9B%BD%E4%BA%A7%E5%89%A7&sort=rank&page_limit=20&page_start={}" # 手机模式下国产剧请求网址

10         self.headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36"}

11

12     # 发送请求，获得json，转化为字典

13     def parse_url(self, url):

14         res = requests.get(url, headers=self.headers)

15         return json.loads(res.content.decode())

16

17     # 保存数据

18     def save(self, dic):

19         with open("doubantv.csv", "a", newline='', encoding="utf8") as f:

20             for data in dic["subjects"]:

21                 writer = csv.writer(f, delimiter=',')

22                 writer.writerow([data["rate"], data["title"], data["url"]])

23

24     # 实现主要逻辑

25     def run(self):

26         page_num = 0

27         while True:

28             # 构造url

29             url = self.url.format(page_num)

30             # 发送请求,获取响应

31             dic = self.parse_url(url)

32             # 因为动态加载,通过判断每页电视剧数量来确定是不是到了尾页

33             if len(dic["subjects"]) < 20:

34                 break

35             self.save(dic)

36             page_num += 20

37             print("ok")

38

39

40 if __name__ == "__main__":

41     doubantv = DoubantvSpider()

42     doubantv.run()