一个豆瓣电影Top250爬虫

一个爬虫

这是我第一次接触爬虫，写的第一个爬虫实例。

https://movie.douban.com/top250

模块

import requests #用于发送请求

import re #使用正则表达式，用于匹配处理文本

import os #用于创建文件夹

from lxml import etree #这里我使用了Xpath表达式用于数据解析，我觉得这个模块比BeautifulSoup好用，强烈推荐

from fake_useragent import UserAgent #反爬虫，随机获取浏览器 UA 信息

代码

import requests

import re

import os

from lxml import etree

from fake_useragent import UserAgent

class doubanSpider(object):

    def __init__(self):

        if not os.path.exists('db/douban'):

            os.makedirs('db/douban')

        else:

            pass

        self.f = open('./db/douban/douban.txt', 'a', encoding='utf-8')

    def start(self):

        for i in range(46):

            headers = {

                'User-Agent': UserAgent().random

            }

            url = 'https://movie.douban.com/top250?start=' + str(i * 25)

            r = requests.get(url, headers=headers)

            html = etree.HTML(r.text)

            li_list = html.xpath('//*[@id="content"]/div/div[1]/ol/li')

            movies = []

            for each in li_list:

                movie = {}

                order = each.xpath('.//div/div[1]/em/text()')[0]

                movie['id'] = order

                cover = each.xpath('.//div/div[1]/a/img/@src')[0]

                movie['cover'] = cover

                name = each.xpath('.//div/div[2]/div[1]/a/span/text()')

                flag = ''

                for mo in name:

                    flag += mo

                movie['name'] = flag

                info = each.xpath('.//div/div[2]/div[2]/p[1]/text()[1]')[0].strip()

                info = info.replace("\n", "")

                info = info.replace("\xa0", "")

                director = re.findall(r'[导演:].+[主演:]', info)[0]

                director = director[4:len(director) - 3]

                movie['director'] = director

                try:

                    role = re.findall(r'主.+', info)[0]

                    role = role[4:]

                except IndexError:

                    role = ''

                movie['role'] = role

                plot = each.xpath('.//div/div[2]/div[2]/p[1]/text()[2]')[0].strip()

                plot = plot.replace("\xa0", "")

                movie['plot'] = plot

                star = each.xpath('.//div/div[2]/div[2]/div/span[2]/text()')[0]

                movie['star'] = star

                try:

                    comment = each.xpath('.//div/div[2]/div[2]/p[2]/span/text()')[0]

                except IndexError:

                    comment = ''

                movie['comment'] = comment

                movies.append(movie)

                self.f.write(str(movie)+'\n')

                print(movie)

    def run(self):

        self.start()

        self.f.close()

if __name__ == '__main__':

    spider = doubanSpider()

    spider.run()

巴特西

一个豆瓣电影Top250爬虫

一个爬虫

模块

代码

最新文章

热门文章