spider文件下的代码

import scrapy

import re

from scrapy.linkextractors import LinkExtractor  # 链接提取器

from scrapy.spiders import CrawlSpider, Rule  # 导入全站爬虫和采集规则

class Ygdy8Spider(CrawlSpider):

    name = 'ygdy8'  # 爬虫名字

    allowed_domains = ['www.ygdy8.com']  # 爬虫只在该域名下爬取数据

    start_urls = ['http://www.ygdy8.com/']  # 开始采集的网址

    # 采集规则集合

    rules = (

        # 具体采集的规则

        # allow 是选择出所有带有index的网址 allow是正则表达式 只要写你想提取的链接的一部分就可以了, deny是不允许提取的

        Rule(LinkExtractor(allow=r'index.html', deny='game')),

        # follow=true 意思是下一次提取的网页中如果包含我们需要提取的信息,是否还要继续提取,True代表继续提取

        Rule(LinkExtractor(allow=r'list_\d+_\d+.html'), follow=True),

        # 提取详情页信息 callback 回调函数, 将相应交给这个函数来处理

        Rule(LinkExtractor(allow=r'/\d+/\d+.html'), follow=True, callback='parse_item'),

    )

    # 解析采集回来的数据

    def parse_item(self, response):

        # 处理网页数据,提取下载链接

        # .*?代表提取除了换行以外的任何信息

        ftp_url = re.findall(r'<a href="(.*?)">ftp', response.text)

        name = re.findall(r'<title>(.*?)</title>', response.text)

        if ftp_url and name:

            items = {

                'name': name,

                'ftp_url': ftp_url

            }

        yield items

管道文件

import json

import os

import csv

class JsonPipeline(object):

    def __init__(self):

        self.file = open('阳光电影.json', 'w+', encoding='utf-8')

    def open_spider(self, spider):

        self.file.write('[')

    def process_item(self, item, spider):

        line = json.dumps(dict(item), ensure_ascii=False) + ",\n"

        self.file.write(line)

        return item

    def close_spider(self, spider):

        self.file.seek(-1, os.SEEK_END)

        self.file.truncate()

        self.file.write(']')

        self.file.close()

class CsvPipeline(object):

    def __init__(self):

        self.f = open("阳光电影.csv", "w", newline='')

        self.writer = csv.writer(self.f)

        self.writer.writerow(['name', 'ftp_url'])

    def process_item(self, item, spider):

        yangguang_list = [item['name'], item['ftp_url']]

        self.writer.writerow(yangguang_list)

        return item

settings

ITEM_PIPELINES = {

   'movie.pipelines.JsonPipeline': 300,

   'movie.pipelines.CsvPipeline': 300,

}