# Define your item pipelines here

#

# Don't forget to add your pipeline to the ITEM_PIPELINES setting

# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html

# useful for handling different item types with a single interface

from itemadapter import ItemAdapter

import json

import os

import urllib.request

class MyfirstScrapydemoPipeline:

    #爬虫启动时会调用此方法

    def open_spider(self,spider):

        print('爬虫启动')

        self.fp= open('qiushibaike.txt','w',encoding='utf8')

    def process_item(self, item, spider):

        #下载头像图片

        self.download(item)

        #将数据转化为字典

        obj= dict(item)

        #将字典数据装换为json格式

        string= json.dumps(obj,ensure_ascii=False)

        #将数据写入文件

        self.fp.write(string+'\n')

        #不注释return item时只会下载第一页的头像

        #return item

    def download(self,item):

        #定义头像图片存放路径

        dirpath= r'F:\python_project\爬虫\myfirst_scrapyDemo\myfirst_scrapyDemo\spiders\头像'

        #获取每一张图片的名称

        name= item['name']+'.jpg'

        #拼接每一张头像图片的存储路径

        filepath=os.path.join(dirpath,name)

        #下载头像图片

        urllib.request.urlretrieve(item['face_src'],filepath)

    # 爬虫结束时会调用此方法

    def close_spider(self,spider):

        print('爬虫结束')

        self.fp.close()