class MeizhuoItem(scrapy.Item):

    # define the fields for your item here like:

    # name = scrapy.Field()

    # 图集的标题

    title = scrapy.Field()

    # 图片的url，需要来进行图片的抓取

    url = scrapy.Field()

    pass

# -*- coding: utf-8 -*-

import scrapy

from scrapy.selector import Selector

from fake_useragent import UserAgent

from meizhuo.items import MeizhuoItem

headers = {

    'user-agent': UserAgent(verify_ssl=False).chrome

}

class MzSpider(scrapy.Spider):

    name = 'mz'

    allowed_domains = ['www.win4000.com']

    start_urls = [

        'http://www.win4000.com/wallpaper_2285_0_0_1.html',

        'http://www.win4000.com/wallpaper_204_0_0_1.html'

    ]

    def parse(self, response):

        sel = Selector(response)

        list = sel.xpath('//*[@class="list_cont Left_list_cont"]/div/div/div/ul/li/a')

        for img in list:

            # 这个是每个图集得到的url

            url = img.xpath('@href').extract_first()

            title = img.xpath('@title').extract_first()

            # 对我的每一个URL进行解析

            yield scrapy.Request(url, callback=self.get_all_img, meta={'title': title})

        # 对于下一页进行定位，如果存在就进行跳转

        next_url = sel.xpath('//*[@class="next"]/@href').extract_first()

        if next_url is not None:

            yield scrapy.Request(next_url, callback=self.parse)

    def get_all_img(self, response):

        item = MeizhuoItem()

        container = []

        sel = Selector(response)

        # 这个是所有照片的所有的总共的页数

        img_list = sel.xpath('//*[@class="scroll-img-cont"]/ul')

        for img in img_list:

            img_url = img.xpath('li/a/img/@data-original').extract()

            for url in img_url:

                # 这个url还是需要经过处理的，所以要循环出来挨个进行修改

                cmp_url = url.split('_')[0] + '.jpg'

                container.append(cmp_url)

            item['url'] = container

            item['title'] = response.meta['title']

            # print(container)

            yield item

            container.clear()

# -*- coding: utf-8 -*-

# Define your item pipelines here

#

# Don't forget to add your pipeline to the ITEM_PIPELINES setting

# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html

import os

import requests

from meizhuo.settings import IMAGES_STORE

from fake_useragent import UserAgent

headers = {

    'user-agent': UserAgent(verify_ssl=False).chrome

}

class MeizhuoPipeline(object):

    def process_item(self, item, spider):

        dir_path = IMAGES_STORE

        title = item['title']

        img_list = item['url']

        print(title, img_list)

        """

        1、我要创建指定的路径

        2、然后我要利用requests模块获取到那一个url的二进制数据保存进去

        """

        if not os.path.exists(IMAGES_STORE):

            os.mkdir(IMAGES_STORE)

        # 如果这个顶头文件夹存在的话

        collection_url = os.path.join(IMAGES_STORE, title)

        print('', collection_url)

        if not os.path.exists(collection_url):

            os.mkdir(collection_url)

        for url_list in range(len(img_list)):

            cmp_url = os.path.join(collection_url, img_list[url_list])

            # print(cmp_url)

            file_path = os.path.join(collection_url, title) + str(url_list) + '.jpg'

            print(file_path)

            with open(file_path, 'wb') as fp:

                res = requests.get(img_list[url_list], headers=headers).content

                # print(img_list[url_list])

                fp.write(res)

                print('insert successfully!!!')

BOT_NAME = 'meizhuo'

SPIDER_MODULES = ['meizhuo.spiders']

NEWSPIDER_MODULE = 'meizhuo.spiders'

ROBOTSTXT_OBEY = True

ITEM_PIPELINES = {

   'meizhuo.pipelines.MeizhuoPipeline': 300,

}

# 下载的顶头路径

IMAGES_STORE = '/Users/mac/Documents/girl'

# 下载延迟

DOWNLOAD_DELAY = 0.3