# -*- coding: utf-8 -*-

import scrapy

from selenium import webdriver

"""

Scrapy 使用 selenium

    1.在spider的构造方法中实例化一个浏览器对象

    2.重写spider下的closed方法,该方法用来关闭浏览器

    3.在下载中间件的process_response方法中使用selenium,通过spider参数获取浏览器对象

    4.在中间件中使用selenium对网站进行信息抓取

    5.实例化一个HtmlResponse对象,且将page_source封装到HtmlResponse对象中

    6.返回该新的相应对象

    7.在parse函数中对新的response对象进行解析.

"""

class NewsSpider(scrapy.Spider):

    name = 'news'

    # allowed_domains = ['www.xxx.com']

    start_urls = ['http://war.163.com/']

    # 在实例化Spider对象时创建一个新的浏览器对象

    def __init__(self):

        self.bro = webdriver.Chrome(executable_path=r'F:\爬虫\chromedriver.exe')

    def parse(self, response):

        div_list = response.xpath('//div[@class="data_row news_article clearfix "]')

        for div in div_list:

            title = div.xpath('.//div[@class="news_title"]/h3/a/text()').extract_first()

            print(title)

    # 爬虫关闭后的执行的函数

    def closed(self,spider):

        print('关闭浏览器')

        self.bro.quit()

    def process_response(self, request, response, spider):

        # Called with the response returned from the downloader.

        # Must either;

        # - return a Response object

        # - return a Request object

        # - or raise IgnoreRequest

        # 在此处使用已经实例化好的浏览器对象.

        bro = spider.bro

        bro.get(url=request.url)

        sleep(3)

        page_text = bro.page_source

        sleep(3)

        # 返回新的response对象

        return HtmlResponse(url=bro.current_url, body=page_text, encoding='utf-8', request=request)