# -*- coding: utf-8 -*-

import scrapy

import json

import os

import urllib

import time

from scrapy.http import Request

from selenium import webdriver

from selenium.webdriver.common.by import By

from selenium.webdriver.support.ui import WebDriverWait

from selenium.webdriver.support import expected_conditions as EC

from selenium.common.exceptions import (

    TimeoutException,

    WebDriverException,

    NoSuchElementException,

    StaleElementReferenceException

)

def gen_browser(driver_path):

    '''实例化一个driver'''

    options = webdriver.ChromeOptions()

    options.add_argument("--headless")

    options.add_argument('--no-sandbox')

    options.add_argument('--disable-gpu')

    options.add_argument('--ignore-certificate-errors')

    options.add_argument('disable-infobars')

    options.add_argument("--disable-plugins-discovery")

    user_agent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36"

    options.add_argument('user-agent="{0}"'.format(user_agent))

    # ############### 专业造假 ***************************

    def send(driver, cmd, params={}):

        '''

        向调试工具发送指令

        from: https://stackoverflow.com/questions/47297877/to-set-mutationobserver-how-to-inject-javascript-before-page-loading-using-sele/47298910#47298910

        '''

        resource = "/session/%s/chromium/send_command_and_get_result" % driver.session_id

        url = driver.command_executor._url + resource

        body = json.dumps({'cmd': cmd, 'params': params})

        response = driver.command_executor._request('POST', url, body)

        if response['status']:

            raise Exception(response.get('value'))

        return response.get('value')

    def add_script(driver, script):

        '''在页面加载前执行js'''

        send(driver, "Page.addScriptToEvaluateOnNewDocument", {"source": script})

    # 给 webdriver.Chrome 添加一个名为 add_script 的方法

    webdriver.Chrome.add_script = add_script  # 这里（webdriver.Chrome）可能需要改，当调用不同的驱动时

    # *************** 专业造假 ###################

    browser = webdriver.Chrome(

        executable_path=driver_path,

        chrome_options=options

    )

    # ################## 辅助调试 *********************

    existed = {

        'executor_url': browser.command_executor._url,  # 浏览器可被远程连接调用的地址

        'session_id': browser.session_id  # 浏览器会话ID

    }

    print(existed)

    # ********************* 辅助调试 ##################

    # ############### 专业造假 ***************************

    browser.add_script("""

    Object.defineProperty(navigator, 'webdriver', {

        get: () => false,

    });

    window.navigator.chrome = {

        runtime: {},

    };

    Object.defineProperty(navigator, 'languages', {

        get: () => ['zh-CN', 'zh']

    });

    Object.defineProperty(navigator, 'plugins', {

        get: () => [0, 1, 2],

    });

    """)

    # *************** 专业造假 ###################

    return browser

class XuexingSpider(scrapy.Spider):

    name = 'xuexing'

    allowed_domains = ['www.manhuatai.com']

    start_urls = ['https://www.mh1234.com/wap/comic/9683/262424.html']

    def parse(self, response):

        driver_path = self.settings.get('DRIVER_PATH')

        # import ipdb; ipdb.set_trace()

        browser = gen_browser(driver_path)

        # 获取当前章节名

        next_url = response.url

        i = 0

        while bool(next_url):

            i += 1

            print(str(i).center(60, '*'))

            next_url = self.get_item(browser, next_url)

    def get_item(self, browser, url=None):

        if url is not None:

            browser.get(url)  # 打开页面

        van1 = browser.find_elements_by_xpath('//a[@class="BarTit"]')

        van = van1[0].text.split('(')[0].strip()

        if '/' in van:

            van = '-'.join(van.split('/'))

        # import ipdb; ipdb.set_trace()

        if not os.path.exists('斗罗大陆'):

            os.mkdir('斗罗大陆')

        if not os.path.exists(van):

            os.mkdir(r'斗罗大陆/{0}'.format(van))

        m = 0

        _url = browser.find_element_by_xpath('//*[@id="qTcms_pic"]')

        img_url = _url.get_attribute('src')

        # 保存图片到指定路径

        if img_url != None:

            m += 1

            #保存图片数据

            data = urllib.request.urlopen(img_url).read()

            f = open('斗罗大陆/{0}/{1}.jpg'.format(van, m), 'wb')

            f.write(data)

            f.close()

        ye = int(browser.find_element_by_xpath('//*[@id="k_total"]').text)

        for yei in range(1, ye):

            time.sleep(0.5)

            browser.find_element_by_xpath('//*[@id="action"]/ul/li[3]/a').click()

            _url = browser.find_element_by_xpath('//*[@id="qTcms_pic"]')

            img_url = _url.get_attribute('src')

            # 保存图片到指定路径

            if img_url != None:

                m += 1

                #保存图片数据

                data = urllib.request.urlopen(img_url).read()

                f = open('斗罗大陆/{0}/{1}.png'.format(van, m), 'wb')

                f.write(data)

                f.close()

        xia = browser.find_element_by_xpath('//*[@id="action"]/ul/li[4]/a').get_attribute('href')

        return xia