# -*- coding: utf-8 -*-

 import json

 import os

 import time

 from multiprocessing import Pool

 import multiprocessing

 import requests

 from selenium import webdriver

 def get_image_links(keyword, num_requested = 1000):

     """get image links with selenium

     """

     number_of_scrolls = int(num_requested/400) + 1

     img_urls = set()#设置为集合，自动去除重复链接

     chrome_options = webdriver.ChromeOptions()

     # chrome_options.add_argument('--headless')#设置无头浏览器

     # chrome_options.add_argument('user-agent="Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"')

     # chrome_options.add_argument("lang=en_US")#设置语言

     # prefs = {"profile.managed_default_content_settings.images":2}

     # chrome_options.add_experimental_option("prefs",prefs)#配置不加载图片

     driver = webdriver.Chrome(chrome_options=chrome_options)

     driver.maximize_window()

     search_query = keyword

     url = "https://www.google.com/search?q="+search_query+"&source=lnms&tbm=isch"

     driver.get(url)

     for _ in range(number_of_scrolls):

         for i in range(5):

             # multiple scrolls needed to show all 400 images

             driver.execute_script("window.scrollBy(0, 100000)")

             time.sleep(1)

         time.sleep(5)#等待页面刷新，否则有可能元素不可见

         try:

             # driver.find_element_by_xpath("//input[@value='Show more results']").click()＃浏览器的中英文版本不同

             driver.find_element_by_xpath("//input[@value='显示更多结果']").click()

         except Exception as e:

             print("reach the end of page ")

             break

     # with open('page.html','w') as f:

     #     f.write(driver.page_source)

     imgs = driver.find_elements_by_xpath('//div[contains(@class,"rg_meta")]')#模糊定位

     for i,img in enumerate(imgs):

         img_url = json.loads(img.get_attribute('innerHTML'))["ou"]

         img_urls.add(img_url)

     driver.quit()

     print("finish getting all image urls!")

     return img_urls

 def download(urls,download_dir):

     '''download images

     '''

     print("start downloading images!")

     for url in urls:

         filename=os.path.join(download_dir,os.path.basename(url))

         try:

             r = requests.get(url, stream=True, timeout=60)

             r.raise_for_status()

             with open(filename, 'wb') as f:

                 f.write(r.content)

         except Exception:

             continue

     print("finish downloading images!")

 keywords = ['girl','boy']

 download_dir = './images/'

 download_dirs = []

 for keyword in keywords:

     path = os.path.join(download_dir,keyword)

     download_dirs.append(path)

     if not os.path.exists(path):

         os.makedirs(path)

 # for keyword in main_keywords:

 #     image_urls = get_image_links(keyword)

 #     download(image_urls,download_dir)

 ###################################

 # get image links/MultiProcess

 ###################################

 img_urls=[]

 multiprocessing.freeze_support()

 p = Pool(4) # default number of process is the number of cores of your CPU, change it by yourself

 for keyword in keywords:

     img_urls.append(p.apply_async(get_image_links, (keyword,)))

 #img_urls:[<multiprocessing.pool.ApplyResult object at 0x7f536925fcc0>, <multiprocessing.pool.ApplyResult object at 0x7f536925fd68>]

 for i,urls in enumerate(img_urls):

     img_urls[i]=urls.get()

 p.close()

 p.join()

 # # ###################################

 # # # download images/MultiProcess

 # # ###################################

 p = Pool(4) # default number of process is the number of cores of your CPU, change it by yourself

 for i,urls in enumerate(img_urls):

     p.apply_async(download, [urls,download_dirs[i]])

 p.close()

 p.join()