from selenium import webdriver

 import requests

 import re

 from lxml import etree

 import time

 from selenium.webdriver.support.ui import WebDriverWait

 from selenium.webdriver.support import expected_conditions as EC

 from selenium.webdriver.common.by import By

 class LagouSpider(object):

     def __init__(self):

         opt = webdriver.ChromeOptions()

         # 把chrome设置成无界面模式

         opt.set_headless()

         self.driver = webdriver.Chrome(options=opt)

         self.url = 'https://www.lagou.com/jobs/list_爬虫?px=default&city=北京'

         self.headers = {

             'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36'

         }

     def run(self):

         self.driver.get(self.url)

         while True:

             html = ''

             links = []

             html = self.driver.page_source

             links = self.get_one_page_links(html)

             for link in links:

                 print('\n' + link+'\n')

                 self.parse_detail_page(link)

             WebDriverWait(self.driver, 10).until(

                 EC.presence_of_element_located((By.CLASS_NAME, 'pager_next')))

             next_page_btn = self.driver.find_element_by_class_name('pager_next')

             if 'pager_next_disabled' in next_page_btn.get_attribute('class'):

                 break

             else:

                 next_page_btn.click()

             time.sleep(1)

     def get_one_page_links(self, html):

         links = []

         hrefs = self.driver.find_elements_by_xpath('//a[@class="position_link"]')

         for href in hrefs:

             links.append(href.get_attribute('href'))

         return links

     def parse_detail_page(self, url):

         job_information = {}

         response = requests.get(url, headers=self.headers)

         time.sleep(2)

         html = response.text

         html_element = etree.HTML(html)

         job_name = html_element.xpath('//div[@class="job-name"]/@title')

         job_description = html_element.xpath('//dd[@class="job_bt"]//p//text()')

         for index, i in enumerate(job_description):

             job_description[index] = re.sub('\xa0', '', i)

         job_address = html_element.xpath('//div[@class="work_addr"]/a/text()')

         job_salary = html_element.xpath('//span[@class="salary"]/text()')

         # 字符串处理去掉不必要的信息

         for index, i in enumerate(job_address):

             job_address[index] = re.sub('查看地图', '', i)

         while '' in job_address:

             job_address.remove('')

         job_information['job_name'] = job_name

         job_information['job_description'] = job_description

         job_information['job_address'] = job_address

         job_information['job_salary'] = job_salary

         print(job_information)

 def main():

     spider = LagouSpider()

     spider.run()

 if __name__ == '__main__':

     main()