# /bin/python

 # author:leozhao

 # author@email: dhzzy88@.com

 """

 这是整个爬虫系统的主程序

 """

 import numpy as np

 import dataFactory

 import plotpy

 import sqlDeal

 import zhilian

 from Constant import JOB_KEY

 #

 # 启动爬虫程序

 zhilian.spidefmain(JOB_KEY)

 """

  爬取数据结束后对数据可视化处理

 """

 # 从数据库读取爬取的数据

 # 先得到的是元组name,salray，demand,welfare

 value = sqlDeal.sqlselect()

 # 工资上限，下限，平均值

 updata = np.array([], dtype=np.int)

 downdata = np.array([], dtype=np.int)

 average = np.array([], dtype=np.int)

 for item in value:

     salray = dataFactory.SarayToInt(item[])

     salray.slove()

     updata = np.append(updata, salray.up)

     downdata = np.append(downdata, salray.down)

     average = np.append(average, (salray.up + salray.down) / )

 # 工资上下限

 average.sort()

 # 匹配城市信息 暂时还未实现

 # 统计信息

 # 两种图形都加载出来 方便查看

 plotpy.plotl(average)

 plotpy.plots(average)

 print(average, average.sum())

 print("平均工资:", average.sum() / len(average))

 print("最高：", average.max())

 print("最低", average.min())

 print("职位数", len(average))

 # 画图

 # /bin/python

 # author:leo

 # author@email : dhzzy88@163.com

 from selenium import webdriver

 from selenium.webdriver.common.by import By

 from selenium.webdriver.common.keys import Keys

 from selenium.webdriver.support import expected_conditions as EC

 from selenium.webdriver.support.ui import WebDriverWait

 import sqlDeal

 from Constant import PAGE_NUMBER

 def init(key="JAVA"):

     # 智联招聘的主页搜索关键字，初始化到采集页面

     url = "https://www.zhaopin.com/"

     opt = webdriver.FirefoxOptions()

     opt.set_headless()           #设置无头浏览器模式

     driver = webdriver.Firefox(options=opt)

     driver.get(url)

     driver.find_element_by_class_name("zp-search-input").send_keys(key)

     # driver.find_element_by_class_name(".zp-search-btn zp-blue-button").click()

     driver.find_element_by_class_name("zp-search-input").send_keys(Keys.ENTER)

     import time

     time.sleep(2)

     all = driver.window_handles

     driver.switch_to_window(all[1])

     url = driver.current_url

     return url

 class ZhiLian:

     def __init__(self, key='JAVA'):

         # 默认key:JAVA

         indexurl = init(key)

         self.url = indexurl

         self.opt = webdriver.FirefoxOptions()

         self.opt.set_headless()

         self.driver = webdriver.Firefox(options=self.opt)

         self.driver.get(self.url)

     def job_info(self):

         # 提取工作信息     可以把详情页面加载出来

         job_names = self.driver.find_elements_by_class_name("job_title")

         job_sarays = self.driver.find_elements_by_class_name("job_saray")

         job_demands = self.driver.find_elements_by_class_name("job_demand")

         job_welfares = self.driver.find_elements_by_class_name("job_welfare")

         for job_name, job_saray, job_demand, job_welfare in zip(job_names, job_sarays, job_demands, job_welfares):

             sqlDeal.sqldeal(str(job_name.text), str(job_saray.text), str(job_demand.text), str(job_welfare.text))

         # 等待页面加载

         print("等待页面加载")

         WebDriverWait(self.driver, 10, ).until(

             EC.presence_of_element_located((By.CLASS_NAME, "job_title"))

         )

     def page_next(self):

         try:

             self.driver.find_elements_by_class_name("btn btn-pager").click()

         except:

             return None

         self.url = self.driver.current_url

         return self.driver.current_url

 def spidefmain(key="JAVA"):

     ZHi = ZhiLian(key)

     ZHi.job_info()

     # 设定一个爬取的页数

     page_count = 0

     while True:

         ZHi.job_info()

         ZHi.job_info()

         page_count += 1

         if page_count == PAGE_NUMBER:

             break

     # 采集结束后把对象清除

     del ZHi

 if __name__ == '__main__':

 spidefmain("python")

 # /bin/python

 # author:leozhao

 # author@email :dhzzy88@163.com

 import mysql.connector

 from Constant import SELECT

 from Constant import SQL_USER

 from Constant import database

 from Constant import password

 def sqldeal(job_name, job_salray, job_demand, job_welfare):

     conn = mysql.connector.connect(user=SQL_USER, password=password, database=database, use_unicode=True)

     cursor = conn.cursor()

     infostring = "insert into zhilian value('%s','%s','%s','%s')" % (

         job_name, job_salray, job_demand, job_welfare) + ";"

     cursor.execute(infostring)

     conn.commit()

     conn.close()

 def sqlselect():

     conn = mysql.connector.connect(user=SQL_USER, password=password, database=database, use_unicode=True)

     print("连接数据库读取信息")

     cursor = conn.cursor()

     cursor.execute(SELECT)

     values = cursor.fetchall()

     conn.commit()

     conn.close()

     return values

# /bin/python

# author:leozhao

# author@email : dhzzy88@163.com

import matplotlib.pyplot as plt

import numpy as np

from Constant import JOB_KEY

# 线型图

def plotl(dta):

    dta.sort()

    print("dta", [dta])

    num = len(dta)

    x = np.linspace(0, num - 1, num)

    print([int(da) for da in dta])

    print(len(dta))

    plt.figure()

    line = plt.plot(x, [sum(dta) / num for i in range(num)], dta)

    # plt.xlim(0, 250)

    plt.title(JOB_KEY + 'Job_Info')

    plt.xlabel(JOB_KEY + 'Job_Salray')

    plt.ylabel('JobNumbers')

    plt.show()

# 条形图

def plots(dta):

    fig = plt.figure()

    ax = fig.add_subplot(111)

    ax.hist(dta, bins=15)

    plt.title(JOB_KEY + 'Job_Info')

    plt.xlabel(JOB_KEY + 'Job_Salray')

    plt.ylabel('JobNumbers')

plt.show()