import requests

import pymongo

import time

import random

mycon = pymongo.MongoClient('127.0.0.1',27017)  # 建立连接

mydb = mycon['lagou_data']                      # 设置库名

class LaGouSpider():

    def __init__(self,city,kd):

        self.headers = {

            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.90 Safari/537.36',

            'Referer': 'https://www.lagou.com/jobs/list_python?labelWords=&fromSearch=true&suginput=',

        }

        self.city = city

        self.max_pn =  1

        self.kd =  kd

    def get_start(self):

        mycol = mydb[self.kd]  # 设置集合名

        url = "https://www.lagou.com/jobs/positionAjax.json?city="+ self.city +"&needAddtionalResult=false"

        for page in range(1,10):

            data = {

                'first': 'true',

                'pn': page,

                'kd': self.kd

            }

            s = requests.Session()

            s.get(url = "https://www.lagou.com/jobs/list_python%E5%BC%80%E5%8F%91%E5%B7%A5%E7%A8%8B%E5%B8%88?labelWords=&fromSearch=true&suginput=",headers = self.headers)

            cookies = s.cookies

            response = s.post(url=url,data=data,cookies = cookies,headers = self.headers).json()

            content = response.get('content')

            if content:

                result = content['positionResult']['result']

                print('岗位名称:{},所在城市:{},开始抓取第:{}页\n'.format(self.kd,self.city,page))

                for i in result:

                    lagou_data = {}

                    lagou_data['positionName'] = i['positionName']        # 岗位名称

                    lagou_data['companyFullName'] = i['companyFullName']  # 公司全名

                    lagou_data['workYear'] = i['workYear']                # 工作经验要求

                    lagou_data['education'] = i['education']              # 学历要求

                    lagou_data['jobNature'] = i['jobNature']              # 工作性质

                    lagou_data['salary'] = i['salary']                    # 薪资

                    lagou_data['city'] = i['city']                        # 所在城市

                    lagou_data['financeStage'] = i['financeStage']        # 金融阶段

                    lagou_data['industryField'] = i['industryField']      # 经营范围

                    lagou_data['companyShortName'] = i['companyShortName']# 公司简名

                    lagou_data['positionAdvantage'] = i['positionAdvantage']# 岗位优势

                    lagou_data['companySize'] = i['companySize']          # 公司规模

                    lagou_data['companyLabelList'] = i['companyLabelList']# 岗位待遇标签

                    lagou_data['district'] = i['district']                # 所在区域

                    lagou_data['positionLables'] = i['positionLables']    # 技术范围标签

                    lagou_data['firstType'] = i['firstType']              # 岗位类型

                    lagou_data['createTime'] = i['createTime']            # 发布时间

                    print(lagou_data)

                    mycol.insert(lagou_data)

            time.sleep(random.uniform(3,7))                               # 随机休眠

if __name__ == '__main__':

    lagou = LaGouSpider('北京','python')

    lagou.get_start()