from urllib import request

from urllib import parse

from urllib.error import URLError

import json

import math

import pymongo

MONGO_URL='localhost'

MONGO_DB='LaGou'

MONGO_TABLE='数据分析'

client = pymongo.MongoClient(MONGO_URL)#连接数据库

db=client[MONGO_DB]#创建数据库的名字

page=0#页码信息

City='深圳'

KeyWord='数据分析师'

base_url='https://www.lagou.com/jobs/positionAjax.json?'

params = {

    'px':'default',

    'city':City,

    'needAddtionalResult':'false'

}

url = base_url+parse.urlencode(params)

headers = {

    'Accept':'application/json, text/javascript, */*; q=0.01',

    'Accept-Encoding':'gzip, deflate, br',

    'Accept-Language':'zh-CN,zh;q=0.9',

    'Connection':'keep-alive',

    'Content-Length':64,

    'Cookie':'_ga=GA1.2.257516341.1522050452; user_trace_token=20180326154732-f0f5196c-30c9-11e8-9f49-525400f775ce; LGUID=20180326154732-f0f520df-30c9-11e8-9f49-525400f775ce; LG_LOGIN_USER_ID=60032b76bd32a5a6a596559eade6d0052f15fc2ab9779cc8; index_location_city=%E6%B7%B1%E5%9C%B3; JSESSIONID=ABAAABAAAIAACBI1A73582AEDC2FDC8335E968502A9F5D2; Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1527150390,1528277552,1528789503,1529461642; _gid=GA1.2.1084200828.1529461642; LGSID=20180620153431-5f3689a6-745c-11e8-aa10-525400f775ce; TG-TRACK-CODE=search_code; LGRID=20180620160449-9a8ba121-7460-11e8-aa14-525400f775ce; Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1529481888; SEARCH_ID=efa038bbf91d47488b5c710088432205',

    'Host':'www.lagou.com',

    'Origin':'https://www.lagou.com',

    'Referer':'https://www.lagou.com/jobs/list_%E6%95%B0%E6%8D%AE%E5%88%86%E6%9E%90%E5%B8%88?px=default&city=%E6%B7%B1%E5%9C%B3',

    'Content-Type':'application/x-www-form-urlencoded; charset=UTF-8',

    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'

}

#请求和获取信息的函数

def get_postion_info(pn=1):

    global page#全局的页码

    data = {

        'first':'true',

        'pn':pn,

        'kd':KeyWord

    }

    try:

        data = bytes(parse.urlencode(data),encoding='utf-8')

        req = request.Request(url=url,data=data,headers=headers,method='POST')#构建Request对象

        html = request.urlopen(req)#发送请求

        if html.status==200:

            html = html.read().decode('utf-8')#将响应体的信息转化为字符串

            res = json.loads(html)#把类json的格式转化为字典

            #如果是第一页，就获取页码信息

            if pn==1:

                pagesize = res['content']['pageSize']

                totalcount=res['content']['positionResult']['totalCount']

                page = math.ceil(totalcount/pagesize)

            #获取招聘信息，并保存到MongoDB

            result = res['content']['positionResult']['result']

            for i in range(len(result)):

                save_mongodb(result[i])

            return res

        else:

            print('出现错误',html.status)

            return None

    except URLError:

        print('出现异常！')

        return None

#保存到MongoDB

def save_mongodb(info):

    if db[MONGO_TABLE].insert(info):

        print('保存成功！',info)

    else:

        print(info,'保存失败！',info)

#主体函数

def main():

    get_postion_info()#获取第一页的信息和页码信息

    #爬取第二页以后的信息

    for p in range(2,page+1):

        get_postion_info(pn=p)

if __name__=='__main__':

    main()