#-*-coding:utf-8-*-

import urllib

import urllib2

import re

import requests

import json

import lxml

from bs4 import BeautifulSoup

import time

from pymongo import MongoClient

from lxml import etree

client = MongoClient('localhost',27017)

db = client.test

House = db.House

headers = {

    'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',

    'Accept-Encoding':'gzip, deflate, br',

    'Accept-Language':'zh-CN,zh;q=0.9',

    'Cache-Control':'max-age=0',

    'Connection':'keep-alive',

    'Cookie':'......',

    'Host':'bj.lianjia.com',

    'Upgrade-Insecure-Requests':'',

    'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36'

}

URL = 'https://bj.lianjia.com/ershoufang/pg'

def download(url):

    num_try = 2

    while num_try > 0:

        num_try -= 1

        try:

            content = requests.get(url,headers = headers)

            return content.text

        except urllib2.URLError as e:

            print 'Download error',e.reason

    return None

def get_message(url):

    html = download(url)

    soup = BeautifulSoup(html,'html.parser')

    prices = soup.find_all('div','priceInfo')

    total_price = []

    for each in prices:

        total_price.append(each.span.string)

    address = []

    house_types = []

    areas = []

    towards = []

    decorates = []

    elevates = []

    message = soup.find_all('div',attrs={'class':'houseInfo'})

    for each in message:

        List = each.get_text().split('|')

        address.append(List[0].strip())

        house_types.append(List[1].strip())

        areas.append(List[2].strip())

        towards.append(List[3].strip())

        decorates.append(List[4].strip())

        if len(List) == 5:

            elevates.append("None")

        else:

            elevates.append(List[5].strip())

    for addres,house_type,area,price,toward,decorate,elevate in zip(address,house_types,areas,total_price,towards,decorates,elevates):

        mess = "{\"Address\":\"%s\",\"House_type\":\"%s\",\"Area\":\"%s\",\"Price\":\"%s\",\"Toward\":\"%s\",\"Decorate\":\"%s\",\"Elevete\":\"%s\"}"%(addres,house_type,area,price,toward,decorate,elevate)

        print mess

        message = json.loads(mess)

        House.insert(message)

if __name__ == '__main__':

    t = time.time()

    print t

    for num in xrange(1,101):

        url = URL + str(num)

        print url

        get_message(url)

        time.sleep(1)

    t1 = time.time()

    print 'Total time:'

    print t1 - t - 100