#-*- coding:utf-8 -*-

from bs4 import BeautifulSoup

import requests,chardet,urllib2

ip_list=[]

def get_ip_list(url, headers):

    web_data = requests.get(url, headers=headers)

    soup = BeautifulSoup(web_data.text, 'lxml')

    ips = soup.find_all('tr')

    ip_list = []

    for i in range(1, len(ips)):

        ip_info = ips[i]

        tds = ip_info.find_all('td')

        ip_list.append('http://' + tds[1].text + ':' + tds[2].text)

    return ip_list

def get_random_ip(ip_list):

    proxies = {'http': ip_list[0]}

    return proxies

def getip():

    global ip_list

    url = 'http://www.xicidaili.com/nn/'

    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36'}

    if not ip_list:

        ip_list = get_ip_list(url, headers=headers)

    print ip_list

    proxies = get_random_ip(ip_list)

    return proxies

def deleteip():

    global ip_list

    ip_list.pop(0)

def urllink(link):  # 网页HTML获取以及编码转换

    for i in range(12) :

        try:

            ip = getip()

            print ip

            proxy_support = urllib2.ProxyHandler(ip)

            opener = urllib2.build_opener(proxy_support)

            urllib2.install_opener(opener)

            html_1 = urllib2.urlopen(link, timeout=10).read()

            break

        except Exception,e:

            deleteip()

            print '错误',i,e

            pass

    if i==11:

        return ''

    encoding_dict = chardet.detect(html_1)

    web_encoding = encoding_dict['encoding']

    if web_encoding == 'utf-8' or web_encoding == 'UTF-8':

        html = html_1

    else:

        html = html_1.decode('gbk', 'ignore').encode('utf-8')

    return html

print urllink("http://ccdas.ipmph.com/pc/clinicalExam/getClinicalExamDetail?articleId=8165")