# -*- coding:utf-8 -*-

 from gevent import monkey

 monkey.patch_all()

 import urllib2

 from gevent.pool import Pool

 import requests

 import re

 class SpiderProxy:

     def __init__(self):

         self.headers = {

             "Host": "www.xicidaili.com",

             "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:47.0) Gecko/20100101 Firefox/47.0",

             "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",

             "Accept-Language": "en-US,en;q=0.5",

             "Accept-Encoding": "gzip, deflate",

             "Referer": "http://www.xicidaili.com/nn/",

             }

         self.url = 'http://www.xicidaili.com/nn/'

         self.proxy_list = []

         self.re_ip = re.compile(r'(?<![\.\d])(?:\d{1,3}\.){3}\d{1,3}(?![\.\d])')

         self.re_port = re.compile(r'<td>(\d+)</td>')

     def get_pagesource(self):

         ''' 取得所有1-n页上的代理IP'''

         try:

             num = int(raw_input('please input 1-'))

             for i in range(1, num + 1):

                 pageurl = self.url + str(i)

                 req = requests.session()

                 html = req.get(pageurl, headers=self.headers)

                 ip_list = self.re_ip.findall(html.text)

                 port_list = self.re_port.findall(html.text)

                 proxy_zip = zip(ip_list, port_list)

                 for i in proxy_zip:

                     self.proxy_list.append({'http':i[0] + ':' + i[1]})

         except ValueError:

             print 'please input a num!'

         return self.proxy_list

 class IsActiveProxyIP:

     def __init__(self):

         self.is_active_proxy_ip = []

     def probe_proxy_ip(self, proxy_ip):

         proxy = urllib2.ProxyHandler(proxy_ip)

         opener = urllib2.build_opener(proxy)

         urllib2.install_opener(opener)

         try:

             html = urllib2.urlopen('http://1212.ip138.com/ic.asp')

             if html:

                 self.is_active_proxy_ip.append(proxy_ip)

                 return True

             else:

                 return False

         except Exception as e:

             return False

 if __name__ == '__main__':

     Proxy = SpiderProxy()

     proxy_list = Proxy.get_pagesource()

     proxy_isactive = IsActiveProxyIP()

     pool = Pool(20)

     pool.map(proxy_isactive.probe_proxy_ip, proxy_list)

     with open(r'E:\python_demo\proxy_ip.txt', 'wb') as f:

         for ip in proxy_isactive.is_active_proxy_ip:

             ip = str(ip)

             f.write(ip[11:-2] + '\n')

     print 'file successed written'