采集代理ip 地址【西刺，快代理】

# 嗯，。。。因为经常需要使用代理去抓一点东西，就有了下面一段代码，第一版不是很好，后面用到了再来优化

import re,pymysql,time,redis

from urllib.request import Request

from urllib.request import urlopen

headers={

    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',

}

url = 'http://www.xicidaili.com/wt/'

db = pymysql.connect(host='127.0.0.1', user='root', password="liu",

                 database='test', port=3306, charset='utf8')

cur = db.cursor()

def url_response(url,cur):

    response = urlopen(Request(url,headers=headers)).read()

    response = response.decode()

    pattern='<td>(.*?)</td>\s+<td>(\d+)</td>\s+<td>\s+<a href="/.*?">[\u4e00-\u9fa5]+</a>\s+</td>\s+<td class="country">高匿</td>\s+<td>(\w+)</td>\s+<td class="country">\s+<div title="(\d.\d+)秒"'

    regex = re.compile(pattern)

    ip_list = regex.findall(response)

    for i in ip_list:

        out_time = float(i[3])

        ip_ = i[0]+':'+i[1]

        sql = 'select ip_ from ip where ip_ = "%s"'%ip_

        cur.execute(sql)

        if cur.fetchone():

            print('重复数据跳过')

            continue

        if out_time < 1:

            sql = "insert into ip(ip_,time_,xy_) values('%s','%s','%s')"%(ip_,out_time,i[2])

            cur.execute(sql)

            print('插入成功，',i)

        else:

            pass

for i in range(1,3):

    _ = url+str(i)

    url_response(_,cur)

    db.commit()

    time.sleep(2)

# 第一版使用 mysql存着也没啥用处 ，然后就给来了redis

# 加入线程池的使用，让抓取更加速度

import re,time,redis

from concurrent.futures import ThreadPoolExecutor

from urllib.request import Request

from urllib.request import urlopen

headers={

    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',

}

url = 'http://www.xicidaili.com/wt/'

class R(object):

    def __init__(self):

        r_pool =redis.ConnectionPool(host='127.0.0.1',db=0,password=None,

                                     port=6379)

        self.redis_obj = redis.Redis(connection_pool=r_pool)

    def setex(self,name,value,time):

        return self.redis_obj.setex(name,value,time)

    def get(self,name):

        return self.redis_obj.get(name)

def url_response(url,redis_obj):

    response = urlopen(Request(url,headers=headers)).read()

    response = response.decode()

    pattern='<td>(.*?)</td>\s+<td>(\d+)</td>\s+<td>\s+<a href="/.*?">[\u4e00-\u9fa5]+</a>\s+</td>\s+<td class="country">高匿</td>\s+<td>(\w+)</td>\s+<td class="country">\s+<div title="(\d.\d+)秒"'

    regex = re.compile(pattern)

    ip_list = regex.findall(response)

    for i in ip_list:

        out_time = float(i[3])

        ip_ = i[0]+':'+i[1]

        if redis_obj.get(ip_):

            print('重复数据跳过')

            continue

        if out_time < 1:

            redis_obj.setex(ip_,1,60*30)

            print('插入成功，',ip_)

        else:

            pass

r = R()

T = ThreadPoolExecutor(4)

for i in range(1,5):

    _ = url+str(i)

    T.submit(url_response,_,r)

print('执行完成 ')

T.shutdown()

更新第二版，使用redis___西刺

2018-12-17:

　　第二版出错更新： 15行：return self.redis_obj.setex(name,time,value) #此处已改正

# python 3.7

from lxml import etree

import requests,time,redis

class Kuai_IP(object):

    def __init__(self):

        self.headers = {

            'Host': 'www.kuaidaili.com',

            'Upgrade-Insecure-Requests': '',

            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36',

            'Referer': 'https://www.kuaidaili.com/free/inha/1/'

            }

        self.static='https://www.kuaidaili.com/free/inha/%s/'

        Con_pool = redis.ConnectionPool(host='127.0.0.1')

        self.r = redis.Redis(connection_pool=Con_pool)

    def getPage(self,page_index):

        if page_index==1:

            self.headers['Referer']='https://www.kuaidaili.com/free/inha/'

        else:

            self.headers['Referer']='https://www.kuaidaili.com/free/inha/'+str(page_index-1)+'/'

        res = requests.get(url=self.static%page_index,headers=self.headers)

        self.parse(res.text)

    def parse(self,res):

        html = etree.HTML(res)

        r_list = html.xpath('//tbody/tr/td/text()')

        if r_list:

            for i in range(1, int(len(r_list) / 7)):

                _ =r_list[i * 7:(i + 1) * 7]

                self.r.setex(_[3]+'://'+_[0]+':'+_[1],30,_[3])

                print(_)

        else:

            print(r_list)

    def work_on(self):

        page_index = 2  # 爬取页数

        for i in range(1,page_index+1):

            self.getPage(i)

            print(i, '---------')

            time.sleep(2)

if __name__ == '__main__':

    ip = Kuai_IP()

    ip.work_on()

______快代理IP——没做过滤

2018-12-20

　　嗯，这次由于用到的代理比较多，就把西刺和快代理的代码合到了一起，没做什么大的改进，

　　1 代理ip格式全部成为 requests代理的形式{'http':'xxx://xx.xx.xx.xx:xxx'} 方便requests的调用

# -*- coding:utf-8 -*-

# @time:2018-12-20 22:23

import re,redis,time,requests

from concurrent.futures import ThreadPoolExecutor

from urllib.request import Request

from urllib.request import urlopen

page = 10 #定义抓取页数，由于快代理一页的ip比较少，然后我就在此的基础上+10

headers={

    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',

}

url = 'http://www.xicidaili.com/wt/'

class R(object):

    def __init__(self):

        r_pool =redis.ConnectionPool(host='127.0.0.1',db=0,password=None,

                                     port=6379)

        self.redis_obj = redis.Redis(connection_pool=r_pool)

    def setex(self,name,value,time):

        return self.redis_obj.setex(name, value, time)

    def get(self,name):

        return self.redis_obj.get(name)

def url_response(url,redis_obj):

    response = urlopen(Request(url,headers=headers)).read()

    response = response.decode()

    pattern='<td>(.*?)</td>\s+<td>(\d+)</td>\s+<td>\s+<a href="/.*?">[\u4e00-\u9fa5]+</a>\s+</td>\s+<td class="country">高匿</td>\s+<td>(\w+)</td>\s+<td class="country">\s+<div title="(\d.\d+)秒"'

    regex = re.compile(pattern)

    ip_list = regex.findall(response)

    for i in ip_list:

        out_time = float(i[3])

        ip_ = i[0]+':'+i[1]

        if redis_obj.get(ip_):

            print('重复数据跳过')

            continue

        if out_time < 1:

            ip_ = "HTTP://"+str(ip_)

            redis_obj.setex(ip_,1,60*30*20)

            print('插入成功，',ip_)

        else:

            pass

from lxml import etree

class Kuai_IP(object):

    def __init__(self):

        self.headers = {

            'Host': 'www.kuaidaili.com',

            'Upgrade-Insecure-Requests': '',

            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36',

            'Referer': 'https://www.kuaidaili.com/free/inha/1/'

            }

        self.static='https://www.kuaidaili.com/free/inha/%s/'

        Con_pool = redis.ConnectionPool(host='127.0.0.1',port=6379,db=0)

        self.r = redis.Redis(connection_pool=Con_pool)

    def getPage(self,page_index):

        if page_index==1:

            self.headers['Referer']='https://www.kuaidaili.com/free/inha/'

        else:

            self.headers['Referer']='https://www.kuaidaili.com/free/inha/'+str(page_index-1)+'/'

        res = requests.get(url=self.static%page_index,headers=self.headers)

        self.parse(res.text)

    def parse(self,res):

        html = etree.HTML(res)

        r_list = html.xpath('//tbody/tr/td/text()')

        if r_list:

            for i in range(1, int(len(r_list) / 7)):

                _ =r_list[i * 7:(i + 1) * 7]

                self.r.setex(_[3]+'://'+_[0]+':'+_[1],_[3],30*60*24)

                print(_)

        else:

            print(r_list)

    def work_on(self):

        page_index = page+10  # 爬取页数

        for i in range(1,page_index+1):

            self.getPage(i)

            print(i, '---------')

            time.sleep(2)

if __name__ == '__main__':

    r = R()

    T = ThreadPoolExecutor(4)

    for i in range(1,page):

        _ = url+str(i)

        T.submit(url_response,_,r)

    print('执行完成 ')

    T.shutdown()

    ip = Kuai_IP()

    ip.work_on()

---第三版融合，总共将近500个ip,应该是够用的

2018-12-24

　　嗯，上面代理在80行出现错误，因为快代理的这个超时时间有时出现的是数字+文字，然后在设置缓存的时候就

　　出现了【 value is not an integer or out of range】

# -*- coding:utf-8 -*-

# @time:2018-12-18 22:23

# @Auther:1043453579@qq.com

import re,redis,time,requests

from concurrent.futures import ThreadPoolExecutor

from urllib.request import Request

from urllib.request import urlopen

page = 10

headers={

    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',

}

url = 'http://www.xicidaili.com/wt/'

class R(object):

    def __init__(self):

        r_pool =redis.ConnectionPool(host='127.0.0.1',db=0,password=None,

                                     port=6379)

        self.redis_obj = redis.Redis(connection_pool=r_pool)

    def setex(self,name,value,time):

        return self.redis_obj.setex(name, time, value)

    def get(self,name):

        return self.redis_obj.get(name)

def url_response(url,redis_obj):

    response = urlopen(Request(url,headers=headers)).read()

    response = response.decode()

    pattern='<td>(.*?)</td>\s+<td>(\d+)</td>\s+<td>\s+<a href="/.*?">[\u4e00-\u9fa5]+</a>\s+</td>\s+<td class="country">高匿</td>\s+<td>(\w+)</td>\s+<td class="country">\s+<div title="(\d.\d+)秒"'

    regex = re.compile(pattern)

    ip_list = regex.findall(response)

    for i in ip_list:

        out_time = float(i[3])

        ip_ = i[0]+':'+i[1]

        if redis_obj.get(ip_):

            print('重复数据跳过')

            continue

        if out_time < 1:

            ip_ = "HTTP://"+str(ip_)

            redis_obj.setex(ip_,1,60*30*20)

            print('插入成功，',ip_)

        else:

            pass

r = R()

T = ThreadPoolExecutor(4)

for i in range(1,page):

    _ = url+str(i)

    T.submit(url_response,_,r)

print('执行完成 ')

T.shutdown()

from lxml import etree

class Kuai_IP(object):

    def __init__(self):

        self.headers = {

            'Host': 'www.kuaidaili.com',

            'Upgrade-Insecure-Requests': '',

            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36',

            'Referer': 'https://www.kuaidaili.com/free/inha/1/'

            }

        self.static='https://www.kuaidaili.com/free/inha/%s/'

        Con_pool = redis.ConnectionPool(host='127.0.0.1',port=6379,db=0)

        self.r = redis.Redis(connection_pool=Con_pool)

    def getPage(self,page_index):

        if page_index==1:

            self.headers['Referer']='https://www.kuaidaili.com/free/inha/'

        else:

            self.headers['Referer']='https://www.kuaidaili.com/free/inha/'+str(page_index-1)+'/'

        res = requests.get(url=self.static%page_index,headers=self.headers)

        self.parse(res.text)

    def parse(self,res):

        html = etree.HTML(res)

        r_list = html.xpath('//tbody/tr/td/text()')

        if r_list:

            for i in range(1, int(len(r_list) / 7)):

                _ =r_list[i * 7:(i + 1) * 7]

                print('',_)

                #exit()

                self.r.setex(_[3]+'://'+_[0]+':'+_[1],30*60*24,30*60*24)

                # print(_)

        else:

            print(r_list)

    def work_on(self):

        page_index = page+10  # 爬取页数

        for i in range(1,page_index+1):

            self.getPage(i)

            print(i, '---------')

            time.sleep(2)

ip = Kuai_IP()

ip.work_on()

---第四版融合，将近500+ip,应该是够用的

巴特西

采集代理ip 地址【西刺，快代理】

最新文章

热门文章