代理ip proxy

import requests
import time
from selenium import webdriver
from selenium.webdriver.chrome.options import Options

# kuaidaili 结果排序默认按响应速度(从快到慢)VIP 按最后检测时间(从近到远)VIP
"""
订单号*
查看我的订单从网页提取
提取数量*
数量该怎么填?
所在地区
多个地区用英文逗号分隔如何筛选地区?
排除地区
多个地区用英文逗号分隔查看实时地区分布>>
筛选端口号
多个端口用英文逗号分隔查看实时端口分布>>
排除端口号
多个端口用英文逗号分隔
筛选IP段
筛选以特定部分开头的IP
排除IP段
排除以特定部分开头的IP
运营商
无所谓联通/网通电信移动铁通教育网阿里云
匿名度
高匿名匿名透明查看实时匿名度分布>>
响应速度
极速(<1秒) 快速(1~3秒) 慢速(>3秒) 查看实时速度分布>>
代理协议
全部 HTTP HTTPS (同时也支持HTTP) 查看实时协议分布>>
POST支持
全部 GET POST (同时也支持GET) 查看GET/POST分布>>
代理稳定性新
不筛选稳定VIP 非常稳定SVIP 了解代理稳定性
结果排序
默认按响应速度(从快到慢)VIP 按最后检测时间(从近到远)VIP
结果定制VIP
IP:Port 地区匿名度代理协议响应速度(秒)
历史提取次数专业版把计数清零
提取结果包含的字段(顺序排列, 英文逗号分隔)
浏览器支持
谷歌浏览器(Chrome) IE / 360浏览器火狐浏览器(Firefox) 其它浏览器如何勾选?

安卓手机浏览器 iPhone手机浏览器 iPad浏览器查看浏览器支持分布>>
IP去重
过滤今天提取过的IP
返回格式
文本 jsonVIP xmlVIP
结果分隔符
\r\n分隔 \n分隔空格分隔 |分隔自定义
"""

"""
结果排序按响应速度(从快到慢)
匿名度高匿名
代理协议 HTTP
#http://dev.kuaidaili.com/api/getproxy/?orderid=123112322452122&num=30&b_pcchrome=1&b_pcie=1&b_pcff=1&protocol=1&method=2&an_ha=1&sp1=1&quality=1&sort=1&sep=2

"""

proxyip_d = {}
def gen_proxyip_dict_qps(
        url_proxyip='http://dev.kuaidaili.com/api/getproxy/?orderid=123112322452122&num=proxyip_num&b_pcchrome=1&b_pcie=1&b_pcff=1&protocol=1&method=2&an_ha=1&sp1=1&quality=1&sort=1&sep=2',
        proxyip_num=30, peroid_seconds=300):
    global proxyip_d

url = url_proxyip.replace('proxyip_num', str(proxyip_num))
    # floating point number
    # Return the current time in seconds since the Epoch.
    current_time = time.time()
    if current_time - proxyip_d['last_request_timestamp'] > peroid_seconds:
        try:
            r = requests.get(url)
            proxyip_d['last_request_timestamp'] = int(time.time())
            if r.status_code == 200:
                # 不校验返回结果，假设合法且可用
                l = r.text.split('\n')
                proxyip_d['ip_list'] = [i for i in set(l)]
        except Exception as e:
            ## log
            s = '%s%s' % ('', e)

def fetch_one_proxyip(url_proxyip='http://dev.kuaidaili.com/api/getproxy/?orderid=123112322452122&num=proxyip_num',
                      proxyip_num=30, peroid_seconds=300):
    global proxyip_d
    if 'ip_list' not in proxyip_d:
        proxyip_d['ip_list'] = []
    if 'last_request_timestamp' not in proxyip_d:
        proxyip_d['last_request_timestamp'] = 0
    gen_proxyip_dict_qps(url_proxyip=url_proxyip, proxyip_num=proxyip_num, peroid_seconds=peroid_seconds)

# 根据商家返回结果的排序规则，优先使用前ip
    try:
        r = proxyip_d['ip_list'][0]
        del proxyip_d['ip_list'][0]
    except Exception as e:
        gen_proxyip_dict_qps(peroid_seconds=0)
        r = 'localhost'
        print(e)
    print(proxyip_d['ip_list'])
    print(r)
   # r = 'localhost'
    return r

from selenium import webdriver
from bs4 import BeautifulSoup
import json
import time

from selenium.webdriver.common.proxy import *

f, s_l = 'biz0.txt', []

with open(f, 'r', encoding='utf-8') as fo:
for i in fo:
s_l.append(i.replace('\n', ''))

#browser = webdriver.Chrome().minimize_window()
# def spider_l_t_w(s_wd='长尾词'):
#     #global browser
#     # browser = webdriver.Chrome()
#
#     myproxyip = fetch_one_proxyip() # IP:PORT or HOST:PORT
#
#     chrome_options = webdriver.ChromeOptions()
#     chrome_options.add_argument('--proxy-server=%s' % myproxyip)
#
#     # chrome = webdriver.Chrome(chrome_options=chrome_options)
#     # chrome_options = Options()
#     # chrome_options.add_experimental_option("mobileEmulation", mobile_emulation)
#     browser = webdriver.Chrome(chrome_options=chrome_options)
#
#     url_seed = 'https://m.baidu.com/s?word=s_wd'
#     url_seed='http://www.zzyjschina.com.cn/shenzhen.html'
#    # url_seed = 'https://www.baidu.com/s?word=s_wd'
#
#     url_seed = url_seed.replace('s_wd', s_wd)
#     print(url_seed)
#     #browser.set_window_size(30, 30)
#
#     browser.get(url_seed)
#     # time.sleep(2)
#     # browser.minimize_window()
#     rd = BeautifulSoup(browser.page_source, 'html.parser').find_all('a', class_='rw-item')
#     res_d_l = [{'contents': d.contents, 'href': d.attrs['href']} for d in rd]
#     browser.quit()
#     #browser.close()
#     return res_d_l

def spider_l_t_w(s_wd='长尾词'):
#global browser
# browser = webdriver.Chrome()

myproxyip = fetch_one_proxyip() # IP:PORT or HOST:PORT

# chrome_options = webdriver.Firefox()
    # chrome_options.add_argument('--proxy-server=%s' % myproxyip)
    #
    # # chrome = webdriver.Chrome(chrome_options=chrome_options)
    # # chrome_options = Options()
    # # chrome_options.add_experimental_option("mobileEmulation", mobile_emulation)
    # browser = webdriver.Chrome(chrome_options=chrome_options)

myProxy =fetch_one_proxyip()

proxy = Proxy({
        'proxyType': ProxyType.MANUAL,
        'httpProxy': myProxy,
        'ftpProxy': myProxy,
        'sslProxy': myProxy,
        'noProxy': '' # set this value as desired
    })

browser = webdriver.Firefox(proxy=proxy)
    print(browser)
    url_seed = 'https://m.baidu.com/s?word=s_wd'
   # url_seed='http://www.zzyjschina.com.cn/shenzhen.html'
   # url_seed = 'https://www.baidu.com/s?word=s_wd'

url_seed = url_seed.replace('s_wd', s_wd)
print(url_seed)
#browser.set_window_size(30, 30)

browser.get(url_seed)
    # time.sleep(2)
    # browser.minimize_window()
    rd = BeautifulSoup(browser.page_source, 'html.parser').find_all('a', class_='rw-item')
    res_d_l = [{'contents': d.contents, 'href': d.attrs['href']} for d in rd]
    browser.quit()
    #browser.close()
    return res_d_l

save_c, save_limit, fw = 0, 30, open('biz0.res.txt', 'a', encoding='utf-8')

for i in s_l:
    try:
        d = spider_l_t_w(i)
        s = json.dumps(d, ensure_ascii=False)
    except Exception as e:
        s = e
    s = '%s\n%s\n\n' % (i, s)
    fw.write(s)
    save_c += 1
    if save_c == save_limit:
        fw.close()
        fw = open('biz0.res.txt', 'a', encoding='utf-8')

d = 9

-- google 不能

-- firefox ok

myProxy = fetch_one_proxyip()

proxy = Proxy({
    'proxyType': ProxyType.MANUAL,
    'httpProxy': myProxy,
    'ftpProxy': myProxy,
    'sslProxy': myProxy,
    'noProxy': ''  # set this value as desired
})
browser = webdriver.Firefox(proxy=proxy)
print(browser)
url_seed = 'https://m.baidu.com/s?word=s_wd'
url_seed = url_seed.replace('s_wd', 'fddf')
print(url_seed)
browser.set_window_size(30, 30)
try:
    browser.get(url_seed)
except Exception as e:
    print('---------', e)

巴特西

代理ip proxy

最新文章

热门文章