爬取代理IP
2024-10-19 21:35:12
现在爬虫好难做啊，有些网站直接封IP，本人小白一个，还没钱，只能找免费的代理IP，于是去爬了西刺免费代理，结果技术值太低，程序还没调试好，
IP又被封了。。。
IP又被封了。。。
IP又被封了。。。
于是又开始找其他的免费代理，找到了89免费代理，这个网站提供了API接口。在我调试过程中发现，这个API接口好像对爬虫没啥限制，哈哈哈！！！
下面附上代码：
1.这次找了好多User-Agent，以防万一。
def __init__(self):
self.get_ip_url = "http://www.89ip.cn/tqdl.html?api=1&num=3000"
self.check_ip_url = "http://www.baidu.com"
self.headers = [
{"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36
(KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36"},
{"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36
(KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60"},
{"User-Agent":"Opera/8.0 (Windows NT 5.1; U; en)"},
{"User-Agent":"Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208
Firefox/2.0.0 Opera 9.50"},
{"User-Agent":"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; en) Opera
9.50"},
{"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101
Firefox/34.0"},
{"User-Agent":"Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10)
Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10"},
{"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.57.2
(KHTML,like Gecko) Version/5.1.7 Safari/534.57.2 "},
{"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML,
like Gecko) Chrome/39.0.2171.71 Safari/537.36"},
{"User-Agent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML,like
Gecko) Chrome/23.0.1271.64 Safari/537.11"},
{"User-Agent":"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US)
AppleWebKit/534.16 (KHTML, like Gecko) Chrome/10.0.648.133 Safari/534.16"},
{"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML,
like Gecko) Chrome/30.0.1599.101 Safari/537.36"},
{"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like
Gecko"},
{"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML,
like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11"},
{"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML,
like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER"},
{"User-Agent":"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64;
Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR
3.0.30729;Media Center PC 6.0; .NET4.0C; .NET4.0E; LBBROWSER)"},
{"User-Agent":"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1;
QQDownload 732; .NET4.0C; .NET4.0E; LBBROWSER)"},
{"User-Agent":"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64;
Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR
3.0.30729;Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)"},
{"User-Agent":"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1;
QQDownload 732; .NET4.0C; .NET4.0E) "},
{"User-Agent":"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like
Gecko) Chrome/17.0.963.84 Safari/535.11 SE 2.X MetaSr 1.0"},
{"User-Agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1;
Trident/4.0; SV1; QQDownload 732; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)"},
{"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML,
like Gecko) Maxthon/4.4.3.4000 Chrome/30.0.1599.101 Safari/537.36"},
{"User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML,
like Gecko) Chrome/38.0.2125.122 UBrowser/4.0.3214.0 Safari/537.36"},
{"User-Agent":"Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X;
en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2
Safari/6533.18.5"},
{"User-Agent":"Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_3 like Mac OS X;
en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2
Safari/6533.18.5"},
{"User-Agent":"Mozilla/5.0 (iPad; U; CPU OS 4_2_1 like Mac OS X; zh-cn)
AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148
Safari/6533.18.5"},
{"User-Agent":"Mozilla/5.0 (iPad; U; CPU OS 4_3_3 like Mac OS X; en-us)
AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2
Safari/6533.18.5"},
{"User-Agent":"Mozilla/5.0 (Linux; U; Android 2.2.1; zh-cn;
HTC_Wildfire_A3333 Build/FRG83D) AppleWebKit/533.1 (KHTML, like Gecko)
Version/4.0 Mobile Safari/533.1"},
{"User-Agent":"Mozilla/5.0 (Linux; U; Android 2.3.7; en-us; Nexus One
Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile
Safari/533.1"},
{"User-Agent":"MQQBrowser/26 Mozilla/5.0 (Linux; U; Android 2.3.7; zh-cn;
MB200 Build/GRJ22; CyanogenMod-7) AppleWebKit/533.1 (KHTML, like Gecko)
Version/4.0 Mobile Safari/533.1"},
{"User-Agent":"Opera/9.80 (Android 2.3.4; Linux; Opera Mobi/build-1107180945;
U; en-GB) Presto/2.8.149 Version/11.10"},
{"User-Agent":"Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39)
AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13"},
{"User-Agent":"Mozilla/5.0 (BlackBerry; U; BlackBerry 9800; en)
AppleWebKit/534.1+ (KHTML, like Gecko) Version/6.0.0.337 Mobile
Safari/534.1+"},
{"User-Agent":"Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.0; U; en-US)
AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/233.70 Safari/534.6
TouchPad/1.0"},
{"User-Agent":"Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/20.0.019;
Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/525 (KHTML, like Gecko)
BrowserNG/7.1.18124"},
{"User-Agent":"Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5;
Trident/5.0; IEMobile/9.0; HTC; Titan)"},
{"User-Agent":"UCWEB7.0.2.37/28/999"},
{"User-Agent":"NOKIA5700/ UCWEB7.0.2.37/28/999"},
{"User-Agent":"Openwave/ UCWEB7.0.2.37/28/999"},
{"User-Agent":"Mozilla/4.0 (compatible; MSIE 6.0;)
Opera/UCWEB7.0.2.37/28/999"}]
self.get_ip_queue = Queue()
self.pass_ip_list = []
2.发送请求，通过正则表达式获取IP列表
def get_ip_list(self,url):
header = random.choice(self.headers)
response = requests.get(url,headers=header)
ip_list = re.findall(r"(.*?)<br>", response.content.decode())[1:]
for ip in ip_list:
self.get_ip_queue.put(ip)
3.检查代理IP是否可用
def check_ip(self):
while not self.get_ip_queue.empty():
ip = self.get_ip_queue.get()
proxy = {"http": "http://" + ip,
"https": "https://" + ip}
header = random.choice(self.headers)
try:
response = requests.get(self.check_ip_url,headers=header,proxies=proxy,timeout=3)
except:
print(ip,"连接超时")
else:
print(ip,response.status_code)
if response.status_code == 200:
self.pass_ip_list.append(ip)
self.get_ip_queue.task_done()
4.保存到csv文件中
def save_csv(self,pass_ip_list):
pass_ip_df = pd.DataFrame({"ip": pass_ip_list})
pass_ip_df.to_csv("./ip.csv", index=False)
5.主运行程序
def run(self):
"""运行程序"""
t_start = datetime.datetime.now()
# 1.获取代理IP列表
self.get_ip_list(self.get_ip_url)
# 2.验证代理IP有效性
for i in range(100):
t = threading.Thread(target=self.check_ip)
t.setDaemon(True)
t.start()
# self.get_ip_queue.join()
# 3.保存代理IP至csv文件
print("*" * 100)
self.save_csv(self.pass_ip_list)
t_end = datetime.datetime.now()
print("保存成功", t_end.strftime('%Y-%m-%d %H:%M:%S'))
print("用时{}".format(t_end-t_start),"获取{}个代理IP".format(len(self.pass_ip_list)))
# self.pass_ip_list = []
if __name__ == '__main__':
daili89 = Daili89()
while True:
schedule.run_pending()
daili89.run()
time.sleep(600)
运行结果如下图：
最后，
人生苦短，还是要积极地走下去！
最新文章
- vtkBoxWidget2Example
- java追加文件的几种方式
- php header函数要点
- C# Winform程序请求管理员权限
- 322. Coin Change
- Linux常用命令总结——文件管理
- Writing clean code is what you must do in order to call yourself a professional.
- Spring-depends on
- 海量数据挖掘MMDS week2: 频繁项集挖掘 Apriori算法的改进：基于hash的方法
- 将xml文件由格式化变为压缩字符串
- Java基础知识➣集合整理（三）
- Eureka自我保护模式——难点重点
- Latex citation using natbib and footnotesize
- MSSQL索引视图（indexed view）之简述及使用
- python3.4学习笔记(一) 基本语法 python3不向下兼容，有些语法跟python2.x不一样
- RxJava Subject
- apache环境 php开启intl扩展
- 10g 升级到11g 失效对象2则
- 【2D游戏引擎】WIP反思
- bzoj1968: [Ahoi2005]COMMON 约数研究（数论）
热门文章
- Oracle学习(十一)：PL/SQL
- luogu P3391 【模板】文艺平衡树（Splay）
- Ingress 暴露tcp端口
- 学记笔记 $\times$ 巩固 · 期望泛做$Junior$
- P1796 汤姆斯的天堂梦
- Linux Shell常用技巧(六)
- 1.1《想成为黑客，不知道这些命令行可不行》(Learn Enough Command Line to Be Dangerous)——运行终端
- 解决：Cannot load ocl.dll library(error code 126). The ocil.dll library may be missing from the system
- JS控制上传文件个数
- Robosup3D平台搭建