自学Python十一 Python爬虫总结
通过几天的学习与尝试逐渐对python爬虫有了一些小小的心得,我们渐渐发现他们有很多共性,总是要去获取一系列的链接,读取网页代码,获取所需内容然后重复上面的工作,当自己运用的越来越熟练之后我们就会尝试着去总结一下爬虫的共性,试着去写个helper类以避免重复性劳动。
1.访问网站 #最简单的得到网页代码的方法
import urllib2
response = urllib2.urlopen("http://www.xx.com")
print response.read()
2.伪装成浏览器(User-Agent,Referer等) #为了不被服务器禁止访问所以还是伪装成浏览器比较好
headers = {
'User-Agent': 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)',
'Referer':'http://www.xx.com/xx',
'Accept':'application/javascript, */*;q=0.8'
}
response = urllib2.Request(url = "http://www.xx.com",data = None,headers = headers)
3.Post数据转码
import urllib,urllib2
values = {
'username':'xxx',
'password':'xxx',
'key':'xxx'
}
postdata = urllib.urlencode(values)
response = urllib2.Request(url,data = postdata)
4.Cookies
import urllib2,cookielib
cookie_handler = urllib2.HTTPCookieProcessor(cookielib.CookieJar())
opener = urllib2.build_opener(cookie_handler)
urllib2.install_opener(opener)
response = urllib2.urlopen(url)
5.代理服务器 #重复多次访问同一网址 结果被封了ip或限制了访问次数
import urllib2
proxy_handler = urllib2.ProxyHandler({"http" : '42.121.6.80:8080'})
opener = urllib2.build_opener(proxy_handler)
urllib2.install_opener(opener)
response = urllib2.urlopen(url)
问:如果想cookie和proxy一起用怎么办?
答:urllib2.build_opener可以放多个参数,即handler 如:BaseHandler,ProxyHandler,HTTPHandler,FileHandler,FTPHandler,CacheFTPHandler等等等等
6.gzip #现在普遍支持gzip压缩,我们默认获取压缩后的网页,大大提高了抓取网页的效率,减少了带宽负荷。
import urllib2,zlib
req = urllib2.Request(url)
req.add_header('Accept-encoding', 'gzip')
response = urllib2.urlopen(req, timeout=120)
html = response.read()
gzipped = response.headers.get('Content-Encoding')
if gzipped:
html = zlib.decompress(html, 16+zlib.MAX_WBITS)
7.其他
设置线程栈大小:栈大小显著影响python的内存占用,方法如下:
from threading import stack_size stack_size(32768*16)
设置超时
import socket
socket.setdefaulttimeout(10) #设置10秒后连接超时
失败后重试
def get(self,req,retries=3):
try:
response = self.opener.open(req)
data = response.read()
except Exception , what:
print what,req
if retries>0:
return self.get(req,retries-1)
else:
print 'GET Failed',req
return ''
return data
根据以上内容,我们可以写出便于配置解决重复性工作的自己的helper类:
# -*- coding: utf-8 -*-
import cookielib, urllib, urllib2, socket
import zlib,StringIO
class HttpClient:
__cookie = cookielib.CookieJar()
#代理设置,需要时添加(后续设置为多代理切换)
#__proxy_handler = urllib2.ProxyHandler({"http" : '42.121.6.80:8080'})
__req = urllib2.build_opener(urllib2.HTTPCookieProcessor(__cookie))#,__proxy_handler)
__req.addheaders = [
('Accept', 'application/javascript, */*;q=0.8'),
('User-Agent', 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)')
]
urllib2.install_opener(__req) def Get(self, url, refer=None,retries=3):
try:
req = urllib2.Request(url)
req.add_header('Accept-encoding', 'gzip')
if not (refer is None):
req.add_header('Referer', refer)
response = urllib2.urlopen(req, timeout=120)
html = response.read()
gzipped = response.headers.get('Content-Encoding')
if gzipped:
html = zlib.decompress(html, 16+zlib.MAX_WBITS)
return html
except Exception,what:
print what
if retries>0:
return self.Get(url,refer,retries-1)
else:
print "Get Failed",url
return ''
#except urllib2.HTTPError, e:
# return e.read()
#except socket.timeout, e:
# return ''
#except socket.error, e:
# return '' def Post(self, url, data, refer=None):
try:
req = urllib2.Request(url, urllib.urlencode(data))
#req = urllib2.Request(url,data)
if not (refer is None):
req.add_header('Referer', refer)
return urllib2.urlopen(req, timeout=120).read()
except urllib2.HTTPError, e:
return e.read()
except socket.timeout, e:
return ''
except socket.error, e:
return '' def Download(self, url, file):
output = open(file, 'wb')
output.write(urllib2.urlopen(url).read())
output.close() def getCookie(self, key):
for c in self.__cookie:
if c.name == key:
return c.value
return '' def setCookie(self, key, val, domain):
ck = cookielib.Cookie(version=0, name=key, value=val, port=None, port_specified=False, domain=domain, domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False)
self.__cookie.set_cookie(ck)
HttpClient
至于多线程就参考网上找的这段代码好了,还支持并发。。。
from threading import Thread
from Queue import Queue
from time import sleep
#q是任务队列
#NUM是并发线程总数
#JOBS是有多少任务
q = Queue()
NUM = 2
JOBS = 10
#具体的处理函数,负责处理单个任务
def do_somthing_using(arguments):
print arguments
#这个是工作进程,负责不断从队列取数据并处理
def working():
while True:
arguments = q.get()
do_somthing_using(arguments)
sleep(1)
q.task_done()
#fork NUM个线程等待队列
for i in range(NUM):
t = Thread(target=working)
t.setDaemon(True)
t.start()
#把JOBS排入队列
for i in range(JOBS):
q.put(i)
#等待所有JOBS完成
q.join()
ThreadDemo
爬虫就靠一段落吧,更深入的爬虫框架以及html解析库暂时放一放,让我考虑考虑接下来的内容,是pygame还是django!
爬虫demo的github地址(刚学着玩git ):http://git.oschina.net/tabei/Python_spider
最新文章
- BZOJ 3809: Gty的二逼妹子序列
- Centos7 修改ssh 默认端口号
- tomcat启动时报错
- Jmeter之csv、用户自定义变量以及Query Type分析(八)
- Hyper-V的使用方法
- [原创]java WEB学习笔记60:Struts2学习之路--Actioin-声明式异常处理
- [翻译] CBStoreHouseTransition
- How to Use Rsync to Sync New or Changed/Modified Files in Linux
- AJAX局部更新演出排期
- oracle编译 失效对象方式
- oracle_根据表名拼装语句
- Hibernate问题浅析
- MySQL优化的一些基础
- SAR指标(转)
- 【Django】数据迁移问题
- gitlab 10安装
- Netty 核心组件笔记
- UWA 技术分享连载 转载
- 编写安全的API接口
- matplotlib 操作子图(subplot,axes)