from scrapy.http import HtmlResponse

 from twisted.internet import defer

 from twisted.internet.error import TimeoutError, DNSLookupError, \

     ConnectionRefusedError, ConnectionDone, ConnectError, \

     ConnectionLost, TCPTimedOutError

 from twisted.web.client import ResponseFailed

 from scrapy.core.downloader.handlers.http11 import TunnelError

 class ProcessAllExceptionMiddleware(object):

     ALL_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError,

                       ConnectionRefusedError, ConnectionDone, ConnectError,

                       ConnectionLost, TCPTimedOutError, ResponseFailed,

                       IOError, TunnelError)

     def process_response(self, request, response, spider):

         # 捕获状态码为40x/50x的response

         if str(response.status).startswith('') or str(response.status).startswith(''):

             # 随意封装，直接返回response，spider代码中根据url==''来处理response

             response = HtmlResponse(url=str(response.status), status=200)

             return response

         # 其他状态码不处理

         return response

     def process_exception(self, request, exception, spider):

         # 捕获几乎所有的异常

         if isinstance(exception, self.ALL_EXCEPTIONS):

             # 在日志中打印异常类型

             print('Got exception: %s' % (exception))

             # 随意封装一个response，返回给spider

             response = HtmlResponse(url='exception')

             return response

         # 打印出未捕获到的异常

         print('not contained exception: %s' % exception)