安装:

pip install fake-useragent

使用:

from fake_useragent import UserAgent

ua = UserAgent()

调用指定ua:

ua.ie

Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)

随机ua:

ua.random

class MiddleproDownloaderMiddleware(object):

    # Not all methods need to be defined. If a method is not defined,

    # scrapy acts as if the downloader middleware does not modify the

    # passed objects.

    @classmethod

    def from_crawler(cls, crawler):

        # This method is used by Scrapy to create your spiders.

        s = cls()

        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)

        return s

    #可以拦截到所有的非异常处理请求

    # 参数spider 就是一个实例化的对象

    #

    from fake_useragent import UserAgent

    ua = UserAgent()

    def process_request(self, request, spider):

        print('开始执行下载中间件')

        # 进行ua伪装

        print('请求头是'+self.ua.random)

        request.headers['User-Agent'] =self.ua.random

        # 测试进行:代理操作看是否生效

        request.meta['proxy'] ='https://218.60.8.83:3129'

        # Called for each request that goes through the downloader

        # middleware.

        # Must either:

        # - return None: continue processing this request

        # - or return a Response object

        # - or return a Request object

        # - or raise IgnoreRequest: process_exception() methods of

        #   installed downloader middleware will be called

        return None

    # 拦截所有的响应

    def process_response(self, request, response, spider):

        # Called with the response returned from the downloader.

        # Must either;

        # - return a Response object

        # - return a Request object

        # - or raise IgnoreRequest

        return response

    # 拦截所有的异常请求对象

    def process_exception(self, request, exception, spider):

        # Called when a download handler or a process_request()

        # (from other downloader middleware) raises an exception.

        # Must either:

        # - return None: continue processing this exception

        # - return a Response object: stops process_exception() chain

        # - return a Request object: stops process_exception() chain

　　　　#更改内容后重新参与调度
　　　　　　return request

    #写日志

    def spider_opened(self, spider):

        spider.logger.info('Spider opened: %s' % spider.name)