import random

 from Tencent.settings import USER_AGENTS_LIST # 注意导入路径,请忽视pycharm的错误提示

 class UserAgentMiddleware(object):

     def process_request(self, request, spider):

         user_agent = random.choice(USER_AGENTS_LIST)

         request.headers['User-Agent'] = user_agent

         # 不写return

 class CheckUA:

     def process_response(self,request,response,spider):

         print(request.headers['User-Agent'])

         return response # 不能少！

 DOWNLOADER_MIDDLEWARES = {

    'Tencent.middlewares.UserAgentMiddleware': 543, # 543是权重值

    'Tencent.middlewares.CheckUA': 600, # 先执行543权重的中间件，再执行600的中间件

 }

 USER_AGENTS_LIST = [

     "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",

     "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",

     "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",

     "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",

     "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5"

 ]

 class RandomProxy(object):

     def process_request(self, request, spider):

         proxy = random.choice(PROXY_LIST)

         print(proxy)

         if 'user_passwd' in proxy:

             # 对账号密码进行编码，基础认证,python3中需要是bytes类型的数据才能编码

             b64_up = base64.b64encode(proxy['user_passwd'].encode())

             # 进行代理认证

             request.headers['Proxy-Authorization'] = 'Basic ' + b64_up.decode()

             # 设置代理

             request.meta['proxy'] = proxy['ip_port']

         else:

             #设置代理

             request.meta['proxy'] = proxy['ip_port']

 class ProxyMiddleware(object):

     ......

     def process_response(self, request, response, spider):

         if response.status != '':

             request.dont_filter = True # 重新发送的请求对象能够再次进入队列

             return requst

 PROXY_LIST = [

     {"ip_port": "139.199.121.163:16818", "user_passwd": "user:password"},#收费代理

     # {"ip_port": "114.234.81.72:9000"} # 免费代理

 ]

 import scrapy

 class Login4Spider(scrapy.Spider):

     name = 'login4'

     allowed_domains = ['github.com']

     start_urls = ['https://github.com/returnes'] # 直接对验证的url发送请求

     def parse(self, response):

         with open('check.html', 'w') as f:

             f.write(response.body.decode())

 import time

 from selenium import webdriver

 def getCookies():

     # 使用selenium模拟登陆，获取并返回cookie

     username = input('输入github账号:')

     password = input('输入github密码:')

     options = webdriver.ChromeOptions()

     options.add_argument('--headless')

     options.add_argument('--disable-gpu')

     driver = webdriver.Chrome('/home/worker/Desktop/driver/chromedriver',

                               chrome_options=options)

     driver.get('https://github.com/login')

     time.sleep(1)

     driver.find_element_by_xpath('//*[@id="login_field"]').send_keys(username)

     time.sleep(1)

     driver.find_element_by_xpath('//*[@id="password"]').send_keys(password)

     time.sleep(1)

     driver.find_element_by_xpath('//*[@id="login"]/form/div[3]/input[3]').click()

     time.sleep(2)

     cookies_dict = {cookie['name']: cookie['value'] for cookie in driver.get_cookies()}

     driver.quit()

     return cookies_dict

 class LoginDownloaderMiddleware(object):

     def process_request(self, request, spider):

         cookies_dict = getCookies()

         print(cookies_dict)

         request.cookies = cookies_dict # 对请求对象的cookies属性进行替换

 class SelMiddleWare(object):

     def process_request(self, request, spider):

         url = request.url

         # 过滤需要渲染的请求对象

         if 'daydata' in url:

             driver = webdriver.Chrome()

             driver.get(url)

             time.sleep(3)

             data = driver.page_source

             driver.close()

             res = HtmlResponse(

                 url=url,

                 body=data,

                 encoding='utf-8',

                 request=request

             )

             return res