如何巧妙的利用selenium和requests组合来进行操作需要登录的页面

一、在这里selenium的作用

（1）模拟的登录。

（2）获取登录成功之后的cookies

代码

    def start_login(self):

        chrome_options = Options()

        # 禁止图片加载,禁止推送通知

        prefs = {

            "profile.default_content_setting_values": {

                "images": 2

            }, "profile.default_content_setting_values.notifications": 2

        }

        chrome_options.add_experimental_option("prefs", prefs)

        if chrome_args().get("headless_flag") == "1":

            chrome_options.add_argument(chrome_args().get("headless"))

        chrome_options.add_argument(chrome_args().get("nogpu"))

        chrome_options.add_argument(chrome_args().get("noinfobars"))

        chrome_options.add_argument(chrome_args().get("max_windows"))

        chrome_options.add_argument(self.Proxy_server)

        driver = webdriver.Chrome(chrome_options=chrome_options)

        try:

            get_logger().info("start login.....")

            try:

                # login info

                self.login_name = self.loginInfo.get("login_id")

                self.password = aes_cbc_decrypt(self.loginInfo.get("login_pwd"))

            except Exception:

                get_logger().error("cant get login info,here are detals".format(traceback.format_exc()))

            wait = WebDriverWait(driver, 30)

            print(u"start login in")

            driver.get(self.login_url)

            try:

                login_id = wait.until(

                    eccd.presence_of_element_located(

                        (By.XPATH, self.Id_xpath))

                )

                login_id.send_keys(self.login_name)

                login_id.send_keys(Keys.ENTER)

                password = wait.until(

                    eccd.presence_of_element_located(

                        (By.XPATH, self.pwd_xpath))

                )

                password.send_keys(self.password)

                submit = wait.until(

                    eccd.presence_of_element_located(

                        (By.XPATH, self.login_btn_xpath))

                )

                submit.click()

                # login signal

                #判断是否显示 右上角是否显示用户名

                login_ok = wait.until(

                    eccd.presence_of_element_located(

                        (By.XPATH, self.login_ok_xpath))

                )

                try:

                    #判断登录成功

                    if login_ok:

                        get_logger().info("get user name successful:"+u"{}".format(login_ok.text))

                        try:

                            get_weibo_info=driver.find_element_by_xpath(self.forward_home_page)

                        except:

                            driver.get(self.first_page)

                        forward_home_page_ok = wait.until(

                              eccd.presence_of_element_located(

                                  (By.XPATH, self.forward_home_page))

                          )

                        forward_home_page_ok.click()

                        time.sleep(5)

                        self.first_page=driver.current_url

                        get_logger().info("get homepage successful,url is {}".format(driver.current_url))

                        cookies = driver.get_cookies()  # 导出cookie

                        get_logger().info("get cookies")

                        get_fansnum_ok= wait.until(

                                eccd.presence_of_element_located(

                                    (By.XPATH, self.follow_xpath))

                        )

                        self.get_followfanshome(driver)

                        get_logger().info("get_followfanshome ok")

                        self.home_page_source=driver.page_source

                        driver.close()  # 关闭chrome

                        #获取主页的

                        # 如果需要保存cookies

                        self.write_cookie(cookies)

                        get_logger().info("get cookies,login ok")

                        return cookies

                    else:

                        raise RuntimeError('login failed')

                except:

                    get_logger().error("login failed")

                    return None

            except:

                if driver is not None:

                    driver.close()

        except Exception as e:

            if driver is not None:

                driver.close()

                get_logger().error("dbs operation error，here are details:{}".format(traceback.format_exc()))

　通过使用cookies = driver.get_cookies() #我们获取了cookie那么我们怎用呢。

二、requests如何使用获取到的cookies

1.创建一个Session对象

req = requests.Session()  # 构建Session

2.转换上面的cookies对象

for cookie in cookies:

            req.cookies.set(cookie['name'], cookie['value'])  # 转换cookies

3.开始访问该网站的需求页面

data = req.get(url, headers=self.headers).text #获取文本网页

jsondata = req.get(url, headers=self.headers).json() #获取json型网页

　　好了就是这么简单，另外补充一个经常使用的库fake-useragent，可以获取随机的useragent，但是其中谷歌的useragent版本比较低，尤其在访问知乎的时候，会提示浏览器版本低的信息，这一点需要注意。

另外拥有一个headers是对爬虫起码的尊重，所以有个构建好的headers是非常必要的。我就经常用下面的例：

from fake_useragent import UserAgent as UA

import random

headers = {

            'Accept': 'text/html, application/xhtml+xml, image/jxr, */*',

            'Accept - Encoding': 'gzip, deflate',

            'Accept-Language': 'zh-Hans-CN, zh-Hans; q=0.5',

            'Connection': 'Keep-Alive',

            'User-Agent': UA().random #获取随机的useragent

        }

巴特西