request-html 简单爬虫

import asyncio

from requests_html import HTMLSession

url  = 'http://www.xiaohuar.com/hua/'

session = HTMLSession( browser_args=[

        '--no-sand',

        '--disable-infobars'

        '--user-agent=Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36'

    ],headless=False)

res = session.request(url=url,method='GET')

script = """

                () => {

                    return {

                        width: document.documentElement.clientWidth,

                        height: document.documentElement.clientHeight,

                        deviceScaleFactor: window.devicePixelRatio,

                    }

                }

               """

try:

    res.html.render(keep_page = True)

    async def main():

        await res.html.page.waitFor(1000)

        await res.html.page.setViewport({'width': 1366, 'height': 768})

        url_list = await  res.html.page.xpath('//div[@class="img"]/a')

        for url in url_list:

            url_link = await (await url.getProperty('href')).jsonValue()

            print(url_link)

    asyncio.get_event_loop().run_until_complete(main())

except Exception as e:

    print(e)

finally:

    session.close()
巴特西

request-html 简单爬虫

最新文章

热门文章
