import requests

import re

#糗事百科爬虫类

class QSBK:

    #初始化方法，定义一些变量

    def __init__(self):

        self.headers={

            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36"

        }

        #存放段子的变量，每一个元素是每一页的段子们

        self.stories=[]

        #存放程序是否继续运行的变量

        self.enable =False

    def getPage(self,page):

        try:

            url = 'http://www.qiushibaike.com/hot/page/' + str(page)

            print(url)

            response = requests.get(url,headers=self.headers)

            html_content = response.content.decode('UTF-8')

            #print(html_content)

            # regex=re.compile('<div class="article block untagged mb15\s\S+" id="\S+">.*?</div>')

            regex = re.compile('<h2>(.*?)</h2>.*?<div class="content">\W+<span>(.*?)</span>', re.S)

            #regex = re.compile('<h2>', re.S)

            regex_content = re.findall(regex, html_content)

            print(regex_content)

            for i in regex_content:

                self.stories.append(i[0].replace('\n', ''),i[1].replace('\n', ''))

            return self.stories

        except Exception as e:

            print('异常：%s' % e)

js = QSBK()

for i in range(100):

    lists = js.getPage(i)

    print('============================================ 第 '+str(i)+' 页 =============================================')

    print(lists)