from bs4 import BeautifulSoup

import urllib.request#导入urllib库

url = 'https://www.p y t hontab.com/html/pythonhexinbiancheng/index.html'#获取网页链接

request = urllib.request.urlopen(url)

html = request.read()#读取网页源代码

soup = BeautifulSoup(html,'html.parser')#解析html

title_links = soup.select('#catlist > li > a')#找到标题与链接

source_list = []#存储标题与链接的字典

for title_link in title_links:

    data = {

        'title' : title_link.get_text(),

        'link' : title_link.get('href')

    }

    source_list.append(data)

for dic in source_list:#遍历每一个字典

    request = urllib.request.urlopen(dic["link"])

    html = request.read()

    soup = BeautifulSoup(html, 'html.parser')

    text_p = soup.select('#Article > div.content > p')#拿到p标签下的数据

    text = []#存储文章内容

    for a in text_p:

        text.append(a.get_text().encode('utf-8'))#取出p标签下的文本部分，即文章的内容

    name = dic["title"]

    with open('lesson/%s.txt' % name, 'wb') as f:#将文章写入文件

        for line in text:

            f.write(line)

from bs4 import BeautifulSoup

import urllib.request#导入urllib库

url_list = ['https://www.p y t hontab.com/html/pythonhexinbiancheng/index.html']#获取网页链接

for i in range(2,20):

    url = 'https://www.py tho ntab.com/html/pythonhexinbiancheng/%s.html' % i

    url_list.append(url)

for url in url_list:

    request = urllib.request.urlopen(url)

    html = request.read()#读取网页源代码

    soup = BeautifulSoup(html,'html.parser')#解析html

    title_links = soup.select('#catlist > li > a')#找到标题与链接

    source_list = []#存储标题与链接的字典

    for title_link in title_links:

        data = {

            'title' : title_link.get_text(),

            'link' : title_link.get('href')

        }

        source_list.append(data)

    for dic in source_list:#遍历每一个字典

        request = urllib.request.urlopen(dic["link"])

        html = request.read()

        soup = BeautifulSoup(html, 'html.parser')

        text_p = soup.select('#Article > div.content > p')#拿到p标签下的数据

        text = []#存储文章内容

        for a in text_p:

            text.append(a.get_text().encode('utf-8'))#取出p标签下的文本部分，即文章的内容

        name = dic["title"]

        directory = '%s.txt' % name

        dir = directory.replace('/','_').replace('*','@').replace('"','o').replace('?','w').replace(':','m')

        with open('lesson/'+dir, 'wb') as f:#将文章写入文件

            for line in text:

                f.write(line)