import urllib.parse

import urllib.request

response = urllib.request.urlopen("https://www.cnblogs.com")

print(response.read())

urlopen(url, data, timeout)

import urllib.parse

import urllib.request

values = {"username":"XXX","password":"XXX"}

data = urllib.parse.urlencode(values)

data = data.encode('utf-8')

url = "https://passport.cnblogs.com/user/signin?ReturnUrl=https://home.cnblogs.com/&AspxAutoDetectCookieSupport=1"

response = urllib.request.urlopen(url,data)

print(response.read())

import urllib.parse

import urllib.request

values = {"itemCount":30}

data = urllib.parse.urlencode(values)

data = data.encode('utf-8')

url = "https://news.cnblogs.com/CommentAjax/GetSideComments"

data = urllib.parse.urlencode(values)

response = urllib.request.urlopen(url+'?'+data)

print(response.read())

#返回pattern对象

re.compile(string[,flag])

#以下为匹配所用函数

re.match(pattern, string[, flags]) #在字符串中查找，是否能匹配正则表达式

re.search(pattern, string[, flags]) #字符串的开头是否能匹配正则表达式

re.split(pattern, string[, maxsplit]) #通过正则表达式将字符串分离

re.findall(pattern, string[, flags]) #找到 RE 匹配的所有子串，并把它们作为一个列表返回

re.finditer(pattern, string[, flags]) #找到 RE 匹配的所有子串，并把它们作为一个迭代器返回

re.sub(pattern, repl, string[, count]) #找到 RE 匹配的所有子串，并将其用一个不同的字符串替换

re.subn(pattern, repl, string[, count])#返回 (sub(repl, string[, count]), 替换次数)

import urllib.parse

import urllib.request

def getHtml(url,values):

    data = urllib.parse.urlencode(values)

    response_result = urllib.request.urlopen(url+'?'+data).read()

    html = response_result.decode('utf-8')

    return html

def requestCnblogs(num):

    print('请求数据page:',num)

    url = 'https://www.cnblogs.com/mvc/AggSite/PostList.aspx'

    values= {

        'CategoryId':808,

        'CategoryType' : 'SiteHome',

        'ItemListActionName' :'PostList',

        'PageIndex' : num,

        'ParentCategoryId' : 0,

        'TotalPostCount' : 4000

    }

    result = getHtml(url,values)

    return result

from bs4 import BeautifulSoup

import request

import re

def blogParser(index):

  cnblogs = request.requestCnblogs(index)

  soup = BeautifulSoup(cnblogs, 'html.parser')

  all_div = soup.find_all('div', attrs={'class': 'post_item_body'}, limit=20)

  blogs = []

  #循环div获取详细信息

  for item in all_div:

      blog = analyzeBlog(item)

      blogs.append(blog)

  return blogs

def analyzeBlog(item):

    result = {}

    a_title = find_all(item,'a','titlelnk')

    if a_title is not None:

        result["title"] = a_title[0].string

        result["link"] = a_title[0]['href']

    p_summary = find_all(item,'p','post_item_summary')

    if p_summary is not None:

        result["summary"] = p_summary[0].text

    footers = find_all(item,'div','post_item_foot')

    footer = footers[0]

    result["author"] = footer.a.string

    str = footer.text

    time = re.findall(r"发布于 .+? .+? ", str)

    result["create_time"] = time[0].replace('发布于 ','')

    return result

def find_all(item,attr,c):

    return item.find_all(attr,attrs={'class':c},limit=1)

from mongoengine import *

connect('test', host='localhost', port=27017)

import datetime

class Blogs(Document):

    title = StringField(required=True, max_length=200)

    link = StringField(required=True)

    author = StringField(required=True)

    summary = StringField(required=True)

    create_time = StringField(required=True)

def savetomongo(contents):

    for content in contents:

        blog = Blogs(

            title=content['title'],

            link= content['link'],

            author=content['author'],

            summary=content['summary'],

            create_time=content['create_time']

        )

        blog.save()

    return "ok"

def haveBlogs():

    blogs = Blogs.objects.all()

    return len(blogs)

import db

import deal

print("start.......")

for i in range(1, 21):

    contents = deal.blogParser(i)

    db.savetomongo(contents)

    print('page',i,' OK.')

counts = db.haveBlogs()

print("have ",counts," blogs")

print("end.......")