其实本章算不上逆向教程只是介绍golang的colly框架而已

列表页分析

根据关键字搜索

通过抓包分析可知

下一页所请求的参数如下

上图标红的代表所需参数

所以其实我们真正需要的也就是SearchSql

但是你多观察即可知这个SearchSql在第一页访问的时候就传给了

所以也就是

爬取第一页获取第二页的请求参数——爬取第二页获取第三页的请求参数

详情页分析

详情页链接在列表页给你的链接是错误的

真是的链接如下图所示

由此图可知我们需要很多参数才能完成爬取

但是列表页给我们的链接也不是毫无用处

我们用正则提取中间所需要的参数即可完成爬取

代码

由于代码是用Go写的用colly 还不是特别熟练

package main

import (

	"fmt"

	"github.com/gocolly/colly"

	"regexp"

	"time"

)

func main() {

	PostUrl := "https://kns.cnki.net/kns8/Brief/GetGridTableHtml"

	c, contentCollector := initCollector()

	c.OnRequest(func(r *colly.Request) {

		fmt.Println(r)

		fmt.Println(r.Method)

		r.Headers.Set("Accept", "text/html, */*; q=0.01")

		r.Headers.Set("Accept-Language", "zh-CN,zh;q=0.9")

		r.Headers.Set("Connection", "keep-alive")

		r.Headers.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")

		r.Headers.Set("Origin", "https://kns.cnki.net")

		r.Headers.Set("Referer", "https://kns.cnki.net/kns8/defaultresult/index")

		r.Headers.Set("Sec-Fetch-Dest", "empty")

		r.Headers.Set("Sec-Fetch-Mode", "cors")

		r.Headers.Set("Sec-Fetch-Site", "same-origin")

		r.Headers.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36")

		r.Headers.Set("X-Requested-With", "XMLHttpRequest")

	})

	// 解析列表页

	c.OnHTML(".result-table-list tbody", func(element *colly.HTMLElement) {

		element.ForEach("tr", func(i int, element *colly.HTMLElement) {

			// 获取链接地址

			titleUrl, _ := element.DOM.Find("a").Attr("href")

			// 获取时间

			titleDate := element.DOM.Find("td.date").Text()

			// 拿到标题名

			s := element.DOM.Find("a").Text()

			// 切片取值

			if len(s) < 0 {

				_ = fmt.Errorf("匹配失败,长度为：%v\n", len(s))

			}

			titleName := s[:len(s)-4]

			// 正则取 decode的值

			//deCodeRe := `DbCode=(.*?)&`

			// 正则 fileName 的值

			fileNameRe := `FileName=(.*?)&`

			// 正则 取DbName 的值

			dbNameRe := `DbName=(.*?)&`

			isMatched, _ := regexp.MatchString(fileNameRe, titleUrl)

			if !isMatched {

				_ = fmt.Errorf("匹配失败,%v\n", isMatched)

			}

			fileName := regexp.MustCompile(fileNameRe).FindStringSubmatch(titleUrl)[1]

			dbName := regexp.MustCompile(dbNameRe).FindStringSubmatch(titleUrl)[1]

			trueUrl := fmt.Sprintf(`https://kns.cnki.net/kcms/detail/detail.aspx?dbcode=CCND&dbname=%s&filename=%s&uniplatform=NZKPT`, dbName, fileName)

			fmt.Println(titleName)

			fmt.Println(titleDate)

			fmt.Println(trueUrl)

			_ = contentCollector.Visit(trueUrl)

		})

		sqlVal, _ := element.DOM.Find("#sqlVal").Attr("value")

		_ = c.Post(PostUrl, map[string]string{

			"IsSearch":          "false",

			"QueryJson":         `{"Platform":"","DBCode":"CFLS","KuaKuCode":"CJFQ,CDMD,CIPD,CCND,CISD,SNAD,BDZK,CCJD,CCVD,CJFN","QNode":{"QGroup":[{"Key":"Subject","Title":"","Logic":1,"Items":[{"Title":"主题","Name":"SU","Value":"矿","Operate":"%=","BlurType":""}],"ChildItems":[]}]}}`,

			"SearchSql":         sqlVal,

			"PageName":          "defaultresult",

			"HandlerId":         "16",

			"DBCode":            "CFLS",

			"KuaKuCodes":        "CJFQ,CDMD,CIPD,CCND,CISD,SNAD,BDZK,CCJD,CCVD,CJFN",

			"CurPage":           "2",

			"RecordsCntPerPage": "20",

			"CurDisplayMode":    "listmode",

			"CurrSortField":     "",

			"CurrSortFieldType": "desc",

			"IsSortSearch":      "false",

			"IsSentenceSearch":  "false",

			"Subject":           "",

		})

	})

	// 解析详情页

	contentCollector.OnHTML(".brief", func(element *colly.HTMLElement) {

		contentHtml := element.Text

		fmt.Printf(contentHtml)

	})

	_ = c.Post(PostUrl, map[string]string{

		"IsSearch":          "true",

		"QueryJson":         `{"Platform":"","DBCode":"CFLS","KuaKuCode":"CJFQ,CDMD,CIPD,CCND,CISD,SNAD,BDZK,CCJD,CCVD,CJFN","QNode":{"QGroup":[{"Key":"Subject","Title":"","Logic":1,"Items":[{"Title":"主题","Name":"SU","Value":"矿","Operate":"%=","BlurType":""}],"ChildItems":[]}]}}`,

		"PageName":          "defaultresult",

		"DBCode":            "CFLS",

		"KuaKuCodes":        "CJFQ,CDMD,CIPD,CCND,CISD,SNAD,BDZK,CCJD,CCVD,CJFN",

		"CurPage":           "1",

		"RecordsCntPerPage": "20",

		"CurDisplayMode":    "listmode",

		"CurrSortField":     "",

		"CurrSortFieldType": "desc",

		"IsSentenceSearch":  "false",

		"Subject":           "",

	})

}

func initCollector() (*colly.Collector, *colly.Collector) {

	// 1. 创建收集器

	c := colly.NewCollector(

		colly.UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.125 Safari/537.36"),

		colly.AllowURLRevisit(),

	)

	// 设置抓取频率限制

	_ = c.Limit(&colly.LimitRule{

		DomainGlob:  "*",

		RandomDelay: 5 * time.Second, // 随机延迟

	})

	// 设置clone内容 解析

	contentCollector := c.Clone()

	return c, contentCollector

}

巴特西

JS逆向实战8——某网实战（基于golang-colly）

列表页分析

详情页分析

代码

最新文章

热门文章