在编写网络爬虫时,HttpWebRequest几乎可以完成绝大多数网站的抓取,为了更好的使用这一技术,我将常用的几个功能进行了封装,以方便调用。这个类已经在多个项目中得到使用,主要解决了Cookies相关的一些问题;如果有其它方面的问题可以提出来,我会进一步完善。

目前HttpHelper包含了以下几个方面:

  • GetHttpContent:通过Get或Post来获取网页的Html
  • SetCookie:根据response中头部的set-cookie对cookie进行设置,能识别httponly
  • GetAllCookies:将CookieContainer转换为键值对,方便存储和跨程序间调用
  • ConvertToCookieContainer:将键值对转换回CookieContainer供程序调用
  • BuildPostData:通过一个需要post的html构建出postdata

代码如下:

 using System;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Text.RegularExpressions;
using System.Collections;
using HtmlAgilityPack; namespace TNIdea.Common.Helper
{
public class HttpHelper
{
public const string CharsetReg = @"(meta.*?charset=""?(?<Charset>[^\s""'>]+)""?)|(xml.*?encoding=""?(?<Charset>[^\s"">]+)""?)"; /// <summary>
/// 获取网页的内容
/// </summary>
/// <param name="url">Url</param>
/// <param name="postData">Post的信息</param>
/// <param name="cookies">Cookies</param>
/// <param name="userAgent">浏览器标识</param>
/// <param name="referer">来源页</param>
/// <param name="cookiesDomain">Cookies的Domian参数,配合cookies使用;为空则取url的Host</param>
/// <param name="encode">编码方式,用于解析html</param>
/// <returns></returns>
public static string GetHttpContent(string url, string postData = null, CookieContainer cookies = null, string userAgent = "", string referer = "", string cookiesDomain = "", Encoding encode = null)
{
try
{
HttpWebResponse httpResponse = null;
if (!string.IsNullOrWhiteSpace(postData))
httpResponse = CreatePostHttpResponse(url, postData, cookies: cookies, userAgent: userAgent, referer: referer);
else
httpResponse = CreateGetHttpResponse(url, cookies: cookies, userAgent: userAgent, referer: referer); #region 根据Html头判断
string Content = null;
//缓冲区长度
const int N_CacheLength = ;
//头部预读取缓冲区,字节形式
var bytes = new List<byte>();
int count = ;
//头部预读取缓冲区,字符串
String cache = string.Empty; //创建流对象并解码
Stream ResponseStream;
switch (httpResponse.ContentEncoding.ToUpperInvariant())
{
case "GZIP":
ResponseStream = new GZipStream(
httpResponse.GetResponseStream(), CompressionMode.Decompress);
break;
case "DEFLATE":
ResponseStream = new DeflateStream(
httpResponse.GetResponseStream(), CompressionMode.Decompress);
break;
default:
ResponseStream = httpResponse.GetResponseStream();
break;
} try
{
while (
!(cache.EndsWith("</head>", StringComparison.OrdinalIgnoreCase)
|| count >= N_CacheLength))
{
var b = (byte)ResponseStream.ReadByte();
if (b < ) //end of stream
{
break;
}
bytes.Add(b); count++;
cache += (char)b;
} if (encode == null)
{
try
{
if (httpResponse.CharacterSet == "ISO-8859-1" || httpResponse.CharacterSet == "zh-cn")
{
Match match = Regex.Match(cache, CharsetReg, RegexOptions.IgnoreCase | RegexOptions.Multiline);
if (match.Success)
{
try
{
string charset = match.Groups["Charset"].Value;
encode = Encoding.GetEncoding(charset);
}
catch { }
}
else
encode = Encoding.GetEncoding("GB2312");
}
else
encode = Encoding.GetEncoding(httpResponse.CharacterSet);
}
catch { }
} //缓冲字节重新编码,然后再把流读完
var Reader = new StreamReader(ResponseStream, encode);
Content = encode.GetString(bytes.ToArray(), , count) + Reader.ReadToEnd();
Reader.Close();
}
catch (Exception ex)
{
return ex.ToString();
}
finally
{
httpResponse.Close();
}
#endregion 根据Html头判断 //获取返回的Cookies,支持httponly
if (string.IsNullOrWhiteSpace(cookiesDomain))
cookiesDomain = httpResponse.ResponseUri.Host; cookies = new CookieContainer();
CookieCollection httpHeaderCookies = SetCookie(httpResponse, cookiesDomain);
cookies.Add(httpHeaderCookies ?? httpResponse.Cookies); return Content;
}
catch
{
return string.Empty;
}
} /// <summary>
/// 创建GET方式的HTTP请求
/// </summary>
/// <param name="url"></param>
/// <param name="timeout"></param>
/// <param name="userAgent"></param>
/// <param name="cookies"></param>
/// <param name="referer"></param>
/// <returns></returns>
public static HttpWebResponse CreateGetHttpResponse(string url, int timeout = , string userAgent = "", CookieContainer cookies = null, string referer = "")
{
HttpWebRequest request = null;
if (url.StartsWith("https", StringComparison.OrdinalIgnoreCase))
{
//对服务端证书进行有效性校验(非第三方权威机构颁发的证书,如自己生成的,不进行验证,这里返回true)
ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CheckValidationResult);
request = WebRequest.Create(url) as HttpWebRequest;
//request.ProtocolVersion = HttpVersion.Version10; //http版本,默认是1.1,这里设置为1.0
}
else
{
request = WebRequest.Create(url) as HttpWebRequest;
} request.Referer = referer;
request.Method = "GET"; //设置代理UserAgent和超时
if (string.IsNullOrWhiteSpace(userAgent))
userAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.116 Safari/537.36"; request.UserAgent = userAgent;
request.Timeout = timeout;
request.KeepAlive = true;
request.AllowAutoRedirect = true; if (cookies == null)
cookies = new CookieContainer();
request.CookieContainer = cookies; return request.GetResponse() as HttpWebResponse;
} /// <summary>
/// 创建POST方式的HTTP请求
/// </summary>
/// <param name="url"></param>
/// <param name="postData"></param>
/// <param name="timeout"></param>
/// <param name="userAgent"></param>
/// <param name="cookies"></param>
/// <param name="referer"></param>
/// <returns></returns>
public static HttpWebResponse CreatePostHttpResponse(string url, string postData, int timeout = , string userAgent = "", CookieContainer cookies = null, string referer = "")
{
HttpWebRequest request = null;
//如果是发送HTTPS请求
if (url.StartsWith("https", StringComparison.OrdinalIgnoreCase))
{
ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CheckValidationResult);
request = WebRequest.Create(url) as HttpWebRequest;
//request.ProtocolVersion = HttpVersion.Version10;
}
else
{
request = WebRequest.Create(url) as HttpWebRequest;
}
request.Referer = referer;
request.Method = "POST";
request.ContentType = "application/x-www-form-urlencoded"; //设置代理UserAgent和超时
if (string.IsNullOrWhiteSpace(userAgent))
request.UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36";
else
request.UserAgent = userAgent;
request.Timeout = timeout;
request.KeepAlive = true;
request.AllowAutoRedirect = true; if (cookies == null)
cookies = new CookieContainer();
request.CookieContainer = cookies; //发送POST数据
if (!string.IsNullOrWhiteSpace(postData))
{
byte[] data = Encoding.UTF8.GetBytes(postData);
request.ContentLength = data.Length;
using (Stream stream = request.GetRequestStream())
{
stream.Write(data, , data.Length);
}
}
//string[] values = request.Headers.GetValues("Content-Type");
return request.GetResponse() as HttpWebResponse;
} /// <summary>
/// 验证证书
/// </summary>
/// <param name="sender"></param>
/// <param name="certificate"></param>
/// <param name="chain"></param>
/// <param name="errors"></param>
/// <returns>是否验证通过</returns>
private static bool CheckValidationResult(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors errors)
{
if (errors == SslPolicyErrors.None)
return true;
return false;
} /// <summary>
/// 根据response中头部的set-cookie对request中的cookie进行设置
/// </summary>
/// <param name="setCookie">The set cookie.</param>
/// <param name="defaultDomain">The default domain.</param>
/// <returns></returns>
private static CookieCollection SetCookie(HttpWebResponse response, string defaultDomain)
{
try
{
string[] setCookie = response.Headers.GetValues("Set-Cookie"); // there is bug in it,the datetime in "set-cookie" will be sepreated in two pieces.
List<string> a = new List<string>(setCookie);
for (int i = setCookie.Length - ; i > ; i--)
{
if (a[i].Substring(a[i].Length - ) == "GMT")
{
a[i - ] = a[i - ] + ", " + a[i];
a.RemoveAt(i);
i--;
}
}
setCookie = a.ToArray<string>();
CookieCollection cookies = new CookieCollection();
foreach (string str in setCookie)
{
NameValueCollection hs = new NameValueCollection();
foreach (string i in str.Split(';'))
{
int index = i.IndexOf("=");
if (index > )
hs.Add(i.Substring(, index).Trim(), i.Substring(index + ).Trim());
else
switch (i)
{
case "HttpOnly":
hs.Add("HttpOnly", "True");
break;
case "Secure":
hs.Add("Secure", "True");
break;
}
}
Cookie ck = new Cookie();
foreach (string Key in hs.AllKeys)
{
switch (Key.ToLower().Trim())
{
case "path":
ck.Path = hs[Key];
break;
case "expires":
ck.Expires = DateTime.Parse(hs[Key]);
break;
case "domain":
ck.Domain = hs[Key];
break;
case "httpOnly":
ck.HttpOnly = true;
break;
case "secure":
ck.Secure = true;
break;
default:
ck.Name = Key;
ck.Value = hs[Key];
break;
}
}
if (ck.Domain == "") ck.Domain = defaultDomain;
if (ck.Name != "") cookies.Add(ck);
}
return cookies;
}
catch
{
return null;
}
} /// <summary>
/// 遍历CookieContainer
/// </summary>
/// <param name="cookieContainer"></param>
/// <returns>List of cookie</returns>
public static Dictionary<string, string> GetAllCookies(CookieContainer cookieContainer)
{
Dictionary<string, string> cookies = new Dictionary<string, string>(); Hashtable table = (Hashtable)cookieContainer.GetType().InvokeMember("m_domainTable",
System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.GetField |
System.Reflection.BindingFlags.Instance, null, cookieContainer, new object[] { }); foreach (string pathList in table.Keys)
{
StringBuilder _cookie = new StringBuilder();
SortedList cookieColList = (SortedList)table[pathList].GetType().InvokeMember("m_list",
System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.GetField
| System.Reflection.BindingFlags.Instance, null, table[pathList], new object[] { });
foreach (CookieCollection colCookies in cookieColList.Values)
foreach (Cookie c in colCookies)
_cookie.Append(c.Name + "=" + c.Value + ";"); cookies.Add(pathList, _cookie.ToString().TrimEnd(';'));
}
return cookies;
} /// <summary>
/// convert cookies string to CookieContainer
/// </summary>
/// <param name="cookies"></param>
/// <returns></returns>
public static CookieContainer ConvertToCookieContainer(Dictionary<string, string> cookies)
{
CookieContainer cookieContainer = new CookieContainer(); foreach (var cookie in cookies)
{
string[] strEachCookParts = cookie.Value.Split(';');
int intEachCookPartsCount = strEachCookParts.Length; foreach (string strCNameAndCValue in strEachCookParts)
{
if (!string.IsNullOrEmpty(strCNameAndCValue))
{
Cookie cookTemp = new Cookie();
int firstEqual = strCNameAndCValue.IndexOf("=");
string firstName = strCNameAndCValue.Substring(, firstEqual);
string allValue = strCNameAndCValue.Substring(firstEqual + , strCNameAndCValue.Length - (firstEqual + ));
cookTemp.Name = firstName;
cookTemp.Value = allValue;
cookTemp.Path = "/";
cookTemp.Domain = cookie.Key;
cookieContainer.Add(cookTemp);
}
}
}
return cookieContainer;
} public static string BuildPostData(string htmlContent)
{
HtmlDocument htmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(htmlContent);
//Get the form node collection.
HtmlNode htmlNode = htmlDoc.DocumentNode.SelectSingleNode("//form");
HtmlNodeCollection htmlInputs = htmlNode.SelectNodes("//input"); StringBuilder postData = new StringBuilder(); foreach (HtmlNode input in htmlInputs)
{
if(input.Attributes["value"] != null)
postData.Append(input.Attributes["name"].Value + "=" + input.Attributes["value"].Value + "&");
}
return postData.ToString().TrimEnd('&');
}
}
}

部分网站需要登录的问题我已经着手通过另一个项目来解决(imitate-login),目前还有许多网页使用了JavaScript或各种基于JS的框架来对网页进行数据加载,如何来模拟执行JavaScript暂时还没找到比较优美的解决方案,如果大家有什么好的方案可以发给我,谢谢!

未经授权,拒绝任何全文及摘要转载!

最新文章

  1. 常用js函数封装
  2. 设置CentOS不启用图形界面
  3. JVM 学习笔记(一)
  4. bzoj 2705: [SDOI2012]Longge的问题 歐拉函數
  5. JAVA DATE解析(时间戳解析为固定格式)
  6. Linux SSH 远程操作与传送文件
  7. 【嵌入式开发】写入开发板Linux系统-模型S3C6410
  8. 201521123069 《Java程序设计》 第13周学习总结
  9. Objc运行时读取和写入plist文件遇到的问题
  10. SpringCloud的分布式配置及消息总线
  11. Spring Boot 2 - 使用CommandLineRunner与ApplicationRunner
  12. 史上最全python面试题详解(三)(附带详细答案(关注、持续更新))
  13. 【资料】Mac OS X 10.9虚拟机镜像及tools
  14. 爬虫之 案列1补充(pipelines优化)
  15. JVM笔记10-性能优化之高级特性
  16. 使用go语言编写IOS和Android程序
  17. jenkins 使用的python 不是指定的python 的解决方法
  18. PixelLink
  19. 【CSS】使用浮动来创建拥有页眉、页脚、左侧目录和主体内容的首页
  20. C++复习14 构造函数初始化调用顺序

热门文章

  1. java.lang.IncompatibleClassChangeError: Implementing class的解决办法,折腾了一天总算解决了
  2. Java接口之间的继承
  3. web iis服务器安全性配置实例
  4. jQuery fullPage.js 全屏滚动
  5. 25个最佳的 WordPress Gallery 画廊插件
  6. 国外经典设计:12个漂亮的移动APP网站案例
  7. NavigationBar隐藏后,返回时表现不完美
  8. DOM事件探秘
  9. URL 路径长度限制(错误:指定的文件或文件夹名称太长)
  10. OpenGL显示图片