C#实现网页爬虫
2024-09-21 09:55:09
HTTP请求工具类(功能：1、获取网页html；2、下载网络图片；)：
using System;
using System.Collections.Generic;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;
using System.Windows.Forms; namespace Utils
{
/// <summary>
/// HTTP请求工具类
/// </summary>
public class HttpRequestUtil
{
/// <summary>
/// 获取页面html
/// </summary>
public static string GetPageHtml(string url)
{
// 设置参数
HttpWebRequest request = WebRequest.Create(url) as HttpWebRequest;
request.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)";
//发送请求并获取相应回应数据
HttpWebResponse response = request.GetResponse() as HttpWebResponse;
//直到request.GetResponse()程序才开始向目标网页发送Post请求
Stream responseStream = response.GetResponseStream();
StreamReader sr = new StreamReader(responseStream, Encoding.UTF8);
//返回结果网页（html）代码
string content = sr.ReadToEnd();
return content;
} /// <summary>
/// Http下载文件
/// </summary>
public static void HttpDownloadFile(string url, int minWidth, int minHeight)
{
int pos = url.LastIndexOf("/") + ;
string fileName = url.Substring(pos);
string path = Application.StartupPath + "\\download";
if (!Directory.Exists(path))
{
Directory.CreateDirectory(path);
}
string filePathName = path + "\\" + fileName;
if (File.Exists(filePathName)) return; // 设置参数
HttpWebRequest request = WebRequest.Create(url) as HttpWebRequest;
request.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)";
request.Proxy = null;
//发送请求并获取相应回应数据
HttpWebResponse response = request.GetResponse() as HttpWebResponse;
//直到request.GetResponse()程序才开始向目标网页发送Post请求
Stream responseStream = response.GetResponseStream(); MemoryStream memoryStream = new MemoryStream();
byte[] bArr = new byte[];
int size = responseStream.Read(bArr, , (int)bArr.Length);
while (size > )
{
memoryStream.Write(bArr, , size);
size = responseStream.Read(bArr, , (int)bArr.Length);
}
Image tempImage = System.Drawing.Image.FromStream(memoryStream, true);
int imageHeight = tempImage.Height;
int imageWidth = tempImage.Width;
if (imageHeight >= minHeight && imageWidth >= minWidth)
{
memoryStream.Seek(, SeekOrigin.Begin);
size = memoryStream.Read(bArr, , (int)bArr.Length);
FileStream fs = new FileStream(filePathName, FileMode.Create);
while (size > )
{
fs.Write(bArr, , size);
size = memoryStream.Read(bArr, , (int)bArr.Length);
}
fs.Close();
}
memoryStream.Close();
responseStream.Close();
}
}
}
VisitedHelper类：
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Windows.Forms; namespace Utils
{
/// <summary>
/// 已访问的网址列表
/// </summary>
public class VisitedHelper
{
private static List<string> m_VisitedList = new List<string>(); #region 判断是否已访问
/// <summary>
/// 判断是否已访问
/// </summary>
public static bool IsVisited(string url)
{
if (m_VisitedList.Exists(a => a == url))
{
return true;
}
return false;
}
#endregion #region 添加已访问
/// <summary>
/// 添加已访问
/// </summary>
public static void Add(string url)
{
m_VisitedList.Add(url);
}
#endregion }
}
多线程爬取网页代码：
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Forms;
using Utils; namespace 爬虫
{
public partial class Form1 : Form
{
private static int m_MinWidth = ;
private static int m_MinHeight = ;
private static int m_CompletedCount = ; public Form1()
{
InitializeComponent();
} private void button1_Click(object sender, EventArgs e)
{
ThreadPool.SetMaxThreads(, );
int.TryParse(txtMinWidth.Text, out m_MinWidth);
int.TryParse(txtMinHeight.Text, out m_MinHeight);
button1.Enabled = false;
lblMsg.Text = "正在爬取图片…";
timer1.Start();
new Thread(new ThreadStart(delegate()
{
Crawling(txtUrl.Text, null);
})).Start();
} /// <summary>
/// 爬取
/// </summary>
private void Crawling(string url, string host)
{
if (!VisitedHelper.IsVisited(url))
{
VisitedHelper.Add(url); if (host == null)
{
host = GetHost(url);
} string pageHtml = HttpRequestUtil.GetPageHtml(url);
Regex regA = new Regex(@"<a[\s]+[^<>]*href=(?:""|')([^<>""']+)(?:""|')[^<>]*>[^<>]+</a>", RegexOptions.IgnoreCase);
Regex regImg = new Regex(@"<img[\s]+[^<>]*src=(?:""|')([^<>""']+(?:jpg|jpeg|png|gif))(?:""|')[^<>]*>", RegexOptions.IgnoreCase); MatchCollection mcImg = regImg.Matches(pageHtml);
foreach (Match mImg in mcImg)
{
string imageUrl = mImg.Groups[].Value;
try
{
int imageWidth = GetImageWidthOrHeight(mImg.Value, true);
int imageHeight = GetImageWidthOrHeight(imageUrl, false);
if (imageWidth >= m_MinWidth && imageHeight >= m_MinHeight)
{
if (imageUrl.IndexOf("javascript") == -)
{
if (imageUrl.IndexOf("http") == )
{
HttpRequestUtil.HttpDownloadFile(imageUrl, m_MinWidth, m_MinHeight);
}
else
{
HttpRequestUtil.HttpDownloadFile(host + imageUrl, m_MinWidth, m_MinHeight);
}
}
}
}
catch { }
} //递归遍历
MatchCollection mcA = regA.Matches(pageHtml);
foreach (Match mA in mcA)
{
try
{
string nextUrl = mA.Groups[].Value;
if (nextUrl.IndexOf("javascript") == -)
{
if (nextUrl.IndexOf("http") == )
{
if (GetHost(url) == host)
{
ThreadPool.QueueUserWorkItem(new WaitCallback(delegate(object obj)
{
try
{
Crawling(nextUrl, host);
m_CompletedCount++;
}
catch { }
}));
}
}
else
{
if (GetHost(url) == host)
{
ThreadPool.QueueUserWorkItem(new WaitCallback(delegate(object obj)
{
try
{
Crawling(host + nextUrl, host);
m_CompletedCount++;
}
catch { }
}));
}
}
}
}
catch { }
}
}
} //end Crawling方法 /// <summary>
/// 获取主机
/// </summary>
private string GetHost(string url)
{
Regex regHost = new Regex(@"(?:http|https)://[a-z0-9\-\.:]+", RegexOptions.IgnoreCase);
Match mHost = regHost.Match(url);
return mHost.Value + "/";
} //计时器事件
private void timer1_Tick(object sender, EventArgs e)
{
int workerThreads;
int completionPortThreads;
ThreadPool.GetAvailableThreads(out workerThreads, out completionPortThreads);
if (workerThreads == && m_CompletedCount > )
{
lblMsg.Text = "已结束";
}
else
{
lblMsg.Text = "正在爬取图片…";
}
} /// <summary>
/// 获取图片宽度或高度
/// </summary>
private int GetImageWidthOrHeight(string imageTagString, bool isWidth)
{
string tag = isWidth ? "width" : "height";
Regex reg = new Regex(string.Format(@"{0}=""([\d\.]+)""", tag), RegexOptions.IgnoreCase);
Match match = reg.Match(imageTagString);
if (match.Success)
{
return (int)Convert.ToDouble(match.Groups[].Value);
}
else
{
reg = new Regex(string.Format(@"{0}[\s]*:[\s]*([\d\.]+)[\s]*px[\s]*;", tag), RegexOptions.IgnoreCase);
match = reg.Match(imageTagString);
if (match.Success)
{
return (int)Convert.ToDouble(match.Groups[].Value);
}
}
return int.MaxValue;
} } //end Form1类 /// <summary>
/// 跨线程访问控件的委托
/// </summary>
public delegate void InvokeDelegate();
}
截图：
最新文章
- 配置java环境 ，安装Android Studio...(ps:用eclipse也不错，但as是趋势 自己凭爱好选择)
- 如何使用Case进行简单判断
- iis上json解析失败404
- 修改oracle重做日志文件大小
- 《sql注入攻击与防御 第2版》的总结 之 如何确定有sql注入漏洞
- ACM2114_S[I](1^3+2^3+3^3）
- 自定义searchview的编辑框，搜索按钮，删除按钮，光标等
- Java获取客户端真实IP地址的两种方法
- (四)SAX方式解析XML数据
- 用ingress的方式部署jenkins，启动后提示没有下载插件，未解决
- JavaWeb学习总结(一)——JavaWeb开发入门(转)
- CSS3 弹性盒子
- JavaBasic_正则表达式
- 《Python》并发编程
- 2.13 C++拷贝构造函数
- os.chmod()--更改目录授权权限
- jQuery中ajax和post处理json的不同
- no_unnest,push_subq,push_pred的用法 (转)
- NLP分词
- JVM内存模型以及垃圾回收
热门文章
- 那些年使用Hive踩过的坑
- Stealth视频教程学习笔记（第一章）
- asp.net mvc跨域filter
- Android消息机制
- IOS Animation-CAShapeLayer、UIBezierPath与Animation的结合
- tr
- Atitit Atitit 图像处理之  Oilpaint油画滤镜 水彩画 源码实现
- Atitti  css   transition Animation differ区别
- paip. 解决php 以及 python 连接access无效的参数量。参数不足，期待是 1”的错误
- 多线程 ThreadPool线程池