Asp.net 使用正则和网络编程抓取网页数据(有用)

Asp.net 使用正则和网络编程抓取网页数据(有用)

        /// <summary>

        /// 抓取网页对应内容

        /// </summary>

        /// <param name="strUrl">採集地址</param>

        /// <param name="Begin">開始字符</param>

        /// <param name="End">结束字符</param>

        /// <returns></returns>

        private static String GetContent(String strUrl, String Begin, String End)

        {

            String result = String.Empty;

            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(strUrl);

            HttpWebResponse response = (HttpWebResponse)request.GetResponse();

            using (StreamReader reader = new StreamReader(response.GetResponseStream(), System.Text.Encoding.Default))

            {

                result = reader.ReadToEnd();

                reader.Close();

                response.Close();

            }

            //抓取内容

            Match table = Regex.Match(result, "(?<=" + Begin + ")[\\s\\S]*?

(?

=" + End + ")", RegexOptions.IgnoreCase);

            result = NoHTML(table.Value);

            return result;

        }

        ///<summary>

        ///去除HTML标记

        ///</summary>

        ///<param   name="NoHTML">包含HTML的源代码   </param>

        ///<returns>已经去除后的文字</returns>

        private static string NoHTML(string Htmlstring)

        {

            //删除脚本

            Htmlstring = Regex.Replace(Htmlstring, @"<script[^>]*?

>.*?

</script>", "",

                RegexOptions.IgnoreCase);

            //删除HTML

            Htmlstring = Regex.Replace(Htmlstring, @"<(.[^>]*)>", "",

                RegexOptions.IgnoreCase);

            Htmlstring = Regex.Replace(Htmlstring, @"([\r\n])[\s]+", "",

                RegexOptions.IgnoreCase);

            Htmlstring = Regex.Replace(Htmlstring, @"-->", "", RegexOptions.IgnoreCase);

            Htmlstring = Regex.Replace(Htmlstring, @"<!--.*", "", RegexOptions.IgnoreCase);

            Htmlstring = Regex.Replace(Htmlstring, @"&(quot|#34);", "\"",

                RegexOptions.IgnoreCase);

            Htmlstring = Regex.Replace(Htmlstring, @"&(amp|#38);", "&",

                RegexOptions.IgnoreCase);

            Htmlstring = Regex.Replace(Htmlstring, @"&(lt|#60);", "<",

                RegexOptions.IgnoreCase);

            Htmlstring = Regex.Replace(Htmlstring, @"&(gt|#62);", ">",

                RegexOptions.IgnoreCase);

            Htmlstring = Regex.Replace(Htmlstring, @"&(nbsp|#160);", "   ",

                RegexOptions.IgnoreCase);

            Htmlstring = Regex.Replace(Htmlstring, @"&(iexcl|#161);", "\xa1",

                RegexOptions.IgnoreCase);

            Htmlstring = Regex.Replace(Htmlstring, @"&(cent|#162);", "\xa2",

                RegexOptions.IgnoreCase);

            Htmlstring = Regex.Replace(Htmlstring, @"&(pound|#163);", "\xa3",

                RegexOptions.IgnoreCase);

            Htmlstring = Regex.Replace(Htmlstring, @"&(copy|#169);", "\xa9",

                RegexOptions.IgnoreCase);

            Htmlstring = Regex.Replace(Htmlstring, @"&#(\d+);", "",

                RegexOptions.IgnoreCase);

            Htmlstring = Regex.Replace(Htmlstring, ">", "");

            Htmlstring = Regex.Replace(Htmlstring, "<", "");

            Htmlstring = Regex.Replace(Htmlstring, "\r\n", "");

            Htmlstring = Htmlstring.Substring(Htmlstring.IndexOf("\n") + 1);

            if (Htmlstring.LastIndexOf("'") >= 0)

                Htmlstring = Htmlstring.Substring(Htmlstring.LastIndexOf("'") + 1);

            if (Htmlstring.IndexOf("class='tdbk'") >= 0)

                Htmlstring = Htmlstring.Substring(Htmlstring.IndexOf("class='tdbk'") + "class='tdbk'".Length);

            return Htmlstring;

        }