public static string HttpGet(string url, Encoding encoding = null,  Dictionary<string,string> headDic=null)

        {

            string html = string.Empty;

            try

            {

                HttpWebRequest request = HttpWebRequest.Create(url) as HttpWebRequest;//模拟请求

                request.Timeout =  * ;

                request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36";

                request.ContentType = "text/html; charset=utf-8";

                if (headDic != null)

                {

                    foreach (var item in headDic)

                    {

                        request.Headers.Add(item.Key, item.Value);

                    }

                }

                if(encoding==null)

                    encoding = Encoding.UTF8; // 如果是乱码就改成 utf-8 / GB2312

                else

                    encoding=Encoding.GetEncoding("GB2312");

                using (HttpWebResponse response = request.GetResponse() as HttpWebResponse)

                {

                    if (response.StatusCode != HttpStatusCode.OK)

                    {

                        log.Warn(string.Format("抓取{0}地址返回失败,response.StatusCode为{1}", url, response.StatusCode));

                    }

                    else

                    {

                        try

                        {

                            StreamReader sr = new StreamReader(response.GetResponseStream(), encoding);

                            html = sr.ReadToEnd();//读取数据

                            sr.Close();

                        }

                        catch (Exception ex)

                        {

                            log.Error(string.Format("DownloadHtml抓取{0}保存失败", url), ex);

                            html = null;

                        }

                    }

                }

            }

            catch (WebException ex)

            {

                if (ex.Message.Equals("远程服务器返回错误: (306)。"))

                {

                    log.Error("远程服务器返回错误: (306)。", ex);

                    return null;

                }

            }

            catch (Exception ex)

            {

                log.Error(string.Format("DownloadHtml抓取{0}出现异常", url), ex);

                html = null;

            }

            return html;

        }

        /// <summary>

        /// Post 调用借口

        /// </summary>

        /// <param name="url">接口地址</param>

        /// <param name="value">接口参数</param>

        /// <returns></returns>

        public static string HttpPost(string url, string value)

        {

            string param = value;

            Stream stream = null;

            byte[] postData = Encoding.UTF8.GetBytes(param);

            try

            {

                HttpWebRequest myRequest = (HttpWebRequest)WebRequest.Create(url);

                myRequest.Method = "POST";

                myRequest.ContentType = "application/x-www-form-urlencoded";

                myRequest.ContentLength = postData.Length;

                stream = myRequest.GetRequestStream();

                stream.Write(postData, , postData.Length);

                HttpWebResponse myResponse = (HttpWebResponse)myRequest.GetResponse();

                if (myResponse.StatusCode == HttpStatusCode.OK)

                {

                    StreamReader sr = new StreamReader(myResponse.GetResponseStream(), Encoding.UTF8);

                    string rs = sr.ReadToEnd().Trim();

                    sr.Close();

                    return rs;

                }

                else

                {

                    return "失败:Status:" + myResponse.StatusCode.ToString();

                }

            }

            catch (Exception ex)

            {

                return "失败:ex:" + ex.ToString();

            }

            finally

            {

                if (stream != null)

                {

                    stream.Close();

                    stream.Dispose();

                }

            }

        }

StreamWriter sw = new StreamWriter("路径.txt", true, Encoding.GetEncoding("utf-8"));

sw.Write("爬取的html字符串");

sw.Close();

Regex reTitle = new Regex(@"<title>(?<html>[\s\S]+?)</title>"/>");

string title = reTitle.Match(html).Groups["html"].Value;

Regex rgInfo = new Regex(@"<td align=""left"">(?<company>[^<>]+)</td><td align=""center"">(?<id>[\dA-Z]+)</td><td align=""center"">(?<cat>[^<>]+)</td><td align=""center"">(?<grade>[A-Z]+)</td><td align=""center"">(?<date>[^\s&]*)");

MatchCollection mchInfos = rgInfo.Matches(strHtml);

foreach (Match m in mchInfos)

{

    string strCompany = m.Groups["company"].Value;

    string strId = m.Groups["id"].Value;

    string strCat = m.Groups["cat"].Value.Replace("&nbsp;", "");

    string grade = m.Groups["grade"].Value;

    string date = m.Groups["date"].Value;

}

List<Task> taskList = new List<Task>();

TaskFactory taskFactory = new TaskFactory();

for(int i=;i<;i++)

{

   taskList.Add(taskFactory.StartNew(Crawler));//将一个执行Crawler方法的线程放到集合里面，创建并启动 任务

   if (taskList.Count > ) //线程池启动15个线程

   {

        taskList = taskList.Where(t => !t.IsCompleted && !t.IsCanceled && !t.IsFaulted).ToList();

        Task.WaitAny(taskList.ToArray());//有线程执行完毕

    }

}

Task.WaitAll(taskList.ToArray());//100个线程全部执行完成

Console.WriteLine("抓取全部完成 - -", DateTime.Now);