using System;

using System.Collections.Generic;

using System.ComponentModel;

using System.Data;

using System.Drawing;

using System.IO;

using System.Linq;

using System.Net;

using System.Text;

using System.Threading.Tasks;

using System.Windows.Forms;

using Newtonsoft.Json.Linq;

using Newtonsoft.Json;

using System.Text.RegularExpressions;

namespace 针对百度图片的动态网页爬虫

{

    public partial class Form1 : Form

    {

        static int count = ;

        public Form1()

        {

            InitializeComponent();

        }

        private void btnDo_Click(object sender, EventArgs e)

        {

            int pageCount=;

            string keyword = this.keyWords.Text;

            for (int i = ; i <pageCount; i++)

            {

                HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create("http://image.baidu.com/search/index?tn=baiduimage&ipn=r&ct=201326592&cl=2&lm=-1&st=-1&fm=result&fr=&sf=1&fmq=1466307565574_R&pv=&ic=0&nc=1&z=&se=1&showtab=0&fb=0&width=&height=&face=0&istype=2&ie=utf-8&word="+keyword.ToString());

                using (HttpWebResponse response = (HttpWebResponse)request.GetResponse())

                {

                    if (response.StatusCode == HttpStatusCode.OK)

                    {

                        using (Stream stream = response.GetResponseStream())

                        {

                            try

                            {

                                // 下载指定页的所有图片

                                DownloadPage(stream);

                            }

                            catch (Exception ex)

                            {

                                // 跨线程访问UI线程的txtLogs

                            }

                        }

                    }

                    else

                    {

                       // MessageBox.Show("获取第" + pageCount + "页失败：" + response.StatusCode);

                    }

                }

            }

            MessageBox.Show("执行成功，共"+count.ToString()+"图片");

        }

        private static string[] getLinks(string html)

        {

            const string pattern = @"http://([\w-]+\.)+[\w-]+(/[\w- ./?%&=]*)?";

            Regex r = new Regex(pattern, RegexOptions.IgnoreCase); //新建正则模式

            MatchCollection m = r.Matches(html); //获得匹配结果

            string[] links = new string[m.Count];

            int count=;

            for (int i = ; i < m.Count; i++)

            {

                if(isValiable(m[i].ToString()))

                {

                      links[count] = m[i].ToString(); //提取出结果

                    count++;

                }

            }

            return links;

        }

        private void DownloadPage(Stream stream)

        {

            using(StreamReader reader=new StreamReader(stream))

            {

                string r1;

                StringBuilder sb = new StringBuilder();

                while((r1=reader.ReadLine())!=null)

                {

                    sb.Append(r1);

                }

                FileStream aFile = new FileStream("../../txt.txt", FileMode.OpenOrCreate);

                StreamWriter sw = new StreamWriter(aFile);//将网页存储到了txt文本文件中

                sw.WriteLine(sb.ToString());

                sw.Close();

                string[] s;

                s = getLinks(sb.ToString());

                int i = ;

                for(i=;i<s.Count();i++)

                {

                    if(s[i]!=null||s[i]!="")

                    {

                        count++;

                        savePicture(s[i]);

                    }

                }

                this.label2.Text = count.ToString();

            }

        }

        private static bool isValiable(string url)

        {

            if (url.Contains(".jpg") || url.Contains(".gif") || url.Contains(".png"))

            {

                return true; //得到一些图片之类的资源

            }

            return false;

        }

        private static void savePicture(string path)

        {

            DataClasses1DataContext db = new DataClasses1DataContext();

            Uri url = new Uri(path);

            HttpWebRequest webRequest = (HttpWebRequest)HttpWebRequest.Create(url);

            webRequest.Referer = "http://image.baidu.com";

            HttpWebResponse webResponse = (HttpWebResponse)webRequest.GetResponse();

            if (isValiable(path))//判断如果是图片，就将其存储到数据库中。

            {

                Bitmap myImage = new Bitmap(webResponse.GetResponseStream());

                MemoryStream ms = new MemoryStream();

                myImage.Save(ms, System.Drawing.Imaging.ImageFormat.Jpeg);

                var p = new pictureUrl

                {

                    pictureUrl1 = ms.ToArray()

                };

                db.pictureUrl.InsertOnSubmit(p);

                db.SubmitChanges();

            }

        }

    }

}