package com.mieba.spiader;

 import us.codecraft.webmagic.Page;

 import us.codecraft.webmagic.Site;

 import us.codecraft.webmagic.Spider;

 import us.codecraft.webmagic.pipeline.FilePipeline;

 import us.codecraft.webmagic.processor.PageProcessor;

 public class SinaPageProcessor implements PageProcessor

 {

     public static final String URL_LIST = "http://blog\\.sina\\.com\\.cn/s/articlelist_1487828712_0_\\d+\\.html";

     public static final String URL_POST = "http://blog\\.sina\\.com\\.cn/s/blog_\\w+\\.html";

     private Site site = Site.me().setDomain("blog.sina.com.cn").setRetryTimes(3).setSleepTime(3000).setUserAgent(

                     "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");

     @Override

     public void process(Page page)

     {

         // 列表页

         if (page.getUrl().regex(URL_LIST).match())

         {

             // 从页面发现后续的url地址来抓取

             page.addTargetRequests(page.getHtml().xpath("//div[@class=\"articleList\"]").links().regex(URL_POST).all());

             page.addTargetRequests(page.getHtml().links().regex(URL_LIST).all());

             // 文章页

         } else

         {

             // 定义如何抽取页面信息，并保存下来

             page.putField("title", page.getHtml().xpath("//div[@class='articalTitle']/h2"));

             page.putField("content", page.getHtml().xpath("//div[@id='articlebody']//div[@class='articalContent']"));

             page.putField("date", page.getHtml().xpath("//div[@id='articlebody']//span[@class='time SG_txtc']").regex("\\((.*)\\)"));

         } 

     }

     @Override

     public Site getSite()

     {

         return site;

     }

     public static void main(String[] args)

     {

         Spider.create(new SinaPageProcessor())

         //从"http://blog.sina.com.cn/s/articlelist_1487828712_0_1.html"开始抓

         .addUrl("http://blog.sina.com.cn/s/articlelist_1487828712_0_1.html")

         //结果用文件的格式保存下来

         .addPipeline(new FilePipeline("E:\\webmagic\\"))

         //开启5个线程抓取

         .thread(5)

         //启动爬虫

         .run();

     }

 }