/**

  * 使用搜狗搜索检索关键字并爬取结果集的标题

  * @author tele

  *

  */

 public class SougouCrawler extends RamCrawler{

     public SougouCrawler() {

     }

     public SougouCrawler(String keyword,int maxnum) {

         for(int i=1;i<=maxnum;i++) {

             //拼接url

             String url ="https://www.sogou.com/web?query="+keyword+"&s_from=result_up&cid=&page="+ i +"&ie=utf8&p=40040100&dp=1&w=01029901&dr=1";

             CrawlDatum crawlDatum = new CrawlDatum(url).meta("pageNum",i);

             addSeed(crawlDatum);

             addRegex(".*");

         }

     }

     @Override

     public void visit(Page page, CrawlDatums next) {

         String pageNum = page.meta("pageNum");

         Elements results = page.doc().select("div.results div[^class] h3 a");

         for(int i=0;i<results.size();i++) {

             System.out.println("第"+ pageNum +"页第"+ (i+1) +"条结果------" + results.get(i).text());

         }

     }

     public static void main(String[] args) throws Exception {

         String keyword="淘宝";

         SougouCrawler crawler  = new SougouCrawler(keyword,3);

         crawler.setThreads(8);

         Configuration conf = Configuration.copyDefault();

         conf.setExecuteInterval(3000);

         conf.setReadTimeout(5000);

         conf.setWaitThreadEndTime(3000);

         crawler.setConf(conf);

         crawler.start(1);//只有一层

     }

 }