java爬虫实现爬取百度风云榜Top10

最近在项目中遇到了java和python爬虫进行程序调用和接口对接的问题, 刚开始也是调试了好久才得出点门道.

而后,自己也发现了爬虫的好玩之处,边想着用java来写个爬虫玩玩,虽说是个不起眼的demo,但还是想记录一下这个小爬虫,便于以后的查阅.

直接上代码:

 import org.jsoup.Connection;

 import org.jsoup.Jsoup;

 import org.jsoup.nodes.Document;

 import org.jsoup.nodes.Element;

 import org.jsoup.select.Elements;

 import org.springframework.util.StringUtils;

 import java.io.IOException;

 import java.util.ArrayList;

 import java.util.List;

 public class MySpider {

     public static void main(String[] args) {

         List<NewsEntity> list = new ArrayList<NewsEntity>();

         Connection connect = Jsoup.connect("http://top.baidu.com/buzz?b=1&fr=tph_right");  //百度风云榜网址

         connect.userAgent("Mozilla/4.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)");  //模拟火狐浏览器访问网页

         try {

             Document document = connect.get();      //建立连接,获取网页内容为文档对象

             Element main = document.getElementById("main");  //获取需要爬去部位的根元素

             Elements url = main.select("div[class=mainBody]").select("table[class=list-table]")

                     .select("tbody").select("tr"); //css选择器

             int i = 0;

             for (Element element : url) {

                 NewsEntity entity = new NewsEntity();

                 String attr_url = element.select("td[class=keyword]").select("a[class=list-title]").attr("href");

                 String text = element.select("td[class=keyword]").select("a[class=list-title]").text();

                 String span = element.select("td[class=last").select("span").text();

                 if (StringUtils.isEmpty(attr_url) || StringUtils.isEmpty(text) || StringUtils.isEmpty(span)) {

                     continue;

                 }

                 entity.setTitle(text);

                 entity.setUrl(attr_url);

                 entity.setHots(span);

                 i++;

                 if (i > 10) {

                     break;

                 }

                 list.add(entity);

             }

             System.out.println(list.toString());

             System.out.println(list.size());

         } catch (IOException e) {

             e.printStackTrace();

             System.out.println("网页元素发生改变或访问被禁止");

         }

     }

 }

简易封装:

 /**

  * @author RYH

  * @description 封装新闻实体

  * @date 2019/2/26

  **/

 public class NewsEntity {

     private String title;

     private String url;

     private String hots;

     public String getTitle() {

         return title;

     }

     public void setTitle(String title) {

         this.title = title;

     }

     public String getUrl() {

         return url;

     }

     public void setUrl(String url) {

         this.url = url;

     }

     public String getHots() {

         return hots;

     }

     public void setHots(String hots) {

         this.hots = hots;

     }

     @Override

     public String toString() {

         return "NewsEntity{" +

                 "title='" + title + '\'' +

                 ", url='" + url + '\'' +

                 ", hots=" + hots +

                 '}';

     }

 }

导入的包也只有jsoup包,功能还是很强大的

<dependency>
　　<groupId>org.jsoup</groupId>
　　<artifactId>jsoup</artifactId>
　　<version>1.12.1</version>
</dependency>

控制台打印也一目了然, 做些简单的爬取还是很容易的

巴特西

java爬虫实现爬取百度风云榜Top10

最新文章

热门文章