java调用Linux执行Python爬虫，并将数据存储到elasticsearch中--（java后台代码）

该篇博客主要是java代码，如需相应脚本及java连接elasticsearch工具类代码，请移步到上一篇博客（https://www.cnblogs.com/chenyuanbo/p/9973685.html）

一、创建连接执行Linux脚本工具类

package com.yjlc.platform.utils.Elasticsearch;

import ch.ethz.ssh2.Connection;

import ch.ethz.ssh2.StreamGobbler;

import java.io.*;

/**

 * --------------------------------------------------------------

 * CopyRights(c)2018,YJLC

 * All Rights Reserved

 * <p>

 * FileName: SingletonUtil.java

 * Description:

 * Author: cyb

 * CreateDate: 2018-11-15

 * --------------------------------------------------------------

 */

public class SingletonUtil {

    //无参构造

    private SingletonUtil(){}

    private volatile static SingletonUtil instance;

    //字符编码默认是utf-8

    public static String  DEFAULTCHART="UTF-8";

    public static Connection conn;

    private String ip;

    private String userName;

    private String userPwd;

    public static Boolean flag=false;

//有参构造

    public SingletonUtil(String ip, String userName, String userPwd) {

        this.ip = ip;

        this.userName = userName;

        this.userPwd = userPwd;

    }

    public SingletonUtil getInstance(String ip, String userName, String userPwd){

        if(instance==null){

            synchronized(SingletonUtil.class){

                //防止多线程多次创建

                if(instance==null){

                    instance=new SingletonUtil(ip,userName, userPwd);

                }

            }

        }

        flag= instance.login();//调用登录方法

        return instance;

    }

    //登录

    public Boolean login(){

        boolean flg=false;

        try {

            System.out.println("进入连接");

            conn = new Connection(ip);

            try {

                conn.connect();//连接

            } catch (IOException e) {

                e.printStackTrace();

            }

            flg=conn.authenticateWithPassword(userName, userPwd);//认证

            if (flg){

                System.out.println("认证成功！");

            }

        } catch (IOException e) {

            e.printStackTrace();

        }

        return flg;

    }

    /**

     *@description:纯文本格式返回

     *@author:cyb

     *@date: 2018-11-15 16:56

    *@param: in

    *@param: charset

     *@return: java.lang.String

     */

    public static String processStdout(InputStream in, String charset){

        InputStream    stdout = new StreamGobbler(in);

        StringBuffer buffer = new StringBuffer();;

        try {

            BufferedReader br = new BufferedReader(new InputStreamReader(stdout,charset));

            String line=null;

            while((line=br.readLine()) != null){

                buffer.append(line+"\n");

            }

        } catch (UnsupportedEncodingException e) {

            e.printStackTrace();

        } catch (IOException e) {

            e.printStackTrace();

        }

        return buffer.toString();

    }

}

二、控制层

/**

     *@description:开启爬虫

     *@author:cyb

     *@date: 2018-11-14 15:59

     *@param: id

     *@param: execute

     *@return: java.util.Map<java.lang.String,java.lang.Object>

     */

    @RequestMapping("openTask")

    @ResponseBody

    public Map<String,Object> openTask(String id,Boolean execute){

        Map<String,Object> map = new HashMap<>();

        //根据id查询任务详细信息

        BsKnowledgeInfoDTO  knowledgeInfoDTO=  knolegeService.getDataInfoById(id);

        if(execute==true){

            execute=false;

        }else {

            execute=true;

        }

        knowledgeInfoDTO.setExecute(execute);//修改任务的状态（开启、关闭）

        int k = knolegeService.updateDataInfo(knowledgeInfoDTO);

//        StringBuilder url = new StringBuilder(knowledgeInfoDTO.getPath()) ;//爬虫目标路径

        StringBuilder url= new StringBuilder("https://mil.news.sina.com.cn/");

        StringBuilder reptileMethod= new StringBuilder("http://192.168.200.8:8000/news");//爬虫方法http://192.168.200.8:8000/news

        StringBuilder themeid= new StringBuilder("hottopic");//存储索引名称

       //http://192.168.200.8:8000/news?themeid=hottopic&url=https://mil.news.sina.com.cn/history/2018-11-15/doc-ihmutuec0443667.shtml

        StringBuilder path =reptileMethod.append("?").append("themid=").append(themeid).append("&").append("url=").append(url);

        String ip="192.168.200.8";//Linux 路径

        String userName ="root";

        String userPwd ="yjlc20148";

        int w = knolegeService.reptile(path.toString(),ip,userName,userPwd);

        if(w==200){

            map.put("code",200);

            map.put("message","爬虫成功！");

        }else if(w==206){

            map.put("code",206);

            map.put("message","连接失败！");

        }

        return map;

    }

三、service层（此处省略了service接口层）

/**

 *@description: 爬虫

 *@author:cyb

 *@date: 2018-11-15 20:52

*@param: path 爬虫方法路径+ES存储索引+爬虫目标url合集

*@param: ip 连接ip地址

*@param: userName ：用户名

*@param: userPwd：用户密码

 *@return: int

 */

@Override

public int reptile(String path,String ip,String userName,String userPwd) {

    SingletonUtil singletonUtil = new SingletonUtil("192.168.200.8", "root","yjlc20148");

    singletonUtil.getInstance(ip, userName,userPwd);

    Boolean b =SingletonUtil.flag;//看是否连接成功

    if(b==true){

        System.out.println("=====第一个步骤=====");

        Session session= null;//打开一个会话

        try {

            session = singletonUtil.conn.openSession();

            session.execCommand("sh /opt/zc/linux_sina.sh");//执行命令

        } catch (IOException e) {

            e.printStackTrace();

        }

        //TODO:多条命令

        String result=singletonUtil.processStdout(session.getStdout(),singletonUtil.DEFAULTCHART);

        //如果为得到标准输出为空，说明脚本执行出错了

        if(StringUtils.isBlank(result)){

            System.out.println("脚本出错");

           result=singletonUtil.processStdout(session.getStderr(),singletonUtil.DEFAULTCHART);

        }

        System.out.println("第一个步骤脚本运行成功"+result);

        ConnectNetworkUtil connectNetworkUtil = new ConnectNetworkUtil();

        connectNetworkUtil.ConnectNetwork(path);

        System.out.println("采集成功！");

        session.close();//关闭session

        singletonUtil.conn.close();//爬虫关闭连接

        return 200;//爬虫成功

    }else {

        return 206;//连接失败

    }

}

以上代码已省略了service接口层和java连接elasticsearch工具类（上一篇博客中已写到），以上代码仅供参考，若代码中有不合理或者不规范的地方，请各位指出，技术在于交流！

巴特西

java调用Linux执行Python爬虫，并将数据存储到elasticsearch中--（java后台代码）

最新文章

热门文章