java爬虫遇到个页面加密的东西，找了些资料学习学习

做了个java运行js的工具类，希望对大家有用，其中用到client（获取js）可以自行换成自己的client。主要是用了

Rhino就是JavaScript引擎，它的目的就是实现Java与JavaScript的互操作性。rhino-1.7R1.jar

Envjs一个纯js方式在无浏览器环境下模拟浏览器的行为。envjs-1.2.js

一般网站js中都会用到jauery，所以还用了jauery.js

import java.io.BufferedReader;

import java.io.ByteArrayInputStream;

import java.io.ByteArrayOutputStream;

import java.io.FileNotFoundException;

import java.io.FileReader;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.io.OutputStream;

import java.io.Reader;

import java.lang.ref.SoftReference;

import java.net.URI;

import java.nio.charset.Charset;

import java.util.Locale;

import java.util.regex.Matcher;

import java.util.regex.Pattern;

import java.util.zip.GZIPInputStream;

import org.apache.commons.io.IOUtils;

import org.apache.commons.lang.StringUtils;

import org.apache.commons.lang.Validate;

import org.apache.http.Header;

import org.apache.http.HeaderElement;

import org.apache.http.HttpEntity;

import org.apache.http.ParseException;

import org.apache.http.client.ClientProtocolException;

import org.apache.http.client.methods.CloseableHttpResponse;

import org.apache.http.client.methods.HttpGet;

import org.apache.http.client.protocol.RequestAcceptEncoding;

import org.apache.http.impl.DefaultConnectionReuseStrategy;

import org.apache.http.impl.client.BasicCookieStore;

import org.apache.http.impl.client.CloseableHttpClient;

import org.apache.http.impl.client.HttpClientBuilder;

import org.apache.http.util.Args;

import org.apache.http.util.ByteArrayBuffer;

import org.jsoup.Jsoup;

import org.mozilla.javascript.Context;

import org.mozilla.javascript.ContextFactory;

import org.mozilla.javascript.Function;

import org.mozilla.javascript.Scriptable;

import com.ibm.icu.text.CharsetDetector;

import com.ibm.icu.text.CharsetMatch;

//import net.sourceforge.htmlunit.corejs.javascript.Context;

//import net.sourceforge.htmlunit.corejs.javascript.ContextFactory;

//import net.sourceforge.htmlunit.corejs.javascript.Function;

//import net.sourceforge.htmlunit.corejs.javascript.Scriptable;

/**

 * 参照http://mybeautiful.iteye.com/blog/1442839

 * http://m.oschina.net/blog/121347

 * http://blog.csdn.net/dwjmantou/article/details/45276967

 * http://lcllcl987.iteye.com/blog/87423

 * ***不可使用htmlunit的包******Cannot call method "setOptimizationLevel" of null

 * @author 5432

 *

 */

public class RhinoScaper {

    private Context context;

    private Scriptable scriptable;

    /**

     * 初始化方法

     */

    public void init(){

        context = ContextFactory.getGlobal().enterContext();

        scriptable =context.initStandardObjects(null);

        context.setOptimizationLevel(-1);

        context.setLanguageVersion(Context.VERSION_1_5);

//        初始化测试用，并定义envjs-1.2.js未定义print

        context.evaluateString(scriptable,

                "var v='sssaass';"

                + "var print = function(v) {"+

                      " java.lang.System.out.println(v);return v ;"+

                " };function hah(){return v }",

                "print",1,null);

//        System.out.println("v == " + scriptable.get("v", scriptable)  );

        Function prf =  (Function)scriptable.get("print", scriptable);

        Object call = prf.call(Context.getCurrentContext(), scriptable, prf, new Object[]{"test"});

//        System.out.println("print == "+call.toString());

        Object invokFunction = invokFunction("hah");

//        System.out.println(invokFunction.toString());

        String[] file = { this.getClass().getResource("/")+"envjs-1.2.js", "./lib/jquery.js" };

        for (String f : file) {

            evaluateJs(f);

        }

    }

    /**

     * 调用函数

     * @param functionName

     * @param functionArags

     * @return

     */

    public Object invokFunction(String functionName,Object... functionArags) {

        Validate.notNull(context, "context is null");

        Validate.notNull(scriptable, "scriptable is null");

        Function function = (Function) scriptable.get(functionName, scriptable);

        Object call = function.call(Context.getCurrentContext(), scriptable, function, functionArags);

//        System.out.println("reslult  = "+call.toString());

        return call;

    }

    /**

     * 加载js文件

     * （当没有找到对应文件，

     * 且要加载文件名路径包含‘envjs-1.2.js’ 会访问  https://raw.githubusercontent.com/ryan-roemer/envjs-1.2/master/env.rhino.1.2.js

     * 文件名路径包含‘jquery.js’ 会访问 http://apps.bdimg.com/libs/jquery/1.6.0/jquery.js

     * 加载js文件 ）

     * @param f 文件名路径

     */

    public void evaluateJs(String f) {

        Validate.notNull(context, "context is null");

        Validate.notNull(scriptable, "scriptable is null");

        FileReader in = null;

        try {

//            FileInputStream fI = new FileInputStream(f);

//            String js = IOUtils.toString(fI, "UTF-8");//设置默认js文件编码为utf-8

//            context.evaluateString(scriptable, js, f, 1, null);

            in = new FileReader(f);

            context.evaluateReader(scriptable, in, f, 1, null);

        } catch (FileNotFoundException e1) {

//            e1.printStackTrace();

            if (f.contains("envjs-1.2.js")) {

                String envjs ="https://raw.githubusercontent.com/ryan-roemer/envjs-1.2/master/env.rhino.1.2.js";

                try {

                    SoftReference<String> htmlString = Client.getHtmlString(envjs);

                    String jqueryStr = htmlString==null?"":htmlString.get();

//                    DefaultClient defaultClient = new DefaultClient();

//                    String jqueryStr =defaultClient.get(envjs).asHtml();

                    context.evaluateString(scriptable, jqueryStr, envjs, 1, null);

                } catch (Exception e) {

                    e.printStackTrace();

                }

            } else if (f.contains("jquery.js")) {

                String jquery = "http://apps.bdimg.com/libs/jquery/1.6.0/jquery.js";

                Reader bufR =null;

                try {

                    SoftReference<Reader> htmlReader = Client.getHtmlReader(jquery);

                    bufR = htmlReader==null?new BufferedReader(null):htmlReader.get();

//                     String js = IOUtils.toString(bufR);

                    context.evaluateReader(scriptable, bufR , jquery, 1, null);

                } catch (IOException e) {

                    e.printStackTrace();

                } catch (Exception e) {

                    e.printStackTrace();

                }finally {

//                    close(bufR);

                    IOUtils.closeQuietly(bufR);

                }

            } else{

                throw new RuntimeException("unknown file "+f);

            }

        } catch (IOException e1) {

            e1.printStackTrace();

        }finally {

//            close(in);

            IOUtils.closeQuietly(in);

        }

    }

    public static void main(String[] args) {

        RhinoScaper rhinoScaper = new RhinoScaper();

        rhinoScaper.init();

//        rhinoScaper.JSloadString("jsString", "jsname");

//        rhinoScaper.evaluateJs("E:/Desktop/loginjs.js");

//        rhinoScaper.loadJS("", classpathURI);

//        电信登录加密测试

        String pwd="111";

        StringBuilder ascending = new StringBuilder();

        SoftReference<String> htmlString = null;

        try {

            htmlString = Client.getHtmlString("http://login.189.cn/bundles/jquery?v=h3Pl8XT8zdNkoI1VbV5sEZOBrSqsxRXX0TIQ9S_lAlM1");

        } catch (Exception e) {

            e.printStackTrace();

        }

        String jsStr =htmlString==null?"":htmlString.get();

        jsStr = jsStr.replaceAll("float:", "floats:").replaceAll("throws", "throwss");

        ascending.append(jsStr);

        ascending.append(";\n var input=document.createElement(\"input\");input.value='"+pwd+"';;input.id= 'pass';input.type='password';");

        ascending.append("\n function getpassword(){ return $(input).valAesEncryptSet()}");

        rhinoScaper.JSloadString(ascending.toString(), "jsname");

        Object result = rhinoScaper.invokFunction("getpassword");

        System.out.println(result);

        try {

            htmlString = Client.getHtmlString("http://www.youdaili.net/Daili/");

            jsStr =htmlString==null?"":htmlString.get();

            String runScript = rhinoScaper.runScript(jsStr);

            System.out.println(runScript);

        } catch (Exception e) {

            e.printStackTrace();

        }

    }

    /**

     * 运行js

     * @param html

     * @return

     */

    private String runScript(String html) {

        String function = null;int jsfrom = 0;

        Pattern p = Pattern.compile("setTimeout\\(\"(.*)\\((.*)\\)\", 200\\);");

        Matcher m = p.matcher(html);

        if(m.find()){

         function = m.group(1);//函数名

         jsfrom = Integer.parseInt(m.group(2));//参数

        }

        JSloadString(Jsoup.parse(html).select("script").html().replace("eval(\"qo=eval;qo(po);\")", "return po"), "jsname");

        Object result = invokFunction(function, jsfrom);

        return result.toString();

    }

    /**

     * 加载js文件

     * @param sourceName 名称

     * @param classpathURI 文件路径

     */

    public void loadJS(String sourceName, String classpathURI) {

        Validate.notNull(context, "context is null");

        Validate.notNull(scriptable, "scriptable is null");

        String js = null;

        InputStream inputStream = null;

        try {

            inputStream = getClass().getResourceAsStream(classpathURI);

            js = IOUtils.toString(inputStream, "UTF-8");//设置默认js文件编码为utf-8

        } catch (IOException e) {

            e.printStackTrace();

        } finally {

            IOUtils.closeQuietly(inputStream);

        }

        context.evaluateString(scriptable, js, sourceName, 1, null);

    }

    /**

     * 加载js字符串

     * @param source js字符串（注意处理js中由于变量名为throws，float类似名称导致的报错）

     * @param sourceName 名称

     */

    public void JSloadString(String source, String sourceName){

        Validate.notNull(context, "context is null");

        Validate.notNull(scriptable, "scriptable is null");

        context.evaluateString(scriptable, source, sourceName, 1, null);

    }

}

class Client{

    public static void close(AutoCloseable close) {

        if (close != null) {

            try {

                close.close();

            } catch (Exception e) {

                e.printStackTrace();

            }

        }

    }

    public static CloseableHttpResponse HttpGetResponse(String url) throws IOException, ClientProtocolException {

        HttpGet httpGet = new HttpGet(URI.create(url));

        BasicCookieStore cookieStore = new BasicCookieStore();

        HttpClientBuilder builder = HttpClientBuilder.create().disableContentCompression()

                .setConnectionReuseStrategy(new DefaultConnectionReuseStrategy()).setUserAgent("Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/537.36");

        builder.addInterceptorLast(new RequestAcceptEncoding());

        builder.setDefaultCookieStore(cookieStore);

        CloseableHttpClient client = builder.build();

        CloseableHttpResponse execute = client.execute(httpGet);

        return execute;

    }

    public static SoftReference<String> getHtmlString(String url)throws Exception {

        CloseableHttpResponse execute = null;

        byte[] binary =null;//初次解析内容

        SoftReference<String> result = null;

        try {

            execute = HttpGetResponse(url);

//            content = execute.getEntity().getContent();

            binary = HttpEntityTOByte(execute.getEntity());

        }finally {

            close(execute);

        }

        String html;

        byte[] decode;

        try {

            System.out.println(execute.getStatusLine().toString());

            System.out.println(execute.getEntity().getContentEncoding());

            Args.notNull(binary, "binary");

            decode= decode(binary,execute.getEntity());

            try {

                String charset = getContentCharSet(execute.getEntity().getContentType().getValue());

                if (charset != null) {

                    html = new String(decode, Charset.forName(charset));

                } else {

                    CharsetMatch match = new CharsetDetector().setText(decode)

                            .detect();

                    html = match.getString();

                }

            } catch (Exception e) {

                throw new Exception(e);

            }

            result = new SoftReference<String>(html);

        }finally {

            binary =null;

            decode =null;

            html=null;

        }

        return result;

    }

    public static SoftReference<Reader> getHtmlReader(String url)throws Exception {

        CloseableHttpResponse execute = null;

        byte[] binary =null;//初次解析内容

        SoftReference<Reader> result = null;

        try {

            execute = HttpGetResponse(url);

            binary = HttpEntityTOByte(execute.getEntity());

        }finally {

            close(execute);

        }

        byte[] decode;

        Reader bufR = null;

        try {

            System.out.println(execute.getStatusLine().toString());

            System.out.println(execute.getEntity().getContentEncoding().toString());

            Args.notNull(binary, "binary");

            decode= decode(binary,execute.getEntity());

            bufR= new BufferedReader(new InputStreamReader(new ByteArrayInputStream(decode)));

            result=new SoftReference<Reader>(bufR);

        }finally {

            binary =null;

            decode =null;

            //close(bufR);

        }

        return result;

    }

    private static String getContentCharSet(String contentType) throws ParseException {

        String charset = null;

        if (StringUtils.isNotEmpty(contentType)) {

            String[] strs = contentType.split(";");

            for (String string : strs) {

                if (string.contains("charset")) {

                    String[] tmp = string.split("=");

                    if (tmp.length == 2) {

                        return tmp[1];

                    }

                }

            }

        }

        return charset;

    }

    public static final int BUFFER = 1024;

    /**

     * 数据解压缩 gizp

     *

     * @param data

     * @return

     * @throws Exception

     * @author http://snowolf.iteye.com/blog/643010

     */

    public static byte[] decompress(byte[] data) throws Exception {

        ByteArrayInputStream bais = new ByteArrayInputStream(data);

        ByteArrayOutputStream baos = new ByteArrayOutputStream();

        // 解压缩

        decompress(bais, baos);

        data = baos.toByteArray();

        baos.flush();

        close(baos);

        close(bais);

//        baos.close();

//        bais.close();

        return data;

    }

    /**

     * 数据解压缩

     *

     * @param is

     * @param os

     * @throws Exception

     */

    public static void decompress(InputStream is, OutputStream os)

            throws Exception {

        GZIPInputStream gis =null;

        byte data[];

        try {

            gis = new GZIPInputStream(is);

            int count;

            data = new byte[BUFFER];

            while ((count = gis.read(data, 0, BUFFER)) != -1) {

                os.write(data, 0, count);

            }

        } finally{

            data = null;

            close(gis);

//            gis.close();

        }

    } 

    /**

     * gizp解压

     * @param binary

     * @param res

     * @param entity

     * @return

     * @throws Exception

     *

     */

    public static byte[] decode(byte[] binary, final HttpEntity entity) throws Exception {

        if (entity != null && entity.getContentLength() != 0) {

            final Header ceheader = entity.getContentEncoding();

            if (ceheader != null) {

                final HeaderElement[] codecs = ceheader.getElements();

                for (final HeaderElement codec : codecs) {

                    final String codecname = codec.getName().toLowerCase(Locale.US);

                    if ("gzip".equals(codecname) || "x-gzip".equals(codecname)) {

                         return decompress(binary);

                    } else if ("deflate".equals(codecname)) {

                        return binary;

                    } else if ("identity".equals(codecname)) {

                        /* Don't need to transform the content - no-op */

                        return binary;

                    } else {

                        throw new Exception("Unsupported Content-Coding: "+codecname );

                    }

                }

            }

        }

        return binary;

    }

    /**

     * 将HttpEntity转换成byte数组

     * @param entity HttpEntity

     * @return byte[]

     * @throws IOException

　　 * @author EntityUtils.toByteArray(entity)

     */

    public static byte[] HttpEntityTOByte(HttpEntity entity) throws IOException{

        final InputStream instream = entity.getContent();

        if (instream == null) {

            return null;

        }

        try {

            Args.check(entity.getContentLength() <= Integer.MAX_VALUE,

                    "HTTP entity too large to be buffered in memory");

            int i = (int)entity.getContentLength();

            if (i < 0) {

                i = 4096;

            }

            final ByteArrayBuffer buffer = new ByteArrayBuffer(i);

            final byte[] tmp = new byte[4096];

            int l;

            while((l = instream.read(tmp)) != -1) {

                buffer.append(tmp, 0, l);

            }

            return buffer.toByteArray();

        } finally {

            instream.close();

        }

    }

}

最新文章

  1. spring 学习之 bean 的注入方式 property和constructor-arg的使用方式
  2. Allegro批量复制Via并保持net属性
  3. POJ2225+BFS
  4. iis最大连接数和队列长度
  5. 全国计算机等级考试二级教程-C语言程序设计_第9章_数组
  6. OpenID倡议：别再创建新的用户名和密码
  7. 安装Oracle，新建组、用户的时候的一个错误
  8. chapter8_4 错误处理
  9. 分别用EasyAR和Vuforia开发AR(入门级）
  10. django 连接mysql
  11. android 减少图片出现oom错误
  12. ansible的modules
  13. HTML+CSS技术实现网页滑动门效果
  14. C#对接----韵达开发平台--取电子面单
  15. js 倒计时10s
  16. 【PAT】B1072 开学寄语（20 分）
  17. RegexHelper.js
  18. 背景图片自适应整个页面CSS+DIV
  19. QoS policy-map class-map
  20. 软件工程第十周psp

热门文章

  1. python数据结构与算法——哈希表
  2. oracle select into 的时候提示未找到数据
  3. 关于Java中的基本数据类型转换
  4. code forces 383 Arpa&#39;s loud Owf and Mehrdad&#39;s evil plan(有向图最小环)
  5. USB
  6. 安装Adobe系列时遇到的问题解决
  7. 一次失败的Selenium chromedriver切换
  8. 【转载】关于OpenGL的图形流水线
  9. LINQ 联表查询 取count 值
  10. virtual file system (VFS)