工作中,遇到把大对象写入Postgresql数据库中

package com.geni_sage.gdme.cws.dic;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.OutputStream;
import java.io.Serializable;
import java.nio.ByteBuffer;
import java.nio.ShortBuffer;
import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.sql.Blob;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.HashMap; import com.geni_sage.gdme.cws.kuromoji.dict.CharacterDefinition;
import com.geni_sage.gdme.cws.kuromoji.dict.ConnectionCosts;
import com.geni_sage.gdme.cws.kuromoji.dict.Dictionaries;
import com.geni_sage.gdme.cws.kuromoji.dict.TokenInfoDictionary;
import com.geni_sage.gdme.cws.kuromoji.dict.UnknownDictionary;
import com.geni_sage.gdme.cws.kuromoji.trie.DoubleArrayTrie; /**
* 将日语词典写入维护库
*
* @author ywf
*
*/
public class WriteObj2DBTest {
static String pathname = "D:\\yuwenfeng\\T_Miner_TextMing\\gdme\\gdm_agent_start\\mode\\segmentdic\\japanesedic";
static String[] dictionaries = { "tid.dat", "tid_map.dat", "unk.dat",
"unk_map.dat", "cd.dat", "cc.dat", "dat.dat" }; /**
* @param args
* @throws ClassNotFoundException
* @throws IOException
*/
public static void main(String[] args) throws IOException,
ClassNotFoundException {
String driver = "org.postgresql.Driver";
String url = "jdbc:postgresql://127.0.0.1:42856/GDM";
String user = "gsdba";
String password = "gsdba";
if (args.length == 4) {
driver = args[0];
url = args[1];
user = args[2];
password = args[3];
}
try {
Class.forName(driver);
} catch (ClassNotFoundException e1) {
System.err.println("链接驱动失败");
}
Connection con = null;
try {
con = DriverManager.getConnection(url, user, password);
} catch (SQLException e1) {
System.err.println("创建连接失败");
}
try {
createTable(con);
write2db(con, dictionaries);
// readfromdb(con, "gs_japanesedic", "tid.dat");
// readfromdb(con, "gs_japanesedic", "tid_map.dat");
// readfromdb(con, "gs_japanesedic", "unk.dat");
// readfromdb(con, "gs_japanesedic", "unk_map.dat");
// readfromdb(con, "gs_japanesedic", "cd.dat");
// readfromdb(con, "gs_japanesedic", "cc.dat");
// readfromdb(con, "gs_japanesedic", "dat.dat");
con.close();
} catch (SQLException e1) {
e1.printStackTrace();
} catch (Exception e) {
e.printStackTrace();
}
} /**
* 建立日语词典表
*
* @param con
* @throws SQLException
*/
private static void createTable(Connection con) throws SQLException {
String sql = "create table gs_japanesedic(dictype varchar(100),japaninfo bytea)";
Statement stmt = con.createStatement();
stmt.execute(sql);
stmt.close();
} /**
* 把流写入维护库
*
* @param con
* @param path
* @param dicname
* @throws Exception
*/
private static void write2db(Connection con, String[] path)
throws Exception {
CharacterDefinition characterDefinition = UnknownDictionary
.getInstance(pathname).getCharacterDefinition();
short[][] cost = ConnectionCosts.getInstance(pathname).getCosts();
String sql = "insert into gs_japanesedic values(?,?)";
PreparedStatement pstmt = con.prepareStatement(sql);
for (int i = 0; i < path.length; i++) {
pstmt.setString(1, path[i]);
byte[] by = null;
if (path[i].equals("cd.dat") || path[i].equals("cc.dat")) {
if (path[i].startsWith("cd")) {// CharacterDefinition
by = getBytes(characterDefinition);
} else {
by = getBytes(cost);// cost
}
} else {
InputStream input = new FileInputStream(pathname + "/"
+ path[i]);
by = getbytes(input);
}
pstmt.setBytes(2, by);
pstmt.addBatch();
}
pstmt.executeBatch();
pstmt.close();
} /**
* convet inputstream into byte[]
*
* @param input
* @return
* @throws IOException
*/
private static byte[] getbytes(InputStream input) throws IOException {
byte[] by = new byte[input.available()];
input.read(by);
return by;
} public static DoubleArrayTrie readtrie(InputStream is) throws IOException {
DoubleArrayTrie trie = new DoubleArrayTrie();
DataInputStream dis = new DataInputStream(new BufferedInputStream(is));
int baseCheckSize = dis.readInt();
int tailSize = dis.readInt();
ReadableByteChannel channel = Channels.newChannel(dis); ByteBuffer tmpBaseBuffer = ByteBuffer.allocateDirect(baseCheckSize * 4);
channel.read(tmpBaseBuffer);
tmpBaseBuffer.rewind();
tmpBaseBuffer.asIntBuffer().asReadOnlyBuffer(); ByteBuffer tmpCheckBuffer = ByteBuffer
.allocateDirect(baseCheckSize * 4);
channel.read(tmpCheckBuffer);
tmpCheckBuffer.rewind();
tmpCheckBuffer.asIntBuffer().asReadOnlyBuffer(); ByteBuffer tmpTailBuffer = ByteBuffer.allocateDirect(tailSize * 2);
channel.read(tmpTailBuffer);
tmpTailBuffer.rewind();
tmpTailBuffer.asCharBuffer().asReadOnlyBuffer(); is.close();
return trie;
} private static void readfromdb(Connection con, String table, String dictype)
throws SQLException, IOException, ClassNotFoundException {
String sql = "select * from " + table + " where dictype = '" + dictype
+ "'";
PreparedStatement stmt = con.prepareStatement(sql);
ResultSet rs = stmt.executeQuery();
while (rs.next()) {
String dic = rs.getString(1);
byte[] by = rs.getBytes(2);
InputStream bi = new ByteArrayInputStream(by);
if (dictype.equals("cd.dat") || dictype.equals("cc.dat")) {
if (dictype.startsWith("cd")) {// CharacterDefinition
CharacterDefinition characterDefinition = (CharacterDefinition) loadObject(bi);
} else {
short[][] cost = (short[][]) loadObject(bi);
} } else if (dictype.startsWith("dat")) {
readtrie(bi);
} else if (dictype.equals("tid.dat") || dictype.equals("unk.dat")) {
loadDictionary(bi);
} else {
loadTargetMap(bi);
}
} } protected static void loadDictionary(InputStream is) throws IOException {
DataInputStream dis = new DataInputStream(is);
int size = dis.readInt();
ByteBuffer tmpBuffer = ByteBuffer.allocateDirect(size);
ReadableByteChannel channel = Channels.newChannel(is);
channel.read(tmpBuffer);
is.close();
tmpBuffer.asReadOnlyBuffer();
} protected static void loadTargetMap(InputStream is) throws IOException,
ClassNotFoundException {
ObjectInputStream ois = new ObjectInputStream(new BufferedInputStream(
is));
int[][] targetMap = (int[][]) ois.readObject();
is.close();
} /**
* 根据序列化对象得到byte[]
*
* @param obj
* @return
* @throws IOException
*/
public static byte[] getBytes(Serializable obj) throws IOException {
ByteArrayOutputStream bout = new ByteArrayOutputStream();
ObjectOutputStream out = new ObjectOutputStream(bout);
out.writeObject(obj);
out.flush();
byte[] bytes = bout.toByteArray();
bout.close();
out.close();
return bytes;
} public static Object getObject(byte[] bytes) throws IOException,
ClassNotFoundException {
ByteArrayInputStream bi = new ByteArrayInputStream(bytes);
ObjectInputStream oi = new ObjectInputStream(bi);
Object obj = oi.readObject();
bi.close();
oi.close();
return obj;
} public static Object loadObject(InputStream in) throws IOException,
ClassNotFoundException {
ObjectInputStream oi = new ObjectInputStream(in);
Object obj = oi.readObject();
in.close();
oi.close();
return obj;
}
}

最新文章

  1. Zookeeper常用命令
  2. Centos 源码安装zabbix 2.4.5
  3. 创建第一个 local network(I) - 每天5分钟玩转 OpenStack(80)
  4. Linux 命令收集
  5. DIV+CSS实现左侧带三角形的提示框
  6. VMWare虚拟机下RedHat 9.0linux的网络设置
  7. 从C中变化过来的各种语言的printf输出格式
  8. POJ 2092
  9. [转载]做一个 App 前需要考虑的几件事
  10. jquery向列表添加新元素
  11. 关于Comparable接口的使用
  12. Ibatis 返回datatable数据类型案例
  13. UI 网页三原色
  14. 使用 neon-wallet-db + neon-js + NEO-cli /rpc 搭建轻钱包服务端
  15. Asp.Net Core 2.0 项目实战(8)Core下缓存操作、序列化操作、JSON操作等Helper集合类
  16. 结合API Gateway和Lambda实现登录时的重定向和表单提交请求(Python3实现)
  17. App性能测试之启动时间(安卓)手动+脚本
  18. 登录views
  19. ruby自动化之selenium webGUI
  20. 各大公司Java面试题超详细总结

热门文章

  1. TextBox取不到值及其TextBox取不到js赋的值
  2. 图片上传封装类【包括图片上传和缩略图上传】.NET
  3. .NET获取URL的各种方式及其区别
  4. Sql Server 2005 中的row_number() 分页技术
  5. linux解决无法打开资源管理器
  6. maven更换阿里云仓库
  7. 【C/C++】快速排序的两种实现思路
  8. hdu 5475(打破固定思维OR线段树)
  9. Android大神 博客
  10. BZOJ—— 3402: [Usaco2009 Open]Hide and Seek 捉迷藏