solr4.0.0学习（二）数据库导入clob与blob为索引

导入clob很简单。但是blob好像没有提供方法，所以改了一下源码，重新编译替换class文件，竟然成功了。

先把配置文件贴上

SCHEMA.XML

<?xml version="1.0" ?>

<schema name="test" version="1.1">

  <types>

   <fieldtype name="string"  class="solr.StrField" sortMissingLast="true" omitNorms="true"/>

   <fieldType name="standard" class="solr.TextField" positionIncrementGap="100">

	  <analyzer type="index">

		<tokenizer class="solr.StandardTokenizerFactory"/>

		<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />

		<filter class="solr.LowerCaseFilterFactory"/>

	  </analyzer>

	  <analyzer type="query">

		<tokenizer class="solr.StandardTokenizerFactory"/>

		<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />

		<filter class="solr.LowerCaseFilterFactory"/>

	  </analyzer>

	</fieldType>

	<fieldType name="ik" class="solr.TextField">

       <analyzer class="org.wltea.analyzer.lucene.IKAnalyzer"/>

	</fieldType>

  </types>

 <fields>

  <field name="blogId"      type="string"   indexed="false"  stored="true"  multiValued="false"/>

  <field name="blogTitle"   type="ik"   indexed="true"  stored="true"  multiValued="false" />

  <field name="blogAuthorName"   type="ik"   indexed="true"  stored="true"  multiValued="false" />

  <field name="blogContent"   type="ik"   indexed="true"  stored="true"  multiValued="false" />

  <field name="TITLE"   type="ik"   indexed="true"  stored="true"  />

  <field name="TEXT"   type="ik"   indexed="true"  stored="true"  />

 </fields>

 <defaultSearchField>blogTitle</defaultSearchField>

 <solrQueryParser defaultOperator="OR"/>

</schema>

这里的field只用到了blogContent一个。

SOLRCONFIG.XML

<?xml version="1.0" encoding="UTF-8" ?>

<config>

  <luceneMatchVersion>LUCENE_34</luceneMatchVersion>

  <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.StandardDirectoryFactory}"/>

  <updateHandler class="solr.DirectUpdateHandler2" />

  <requestDispatcher handleSelect="true" >

    <requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" />

  </requestDispatcher>

  <requestHandler name="standard" class="solr.StandardRequestHandler" default="true" />

  <requestHandler name="/update" class="solr.XmlUpdateRequestHandler" />

  <requestHandler name="/admin/" class="org.apache.solr.handler.admin.AdminHandlers" />

 <!-- the dataimport requestHandler -->

       <requestHandler name="/dataimport" class="org.apache.solr.handler.dataimport.DataImportHandler">

               <lst name="defaults">

              <str name="config">db-data-config.xml</str>

             </lst>

       </requestHandler> 

  <admin>

    <defaultQuery>solr</defaultQuery>

  </admin>

  <unlockOnStartup>true</unlockOnStartup>

	<lockType>simple</lockType>

  <requestHandler name="/analysis/field"

                  startup="lazy"

                  class="solr.FieldAnalysisRequestHandler" />

</config>

db-data-config.xml

<dataConfig>

<dataSource name="f1" type="FieldStreamDataSource"/>

 <dataSource driver="oracle.jdbc.driver.OracleDriver"

 url="jdbc:oracle:thin:@127.0.0.1:1521:orcl" user="HT" password="HT"/>

 <document>

		<entity name="blog" query="SELECT BLOG_CONTENT from  TB_ENT_BLOG" transformer="ClobTransformer">

				 <field column="BLOG_CONTENT" name="blogContent" clob="true"/>

		</entity>

 </document>

</dataConfig>

然后修改了ClobTransformer.java。使其同时支持BLOG格式。

package org.apache.solr.handler.dataimport;

import java.io.BufferedReader;

import java.io.IOException;

import java.io.InputStream;

import java.io.InputStreamReader;

import java.io.Reader;

import java.sql.Blob;

import java.sql.Clob;

import java.sql.SQLException;

import java.util.ArrayList;

import java.util.List;

import java.util.Map;

public class ClobTransformer extends Transformer

{

  public static final String CLOB = "clob";

  public Object transformRow(Map<String, Object> aRow, Context context)

  {

    for (Map map : context.getAllEntityFields()) {

      if ("true".equals(map.get("clob"))) {

        String column = (String)map.get("column");

        String srcCol = (String)map.get("sourceColName");

        if (srcCol == null)

          srcCol = column;

        Object o = aRow.get(srcCol);

        if ((o instanceof List)) {

          List inputs = (List)o;

          List results = new ArrayList();

          for (Object input : inputs) {

            if ((input instanceof Clob)) {

              Clob clob = (Clob)input;

              results.add(readFromClob(clob));

            }else if(input instanceof Blob){

            	Blob blob = (Blob)input;

            	results.add(readFromBlob(blob));

            }

          }

          aRow.put(column, results);

        }

        else if ((o instanceof Clob)) {

          Clob clob = (Clob)o;

          aRow.put(column, readFromClob(clob));

        }else if(o instanceof Blob){

        	Blob blob = (Blob)o;

        	aRow.put(column, readFromBlob(blob));

        }

      }

    }

    return aRow;

  }

  private String readFromBlob(Blob blob) {

	  try{

		  	InputStream is = blob.getBinaryStream();

			BufferedReader br = new BufferedReader(new InputStreamReader(is));

			String str = "";

			String res = "";

			while((str=br.readLine())!=null){

				res += str;

			}

			return res;

	  }catch (Exception e) {

		  e.printStackTrace();

		  return "";

	}

}

private String readFromClob(Clob clob) {

    Reader reader = null;

	try {

		reader = clob.getCharacterStream();

	} catch (SQLException e1) {

		e1.printStackTrace();

	}

    StringBuilder sb = new StringBuilder();

    char[] buf = new char[1024];

    try

    {

      int len;

      while ((len = reader.read(buf)) != -1)

        sb.append(buf, 0, len);

    }

    catch (IOException e) {

      DataImportHandlerException.wrapAndThrow(500, e);

    }

    return sb.toString();

  }

}

这里加了一个readFromBlob方法，加了两个else if。异常的处理很粗糙。

这样替换class文件，导入索引就正常了。在query ":" 页面的response会出现所有blob内容。

如果response没有blob字段或者显示为对象地址，都是错了。

巴特西

solr4.0.0学习（二）数据库导入clob与blob为索引

最新文章

热门文章

巴特西

solr4.0.0学习（二） 数据库导入clob与blob为索引

最新文章

热门文章

solr4.0.0学习（二）数据库导入clob与blob为索引