solr4.0.0学习（二） 数据库导入clob与blob为索引

导入clob很简单。但是blob好像没有提供方法，所以改了一下源码，重新编译替换class文件，竟然成功了。

先把配置文件贴上

SCHEMA.XML

<?xml version="1.0" ?>
<schema name="test" version="1.1">
<types>
<fieldtype name="string"  class="solr.StrField" sortMissingLast="true" omitNorms="true"/>

<fieldType name="standard" class="solr.TextField" positionIncrementGap="100">
<analyzer type="index">
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
<filter class="solr.LowerCaseFilterFactory"/>
</analyzer>
<analyzer type="query">
<tokenizer class="solr.StandardTokenizerFactory"/>
<filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
<filter class="solr.LowerCaseFilterFactory"/>
</analyzer>
</fieldType>

<fieldType name="ik" class="solr.TextField">
<analyzer class="org.wltea.analyzer.lucene.IKAnalyzer"/>
</fieldType>

</types>

<fields>
<field name="blogId"      type="string"   indexed="false"  stored="true"  multiValued="false"/>
<field name="blogTitle"   type="ik"   indexed="true"  stored="true"  multiValued="false" />
<field name="blogAuthorName"   type="ik"   indexed="true"  stored="true"  multiValued="false" />
<field name="blogContent"   type="ik"   indexed="true"  stored="true"  multiValued="false" />
<field name="TITLE"   type="ik"   indexed="true"  stored="true"  />
<field name="TEXT"   type="ik"   indexed="true"  stored="true"  />
</fields>
<defaultSearchField>blogTitle</defaultSearchField>
<solrQueryParser defaultOperator="OR"/>

</schema>

这里的field只用到了blogContent一个。

SOLRCONFIG.XML

<?xml version="1.0" encoding="UTF-8" ?>
<config>
<luceneMatchVersion>LUCENE_34</luceneMatchVersion>
<directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.StandardDirectoryFactory}"/>
<updateHandler class="solr.DirectUpdateHandler2" />

<requestDispatcher handleSelect="true" >
<requestParsers enableRemoteStreaming="false" multipartUploadLimitInKB="2048" />
</requestDispatcher>

<requestHandler name="standard" class="solr.StandardRequestHandler" default="true" />
<requestHandler name="/update" class="solr.XmlUpdateRequestHandler" />
<requestHandler name="/admin/" class="org.apache.solr.handler.admin.AdminHandlers" />

<!-- the dataimport requestHandler -->
<requestHandler name="/dataimport" class="org.apache.solr.handler.dataimport.DataImportHandler">
<lst name="defaults">
<str name="config">db-data-config.xml</str>
</lst>
</requestHandler>

<admin>
<defaultQuery>solr</defaultQuery>
</admin>
<unlockOnStartup>true</unlockOnStartup>
<lockType>simple</lockType>
<requestHandler name="/analysis/field"
startup="lazy"
class="solr.FieldAnalysisRequestHandler" />

</config>

db-data-config.xml

<dataConfig>
<dataSource name="f1" type="FieldStreamDataSource"/>
<dataSource driver="oracle.jdbc.driver.OracleDriver"
url="jdbc:oracle:thin:@127.0.0.1:1521:orcl" user="HT" password="HT"/>
<document>
<entity name="blog" query="SELECT BLOG_CONTENT from  TB_ENT_BLOG" transformer="ClobTransformer">
<field column="BLOG_CONTENT" name="blogContent" clob="true"/>
</entity>
</document>
</dataConfig>

然后修改了ClobTransformer.java。使其同时支持BLOG格式。

package org.apache.solr.handler.dataimport;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.sql.Blob;
import java.sql.Clob;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

public class ClobTransformer extends Transformer
{
public static final String CLOB = "clob";

public Object transformRow(Map<String, Object> aRow, Context context)
{
for (Map map : context.getAllEntityFields()) {
if ("true".equals(map.get("clob"))) {
String column = (String)map.get("column");
String srcCol = (String)map.get("sourceColName");
if (srcCol == null)
srcCol = column;
Object o = aRow.get(srcCol);
if ((o instanceof List)) {
List inputs = (List)o;
List results = new ArrayList();
for (Object input : inputs) {
if ((input instanceof Clob)) {
Clob clob = (Clob)input;
results.add(readFromClob(clob));
}else if(input instanceof Blob){
Blob blob = (Blob)input;
results.add(readFromBlob(blob));
}
}
aRow.put(column, results);
}
else if ((o instanceof Clob)) {
Clob clob = (Clob)o;
aRow.put(column, readFromClob(clob));
}else if(o instanceof Blob){
Blob blob = (Blob)o;
aRow.put(column, readFromBlob(blob));
}
}
}
return aRow;
}

private String readFromBlob(Blob blob) {
try{
InputStream is = blob.getBinaryStream();
BufferedReader br = new BufferedReader(new InputStreamReader(is));
String str = "";
String res = "";
while((str=br.readLine())!=null){
res += str;
}
return res;
}catch (Exception e) {
e.printStackTrace();
return "";
}
}

private String readFromClob(Clob clob) {
Reader reader = null;
try {
reader = clob.getCharacterStream();
} catch (SQLException e1) {
e1.printStackTrace();
}
StringBuilder sb = new StringBuilder();
char[] buf = new char[1024];
try
{
int len;
while ((len = reader.read(buf)) != -1)
sb.append(buf, 0, len);
}
catch (IOException e) {
DataImportHandlerException.wrapAndThrow(500, e);
}
return sb.toString();
}
}

这里加了一个readFromBlob方法，加了两个else if。异常的处理很粗糙。

这样替换class文件，导入索引就正常了。在query ":" 页面的response会出现所有blob内容。

如果response没有blob字段或者显示为对象地址，都是错了。