lucene 构建索引

索引过程：

提取文本和创建文档
分析文档
向索引添加文档（倒排索引）
基本索引操作

向索引添加文档
addDocument(Document 使用默认分析器添加文档
addDocument(Document,Analyzer) -- 使用指定的分析器添加文档)

删除索引中的文档
---deleteDocuments(Term)
---deleteDocuments(Term[])
---deleteDocuments(Query)
---deleteDocuments(Query[])
maxDoc 和 numDocs()方法的区别：maxDoc（）返回索引中删除和未被删除的文档总数，后者返回索引中未被删除的文档总数。

更新索引中的文档
步骤：首先删除旧文档，再插入新文档。新文档必须包含旧文档中的所有域。
---updateDocument(Term,Document)
---updateDocument(Term,Doducment,Analyzer)

域选项
域索引选项(Field.Index.*)
Index.ANALYZED :使用分析器将域值分解成单独的词汇单元流，并使每个词汇单元能被单独搜索。适用于 普通文本区域。
Index.NOT_ANALYZED: 对域索引，但不对String 值进行分析。适用于分析哪些不能被拆分的域值，如URL,文件路径，日期，人名，使用于“精确匹配”搜索。
Index.ANALYZED_NORMS:一个扁体，不会在索引中存储 norms 信息。norm记录索引中的index-time boost信息。
Index.NOT_ANALYZED_NO_NORMS 不存储norms。
Index.NO：不被索引.
域存储选项 （Field.Store.*)
Store.YES: 指定存储域值，用于搜索结果显示域值。
Store.NO：不存储域值。
CompressionTools：提供静态方法压缩和解压字节数组。
域项向量选项

Field对象其它初始化方法
Field(String name,Reader value,TermVector termVector): 使用reader来表示域值

域排序选项

对文档和域进行加权操作
文档加权： setBoost(float)
域加权： Field.setBoost(float);

索引数字：NumericField(name).set<Type>Value;

近实时搜索（near-real-time search)

索引优化：
indexwriter优化方法
optimize() 将索引压缩至一个段，操作完成再返回
optimize(int maxNumSegments) ：部分优化，将所以压缩至最多 max个段。
optimize(boolean doWait)：true立即执行；false 后台线程调用合并程序
optimize(int maxNumSegments,boolean doWaint)：组合
消耗大量CPU和I/O资源。

Directory子类
SimpleFSDirectory
NIOFSDirectory: java.nio.*
MMapDirectory: 内存映射I/O进行文件访问。64位jre是好选择。
RAMDirectory：存入内存
FileSwitchDirectory:使用两个文件目录，根据文件扩展名在两个目录间切换使用。

package com.bit.section2;

import static org.junit.Assert.*;

import java.io.IOException;

import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldSelectorResult;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.Version;
import org.junit.Before;
import org.junit.Test;

import com.bit.util.TestUtil;

public class IndexingTest {

protected String[] ids = {"1","2"};
protected String[] unindexed = {"Netherlands","Italy"};
protected String[] unsorted = {"Amesterdam has lots of canals","Venice has lots of canals"};
protected String[] text = {"Amsterdam","Venice"};

private Directory directory;

@Before
public void setUp() throws Exception {
directory = new RAMDirectory();

IndexWriter writer = getWriter();

for(int i=0;i<ids.length;i++){
Document doc = new Document();
doc.add(new Field("id", ids[i],Field.Store.YES,Field.Index.NOT_ANALYZED));
doc.add(new Field("country",unindexed[i],Field.Store.YES,Field.Index.NO));
doc.add(new Field("contents",unsorted[i],Field.Store.NO,Field.Index.ANALYZED));
doc.add(new Field("city",text[i],Field.Store.YES,Field.Index.ANALYZED));
writer.addDocument(doc);
}
writer.close();
}

/**
* 生成索引生成器
* @return
* @throws IOException
* @throws LockObtainFailedException
* @throws CorruptIndexException
*/
private IndexWriter getWriter() throws CorruptIndexException, LockObtainFailedException, IOException {
return new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_36,new WhitespaceAnalyzer()));
}
@Test
public void test() {
fail("Not yet implemented");
}

protected int getHitCount(String fieldName,String searchString) throws IOException, Exception {
IndexSearcher searcher = new IndexSearcher(directory);
Term term = new Term(fieldName, searchString);
Query query = new TermQuery(term);
int hitCount = TestUtil.hitCount(searcher,query);
searcher.close();
return hitCount;
}

@Test
public void testIndexWriter() throws CorruptIndexException, LockObtainFailedException, IOException{
IndexWriter writer = getWriter();
assertEquals(ids.length, writer.numDocs());
writer.close();
}

@Test
public void testIndexReader() throws CorruptIndexException, IOException{
IndexReader reader = IndexReader.open(directory);
assertEquals(ids.length, reader.maxDoc());
assertEquals(ids.length, reader.numDocs());
reader.close();
}

/**
* 测试删除文档
*/
@Test
public void testDeleteBeforeOptimize() throws IOException{
IndexWriter writer = getWriter();
assertEquals(2, writer.numDocs());

//确认被标记删除的文档
writer.deleteDocuments(new Term("id", "1"));
writer.commit();
assertTrue(writer.hasDeletions());

//确认删除一个文档并剩余一个文档
assertEquals(2, writer.maxDoc());
assertEquals(1, writer.numDocs());
writer.close();
}

/**
* 优化后的删除
* @throws IOException
*/
@Test
public void testDeleteAfterOptimize() throws IOException{
IndexWriter writer = getWriter();
assertEquals(2, writer.numDocs());

writer.deleteDocuments(new Term("id", "1"));
writer.optimize();
writer.commit();

assertFalse(writer.hasDeletions());
assertEquals(1, writer.maxDoc());
assertEquals(1, writer.numDocs());
writer.close();
}

/**
* 更新文档
* @throws Exception
*/
@Test
public void testUpdate() throws Exception{
assertEquals(1, getHitCount("city", "Amsterdam"));

IndexWriter writer = getWriter();

Document doc = new Document();
doc.add(new Field("id", "1",Field.Store.YES,Field.Index.NOT_ANALYZED));
doc.add(new Field("country","Netherlands",Field.Store.YES,Field.Index.NO));
doc.add(new Field("contents","Den Haag has lots of museums",Field.Store.NO,Field.Index.ANALYZED));
doc.add(new Field("city","DenHaag",Field.Store.YES,Field.Index.ANALYZED));

writer.updateDocument(new Term("id","1"),doc );

writer.commit();

assertEquals(0, getHitCount("city", "Amsterdam"));
assertEquals(1, getHitCount("city", "DenHaag"));
}

}