一种中文文本的快速分词方法（二）

package org.zhukovasky.chineseSeg;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.ObjectOutputStream;
import java.io.OutputStream;
import java.io.Reader;

import org.zhukovasky.HashBinaryClass.HashBinaryContainer;
import org.zhukovasky.HashBinaryClass.Maps;
import org.zhukovasky.fileutil.WordCount;
import org.zhukovasky.fileutil.WordDictUtil;
import org.zhukovasky.invertedindex.MapWords;

/**
* 以下是中文文本的分词工具，
* 文本的编码为UTF-8
* @author zhukovasky
* @version 1.0
* @since 2013.12
* @email zhukovasky@163.com
* */
public class chineseSeg {
/**
* 以下方法为对中文文本的分词写入到倒排索引中
* @param afterprocess经过预处理后的文本
* @param invertedIndex 存放倒排索引的地址
* @param 字典所在的地址
*
* */
public final static int MAXLENGTH=10;
public static void FileSeg(File afterprocess,File invertedIndex,File dict){
MapWords mapwords=new MapWords();
Reader r=null;
BufferedReader bf=null;
ObjectOutputStream oos=null;
OutputStream output=null;
String Line=null;
Maps map=WordCount.getDict(dict);
int i=0;
try {
r=new FileReader(afterprocess);
bf=new BufferedReader(r);
Line=bf.readLine();
int Kase=0;
if(Line.length()<=MAXLENGTH+1){
Kase=1;
}else{
Kase=2;
}
switch(Kase){
case 1:{
while(i<=Line.length()-1){
String str=null;
String str1=null;
String str2=null;
str=Line.substring(0);
str1=Line.substring(0, 1);
str2=Line.substring(1, 2);
int seek=0;
if(map.isCwordExist(str1)){
if(map.getHBC(str1).isSecondWordExist(str2)){
HashBinaryContainer hbc=map.getHBC(str1);
String[] temp=hbc.getMatchArray(str2);
String[] maxletemp=WordDictUtil.getStringLengthArray(temp);
if(maxletemp[0].length()==1){
String segword=str1+str2;
seek=2;
mapwords.addNewNodeElement(segword, afterprocess.getName(), i);
i=i+2;
}else{
int length=maxletemp[0].length();
String str3=str.substring(1, length+1);
String segword=str1+str3;
if(WordDictUtil.isWordMatched(str3, maxletemp)){
mapwords.addNewNodeElement(segword, afterprocess.getName(), i);
seek=segword.length();
i=i+seek;
}else{
i=i+2;
segword=str1+str2;
mapwords.addNewNodeElement(segword, afterprocess.getName(), i);
}
}
}else{
mapwords.addNewNodeElement(str1, afterprocess.getName(), i);
i++;
}
}else{
mapwords.addNewNodeElement(str1, afterprocess.getName(), i);
i++;
}
}
};
break;
case 2:{
while(i<=Line.length()-1){
String str=null;
String str1=null;
String str2=null;
if(i>=Line.length()-1-MAXLENGTH&&Line.length()-1-MAXLENGTH>0){
str=Line.substring(i);
if(i>=Line.length()-1&&Line.length()-1>0){
if(i>Line.length()){
break;
}
str1=Line.substring(i);
str2=null;
break;
}else{
str1=str.substring(0,1);
str2=str.substring(1,2);
int seek=0;
if(map.isCwordExist(str1)){
if(map.getHBC(str1).isSecondWordExist(str2)){
HashBinaryContainer hbc=map.getHBC(str1);
String[] temp=hbc.getMatchArray(str2);
String[] MaxLeTemp=WordDictUtil.getStringLengthArray(temp);
if(MaxLeTemp[0].length()==1){
String segword=str1+str2;
seek=2;
mapwords.addNewNodeElement(segword, afterprocess.getName(), i);
i=i+2;
}else{
int length=MaxLeTemp[0].length();
if(str.length()<length){
break;
}
String str3=str.substring(1, length+1);

String segword=str1+str3;
if(WordDictUtil.isWordMatched(str3, MaxLeTemp)){
mapwords.addNewNodeElement(segword, afterprocess.getName(), i);
seek=segword.length();
i=i+seek;
}else{
i=i+2;
segword=str1+str2;
mapwords.addNewNodeElement(segword, afterprocess.getName(), i);
}
}
}else{
mapwords.addNewNodeElement(str1, afterprocess.getName(), i);
seek=1;
i=i+seek;
}
}else{
mapwords.addNewNodeElement(str1, afterprocess.getName(), i);
seek=1;
i=i+seek;
}
}
}else{
str=Line.substring(i, i+MAXLENGTH);
str1=str.substring(0, 1);
str2=str.substring(1, 2);
int seek=0;
if(map.isCwordExist(str1)){
if(map.getHBC(str1).isSecondWordExist(str2)){
HashBinaryContainer hbc=map.getHBC(str1);
String[] temp=hbc.getMatchArray(str2);
String[] MaxLeTemp=WordDictUtil.getStringLengthArray(temp);
if(MaxLeTemp[0].length()==1){
String segword=str1+str2;
seek=2;
mapwords.addNewNodeElement(segword, afterprocess.getName(), i);
i=i+2;
}else{
int length=MaxLeTemp[0].length();
String str3=str.substring(1, length+1);
String segword=str1+str3;
if(WordDictUtil.isWordMatched(str3, MaxLeTemp)){
mapwords.addNewNodeElement(segword, afterprocess.getName(), i);
seek=segword.length();
i=i+seek;
}else{
i=i+2;
segword=str1+str2;
mapwords.addNewNodeElement(segword, afterprocess.getName(), i);
}
}
}else{
mapwords.addNewNodeElement(str1, afterprocess.getName(), i);
seek=1;
i=i+seek;
}
}else{
mapwords.addNewNodeElement(str1, afterprocess.getName(), i);
seek=1;
i=i+seek;
}
}
}
};
break;
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}finally{
try {
r.close();
bf.close();
} catch (IOException e) {
e.printStackTrace();
}
}

try {
output=new FileOutputStream(invertedIndex);
oos=new ObjectOutputStream(output);
oos.writeObject(mapwords);
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {

e.printStackTrace();
}finally{
try {
oos.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
/**
* 以下方法为对中文文本的分词写入到倒排索引中
* @param afterprocess[k]经过预处理后的文本
* @param invertedIndex 存放倒排索引的地址
* @param 字典所在的地址
* */
public static void FileArraysSeg(File[] afterprocess,File invertedIndex,File dict){
MapWords mapwords=new MapWords();
Reader r=null;
BufferedReader bf=null;
ObjectOutputStream oos=null;
OutputStream output=null;
String Line=null;
Maps map=WordCount.getDict(dict);
int i=0;
int MAXLENGTH=9;			//取决于词典中最大长度词条
for(int k=0;k<afterprocess.length;k++){
try {
r=new FileReader(afterprocess[k]);
bf=new BufferedReader(r);
Line=bf.readLine();
int Kase=0;
if(Line.length()<=MAXLENGTH+1){
Kase=1;
}else{
Kase=2;
}
switch(Kase){
case 1:{
while(i<=Line.length()-1){
String str=null;
String str1=null;
String str2=null;
str=Line.substring(0);
str1=Line.substring(0, 1);
str2=Line.substring(1, 2);
int seek=0;
if(map.isCwordExist(str1)){
if(map.getHBC(str1).isSecondWordExist(str2)){
HashBinaryContainer hbc=map.getHBC(str1);
String[] temp=hbc.getMatchArray(str2);
String[] maxletemp=WordDictUtil.getStringLengthArray(temp);
if(maxletemp[0].length()==1){
String segword=str1+str2;
seek=2;
mapwords.addNewNodeElement(segword, afterprocess[k].getName(), i);
i=i+2;
}else{
int length=maxletemp[0].length();
String str3=str.substring(1, length+1);
String segword=str1+str3;
if(WordDictUtil.isWordMatched(str3, maxletemp)){
mapwords.addNewNodeElement(segword, afterprocess[k].getName(), i);
seek=segword.length();
i=i+seek;
}else{
i=i+2;
segword=str1+str2;
mapwords.addNewNodeElement(segword, afterprocess[k].getName(), i);
}
}
}else{
mapwords.addNewNodeElement(str1, afterprocess[k].getName(), i);
i++;
}
}else{
mapwords.addNewNodeElement(str1, afterprocess[k].getName(), i);
i++;
}
}
};
break;
case 2:{
while(i<=Line.length()-1){
String str=null;
String str1=null;
String str2=null;
if(i>=Line.length()-1-MAXLENGTH&&Line.length()-1-MAXLENGTH>0){
str=Line.substring(i);
if(i>=Line.length()-1&&Line.length()-1>0){
if(i>Line.length()){
break;
}
str1=Line.substring(i);
str2=null;
break;
}else{
str1=str.substring(0,1);
str2=str.substring(1,2);
int seek=0;
if(map.isCwordExist(str1)){
if(map.getHBC(str1).isSecondWordExist(str2)){
HashBinaryContainer hbc=map.getHBC(str1);
String[] temp=hbc.getMatchArray(str2);
String[] MaxLeTemp=WordDictUtil.getStringLengthArray(temp);
if(MaxLeTemp[0].length()==1){
String segword=str1+str2;
seek=2;
mapwords.addNewNodeElement(segword, afterprocess[k].getName(), i);
i=i+2;
}else{
int length=MaxLeTemp[0].length();
if(str.length()<length){
break;
}
String str3=str.substring(1, length+1);

String segword=str1+str3;
if(WordDictUtil.isWordMatched(str3,
ad0e
MaxLeTemp)){
mapwords.addNewNodeElement(segword, afterprocess[k].getName(), i);
seek=segword.length();
i=i+seek;
}else{
i=i+2;
segword=str1+str2;
mapwords.addNewNodeElement(segword, afterprocess[k].getName(), i);
}
}
}else{
mapwords.addNewNodeElement(str1, afterprocess[k].getName(), i);
seek=1;
i=i+seek;
}
}else{
mapwords.addNewNodeElement(str1, afterprocess[k].getName(), i);
seek=1;
i=i+seek;
}
}
}else{
str=Line.substring(i, i+MAXLENGTH);
str1=str.substring(0, 1);
str2=str.substring(1, 2);
int seek=0;
if(map.isCwordExist(str1)){
if(map.getHBC(str1).isSecondWordExist(str2)){
HashBinaryContainer hbc=map.getHBC(str1);
String[] temp=hbc.getMatchArray(str2);
String[] MaxLeTemp=WordDictUtil.getStringLengthArray(temp);
if(MaxLeTemp[0].length()==1){
String segword=str1+str2;
seek=2;
mapwords.addNewNodeElement(segword, afterprocess[k].getName(), i);
i=i+2;
}else{
int length=MaxLeTemp[0].length();
String str3=str.substring(1, length+1);
String segword=str1+str3;
if(WordDictUtil.isWordMatched(str3, MaxLeTemp)){
mapwords.addNewNodeElement(segword, afterprocess[k].getName(), i);
seek=segword.length();
i=i+seek;
}else{
i=i+2;
segword=str1+str2;
mapwords.addNewNodeElement(segword, afterprocess[k].getName(), i);
}
}
}else{
mapwords.addNewNodeElement(str1, afterprocess[k].getName(), i);
seek=1;
i=i+seek;
}
}else{
mapwords.addNewNodeElement(str1, afterprocess[k].getName(), i);
seek=1;
i=i+seek;
}
}
}
};
break;
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}finally{
try {
r.close();
bf.close();
} catch (IOException e) {
e.printStackTrace();
}
}

}
try {
output=new FileOutputStream(invertedIndex);
oos=new ObjectOutputStream(output);
oos.writeObject(mapwords);
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {

e.printStackTrace();
}finally{
try {
oos.close();
} catch (IOException e) {
e.printStackTrace();
}
}

}
}