一种中文文本的快速分词方法（一）（未完待续）

package org.zhukovasky.fileutil;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.Reader;
import java.io.Writer;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang3.StringU
4000
tils;
/**
* 以下是中文文本的处理工具，用来清洗文本的各种乱码
* 文本的编码为UTF-8
* @author zhukovasky
* @version 1.0
* @since 2013.12
* @email zhukovasky@163.com
* */
public class FileProcess {
public final static Map<String,String> mapDemo;
static{
mapDemo=new HashMap<String,String>();
String punctuation1[]={"㈠",
"㈡","㈢","㈣","㈤",
"㈥","㈦","㈧","㈨","㈩"};
String punctuation2[]={"⑴","⑵","⑶","⑷","⑸","⑹","⑺","⑻","⑼","⑽"};
String punctuation3[]={"⒈","⒉","⒊","⒋","⒌","⒍","⒎","⒏","⒐","⒑"};
String punctuation4[]={"Ⅰ","Ⅱ","Ⅲ",
"Ⅳ","Ⅴ","Ⅵ","Ⅶ","Ⅷ","Ⅸ","Ⅹ"};
String punctuation5[]={"ⅰ","ⅱ","ⅲ","ⅳ","ⅴ","ⅵ","ⅶ","ⅷ","ⅸ","ⅹ"};
String punctuation6[]={"１","２","３","４","５","６","７","８","９","１０"};
mapDemo.put("，", ",");
mapDemo.put("。", ".");
mapDemo.put("〈", "<");
mapDemo.put("〉", ">");
mapDemo.put("‖", "|");
mapDemo.put("《", "<");
mapDemo.put("》", ">");
mapDemo.put("〔", "[");
mapDemo.put("〕", "]");
mapDemo.put("﹖", "?");
mapDemo.put("？", "?");
mapDemo.put("“", "\"");
mapDemo.put("”", "\"");
mapDemo.put("：", ":");
mapDemo.put("、", ",");
mapDemo.put("（", "(");
mapDemo.put("）", ")");
mapDemo.put("【", "[");
mapDemo.put("】", "]");
mapDemo.put("—", "-");
mapDemo.put("～", "~");
mapDemo.put("！", "!");
mapDemo.put("‵", "'");
mapDemo.put("①", "1");
mapDemo.put("②", "2");
mapDemo.put("③", "3");
mapDemo.put("④", "4");
mapDemo.put("⑤", "5");
mapDemo.put("⑥", "6");
mapDemo.put("⑦", "7");
mapDemo.put("⑧", "8");
mapDemo.put("⑨", "9");
mapDemo.put("＼", "\\");
mapDemo.put("０", "0");
mapDemo.put("／", "/");
mapDemo.put(".",  ".");
mapDemo.put("7", "7");
String[] numberArray={"1","2","3","4","5","6","7","8","9","10"};
for(int i=0;i<10;i++){
mapDemo.put(punctuation1[i], numberArray[i]);
}
for(int i=0;i<10;i++){
mapDemo.put(punctuation2[i], numberArray[i]);
}
for(int i=0;i<10;i++){
mapDemo.put(punctuation3[i], numberArray[i]);
}
for(int i=0;i<10;i++){
mapDemo.put(punctuation4[i], numberArray[i]);
}
for(int i=0;i<10;i++){
mapDemo.put(punctuation5[i], numberArray[i]);
}
for(int i=0;i<10;i++){
mapDemo.put(punctuation6[i], numberArray[i]);
}

}
private static String replace(String line){
for(int i=0;i<line.length();i++){
String charat=line.substring(i, i+1);
if(mapDemo.get(charat) != null){
line=line.replace(charat,(String)mapDemo.get(charat));
}
}
return line;
}
private static String replacePunctuation(String line){
Map<Integer,String> mapDemo=new HashMap<Integer,String>();
mapDemo.put(1, ",");
mapDemo.put(2, ".");
mapDemo.put(3, "<");
mapDemo.put(4, ">");
mapDemo.put(5, "|");
mapDemo.put(6, "<");
mapDemo.put(7, ">");
mapDemo.put(8, "[");
mapDemo.put(9, "]");
mapDemo.put(10, "?");
mapDemo.put(11, "?");
mapDemo.put(12, "\"");
mapDemo.put(13, "\"");
mapDemo.put(14, ":");
mapDemo.put(15, ",");
mapDemo.put(16, "(");
mapDemo.put(17, ")");
mapDemo.put(18, "[");
mapDemo.put(19, "]");
mapDemo.put(20, "-");
mapDemo.put(21, "~");
mapDemo.put(22, "!");
mapDemo.put(23, "'");
mapDemo.put(24, "1");
mapDemo.put(25, "2");
mapDemo.put(26, "3");
mapDemo.put(27, "4");
mapDemo.put(28, "5");
mapDemo.put(29, "6");
mapDemo.put(30, "7");
mapDemo.put(31, "8");
mapDemo.put(32, "9");
mapDemo.put(33, "0");
mapDemo.put(34, "●");
mapDemo.put(35, "→");
mapDemo.put(36, "※");
mapDemo.put(37,"·");
mapDemo.put(38,"=");
mapDemo.put(39, "==");
mapDemo.put(40, "'");
mapDemo.put(41, "'");
mapDemo.put(42, ";");
mapDemo.put(43, "(");
mapDemo.put(44, "－");
mapDemo.put(45, "");
mapDemo.put(46, " ");
mapDemo.put(47, "　");
mapDemo.put(48, "，");
mapDemo.put(49, "（");
mapDemo.put(50, "）");
mapDemo.put(51, "｛");
mapDemo.put(52, "｝");
mapDemo.put(53, "★");
mapDemo.put(54, "㊣");
mapDemo.put(55, "¶");
mapDemo.put(56, "∮");
mapDemo.put(57, "€");
mapDemo.put(58, "☀");
mapDemo.put(59, "Θ");
mapDemo.put(60, "○");
mapDemo.put(61, "№");
mapDemo.put(62,"∷");
mapDemo.put(63, "♂");
mapDemo.put(64,"♀");
mapDemo.put(65, "§");
mapDemo.put(66,"；");
mapDemo.put(67, "「");
mapDemo.put(68, "」");
mapDemo.put(69, "!");
mapDemo.put(70, "！");
mapDemo.put(71, "│");
mapDemo.put(72,"｜");
mapDemo.put(73, " (");
mapDemo.put(74, " )");
mapDemo.put(75,"％");
mapDemo.put(76, "——");
mapDemo.put(77, "+");
mapDemo.put(78, "×");
mapDemo.put(79, "☆");
mapDemo.put(80,"．");
mapDemo.put(81, "’");
mapDemo.put(82, "…");
mapDemo.put(83, "‘");
mapDemo.put(84, ",");
mapDemo.put(85, "?");
for(int i=0;i<line.length();i++){
String charat=line.substring(i, i+1);
if(mapDemo.containsValue(charat)){
line=line.replace(charat,"");
}
}
return line;
}
private static String dropBlank(String line){
String dest="";
if(line!=null){
Pattern p=Pattern.compile("\\s*|\t|\r|\n");
Matcher m=p.matcher(line);
dest=m.replaceAll("");
}
return dest;
}
private static String dropNumber(String line){
String dest="";
if(line!=null){
Pattern p=Pattern.compile("[a-zA-Z0-9]");
Matcher m=p.matcher(line);
dest=m.replaceAll("");
}
return dest;
}
/**
* 该方法实现清除旧文本中的字符标点等处理
* @param origin	旧文本
* @param newFile	新文本
* @return 返回新文本
* */
public static File processFile(File origin,File newFile){
Reader fr=null;
Writer wr=null;
BufferedReader reader=null;
BufferedWriter writer=null;
try {
fr = new FileReader(origin);
wr=new FileWriter(newFile);
reader=new BufferedReader(fr);
writer=new BufferedWriter(wr);
String line=reader.readLine();
while(line!=null){
String newLine1=replace(line);
String newLine2=newLine1.trim();
String newLine3=dropBlank(newLine2);
String newLine4=dropNumber(newLine3);
String newLine6=newLine4.replace("●", "");
String newLine7=newLine6.replace("[", "");
String newLine8=newLine7.replace("]","");
String newLine9=newLine8.replace("/", "");
String newLine10=replacePunctuation(newLine9.replaceAll(" +",""));
String newLine=StringUtils.trim(newLine10);
writer.write(newLine);
line=reader.readLine();
}
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}finally{
try {
reader.close();
writer.close();
} catch (IOException e) {
e.printStackTrace();
}

}

return newFile;
}
}