JAVA xml 流方式读取。数据挖掘大文件预处理。

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;

/**
* @author gjf
*db_pre.arff存储的是从xml文件中读取的xml信息
*/
public class ElmAuth {
Map<String, Integer> map = new HashMap<String, Integer>();

//第一步
//从xml文件中提取  读取xml文件中的author信息，写到db_pre.arff，并且替换特殊字符
public void settleXml(String src, String dst){//src=dblp.xml dst=db_pre.arff
File file = new File(src);
File fl = new File(dst);
FileReader fr;
try {
fr = new FileReader(file);
FileWriter fw = new FileWriter(fl);
BufferedReader br = new BufferedReader(fr);
BufferedWriter bw = new BufferedWriter(fw);
String line = null;
boolean flag = true;
int loc_st;
int loc_end;
int len = 0, max = 0;
while((line = br.readLine()) != null){
if(line == null)
break;
loc_st = line.indexOf("<author>");
if(loc_st != -1){
loc_end = line.indexOf("</author>");
line = line.substring(loc_st + 8, loc_end);//在<author></author>之间的数据,一个作者的名字
line=line.replace('&', ' ');
line=line.replace('$', ' ');
line=line.replace("' "," ");
line=line.replace("'", " ");
/*flag以文章为界限,在同一篇文章内,flag=false,写入在同一行*/
if(flag){
bw.write("\n");
bw.write(line);
} else {
bw.write(",");
bw.write(line);
}
len++;//每写一个作者,计数加 +1
flag = false;
} else {
flag = true;
if(max < len) max = len;//选择最大的len;
len = 0;
bw.flush();
}
}
System.out.println("第一步 论文中具有最大的作者数:" + max);
} catch (IOException e) {
e.printStackTrace();
}
}

//消除只有单个作item
//第二步:将作者的信息db_pre.arff中只有一个作者的数据删除
public void elimate_one(String src, String dst){//src=db_pre.arff dst=db_elone.arff
try {
File file = new File(src);
FileReader fr = new FileReader(file);
BufferedReader br = new BufferedReader(fr);
File filew = new File(dst);
FileWriter fw = new FileWriter(filew);
BufferedWriter bw = new BufferedWriter(fw);
Map<String, Integer> map = new HashMap<String, Integer>();
String line = null;
int k = 1;
int res = 0;
while((line = br.readLine()) != null){
String[] arrLine = line.split(",");
//作者之间用","隔离,","的数量表示作者的个数,数量比一少,则不写入.
if(arrLine.length > 1){
bw.write(line);
bw.write("\n");
res ++;
}
}
bw.flush();
br.close();
bw.close();
fr.close();
//System.out.println("The Row of the file is:" + res);
System.out.println("这篇论文中去除单个作者后的行数:" + res);
}catch (IOException e) {
e.printStackTrace();
}
}

//将剩余的作储再hashMap中，key值为人名，value为出现的次数,支持度数
public void createMap(String src){//srr=db_elone.arff
try {
File file = new File(src);
FileReader fr = new FileReader(file);
BufferedReader br = new BufferedReader(fr);

String line = null;
while((line = br.readLine()) != null){
if(line == null)
break;
String[] arrLine = line.split(",");
for(int i = 0; i < arrLine.length; ++i){
if(map.get(arrLine[i]) == null){
map.put(arrLine[i], 1);
} else {
map.put(arrLine[i], map.get(arrLine[i]) + 1);
}
}
}
fr.close();
br.close();
} catch (IOException e) {
e.printStackTrace();
}
}

//从hashMap中删除小于支持度minsup的作者,本次的支持度数为100;
public void settleMap(int minsup){
Iterator it = map.keySet().iterator();
while(it.hasNext()){
String str = (String) it.next();
if(map.get(str) < minsup){
it.remove();
}
}
System.out.println("Map的大小,支持度大于100的作者个数:" + map.size());
}

//将大于minsup的作者存储到文件 db_minsup.arff,存储的是符合筛选的作者
public void updateMap(String src, String dst){//src=db_elone.arff dst=db_minsup.arff
try {
File filer = new File(src);
FileReader fr = new FileReader(filer);
BufferedReader br = new BufferedReader(fr);

File filew = new File(dst);
FileWriter fw = new FileWriter(filew);
BufferedWriter bw = new BufferedWriter(fw);

String line = null;
int res = 0;
boolean flag = true;
while((line = br.readLine()) != null){
if(line == null)break;
String[] arrLine = line.split(",");
if(flag == false)res++;
flag = true;
for(int i = 0; i < arrLine.length; ++i){
if(map.get(arrLine[i]) != null){
if(flag == true){
bw.write("\n" + arrLine[i]);
flag = false;
} else {
bw.write("," + arrLine[i]);
}
}
}
}
bw.flush();
System.out.println("符合筛选的作者合作写的论文篇数:" + res);
fw.close();
bw.close();
fr.close();
br.close();
} catch (IOException e) {
e.printStackTrace();
}
}

//生成weka识别的文  dst=db
public void createWekaFile(String src, String dst){//src=db_minsup.arff dst=db
try {
File filer = new File(src);
FileReader fr = new FileReader(filer);
BufferedReader br = new BufferedReader(fr);

File filew = new File(dst);
FileWriter fw = new FileWriter(filew);
BufferedWriter bw = new BufferedWriter(fw);
bw.write("@relation db" + "\n");
Iterator it = map.keySet().iterator();
while(it.hasNext()){
String str = (String) it.next();
str.replace("'", "\'");
bw.write("@attribute '" + str + "' { t}\n");
}
bw.write("@data" + "\n");

String line = null;
boolean flag = true;
while((line = br.readLine()) != null){
if(line == null)break;
flag = true;
char ch;
it = map.keySet().iterator();
while(it.hasNext()){
String str = (String)it.next();
if(line.indexOf(str) >= 0){
ch = 't';
} else {
ch = '?';
}
if(flag == true){
bw.write(ch);
} else {
bw.write("," + ch);
}
flag = false;
}
bw.write("\n");
}
bw.flush();
fw.close();
bw.close();
fr.close();
br.close();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}

public void clearMap(){
map.clear();
}

public static void main(String args[]){
ElmAuth elmauth = new ElmAuth();
elmauth.settleXml("dblp.xml", "db_pre.arff");
elmauth.elimate_one("db_pre.arff", "db_elone.arff");
elmauth.createMap("db_elone.arff");
elmauth.settleMap(100);//确定最小支持度数
elmauth.updateMap("db_elone.arff", "db_minsup.arff");

for(int i = 0; i < 20; ++i){
System.out.println();
elmauth.elimate_one("db_minsup.arff", "db_minsup_elone.arff");
elmauth.clearMap();
elmauth.createMap("db_minsup_elone.arff");
elmauth.settleMap(100);
elmauth.updateMap("db_minsup_elone.arff", "db_minsup.arff");
}

elmauth.createWekaFile("db_minsup.arff", "db.arff");
}
}