一个简易的网页爬虫，可用于下载在线API文档

package wkx;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.PostMethod;

public class Main {

private static Set<String> filenames = new HashSet<String>();

public static String getSource(String url) {
String response = null;
HttpClient client = new HttpClient();
HttpMethod method = null;
try {
method = new PostMethod(url);
client.executeMethod(method);
if (method.getStatusCode() == HttpStatus.SC_OK) {
response = method.getResponseBodyAsString();
}
} catch (IOException e) {
System.out.println("Get Source Error!");
} finally {
if (method != null)
method.releaseConnection();
}
return response;
}

public static void create(String url, String froot, String cur) {
String curUrl = url + "/" + cur;
if (filenames.contains(curUrl)) {
return;
}
filenames.add(curUrl);
String cont = getSource(curUrl);
if (cont == null) {
return;
}
File f = null;
FileOutputStream fos = null;
try {
f = new File(froot);
if (!f.exists()) {
f.mkdirs();
}
f = new File(froot + "\\" + cur);
f.createNewFile();
fos = new FileOutputStream(f);
fos.write(cont.getBytes());
fos.flush();
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
if (fos != null) {
try {
fos.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
String[] files = cont.split("\"");
for (String file : files) {
if (file.matches(".*\\.html$") && !file.startsWith("http")) {
if (file.contains(" "))
continue;
String turl = url;
String tfroot = froot;
file.replaceAll("//", "/");
int tindex = file.lastIndexOf("?");
if (tindex != -1) {
file = file.substring(0, tindex);
}
int index = 0;
while ((index = file.indexOf("/")) != -1) {
if (file.charAt(index - 1 < 0 ? 0 : index - 1) == '.') {
turl = turl.substring(0, turl.lastIndexOf("/"));
tfroot = tfroot.substring(0, tfroot.lastIndexOf("\\"));
} else {
turl = turl + "/" + file.substring(0, index);
tfroot = tfroot + "\\" + file.substring(0, index);
}
file = file.substring(index + 1);
}
create(turl, tfroot, file);
}
}
}

public static void main(String[] args) {
String url = "http://localhost";
String froot = "C:\\Users\\Jack_Wong\\Desktop\\api";
create(url, froot, "index.html");
}
}