java抓取带验证码登陆后的页面

import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PrintWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.net.URLEncoder;
import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;

public class CookieUtil {

public final static String CONTENT_TYPE = "Content-Type";

public static void main(String[] args) {
// login
//验证码的位置
//Content content = getRandom("GET", "http://localhost:8080/back/random.action", null, null, false,"d:/");
Content content = getRandom("GET", "http://localhost:9090/w/random.action", null, null, false,"d:/");

// build request headers & do rate of user review
List<String> lsit = content.getHeaders().get("Set-Cookie");
Map<String, String> resmap = new HashMap<String, String>();
if (lsit != null) {
StringBuffer sb = new StringBuffer();
boolean isLast = false;
int i = 0;
for (String val : lsit) {
i++;
if (i == lsit.size()) {
isLast = true;
}
int pos = val.indexOf("=");
if (pos != -1) {
String cookieName = val.substring(0, pos);
String cookieVal = val.substring(pos + 1);
System.out.println(cookieName+":"+cookieVal);
cookieVal = cookieVal.split(";")[0];
if (isLast) {
sb.append(cookieName + "=" + cookieVal);
} else {
sb.append(cookieName + "=" + cookieVal + ";");
}
}
}
System.out.println("sb.toString() = "+sb.toString());
resmap.put("Cookie", sb.toString());
}
String a="";
System.out.print("请输入验证码：");
BufferedReader strin=new BufferedReader(new InputStreamReader(System.in));
try {
a=strin.readLine();
}  catch (IOException e) {
e.printStackTrace();
}
System.out.println("输入的数是："+a);

String userCode = "admin";
String password = "123456";
String loginUrl = "http://localhost:9090/w/login.action";
String rateReviewUrl = "http://localhost:9090/w/system/role_list.action";
Map<String, String> paramMap = new HashMap<String, String>();
paramMap.put("userCode", userCode);
paramMap.put("password", password);
paramMap.put("random", a+"");
content = curl("POST", loginUrl, paramMap, resmap, false,"");
System.out.println("第一次 content.getBody()= " + content==null?"no body":content.getBody());
// build request headers & do rate of user review
paramMap = new HashMap<String, String>();

content = curl("POST", rateReviewUrl, paramMap, resmap, false,"");
inFile(content.getBody(), "D:/sss.txt");
System.out.println("第二次content.getBody() = " + content==null?"no body":content.getBody());
}

public static Content curl(String method, //方法类型
String sUrl,//要解析的URL
Map<String, String> paramMap, //存放用户名和密码的map
Map<String, String> requestHeaderMap,//存放COOKIE的map
boolean isOnlyReturnHeader,
String path) {//存放文件路径
System.out.println("-------------"+sUrl+"-------------------");
Content content = null;
HttpURLConnection httpUrlConnection = null;
InputStream in = null;
try {
URL url = new URL(sUrl);
boolean isPost = "POST".equals(method);

if (method == null || (!"GET".equalsIgnoreCase(method) && !"POST".equalsIgnoreCase(method))) {
method = "POST";
}

URL resolvedURL = url;
URLConnection urlConnection = resolvedURL.openConnection();
httpUrlConnection = (HttpURLConnection) urlConnection;
httpUrlConnection.setRequestMethod(method);
httpUrlConnection.setRequestProperty("Accept-Language", "zh-cn,zh;q=0.5");

// Do not follow redirects, We will handle redirects ourself
httpUrlConnection.setInstanceFollowRedirects(false);
urlConnection.setDoOutput(true);
urlConnection.setDoInput(true);
urlConnection.setConnectTimeout(5000);
urlConnection.setReadTimeout(5000);
urlConnection.setUseCaches(false);
urlConnection.setDefaultUseCaches(false);
// set request header
if (requestHeaderMap != null) {
for (Map.Entry<String, String> entry : requestHeaderMap.entrySet()) {
String key = entry.getKey();
String val = entry.getValue();
if (key != null && val != null) {
urlConnection.setRequestProperty(key, val);
}
}
}
if (isPost) {
urlConnection.setDoOutput(true);
ByteArrayOutputStream bufOut = new ByteArrayOutputStream();
boolean firstParam = true;
for (Map.Entry<String, String> entry : paramMap.entrySet()) {
String encName = URLEncoder.encode(entry.getKey(), "UTF-8");
if (firstParam) {
firstParam = false;
} else {
bufOut.write((byte) '&');
}
String encValue = URLEncoder.encode(entry.getValue(),"UTF-8");
bufOut.write(encName.getBytes("UTF-8"));
bufOut.write((byte) '=');
bufOut.write(encValue.getBytes("UTF-8"));
}
byte[] postContent = bufOut.toByteArray();
if (urlConnection instanceof HttpURLConnection) {
((HttpURLConnection) urlConnection).setFixedLengthStreamingMode(postContent.length);
}
OutputStream postOut = urlConnection.getOutputStream();
postOut.write(postContent);
postOut.flush();
postOut.close();
}
httpUrlConnection.connect();
int responseCode = httpUrlConnection.getResponseCode();

// We handle redirects ourself
if (responseCode == HttpURLConnection.HTTP_MOVED_PERM || responseCode == HttpURLConnection.HTTP_MOVED_TEMP) {
String location = httpUrlConnection.getHeaderField("Location");
URL newAction = new URL(url, location);
// Recurse
StringBuffer newUrlSb = new StringBuffer(newAction.getProtocol() + "://" + newAction.getHost());
if (newAction.getPort() != -1) {
newUrlSb.append(":" + newAction.getPort());
}
if (newAction.getPath() != null) {
newUrlSb.append(newAction.getPath());
}
if (newAction.getQuery() != null) {
newUrlSb.append("?" + newAction.getQuery());
}
if (newAction.getRef() != null) {
newUrlSb.append("#" + newAction.getRef());
}

return curl("POST", newUrlSb.toString(), paramMap, requestHeaderMap,isOnlyReturnHeader,path);
} else if (responseCode == HttpURLConnection.HTTP_OK || responseCode == HttpURLConnection.HTTP_CREATED) {
byte[] bytes = new byte[0];
if (!isOnlyReturnHeader) {
if(isPost){
in = httpUrlConnection.getInputStream();
ByteArrayOutputStream bout = new ByteArrayOutputStream();
byte[] buf = new byte[1024];
while (true) {
int rc = in.read(buf);
if (rc <= 0) {
break;
} else {
bout.write(buf, 0, rc);
}
}
bytes = bout.toByteArray();
in.close();
}
}
// only fetch Content-Length and Last-Modified header
String encoding = null;
if (encoding == null) {
encoding = getEncodingFromContentType(httpUrlConnection.getHeaderField(CONTENT_TYPE));
}
content = new Content(sUrl, new String(bytes, encoding),httpUrlConnection.getHeaderFields());
}
} catch (Exception e) {
return null;
} finally {
if (httpUrlConnection != null) {
httpUrlConnection.disconnect();
}
}
return content;
}

public static Content getRandom(String method,
String sUrl,//要解析的url
Map<String, String> paramMap, //存放用户名和密码的map
Map<String, String> requestHeaderMap,//存放COOKIE的map
boolean isOnlyReturnHeader,
String path) {

Content content = null;
HttpURLConnection httpUrlConnection = null;
InputStream in = null;
try {
URL url = new URL(sUrl);
boolean isPost = "POST".equals(method);
if (method == null || (!"GET".equalsIgnoreCase(method) && !"POST".equalsIgnoreCase(method))) {
method = "POST";
}
URL resolvedURL = url;
URLConnection urlConnection = resolvedURL.openConnection();
httpUrlConnection = (HttpURLConnection) urlConnection;
httpUrlConnection.setRequestMethod(method);
httpUrlConnection.setRequestProperty("Accept-Language", "zh-cn,zh;q=0.5");
// Do not follow redirects, We will handle redirects ourself
httpUrlConnection.setInstanceFollowRedirects(false);
httpUrlConnection.setDoOutput(true);
httpUrlConnection.setDoInput(true);
httpUrlConnection.setConnectTimeout(5000);
httpUrlConnection.setReadTimeout(5000);
httpUrlConnection.setUseCaches(false);
httpUrlConnection.setDefaultUseCaches(false);
httpUrlConnection.connect();

int responseCode = httpUrlConnection.getResponseCode();

if (responseCode == HttpURLConnection.HTTP_OK || responseCode == HttpURLConnection.HTTP_CREATED) {
byte[] bytes = new byte[0];
if (!isOnlyReturnHeader) {
DataInputStream ins = new DataInputStream(httpUrlConnection.getInputStream());
//验证码的位置
DataOutputStream out = new DataOutputStream(new FileOutputStream(path+"/code.bmp"));
byte[] buffer = new byte[4096];
int count = 0;
while ((count = ins.read(buffer)) > 0) {
out.write(buffer, 0, count);
}
out.close();
ins.close();
}
String encoding = null;
if (encoding == null) {
encoding = getEncodingFromContentType(httpUrlConnection.getHeaderField(CONTENT_TYPE));
}
content = new Content(sUrl, new String(bytes, encoding),httpUrlConnection.getHeaderFields());
}
} catch (Exception e) {
return null;
} finally {
if (httpUrlConnection != null) {
httpUrlConnection.disconnect();
}
}
return content;
}

public static String getEncodingFromContentType(String contentType) {
String encoding = null;
if (contentType == null) {
return null;
}
StringTokenizer tok = new StringTokenizer(contentType, ";");
if (tok.hasMoreTokens()) {
tok.nextToken();
while (tok.hasMoreTokens()) {
String assignment = tok.nextToken().trim();
int eqIdx = assignment.indexOf('=');
if (eqIdx != -1) {
String varName = assignment.substring(0, eqIdx).trim();
if ("charset".equalsIgnoreCase(varName)) {
String varValue = assignment.substring(eqIdx + 1).trim();
if (varValue.startsWith("\"") && varValue.endsWith("\"")) {
// substring works on indices
varValue = varValue.substring(1,varValue.length() - 1);
}
if (Charset.isSupported(varValue)) {
encoding = varValue;
}
}
}
}
}
if (encoding == null) {
return "UTF-8";
}
return encoding;
}

// 这个是输出
public static boolean inFile(String content, String path) {
PrintWriter out = null;
File file = new File(path);
try {
if (!file.exists()) {
file.createNewFile();
}
out = new PrintWriter(new FileWriter(file));

out.write(content);
out.flush();
return true;
} catch (Exception e) {
e.printStackTrace();
} finally {
out.close();
}
return false;
}

public static String getHtmlReadLine(String httpurl){
String CurrentLine="";
String TotalString="";
InputStream urlStream;
String content="";

try {
URL url = new URL(httpurl);

HttpURLConnection connection = (HttpURLConnection)url.openConnection();

connection.connect();
System.out.println(connection.getResponseCode());
urlStream = connection.getInputStream();

BufferedReader reader = new BufferedReader(

new InputStreamReader(urlStream,"utf-8"));

while ((CurrentLine = reader.readLine()) != null) {
TotalString += CurrentLine+"\n";
}

content = TotalString;

} catch (Exception e) {}

return content;
}
}

class Content {
private String url;
private String body;
private Map<String, List<String>> m_mHeaders = new HashMap<String, List<String>>();

public Content(String url, String body, Map<String, List<String>> headers) {
this.url = url;
this.body = body;
this.m_mHeaders = headers;
}

public String getUrl() {
return url;
}

public String getBody() {
return body;
}

public Map<String, List<String>> getHeaders() {
return m_mHeaders;
}

}