java模拟访问web页面

import java.io.BufferedReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.httpclient.methods.PostMethod;

public class text {

/**
* @param args
* @throws IOException
*/
public static void main(String[] args) throws IOException {
String url = "http://www.12306.cn/otsweb/loginAction.do?method=login";

login(url);
}

public static void doPost(String urls) throws IOException {
try {
//连接地址
String surl = "http://mc.dragontv.cn/get_vote.php?id=120&classid=118";

/**
* 首先要和URL下的URLConnection对话。 URLConnection可以很容易的从URL得到。比如： // Using
*  java.net.URL and //java.net.URLConnection
*/
URL url = new URL(urls);
URLConnection connection = url.openConnection();

/**
* 然后把连接设为输出模式。URLConnection通常作为输入来使用，比如下载一个Web页。
* 通过把URLConnection设为输出，你可以把数据向你个Web页传送。下面是如何做：
*/
connection.setDoOutput(true);
connection.setDoInput(true);
connection.setRequestProperty("User-Agent","Mozilla/4.0 (compatible; MSIE 6.0; Windows 2000)");
connection.connect();

/**
* 最后，为了得到OutputStream，简单起见，把它约束在Writer并且放入POST信息中，例如： ...
*/
OutputStreamWriter out = new OutputStreamWriter(connection.getOutputStream(), "UTF-8");
out.flush();
out.close();

List<String> aa = connection.getHeaderFields().get("Set-Cookie");
String cookieVal = "";
for(int i=0;i<aa.size();i++){
cookieVal += aa.get(i);
System.out.println("#Log Cookie ["+aa.get(i)+"] ");
}

//testPost(urls,cookieVal);
/**
* 这样就可以发送一个看起来象这样的POST：
* POST /jobsearch/jobsearch.cgi HTTP 1.0 ACCEPT:
* text/plain Content-type: application/x-www-form-urlencoded
* Content-length: 99 username=bob password=someword
*/
// 一旦发送成功，用以下方法就可以得到服务器的回应：
String sCurrentLine="";
String sTotalString="";

InputStream urlStream= connection.getInputStream();
// 传说中的三层包装阿！
BufferedReader reader = new BufferedReader(new InputStreamReader(urlStream,"utf-8"));
while ((sCurrentLine = reader.readLine()) != null) {
sTotalString += sCurrentLine + "\r\n";
}
setLocalHtmlStream("C:\\Documents and Settings\\Administrator\\桌面\\demo\\demo.html",sTotalString);
} catch (RuntimeException e) {
System.out.println("#Log Error ["+e.getMessage()+"] ");
}
}
public static void testPost(String urls,String cookieVal) throws IOException {
try {
//重新打开一个连接
URL url = new URL(urls);
HttpURLConnection resumeConnection = (HttpURLConnection)url.openConnection();
if (cookieVal != null) {
//发送cookie信息上去，以表明自己的身份，否则会被认为没有权限
resumeConnection.setRequestProperty("Cookie", cookieVal);
}
resumeConnection.connect();

InputStream urlStream = resumeConnection.getInputStream();
BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(urlStream));
String ss = null;
String total = "";
while ((ss = bufferedReader.readLine()) != null) {
total += ss;
}
System.out.println("#Log Cookie ["+total+"] ");
bufferedReader.close();
}catch (Exception e) {
System.out.println("#Log Error ["+e.getMessage()+"] ");
}
}

/**
* Java模拟Post提交
* @param url 要提交到的位置
* @param data 例如：NameValuePair[] data = {new NameValuePair("key", "nike"),new NameValuePair("proClass", "")};
* @return 返回HTML代码
*/
public static String methodPost(){
String url = "http://www.shopin.net/keysearch.html";
Map<String,String> m = new HashMap<String,String>();
m.put("key", "nike");
m.put("proClass", "");
NameValuePair[] data = {new NameValuePair("key", "nike"),new NameValuePair("proClass", "")};

String response= "";//要返回的response信息
HttpClient httpClient = new HttpClient();
PostMethod postMethod = new PostMethod(url);
// 将表单的值放入postMethod中
postMethod.setRequestBody(data);
// 执行postMethod
int statusCode = 0;
try {
statusCode = httpClient.executeMethod(postMethod);
// HttpClient对于要求接受后继服务的请求，象POST和PUT等不能自动处理转发
// 301或者302
if (statusCode == HttpStatus.SC_MOVED_PERMANENTLY|| statusCode == HttpStatus.SC_MOVED_TEMPORARILY) {
// 从头中取出转向的地址
Header locationHeader = postMethod.getResponseHeader("location");
String location = null;
if (locationHeader != null) {
location = locationHeader.getValue();
System.out.println("The page was redirected to:" + location);
//response= methodPost(location,data);//用跳转后的页面重新请求。
} else {
System.err.println("Location field value is null.");
}
} else {
System.out.println("#Log StatusLine ["+postMethod.getStatusLine()+"] ");

response= postMethod.getResponseBodyAsString();
postMethod.releaseConnection();
}
} catch (Exception e) {
System.out.println("#Log Error ["+e.getMessage()+"] ");
}

return response;
}

//测试登录功能，返回“自动”登录后的页面
public static String login(String urls) throws IOException
{
String responseCookie;//标示Session必须
StringBuilder sbR = new StringBuilder();

//访问URL，并把信息存入sb中
//如果服务端登录成功后，服务端的代码调用下面的代码
//response.sendRedirect("welcome.jsp");
//则会不成功，原因(Step2，没有上传jsessionid值，导致没session)如下
//Step1[login.jsp登录成功]->转到->
//Step2[welcome.jsp不能得到session，判断没有登录成功]->转到->Step3[login.jsp要求用户登录]

URL url = new URL(urls);
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setDoInput(true);
connection.setDoOutput(true);//允许连接提交信息
connection.setRequestMethod("POST");//网页默认“GET”提交方式

StringBuffer sb = new StringBuffer();
//sb.append("Name="+usr);
//sb.append("&Password="+pwd);
connection.setRequestProperty("Content-Length",String.valueOf(sb.toString().length()));

OutputStream os = connection.getOutputStream();
//os.write(sb.toString().getBytes());
os.close();

//取Cookie
BufferedReader br = new BufferedReader(new InputStreamReader(connection.getInputStream()));
responseCookie = connection.getHeaderField("Set-Cookie");//取到所用的Cookie
System.out.println("cookie:" + responseCookie);

//取返回的页面
String line = br.readLine();
while (line != null) {
sbR.append(line);
line = br.readLine();
}

return sbR.toString();
}
//返回页面
public static String viewPage(String urls,String cookie) throws IOException
{
StringBuilder sbR = new StringBuilder();

//打开URL连接
URL url1 = new URL(urls);
HttpURLConnection connection1 = (HttpURLConnection) url1.openConnection();

//给服务器送登录后的cookie
connection1.setRequestProperty("Cookie", cookie);

//读取返回的页面信息到br1
BufferedReader br1 = new BufferedReader(new InputStreamReader(connection1.getInputStream()));
//取返回的页面,br1转sbR
String line1= br1.readLine();
while (line1 != null) {
sbR.append(line1);
line1 = br1.readLine();
}

return sbR.toString();
}

//写入html文件内容
public static void setLocalHtmlStream(String path,String html) {
FileWriter fw;
try {
fw = new FileWriter(path);

fw.write(html);
fw.close();
} catch (Exception e) {}
}
}