使用Gzip加速网页的传输

使用Gzip加速网页的传输

分类：
JAVA

2009-10-26 10:33
368人阅读
评论(3)
收藏
举报

博学，切问，近思--詹子知(http://blog.csdn.net/zhiqiangzhan)

日前笔者在使用HttpClient在处理大数据请求的时候，在连续发请求的时候经常会出现异常 java.io.IOException: chunked stream ended unexpectedly。使用HttpMethod的abort方法也不能完全避免这种异常的出现，但是对于小数据的请求，这种异常就基本上难得一见了。对于同样的页面请求，如何减少网络的数据传输量呢。众所周知，现在大部分的Web Server都是支持数据的压缩传输的。要知道，一般的网页内容经过压缩，大小可以减少到原来的20%以下，而对于纯英文为网站，网页内容更是可以减少到原来内容的5%以下。而要使Web Server对数据进行压缩传输，只需要在请求头上加入Accept-Encoding:gzip, deflate。

[java] view plaincopyprint?
public HttpMethod createHttpMethod(String url, String type, NameValuePair[] params, String contentType) {
HttpMethod method = null;

if (type.equalsIgnoreCase("POST")) {
method = new PostMethod(url);
method.setRequestHeader("Content-Type", contentType);
if(params != null){
((PostMethod) method).setRequestBody(params);
}
} else {
method = new GetMethod(url);
if(params != null){
method.setQueryString(params);
}
}
method.setRequestHeader("Accept-Encoding", "gzip, deflate");
return method;
}

[java] view plaincopyprint?
protected String doSuccess(HttpMethod method) throws IOException {
InputStream in = method.getResponseBodyAsStream();
Header contentEncodingHeader = method.getResponseHeader("Content-Encoding");
if (contentEncodingHeader != null) {
String contentEncoding = contentEncodingHeader.getValue();
if (contentEncoding.toLowerCase(Locale.US).indexOf("gzip") != -1) {
in = new GZIPInputStream(in);
}
}
return decoder.decode(in);
}
protected String doSuccess(HttpMethod method) throws IOException {<br /> 		InputStream in = method.getResponseBodyAsStream();<br /> 		Header contentEncodingHeader = method.getResponseHeader("Content-Encoding");<br /> 		if (contentEncodingHeader != null) {<br /> 			String contentEncoding = contentEncodingHeader.getValue();<br /> 			if (contentEncoding.toLowerCase(Locale.US).indexOf("gzip") != -1) {<br /> 				in = new GZIPInputStream(in);<br /> 			}<br /> 		}<br /> 		return decoder.decode(in);<br /> 	}

上一篇文章，我们介绍了如何检查文档输入流的编码，本节我们就可以利用上文的HtmlInputStreamDecoder类来把文档流来解析文档内容。完整的代码如下：

[java] view plaincopyprint?
import java.io.IOException;
import java.io.InputStream;
import java.util.Locale;
import java.util.zip.GZIPInputStream;

import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.httpclient.URI;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.log4j.Logger;

public class HttpRequest {
private static final Logger LOGGER = Logger.getLogger(HttpRequest.class);
private HttpClient client;
private HtmlInputStreamDecoder decoder;

public HttpRequest() {
this(new HttpClient(new MultiThreadedHttpConnectionManager()));
}

public HttpRequest(HttpClient client) {
this.client = client;
this.decoder = new HtmlInputStreamDecoder();
}

public String doRequest(HttpMethod method) {
String html = null;
try {
int statusCode = client.executeMethod(method);
switch (statusCode) {
case HttpStatus.SC_OK:
html = doSuccess(method);
break;
case HttpStatus.SC_MOVED_PERMANENTLY:
case HttpStatus.SC_MOVED_TEMPORARILY:
case HttpStatus.SC_SEE_OTHER:
case HttpStatus.SC_TEMPORARY_REDIRECT:
doRedirect(method);
break;
default:
html = doError(method);
}
} catch (HttpException e) {
LOGGER.error("Http error occur while visit the url.", e);
} catch (IOException e) {
LOGGER.error("IO error occur while visit the url.", e);
} finally {
method.abort();
method.releaseConnection();
}
return html;
}

protected String doSuccess(HttpMethod method) throws IOException {
InputStream in = method.getResponseBodyAsStream();
Header contentEncodingHeader = method.getResponseHeader("Content-Encoding");
if (contentEncodingHeader != null) {
String contentEncoding = contentEncodingHeader.getValue();
if (contentEncoding.toLowerCase(Locale.US).indexOf("gzip") != -1) {
in = new GZIPInputStream(in);
}
}
return decoder.decode(in);
}

protected String doError(HttpMethod method) {
LOGGER.error("Error Response: " + method.getStatusLine());
return method.getStatusText();
}

protected void doRedirect(HttpMethod method) throws URIException {
Header locationHeader = method.getResponseHeader("location");
if (locationHeader != null) {
String location = locationHeader.getValue();
if (location == null) {
location = "/";
}
doRequest(new GetMethod(getRedirectUrl(method.getURI(), location)));
}
}

public HttpMethod createHttpMethod(String url, String type, NameValuePair[] params, String contentType) {
HttpMethod method = null;

if (type.equalsIgnoreCase("POST")) {
method = new PostMethod(url);
method.setRequestHeader("Content-Type", contentType);
if(params != null){
((PostMethod) method).setRequestBody(params);
}
} else {
method = new GetMethod(url);
if(params != null){
method.setQueryString(params);
}
}
method.setRequestHeader("Accept-Encoding", "gzip, deflate");
return method;
}

protected static String getRedirectUrl(URI origin, String location) throws URIException {
String redirect = null;
if (location.startsWith("http:")) {
redirect = location;
} else if (location.startsWith("/")) {
origin.setPath(location);
redirect = origin.getURI();
} else {
redirect = origin.getURI().replaceAll("(?<=/)[^/]+$", location);
}
return redirect;
}

}

[java] view plaincopyprint?
public static void main(String[] args) {

HttpRequest request = new HttpRequest();
HttpMethod method = request.createHttpMethod("http://www.csdn.com", "GET", null, "text/html");

String html = request.doRequest(method);
System.out.println(html);
}
public static void main(String[] args) {</p> <p>		HttpRequest request = new HttpRequest();<br /> 		HttpMethod method = request.createHttpMethod("http://www.csdn.com", "GET", null, "text/html");</p> <p>		String html = request.doRequest(method);<br /> 		System.out.println(html);<br /> 	}