使用httpclient实现http链接池与使用HttpURLConnection发送http请求的方法与性能对比

使用httpclient实现http链接池与使用HttpURLConnection发送http请求的方法与性能对比

在项目中需要使用http调用接口，实现了两套发送http请求的方法，一个是使用apache的httpclient提供的http链接池来发送http请求，另一个是使用java原生的HttpURLConnection来发送http请求，并对两者性能进行了对比。

使用httpclient中的链接池发送http请求

使用最新的4.5.2版httpclient进行实现。在maven中引入

<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.2</version>
</dependency>

实现代码如下

package util;

import org.apache.http.*;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.ConnectionKeepAliveStrategy;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.LayeredConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicHeaderElementIterator;
import org.apache.http.protocol.HTTP;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.CollectionUtils;
import java.io.IOException;
import java.util.Map;

/**
* Created by xugang on 16/7/11.
*/
public class HttpClientUtil {
private final static Logger logger = LoggerFactory.getLogger(HttpClientUtil.class);
private int maxTotal = 1;//默认最大连接数
private int defaultMaxPerRoute = 1;//默认每个主机的最大链接数
private int connectionRequestTimeout = 3000;//默认请求超时时间
private int connectTimeout = 3000;//默认链接超时时间
private int socketTimeout = 3000;//默认socket超时时间
private HttpRequestRetryHandler httpRequestRetryHandler = new DefaultHttpRequestRetryHandler();//默认不进行重试处理
private CloseableHttpClient httpClient;
public  HttpClientUtil(){
init();
}

public  String sendGet(String url, Map<String, Object> params) throws Exception {
StringBuffer sb = new StringBuffer(url);
if(!CollectionUtils.isEmpty(params)) {
for (Map.Entry<String, Object> entry : params.entrySet()) {
sb.append(entry.getKey())
.append("=")
.append(entry.getValue())
.append("&");
}
}
// no matter for the last '&' character
HttpGet httpget = new HttpGet(sb.toString());
config(httpget);
CloseableHttpResponse response = null;
try {
response = httpClient.execute(httpget, HttpClientContext.create());
} catch (IOException e) {
logger.error("httpclient error:"+e.getMessage());
e.printStackTrace();
}
HttpEntity entity = response.getEntity();
return EntityUtils.toString(entity, "utf-8");
}

private  void init() {
ConnectionSocketFactory plainsf = PlainConnectionSocketFactory.getSocketFactory();
LayeredConnectionSocketFactory sslsf = SSLConnectionSocketFactory.getSocketFactory();
Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", plainsf)
.register("https", sslsf)
.build();
PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registry);
// 设置最大连接数
cm.setMaxTotal(maxTotal);
// 设置每个路由的默认连接数
cm.setDefaultMaxPerRoute(defaultMaxPerRoute);

//连接保持时间
ConnectionKeepAliveStrategy myStrategy = new ConnectionKeepAliveStrategy() {
public long getKeepAliveDuration(HttpResponse response, HttpContext context) {
// Honor 'keep-alive' header
HeaderElementIterator it = new BasicHeaderElementIterator(
response.headerIterator(HTTP.CONN_KEEP_ALIVE));
while (it.hasNext()) {
HeaderElement he = it.nextElement();
String param = he.getName();
String value = he.getValue();
if (value != null && param.equalsIgnoreCase("timeout")) {
try {
return Long.parseLong(value) * 1000;
} catch(NumberFormatException ignore) {
}
}
}
return 30 * 1000;
}
};

this.httpClient = HttpClients.custom()
.setConnectionManager(cm)
.setRetryHandler(httpRequestRetryHandler)
.setKeepAliveStrategy(myStrategy)
.build();

}

/**
* http头信息的设置
*
* @param httpRequestBase
*/
private void config(HttpRequestBase httpRequestBase) {
httpRequestBase.setHeader("User-Agent", "Mozilla/5.0");
httpRequestBase.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
httpRequestBase.setHeader("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3");//"en-US,en;q=0.5");
httpRequestBase.setHeader("Accept-Charset", "ISO-8859-1,utf-8,gbk,gb2312;q=0.7,*;q=0.7");
httpRequestBase.setHeader("connection", "Keep-Alive");
// 配置请求的超时设置
RequestConfig requestConfig = RequestConfig.custom()
.setConnectionRequestTimeout(connectionRequestTimeout)
.setConnectTimeout(connectTimeout)
.setSocketTimeout(socketTimeout)
.build();
httpRequestBase.setConfig(requestConfig);
}

/**
* 请求重试处理
* 默认不进行任何重试
* 如需进行重试可参考下面进行重写
* if (executionCount >= 5) {// 如果已经重试了5次，就放弃
return false;
}
if (exception instanceof NoHttpResponseException) {// 如果服务器丢掉了连接，那么就重试
return true;
}
if (exception instanceof SSLHandshakeException) {// 不要重试SSL握手异常
return false;
}
if (exception instanceof InterruptedIOException) {// 超时
return false;
}
if (exception instanceof UnknownHostException) {// 目标服务器不可达
return false;
}
if (exception instanceof ConnectTimeoutException) {// 连接被拒绝
return false;
}
if (exception instanceof SSLException) {// ssl握手异常
return false;
}

HttpClientContext clientContext = HttpClientContext.adapt(context);
HttpRequest request = clientContext.getRequest();
// 如果请求是幂等的，就再次尝试
if (!(request instanceof HttpEntityEnclosingRequest)) {
return true;
}
*/
private class DefaultHttpRequestRetryHandler implements HttpRequestRetryHandler

{
public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
return false;
}
}

}

使用了spring和slf4j，所以要直接用的话还需在你的pom中添加相关依赖，不想添加的话稍微改一下代码也很简单，就不说了。

<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.13</version>
</dependency>

<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-core</artifactId>
<version>4.1.1.RELEASE</version>
</dependency>

<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-beans</artifactId>
<version>4.1.1.RELEASE</version>
</dependency>

使用java原生的HttpURLConnection发送http链接

代码如下

package util;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.StringUtils;

import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import java.util.Map;

public class HttpUtils {

final static Logger logger = LoggerFactory.getLogger(HttpUtils.class);

/**
* Post 请求超时时间和读取数据的超时时间均为2000ms。
*
* @param urlPath       post请求地址
* @param parameterData post请求参数
* @return String json字符串，成功：code=1001，否者为其他值
* @throws Exception 链接超市异常、参数url错误格式异常
*/
public static String doPost(String urlPath, String parameterData, String who, String ip) throws Exception {

if (null == urlPath || null == parameterData) { // 避免null引起的空指针异常
return "";
}
URL localURL = new URL(urlPath);
URLConnection connection = localURL.openConnection();
HttpURLConnection httpURLConnection = (HttpURLConnection) connection;

httpURLConnection.setDoOutput(true);
if (!StringUtils.isEmpty(who)) {
httpURLConnection.setRequestProperty("who", who);
}
if (!StringUtils.isEmpty(ip)) {
httpURLConnection.setRequestProperty("clientIP", ip);
}
httpURLConnection.setRequestMethod("POST");
httpURLConnection.setRequestProperty("Accept-Charset", "utf-8");
httpURLConnection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
httpURLConnection.setRequestProperty("Content-Length", String.valueOf(parameterData.length()));
httpURLConnection.setConnectTimeout(18000);
httpURLConnection.setReadTimeout(18000);

OutputStream outputStream = null;
OutputStreamWriter outputStreamWriter = null;
InputStream inputStream = null;
InputStreamReader inputStreamReader = null;
BufferedReader reader = null;
StringBuilder resultBuffer = new StringBuilder();
String tempLine = null;

try {
outputStream = httpURLConnection.getOutputStream();
outputStreamWriter = new OutputStreamWriter(outputStream);

outputStreamWriter.write(parameterData.toString());
outputStreamWriter.flush();

if (httpURLConnection.getResponseCode() >= 300) {
throw new Exception("HTTP Request is not success, Response code is " + httpURLConnection.getResponseCode());
}

inputStream = httpURLConnection.getInputStream(); // 真正的发送请求到服务端
inputStreamReader = new InputStreamReader(inputStream);
reader = new BufferedReader(inputStreamReader);

while ((tempLine = reader.readLine()) != null) {
resultBuffer.append(tempLine);
}

} finally {

if (outputStreamWriter != null) {
outputStreamWriter.close();
}

if (outputStream != null) {
outputStream.close();
}

if (reader != null) {
reader.close();
}

if (inputStreamReader != null) {
inputStreamReader.close();
}

if (inputStream != null) {
inputStream.close();
}
}
return resultBuffer.toString();
}

public static String doPost(String url, Map<String, Object> params) throws Exception {
StringBuffer sb = new StringBuffer();
for (Map.Entry<String, Object> entry : params.entrySet()) {
sb.append(entry.getKey())
.append("=")
.append(entry.getValue())
.append("&");
}

// no matter for the last '&' character

return doPost(url, sb.toString(), "", "");
}

/**
* 向指定URL发送GET方法的请求
*
* @param url   发送请求的URL
* @param param 请求参数，请求参数应该是 name1=value1&name2=value2 的形式。
* @return URL 所代表远程资源的响应结果
*/
public static String sendGet(String url, String param, String who, String ip) {
String result = "";
BufferedReader in = null;
try {
String urlNameString = url;
if (!"".equals(param)) {
urlNameString = urlNameString + "?" + param;
}
URL realUrl = new URL(urlNameString);
// 打开和URL之间的连接
URLConnection connection = realUrl.openConnection();
// 设置通用的请求属性
if (!StringUtils.isEmpty(who)) {
connection.setRequestProperty("who", who);
}
if (!StringUtils.isEmpty(ip)) {
connection.setRequestProperty("clientIP", ip);
}
connection.setRequestProperty("accept", "*/*");
connection.setRequestProperty("connection", "Keep-Alive");
connection.setRequestProperty("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;SV1)");
// 建立实际的连接
connection.connect();

// 定义 BufferedReader输入流来读取URL的响应
in = new BufferedReader(new InputStreamReader(
connection.getInputStream()));
String line;
while ((line = in.readLine()) != null) {
result += line;
}
} catch (Exception e) {
logger.warn("发送GET请求出现异常！", e);
}
// 使用finally块来关闭输入流
finally {
try {
if (in != null) {
in.close();
}
} catch (Exception e2) {
logger.warn("fail to close inputStream.", e2);
}
}
return result;
}

public static String sendGet(String url, Map<String, Object> params) throws Exception {
StringBuffer sb = new StringBuffer();
for (Map.Entry<String, Object> entry : params.entrySet()) {
sb.append(entry.getKey())
.append("=")
.append(entry.getValue())
.append("&");
}

// no matter for the last '&' character

return sendGet(url, sb.toString(), "", "");
}

}

也使用了sl4j打日志，不需要的话直接删掉吧。

两种链接方式的性能对比

测试代码如下

HttpClientUtil httpClientUtil = new HttpClientUtil();
long start1 = System.currentTimeMillis();
for(int i = 0;i<1000;i++){
try {
HttpUtils.sendGet("http://yop-console-qa.s.qima-inc.com/app/list", new HashMap<String, Object>());
} catch (Exception e) {
e.printStackTrace();
}
}
long end1 = System.currentTimeMillis();
long start2 = System.currentTimeMillis();
for (int i = 0;i<1000;i++) {
try {               httpClientUtil.sendGet("http://yop-console-qa.s.qima-inc.com/app/list", null);
} catch (Exception e) {
e.printStackTrace();
}
}
long end2 = System.currentTimeMillis();
System.out.println("时间1:" + (end1 - start1) + "时间2:" + (end2 - start2));

两个都向同一地址发送1000个请求，最后输出为：

时间1：35637时间2：34868

即使用java原生的用时35637ms ，使用httpclient用时 34868ms。是不是感觉区别不大？但是要考虑到网络本来就不稳定，你下个片还时而100k时而50k呢，这种测试不能完全说明问题，下面我们来看看使用http链接池和直接使用java原生的http方式有什么区别

http链接池与直接链接的区别

首先我们要知道http是建立在tcp之上的应用层协议，因此在建立http请求前首先要进行tcp的三次握手过程：



在http传输结束断开链接时要进行tcp的四次握手断开链接：



在看一下抓包得到的一次http请求过程可以更直观展现：



首先3次握手建立链接，链接建立后客服端提交一个http请求到服务器，然后是图中高亮的部分，服务器在接受到http请求后先回复一个tcp的确认请求给客服端，再回复http请求到客服端。最后四次握手断开链接。当使用java原生方式发送http请求时，是不是每一次请求都要经历3次握手建立链接－发送http请求并获取结果－4次握手释放链接这样的过程呢？来看图：



忽略图中的tcp segment和tcp window update。可以看到，第一次http结束后并没有断开链接，直接复用了，为了测试我还特意将HttpUtil中的这段代码注释掉了

//connection.setRequestProperty("connection", "Keep-Alive");

那么。。why？为什么这种情况下http链接还能被复用呢？

细心的读者可能注意到了，使用的http是1.1，在Http /1.1中：

在HTTP/1.1版本中，官方规定的Keep-Alive使用标准和在HTTP/1.0版本中有些不同，默认情况下所在HTTP1.1中所有连接都被保持，除非在请求头或响应头中指明要关闭：Connection:Close ，这也就是为什么Connection:

Keep-Alive字段再没有意义的原因。另外，还添加了一个新的字段Keep-Alive,但是因为这个字段并没有详细描述用来做什么，可忽略它

ok，答案很明显了，默认情况下该http链接便是可以被复用的，并且在java的客服端中有：

HttpURLConnection类自动实现了Keep-Alive，如果程序员没有介入去操作Keep-Alive，Keep-Alive会通过客户端内部的一个HttpURLConnection类的实例对象来自动实现。

在java的服务器端有：

HttpServlet、HttpServletRequest和HttpServletResponse类自动实现 了Keep-Alive

原来jdk已经帮我们做了这么多了^ ^

接下来还是再来看看使用链接池时进行多次http请求的情况吧：



看起来少了不少，但其实主流程一样，少的部分只是tcp segment和tcp window update，但是在使用链接池时几乎不会出现tcp segment和tcp window update，使用原生连接时有大量tcp segment出现，虽然对时间影响很小。

因此，单从时间上来说两种http方式耗时相差不大，如果只需简单的发送http请求用原生方法就够了，需要更强大的功能可考虑使用链接池。