Http请求连接池 - HttpClient 的 PoolingHttpClientConnectionManager

两个主机建立连接的过程是很复杂的一个过程，涉及到多个数据包的交换，并且也很耗时间。Http连接需要的三次握手开销很大，这一开销对于比较小的http消息来说更大。但是如果我们直接使用已经建立好的http连接，这样花费就比较小，吞吐率更大。 

传统的HttpURLConnection并不支持连接池，如果要实现连接池的机制，还需要自己来管理连接对象。对于网络请求这种底层相对复杂的操作，个人以为如果有可用的其他方案，也没有必要自己去管理连接对象。

除了HttpURLConnection，大家肯定还知道HttpClient。一般情况下，普通使用HttpClient已经能满足我们的需求，不过有时候，在我们需要高并发大量的请求网络的时候，还是用“连接池”这样的概念能提升吞吐量。

我们来看下怎么使用 org.apache.httpcomponents.httpclient（版本4.4）提供的连接池来实现我们的高并发网络请求。

使用到的jar包： 

org\apache\httpcomponents\httpclient\4.4-beta1\httpclient-4.4-beta1.jar 

org\apache\httpcomponents\httpclient-cache\4.4-beta1\httpclient-cache-4.4-beta1.jar 

org\apache\httpcomponents\httpcore\4.4-beta1\httpcore-4.4-beta1.jar

下面代码实例中主要使用到 PoolingHttpClientConnectionManager

package com.hvgroup.zhuhai10086.jms.utils;

import java.io.IOException;
import java.io.InterruptedIOException;
import java.io.UnsupportedEncodingException;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import javax.net.ssl.SSLException;
import javax.net.ssl.SSLHandshakeException;

import org.apache.http.HttpEntity;
import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpHost;
import org.apache.http.HttpRequest;
import org.apache.http.NameValuePair;
import org.apache.http.NoHttpResponseException;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.ConnectTimeoutException;
import org.apache.http.conn.routing.HttpRoute;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.LayeredConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;

/**
* HttpClient工具类
*
* @return
* @author SHANHY
* @create 2015年12月18日
*/
public class HttpClientUtil {

static final int timeOut = 10 * 1000;

private static CloseableHttpClient httpClient = null;

private final static Object syncLock = new Object();

private static void config(HttpRequestBase httpRequestBase) {
// 设置Header等
// httpRequestBase.setHeader("User-Agent", "Mozilla/5.0");
// httpRequestBase
// .setHeader("Accept",
// "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
// httpRequestBase.setHeader("Accept-Language",
// "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3");// "en-US,en;q=0.5");
// httpRequestBase.setHeader("Accept-Charset",
// "ISO-8859-1,utf-8,gbk,gb2312;q=0.7,*;q=0.7");

// 配置请求的超时设置
RequestConfig requestConfig = RequestConfig.custom()
.setConnectionRequestTimeout(timeOut)
.setConnectTimeout(timeOut).setSocketTimeout(timeOut).build();
httpRequestBase.setConfig(requestConfig);
}

/**
* 获取HttpClient对象
*
* @return
* @author SHANHY
* @create 2015年12月18日
*/
public static CloseableHttpClient getHttpClient(String url) {
String hostname = url.split("/")[2];
int port = 80;
if (hostname.contains(":")) {
String[] arr = hostname.split(":");
hostname = arr[0];
port = Integer.parseInt(arr[1]);
}
if (httpClient == null) {
synchronized (syncLock) {
if (httpClient == null) {
httpClient = createHttpClient(200, 40, 100, hostname, port);
}
}
}
return httpClient;
}

/**
* 创建HttpClient对象
*
* @return
* @author SHANHY
* @create 2015年12月18日
*/
public static CloseableHttpClient createHttpClient(int maxTotal,
int maxPerRoute, int maxRoute, String hostname, int port) {
ConnectionSocketFactory plainsf = PlainConnectionSocketFactory
.getSocketFactory();
LayeredConnectionSocketFactory sslsf = SSLConnectionSocketFactory
.getSocketFactory();
Registry<ConnectionSocketFactory> registry = RegistryBuilder
.<ConnectionSocketFactory> create().register("http", plainsf)
.register("https", sslsf).build();
PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(
registry);
// 将最大连接数增加
cm.setMaxTotal(maxTotal);
// 将每个路由基础的连接增加
cm.setDefaultMaxPerRoute(maxPerRoute);
HttpHost httpHost = new HttpHost(hostname, port);
// 将目标主机的最大连接数增加
cm.setMaxPerRoute(new HttpRoute(httpHost), maxRoute);

// 请求重试处理
HttpRequestRetryHandler httpRequestRetryHandler = new HttpRequestRetryHandler() {
public boolean retryRequest(IOException exception,
int executionCount, HttpContext context) {
if (executionCount >= 5) {// 如果已经重试了5次，就放弃
return false;
}
if (exception instanceof NoHttpResponseException) {// 如果服务器丢掉了连接，那么就重试
return true;
}
if (exception instanceof SSLHandshakeException) {// 不要重试SSL握手异常
return false;
}
if (exception instanceof InterruptedIOException) {// 超时
return false;
}
if (exception instanceof UnknownHostException) {// 目标服务器不可达
return false;
}
if (exception instanceof ConnectTimeoutException) {// 连接被拒绝
return false;
}
if (exception instanceof SSLException) {// SSL握手异常
return false;
}

HttpClientContext clientContext = HttpClientContext
.adapt(context);
HttpRequest request = clientContext.getRequest();
// 如果请求是幂等的，就再次尝试
if (!(request instanceof HttpEntityEnclosingRequest)) {
return true;
}
return false;
}
};

CloseableHttpClient httpClient = HttpClients.custom()
.setConnectionManager(cm)
.setRetryHandler(httpRequestRetryHandler).build();

return httpClient;
}

private static void setPostParams(HttpPost httpost,
Map<String, Object> params) {
List<NameValuePair> nvps = new ArrayList<NameValuePair>();
Set<String> keySet = params.keySet();
for (String key : keySet) {
nvps.add(new BasicNameValuePair(key, params.get(key).toString()));
}
try {
httpost.setEntity(new UrlEncodedFormEntity(nvps, "UTF-8"));
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
}

/**
* GET请求URL获取内容
*
* @param url
* @return
* @author SHANHY
* @throws IOException
* @create 2015年12月18日
*/
public static String post(String url, Map<String, Object> params) throws IOException {
HttpPost httppost = new HttpPost(url);
config(httppost);
setPostParams(httppost, params);
CloseableHttpResponse response = null;
try {
response = getHttpClient(url).execute(httppost,
HttpClientContext.create());
HttpEntity entity = response.getEntity();
String result = EntityUtils.toString(entity, "utf-8");
EntityUtils.consume(entity);
return result;
} catch (Exception e) {
//          e.printStackTrace();
throw e;
} finally {
try {
if (response != null)
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}

/**
* GET请求URL获取内容
*
* @param url
* @return
* @author SHANHY
* @create 2015年12月18日
*/
public static String get(String url) {
HttpGet httpget = new HttpGet(url);
config(httpget);
CloseableHttpResponse response = null;
try {
response = getHttpClient(url).execute(httpget,
HttpClientContext.create());
HttpEntity entity = response.getEntity();
String result = EntityUtils.toString(entity, "utf-8");
EntityUtils.consume(entity);
return result;
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
if (response != null)
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
return null;
}

public static void main(String[] args) {
// URL列表数组
String[] urisToGet = {
"http://blog.csdn.net/catoop/article/details/38849497",
"http://blog.csdn.net/catoop/article/details/38849497",
"http://blog.csdn.net/catoop/article/details/38849497",
"http://blog.csdn.net/catoop/article/details/38849497",

"http://blog.csdn.net/catoop/article/details/38849497",
"http://blog.csdn.net/catoop/article/details/38849497",
"http://blog.csdn.net/catoop/article/details/38849497",
"http://blog.csdn.net/catoop/article/details/38849497",

"http://blog.csdn.net/catoop/article/details/38849497",
"http://blog.csdn.net/catoop/article/details/38849497",
"http://blog.csdn.net/catoop/article/details/38849497",
"http://blog.csdn.net/catoop/article/details/38849497",

"http://blog.csdn.net/catoop/article/details/38849497",
"http://blog.csdn.net/catoop/article/details/38849497",
"http://blog.csdn.net/catoop/article/details/38849497",
"http://blog.csdn.net/catoop/article/details/38849497",

"http://blog.csdn.net/catoop/article/details/38849497",
"http://blog.csdn.net/catoop/article/details/38849497",
"http://blog.csdn.net/catoop/article/details/38849497",
"http://blog.csdn.net/catoop/article/details/38849497",

"http://blog.csdn.net/catoop/article/details/38849497",
"http://blog.csdn.net/catoop/article/details/38849497",
"http://blog.csdn.net/catoop/article/details/38849497",
"http://blog.csdn.net/catoop/article/details/38849497" };

long start = System.currentTimeMillis();
try {
int pagecount = urisToGet.length;
ExecutorService executors = Executors.newFixedThreadPool(pagecount);
CountDownLatch countDownLatch = new CountDownLatch(pagecount);
for (int i = 0; i < pagecount; i++) {
HttpGet httpget = new HttpGet(urisToGet[i]);
config(httpget);
// 启动线程抓取
executors
.execute(new GetRunnable(urisToGet[i], countDownLatch));
}
countDownLatch.await();
executors.shutdown();
} catch (InterruptedException e) {
e.printStackTrace();
} finally {
System.out.println("线程" + Thread.currentThread().getName() + ","
+ System.currentTimeMillis() + ", 所有线程已完成，开始进入下一步！");
}

long end = System.currentTimeMillis();
System.out.println("consume -> " + (end - start));
}

static class GetRunnable implements Runnable {
private CountDownLatch countDownLatch;
private String url;

public GetRunnable(String url, CountDownLatch countDownLatch) {
this.url = url;
this.countDownLatch = countDownLatch;
}

@Override
public void run() {
try {
System.out.println(HttpClientUtil.get(url));
} finally {
countDownLatch.countDown();
}
}
}
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332

Post使用方法

// 其中 params 为 Map<String, Object> params
String ret = HttpClientUtil.post(url, params);
jsonRet = new JSONObject(ret);

1
2
3
1
2
3

一开始我是使用传统的 HttpURLConnection 来做网络请求的，查了很多资料，有不少说 HttpURLConnection 效率高的。可是经过我修改实现方法后，HttpClient 连接池版本的网络请求相对比较稳定。这也说明，我们并不请尽信他人解说，凡事还是要寻找适合自己的方法，真正的解决自己的问题，才是王道。

===========================================

在使用 HttpURLConnection 的时候，大并发对外做网络请求的时候，前期请求耗时还好，后面耗时越来越高。下面是我之前的实现代码：

@Deprecated
protected JSONObject callRestfulOld(String url, Map<String, Object> params)
{
String temp;
String ret="";
JSONObject jsonRet=null;
String sign = generateSign("POST", url, params);// 对参数进行加密签名
if(sign.isEmpty()) return new JSONObject("{\"ret_code\":-1,\"err_msg\":\"generateSign error\"}");
params.put("sign", sign);
try{
URL u = new URL(url);
HttpURLConnection conn = (HttpURLConnection)u.openConnection();
conn.setRequestMethod("POST");
conn.setConnectTimeout(10000);
conn.setDoOutput(true);
conn.setDoInput(true);
conn.setUseCaches(false);
StringBuffer param = new StringBuffer();
for (String key: params.keySet())
{
param.append(key).append("=").append(URLEncoder.encode(params.get(key).toString(), "UTF-8")).append("&");
}
conn.getOutputStream().write(param.toString().getBytes("UTF-8"));

//System.out.println(param);
conn.getOutputStream().flush();
conn.getOutputStream().close();
InputStreamReader isr = new InputStreamReader(conn.getInputStream());
BufferedReader br = new BufferedReader(isr);
while((temp = br.readLine()) != null){
ret += temp;
}
br.close();
isr.close();
conn.disconnect();
//System.out.println(ret);
jsonRet = new JSONObject(ret);

} catch(java.net.SocketTimeoutException e) {
//e.printStackTrace();
jsonRet = new JSONObject("{\"ret_code\":-1,\"err_msg\":\"call restful timeout\"}");
} catch(Exception e) {
//e.printStackTrace();
jsonRet = new JSONObject("{\"ret_code\":-1,\"err_msg\":\"call restful error\"}");
}
return jsonRet;
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47

（完）