WebMagic中使用POST请求并传递参数

Maven依赖

尽量使用最新版本的webmagic

<dependency>
<groupId>us.codecraft</groupId>
<artifactId>webmagic-core</artifactId>
<version>0.6.1</version>
</dependency>
<dependency>
<groupId>us.codecraft</groupId>
<artifactId>webmagic-extension</artifactId>
<version>0.6.1</version>
</dependency>

发起POST请求

//设置POST请求
Request request = new Request("http://PostRequestUrl.com");
//只有POST请求才可以添加附加参数
request.setMethod(HttpConstant.Method.POST);

//设置POST参数
List<NameValuePair> nvs = new ArrayList<NameValuePair>();
nvs.add(new BasicNameValuePair("key1", "value1"));
nvs.add(new BasicNameValuePair("key2", "value2"));
nvs.add(......);

//转换为键值对数组
NameValuePair[] values = nvs.toArray(new NameValuePair[] {});

//将键值对数组添加到map中
Map<String, Object> params = new HashMap<String, Object>();
//key必须是：nameValuePair
params.put("nameValuePair", values);

//设置request参数
request.setExtras(params);

// 开始执行
try {
Spider.create(new MyPageProcessor()).addRequest(request).thread(5).run();
} catch (Exception e) {
e.printStackTrace();
}

为什么参数map的key必须要用”nameValuePair”

类 us.codecraft.webmagic.downloader.HttpClientDownloader 中的selectRequestMethod(us.codecraft.webmagic.Request)方法

protected RequestBuilder selectRequestMethod(Request request) {
String method = request.getMethod();
if (method == null || method.equalsIgnoreCase(HttpConstant.Method.GET)) {
//default get
return RequestBuilder.get();
} else if (method.equalsIgnoreCase(HttpConstant.Method.POST)) {
//只有POST方法才会对request中的附加参数做处理
RequestBuilder requestBuilder = RequestBuilder.post();
//这里，从request中get时，key为：nameValuePair
NameValuePair[] nameValuePair = (NameValuePair[]) request.getExtra("nameValuePair");
if (nameValuePair != null && nameValuePair.length > 0) {
requestBuilder.addParameters(nameValuePair);
}
return requestBuilder;
} else if (method.equalsIgnoreCase(HttpConstant.Method.HEAD)) {
return RequestBuilder.head();
} else if (method.equalsIgnoreCase(HttpConstant.Method.PUT)) {
return RequestBuilder.put();
} else if (method.equalsIgnoreCase(HttpConstant.Method.DELETE)) {
return RequestBuilder.delete();
} else if (method.equalsIgnoreCase(HttpConstant.Method.TRACE)) {
return RequestBuilder.trace();
}
throw new IllegalArgumentException("Illegal HTTP Method " + method);
}