使用gecco获取代理IP，仅测试代码用，不要用于非法用途

这里用到了三个类

首先是gecco开始的地方，抓取ip列表

package com.geccocrawler.gecco.demo.ipcatch;

import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.List;

import com.geccocrawler.gecco.GeccoEngine;
import com.geccocrawler.gecco.annotation.Gecco;
import com.geccocrawler.gecco.annotation.HtmlField;
import com.geccocrawler.gecco.annotation.Request;
import com.geccocrawler.gecco.annotation.RequestParameter;
import com.geccocrawler.gecco.request.HttpRequest;
import com.geccocrawler.gecco.spider.HtmlBean;

@Gecco(matchUrl="http://www.kuaidaili.com/free/inha/{currPage}/", pipelines={"consolePipeline", "IpListPipeline"})
public class IpList implements HtmlBean {

private static final long serialVersionUID = 4544492736813943899L;

@Request
private HttpRequest request;

@RequestParameter("currPage")
private String currPage;

/**
* 抓取列表项的详细内容，包括ip，端口等
*/
@HtmlField(cssPath="#list")
private List<IpDetail> ips;

public HttpRequest getRequest() {
return request;
}

public void setRequest(HttpRequest request) {
this.request = request;
}

public String getCurrPage() {
return currPage;
}

public void setCurrPage(String currPage) {
this.currPage = currPage;
}

public List<IpDetail> getIps() {
return ips;
}

public void setIps(List<IpDetail> ips) throws IOException {
String path = "E:/ip/ip.txt";
File file = new File(path);
if (!file.getParentFile().exists()) {
file.getParentFile().mkdirs();
}
try {
file.createNewFile();
} catch (IOException e) {
e.printStackTrace();
}
FileWriter  pw = new FileWriter (file,true);
String[] ipGroup = ips.get(0).getIp().split("\n");
String[] portGroup = ips.get(0).getPort().split("\n");
for(int i = 0;i<ipGroup.length;i++){
String ip = ipGroup[i] + ":" + portGroup[i];
System.out.println(ipGroup[i]);
pw.write(ip);
pw.write(13);
pw.write(10);
pw.flush();
}
pw.close();
this.ips = ips;
}

public static void main(String[] args) {
GeccoEngine.create()
.classpath("com.geccocrawler.gecco.demo.ipcatch")
//开始抓取的页面地址
.start("http://www.kuaidaili.com/free/inha/1/")
//开启几个爬虫线程,线程数量最好不要大于start request数量
.thread(1)
//单个爬虫每次抓取完一个请求后的间隔时间
.interval(5000)
.run();
}
}

难点是jsoup无法获取没有table标签的td标签，只能得到一个字符串

下面是详细信息

package com.geccocrawler.gecco.demo.ipcatch;

import com.geccocrawler.gecco.annotation.HtmlField;
import com.geccocrawler.gecco.spider.HtmlBean;

public class IpDetail implements HtmlBean {

private static final long serialVersionUID = 2555530396237160927L;

@HtmlField(cssPath="td:nth-child(1)")
private String ip;

@HtmlField(cssPath="td:nth-child(2)")
private String port;

public String getIp() {
return ip;
}

public void setIp(String ip) {
this.ip = ip;
}

public String getPort() {
return port;
}

public void setPort(String port) {
this.port = port;
}

public void text(String ip,String port){

}

}

然后是分页抓取

package com.geccocrawler.gecco.demo.ipcatch;

import com.geccocrawler.gecco.annotation.PipelineName;
import com.geccocrawler.gecco.pipeline.Pipeline;
import com.geccocrawler.gecco.request.HttpRequest;
import com.geccocrawler.gecco.scheduler.SchedulerContext;

@PipelineName("IpListPipeline")
public class IpListPipeline implements Pipeline<IpList> {

@Override
public void process(IpList ipList) {
HttpRequest currRequest = ipList.getRequest();
//下一页继续抓取
String currPage = ipList.getCurrPage();
int nextPage = Integer.parseInt(currPage) + 1;
if(nextPage <= 100) {
String nextUrl = "";
String currUrl = currRequest.getUrl();
if(currUrl.indexOf("inha") != -1) {
nextUrl = "http://www.kuaidaili.com/free/inha/"+nextPage+"/";
} else {
nextUrl = currUrl + "/" + nextPage +"/";
}
SchedulerContext.into(currRequest.subRequest(nextUrl));
}
}

}

当然，这是基于gecco的，首先要引入gecco,地址：https://github.com/xtuhcy/gecco