使用线程池和CountDownLatch计数器来进行抓取的简单事例2

import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.apache.log4j.Logger;
import util.ProxyUtil;
import util.StringUtil;
public class DictEnToChar implements Runnable{
static Logger log = Logger.getLogger(DictEnToChar.class);
static int threadnum = 5;
public static void main(String[] args) {
if(args.length==1){
try{
threadnum = Integer.parseInt(args[0]);
}catch(Exception e){
e.printStackTrace();
System.out.println("请输入线程个数.为整型参数....");
}
}
DictEnToChar dic = new DictEnToChar();
for(char c='A'; c<='Z'; c++){
dic.forTotalPage("http://www.*****.com/hotproduct_list_"+c+"_(*).html",c);
}
DictEn.main(args);
}
/**
* 对每个列表页进行循环
*/
static int totalPage;//每个cas下标的总页数
static int startpage;//每个cas下标的开始页
static String threadurl;//列表页的地址
static CountDownLatch cdl = new CountDownLatch(threadnum);//计数器,每个cas的下标全部抓完才能抓取下一个下标
static ExecutorService threadpool = Executors.newCachedThreadPool();//线程池
public void forTotalPage(String url,int index){
String content = ProxyUtil.getProxyStr(StringUtil.changeURL(url, 1), "utf-8", "", "CAS Number");
String maxstr = StringUtil.getFirststr(content, "Page 1 of", "<span", 1).trim();
try{
totalPage = Integer.parseInt(maxstr);
}catch(Exception e){
e.printStackTrace();
System.out.println(index+"获得最大页数失败............");
}
System.out.println("产品名称的下标为"+index+"的最大页数是:"+totalPage);

cdl = new CountDownLatch(threadnum);//计数重置
startpage=0;//赋初值
threadurl = url;//赋初值
DictEnToChar dicrun = new DictEnToChar();
for(int num=0; num<threadnum; num++){
threadpool.execute(dicrun);//在线程池中启动线程
}
try {
cdl.await();//如果不使用await,那么这个方法便会执行结束
} catch (InterruptedException e) {
e.printStackTrace();}
System.out.println("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$");
}
//线程执行方法
public void run(){
while(startpage < totalPage){
synchronized(DictEn.class){
try {
Thread.sleep(50);//进行睡眠，防止下面的线程不安全
} catch (InterruptedException e) {
e.printStackTrace();}
startpage++;
System.out.println(Thread.currentThread().getName()+"#页数#"+startpage);
}//在这里会有线程安全问题，故在上面的锁内进行睡眠
getLinkList(StringUtil.changeURL(threadurl, startpage));
}
cdl.countDown();
System.out.println(Thread.currentThread().getName()+"-----------------线程结束-----------------还有线程个数："+cdl.getCount());
}
/**
* 得到每个列表页的链接
*/
public void getLinkList(String url){
System.out.println(Thread.currentThread().getName()+"#当前列表页:"+url);
log.info("lookchem列表:"+url);

String content=ProxyUtil.getProxyStr(url, "utf-8", "", "html");
List<String> list=StringUtil.getalltagslist(content, "<table width=\"100%\" border=\"0\" cellspacing=\"0\"", "</table>");
for(String s:list){
String cas = StringUtil.getFirststr(s, "<td width=\"120\" valign=\"middle\" >", "</td>", 1).trim();
//在这里先判断数据库中是否存在该数据
if(!StringUtil.isCasNo(cas) || !DictEnDao.checkCas(cas)){
System.out.println(cas+":Cas重复..............");
continue;
}
String u = StringUtil.getFirststr(s, "href=\"", "\"", 1);
getContent(u);//访问详细页面
}
}