使用线程池和CountDownLatch计数器来进行抓取的简单事例2

import java.util.List;import java.util.concurrent.CountDownLatch;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;import org.apache.log4j.Logger;import util.ProxyUtil;import util.StringUtil;public class DictEnToChar implements Runnable{static Logger log = Logger.getLogger(DictEnToChar.class);static int threadnum = 5;public static void main(String[] args) {if(args.length==1){try{threadnum = Integer.parseInt(args[0]);}catch(Exception e){e.printStackTrace();System.out.println("请输入线程个数.为整型参数....");}}DictEnToChar dic = new DictEnToChar();for(char c='A'; c<='Z'; c++){dic.forTotalPage("http://www.*****.com/hotproduct_list_"+c+"_(*).html",c);}DictEn.main(args);}/** * 对每个列表页进行循环 */static int totalPage;//每个cas下标的总页数static int startpage;//每个cas下标的开始页static String threadurl;//列表页的地址static CountDownLatch cdl = new CountDownLatch(threadnum);//计数器,每个cas的下标全部抓完才能抓取下一个下标static ExecutorService threadpool = Executors.newCachedThreadPool();//线程池public void forTotalPage(String url,int index){String content = ProxyUtil.getProxyStr(StringUtil.changeURL(url, 1), "utf-8", "", "CAS Number");String maxstr = StringUtil.getFirststr(content, "Page 1 of", "<span", 1).trim();try{totalPage = Integer.parseInt(maxstr);}catch(Exception e){e.printStackTrace();System.out.println(index+"获得最大页数失败............");}System.out.println("产品名称的下标为"+index+"的最大页数是:"+totalPage);cdl = new CountDownLatch(threadnum);//计数重置startpage=0;//赋初值threadurl = url;//赋初值DictEnToChar dicrun = new DictEnToChar();for(int num=0; num<threadnum; num++){threadpool.execute(dicrun);//在线程池中启动线程}try {cdl.await();//如果不使用await,那么这个方法便会执行结束} catch (InterruptedException e) {e.printStackTrace();}System.out.println("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$");}//线程执行方法public void run(){while(startpage < totalPage){synchronized(DictEn.class){try {Thread.sleep(50);//进行睡眠，防止下面的线程不安全} catch (InterruptedException e) {e.printStackTrace();}startpage++;System.out.println(Thread.currentThread().getName()+"#页数#"+startpage);}//在这里会有线程安全问题，故在上面的锁内进行睡眠getLinkList(StringUtil.changeURL(threadurl, startpage));}cdl.countDown();System.out.println(Thread.currentThread().getName()+"-----------------线程结束-----------------还有线程个数："+cdl.getCount());}/** * 得到每个列表页的链接 */public void getLinkList(String url){System.out.println(Thread.currentThread().getName()+"#当前列表页:"+url);log.info("lookchem列表:"+url);String content=ProxyUtil.getProxyStr(url, "utf-8", "", "html");List<String> list=StringUtil.getalltagslist(content, "<table width=\"100%\" border=\"0\" cellspacing=\"0\"", "</table>");for(String s:list){String cas = StringUtil.getFirststr(s, "<td width=\"120\" valign=\"middle\" >", "</td>", 1).trim();//在这里先判断数据库中是否存在该数据if(!StringUtil.isCasNo(cas) || !DictEnDao.checkCas(cas)){System.out.println(cas+":Cas重复..............");continue;}String u = StringUtil.getFirststr(s, "href=\"", "\"", 1);getContent(u);//访问详细页面}}