webcollector爬虫框架使用案例

网站为中国政招标网 流程为定时任务触发service层 查询需要查询的网站关键词，爬虫根据关键字爬取当天的数据 条件符合的放入list中结束后返回并保存

package gov.zb.data.webcollector.tender;import gov.zb.data.entity.monitor.Monitor;import gov.zb.data.enums.GeneralEnums;import gov.zb.data.enums.GeneralEnums.WebsiteType;import gov.zb.data.util.DateUtils;import java.io.UnsupportedEncodingException;import java.net.URLEncoder;import java.text.SimpleDateFormat;import java.util.ArrayList;import java.util.Date;import java.util.List;import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;import cn.edu.hfut.dmic.webcollector.model.Page;import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler;public class BidcenterCralwer extends BreadthCrawler{ private WebsiteType website_Type; private String monitorFrom; private List<Monitor>listMonitor = new ArrayList<Monitor>(); private String nowTime ; private String[] keys ; private Monitor monitor;   public BidcenterCralwer(String crawlPath, boolean autoParse,String[] keys){        super(crawlPath, autoParse);        /*对网站前N页进行爬取*/        for(String keyword:keys){//因为网站支持全局搜索 所以直接在url里拼入关键词           String eStr="";   try {   eStr = URLEncoder.encode(keyword, "utf-8");//关键词url加密   } catch (UnsupportedEncodingException e) {   System.out.println("编码出错="+e);   }         for (int i = 1; i <= 1; i++) {//for循环可以根据关键字数量加减              this.addSeed(new CrawlDatum("http://search.bidcenter.com.cn/search?keywords="+eStr+"&type=1&page="+i+"").meta("pageNum", i + ""));         }                }                        this.addRegex("http://www.bidcenter.com.cn/news-\\d+-1.html");//文章页                /*不要爬取包含 # 的URL*/        this.addRegex("-.*#.*");    }@Overridepublic void visit(Page page, CrawlDatums next) {SimpleDateFormat sDate = new SimpleDateFormat("yyyy-MM-dd");nowTime=sDate.format(DateUtils.minus());//获取当前日期减1天并转换格式    try {      String url = page.getUrl();      System.out.println(url+","+page.matchUrl("http://www.bidcenter.com.cn/news-\\d+-1.html"));            /*判断是否为文章页*/            if (page.matchUrl("http://www.bidcenter.com.cn/news-\\d+-1.html")) {            String time = page.select("table[class=table_content] tr + tr td[width=213] span[class=d2] ").first().text();//获取文章日期                String title = page.select("h1[class=content_title2]").text();//标题                String content = page.select("div[class=zdynr]").first().text();//内容                String Jcontent =content.substring(0,content.indexOf("。")+1).replaceAll("【\\d】", "");            for(String keyword:keys){    if((-1!=title.indexOf(keyword)||-1!=content.indexOf(keyword)) && nowTime.equals(time)){    System.out.println(title+","+keyword);    monitor = new Monitor();    monitor.setMonitorUrl(url);//url    monitor.setMonitorTitle(title);//标题    monitor.setMonitorContent(Jcontent);//内容简介    monitor.setKeyword(keyword);//关键字    monitor.setMonitorFrom(monitorFrom);//数据来源    monitor.setWebsiteType(website_Type);    monitor.setCreateDate(new Date());    monitor.setDelState(GeneralEnums.DelState.正常);    listMonitor.add(monitor);    break;    }    }                                                                           }} catch (Exception e) {System.out.println(e);}} public List<Monitor> doItCralwer(String [] keyworlds ,BidcenterCralwer crawler,WebsiteType websiteType,String from){ keys=keyworlds; website_Type = websiteType;//网站类型 monitorFrom = from;//来自那个网站        try {        if(null != crawler){        crawler.setThreads(40);        crawler.setTopN(5000);    crawler.start(2);             }} catch (Exception e) {e.printStackTrace();}        return this.listMonitor;}}

//service@Transactional(propagation = Propagation.REQUIRED)public Boolean BidcenterCralwer(){website= websiteService.getById("xxxxxxxxxxxid");//根据id查询网站信息//取出关键词组 放入数组中String[] keys = website.getWebsiteKey().split("、");//初始化爬虫BidcenterCralwer crawler = new BidcenterCralwer("bidcenterCralwer", true,keys);List<Monitor> listMonitor = crawler.doItCralwer(keys,crawler,website.getWebsiteType(),website.getWebsiteName());if(null != listMonitor && listMonitor.size()>0){for(Monitor monitor:listMonitor){saveMonitor(monitor);}return true;}return false;}