[仗贱天涯]Code-URLParseThread

/* * 创建日期 2005-8-25 * @author 猪神 */package com.ty.parse;import java.io.IOException;import java.util.ArrayList;import java.util.List;import com.ty.cc.Lexer;import com.ty.cfg.TYConfig;import com.ty.dao.DocumentDao;import com.ty.domain.Document;import com.ty.net.PageConnector;import com.ty.util.Logger;import com.ty.util.StringUtil;/** * @author Administrator * * TODO 要更改此生成的类型注释的模板，请转至 * 窗口 － 首选项 － Java － 代码样式 － 代码模板 */public class URLParseThread extends AbstractParseThread{ private String url; private String pattern; private int maxLevel=1; private boolean forceUpdate=false; public URLParseThread(String url,int maxLevel){ this(url,TYConfig.getInstance().getValue("/ty/url/station"),maxLevel, Boolean.valueOf(TYConfig.getInstance().getNode("/ty/url").getAttribute("forceUpdate")).booleanValue()); } public URLParseThread(String url,String pattern,int maxLevel,boolean forceUpdate){ this.url=url; this.pattern=pattern; this.maxLevel=maxLevel; this.forceUpdate=forceUpdate; } public Object parse() throws ParseException { myParse(url,0,-1); return null; } private void myParse(String url,int level,int fromDoc)throws ParseException { if(this.isPostedStop()){//中断 return; } List list=new ArrayList(); try{ if(url==null||!url.startsWith("http://")){ return; } Lexer lex=new Lexer(PageConnector.open(url)); Document doc=DocumentDao.getDocument(url.hashCode()); if(!forceUpdate&&doc!=null&&!doc.isNeedUpdate(maxLevel-level)){ return; } String base=null; String station=null; int i=url.substring("http://".length()).lastIndexOf('/'); if(i!=-1){ base=url.substring(0,"http://".length()+i); }else{ base=url; } station=base; i=base.substring("http://".length()).indexOf('/'); if(i!=-1){ station=base.substring(0,"http://".length()+i); }// Logger.log("station="+station);// Logger.log("base="+base); String w; String title=null; while((w=lex.nextToken())!=null){ // if(w.startsWith("")!=-1){//TODO 大小写 title=lex.nextToken();//文档标题 } } //记录url为已经解析 if(doc==null){ doc=Document.create(title,url,lex.getSourceText(),maxLevel-level,fromDoc); DocumentDao.createDocument(doc); }else{ doc.setContent(lex.getSourceText()); doc.setFromDoc(fromDoc); doc.setLastUpdated(StringUtil.currentTimestamp()); doc.setParseSubLevel(maxLevel-level); DocumentDao.updateDocument(doc); } }catch(IOException ex){ }catch(Exception ex){ ex.printStackTrace(); // throw new ParseException(ex); } //搜索深度 int nextFromDoc=url.hashCode(); level++; if(level<maxLevel||maxLevel<=0){ for(int i=0,n=list.size();i<n;i++){ myParse((String)list.get(i),level,nextFromDoc); } } } private String buildURL(String w,String station,String base){ int b=w.indexOf("href="); if(b==-1){ return null; } StringBuffer sb=new StringBuffer(); boolean begin=false; boolean end=false; char ch; for(int i=b+"href=".length(),n=w.length();i<n;i++){ ch=w.charAt(i); switch(ch){ case ' ': case '/t': case '/n': if(begin){ end=true; break; } continue; case '/'': case '"': case '>': if(begin){ end=true; break; }else{ begin=true; continue; } default: if(!begin){ begin=true; } } if(end){ break; } sb.append(ch); } if(sb.length()==0){ return null; } if(sb.toString().startsWith("http://")){ }else if(sb.charAt(0)=='/'){ sb.insert(0,station); }else{ sb.insert(0,base+'/'); } return sb.toString(); }}//TODO 优化：小规模搜索，URL在内存中访问代替数据库，自动分段持久化。