抓取网页读写文件存入map

工作中面对要和别的公司游戏下载做接口（下面是做获取的部分）：

网上找了些资料自己又针对性的加了些方法及一部分类方法没有重构，XML分解写的不好，有时间修改一下

文件结构片段如入：
CacheManager.java           是方法类主要作用是操作缓存
FileType.java               是获取文件类型
KoXmlCache.java             缓存BEAN
KoXmlSetMapCache.java       分解XML存入MAP
KoXmlSetNameToCache.java    分析xml文件，遍历每个节点，放入MAP里
XmlBean.java                文件节点BEAN

package xml2system;import java.util.*;  /**  * <p>Title: </p>  *  * <p>Description: 管理缓存</p>  * Deep blue 2011-12-20 think  * 没有加的功能 ：1.提供控制cache内存消耗的阀值，并实现LRU淘汰策略 2.统计cache命中率 * 可扩展的功能：当chche到内存溢出时必须清除掉最早期的一些缓存对象，这就要求对每个缓存对象保存创建时间  * <p>Copyright: Copyright (c) 2011</p>  *  * <p>Company: </p>  *  * @author huangyzh  2011-12-20  * @version 1.0  */  public class CacheManager {      private static HashMap cacheMap = new HashMap();        //单实例构造方法      private CacheManager() {          super();      }      //获取布尔值的缓存      public static boolean getSimpleFlag(String key){          try{              return (Boolean) cacheMap.get(key);          }catch(NullPointerException e){              return false;          }      }      public static long getServerStartdt(String key){          try {              return (Long)cacheMap.get(key);          } catch (Exception ex) {              return 0;          }      }      //设置布尔值的缓存      public synchronized static boolean setSimpleFlag(String key,boolean flag){          if (flag && getSimpleFlag(key)) {//假如为真不允许被覆盖              return false;          }else{              cacheMap.put(key, flag);              return true;          }      }      public synchronized static boolean setSimpleFlag(String key,long serverbegrundt){          if (cacheMap.get(key) == null) {              cacheMap.put(key,serverbegrundt);              return true;          }else{              return false;          }      }          //得到缓存。同步静态方法      private synchronized static KoXmlCache getCache(String key) {          return (KoXmlCache) cacheMap.get(key);      }        //判断是否存在一个缓存      private synchronized static boolean hasCache(String key) {          return cacheMap.containsKey(key);      }        //清除所有缓存      public synchronized static void clearAll() {          cacheMap.clear();      }        //清除某一类特定缓存,通过遍历HASHMAP下的所有对象，来判断它的KEY与传入的TYPE是否匹配      public synchronized static void clearAll(String type) {          Iterator i = cacheMap.entrySet().iterator();          String key;          ArrayList<String> arr = new ArrayList<String>();          try {              while (i.hasNext()) {                  java.util.Map.Entry entry = (java.util.Map.Entry) i.next();                  key = (String) entry.getKey();                  if (key.startsWith(type)) { //如果匹配则删除掉                      arr.add(key);                  }              }              for (int k = 0; k < arr.size(); k++) {                  clearOnly(arr.get(k));              }          } catch (Exception ex) {              ex.printStackTrace();          }      }        //清除指定的缓存      public synchronized static void clearOnly(String key) {          cacheMap.remove(key);      }        //载入缓存      public synchronized static void putCache(String key, KoXmlCache obj) {          cacheMap.put(key, obj);      }        //获取缓存信息      public static KoXmlCache getCacheInfo(String key) {            if (hasCache(key)) {          KoXmlCache cache = getCache(key);              if (cacheExpired(cache)) { //调用判断是否终止方法                  cache.setExpired(true);              }              return cache;          }else              return null;      }        //载入缓存信息      public static void putCacheInfo(String key, KoXmlCache obj, long dt,boolean expired) {      KoXmlCache cache = new KoXmlCache();          cache.setKey(key);          cache.setTimeOut(dt + System.currentTimeMillis()); //设置多久后更新缓存          cache.setValue(obj);          cache.setExpired(expired); //缓存默认载入时，终止状态为FALSE          cacheMap.put(key, cache);      }      //重写载入缓存信息方法      public static void putCacheInfo(String key,KoXmlCache obj,long dt){      KoXmlCache cache = new KoXmlCache();          cache.setKey(key);          cache.setTimeOut(dt+System.currentTimeMillis());          cache.setValue(obj);          cache.setExpired(false);          cacheMap.put(key,cache);      }        //判断缓存是否终止      public static boolean cacheExpired(KoXmlCache cache) {          if (null == cache) { //传入的缓存不存在              return false;          }          long nowDt = System.currentTimeMillis(); //系统当前的毫秒数          long cacheDt = cache.getTimeOut(); //缓存内的过期毫秒数          if (cacheDt <= 0||cacheDt>nowDt) { //过期时间小于等于零时,或者过期时间大于当前时间时，则为FALSE              return false;          } else { //大于过期时间 即过期              return true;          }      }        //获取缓存中的大小      public static int getCacheSize() {          return cacheMap.size();      }        //获取指定的类型的大小      public static int getCacheSize(String type) {          int k = 0;          Iterator i = cacheMap.entrySet().iterator();          String key;          try {              while (i.hasNext()) {                  java.util.Map.Entry entry = (java.util.Map.Entry) i.next();                  key = (String) entry.getKey();                  if (key.indexOf(type) != -1) { //如果匹配则删除掉                      k++;                  }              }          } catch (Exception ex) {              ex.printStackTrace();          }            return k;      }        //获取缓存对象中的所有键值名称      public static ArrayList<String> getCacheAllkey() {          ArrayList a = new ArrayList();          try {              Iterator i = cacheMap.entrySet().iterator();              while (i.hasNext()) {                  java.util.Map.Entry entry = (java.util.Map.Entry) i.next();                  a.add((String) entry.getKey());              }          } catch (Exception ex) {} finally {              return a;          }      }        //获取缓存对象中指定类型 的键值名称      public static ArrayList<String> getCacheListkey(String type) {          ArrayList a = new ArrayList();          String key;          try {              Iterator i = cacheMap.entrySet().iterator();              while (i.hasNext()) {                  java.util.Map.Entry entry = (java.util.Map.Entry) i.next();                  key = (String) entry.getKey();                  if (key.indexOf(type) != -1) {                      a.add(key);                  }              }          } catch (Exception ex) {} finally {              return a;          }      }    }  

package xml2system;/** * 缓存BEAN * @author huangyzh * */public class KoXmlCache {      private String key;//缓存ID      private Object value;//缓存数据      private long timeOut;//更新时间      private boolean expired; //是否终止      public KoXmlCache() {              super();      }      public KoXmlCache(String key, Object value, long timeOut, boolean expired) {              this.key = key;              this.value = value;              this.timeOut = timeOut;              this.expired = expired;      }      public String getKey() {              return key;      }      public long getTimeOut() {              return timeOut;      }      public Object getValue() {              return value;      }      public void setKey(String string) {              key = string;      }      public void setTimeOut(long l) {              timeOut = l;      }      public void setValue(Object object) {              value = object;      }      public boolean isExpired() {              return expired;      }      public void setExpired(boolean b) {              expired = b;      }        //测试类，      static class Test {          public static void main(String[] args) {  //写些自己需要的东西        }      }}  

package xml2system;public class FileType {/** * 得文件名字类型 * @param args */public static String getPicfileType(String str){String picfiletype = "";picfiletype = str;if (!picfiletype.equals("")&&picfiletype != null) {picfiletype = picfiletype.substring(picfiletype.length()-6, picfiletype.length());picfiletype = picfiletype.substring(picfiletype.indexOf(".")+1, picfiletype.length());System.out.println(picfiletype);}return picfiletype;}public static void main(String[] args) {//str 可能是多个，要分折XML节点结构信息String str = "http://网站地址/game/pre/jar/2011/08/01/05/gq3Dzwb_2_bH52cq.jar";FileType fileType = new FileType();fileType.getPicfileType(str);}}

package xml2system;import java.io.UnsupportedEncodingException;import java.util.ArrayList;import java.util.HashMap;import java.util.List;import java.util.Map;public class KoXmlSetMapCache {static CacheManager cacheManager ;public static Boolean setMapListValue(String str,String gid){str = str.replace("[", "").replace("]", "");String[] strArray = str.split("");//注意通过TAB键分隔，不是空格String[] strArrays = null ;String list2str = "";List listAddValue = new ArrayList();//取手机机型名称Map mapname = new HashMap();Map mapstack = new HashMap();for (int i = 0; i < strArray.length; i++) {strArrays=strArray[i].split("=");listAddValue  = new ArrayList(strArrays.length);String name = strArrays[0].replace(",", "").replace(" ", "");if (list2str.equals("")) {//!name.isEmpty()&&mapstack.get(name)!=nulllist2str = strArrays[0].replace("[", "").replace(",", "").replace(" ", "");//手机品牌名称}list2str +="=";list2str += strArrays[1].replace(", ", "");//机型list2str +="=";list2str += strArrays[2].replace(" ", "").replace(",", "");//下载大小list2str +="=";list2str += strArrays[3].replace(", ", "");//下载链接list2str +="";String[] mblname = list2str.split("=");if (name.equals(mapname.get(name))) {KoXmlCache koXmlCache = new KoXmlCache();KoXmlCache cache = cacheManager.getCacheInfo(gid+mblname[0]);KoXmlCache kxcache =  (KoXmlCache)cache.getValue();listAddValue.add(kxcache.getValue());listAddValue.add(list2str);koXmlCache.setValue(listAddValue);cacheManager.putCacheInfo(gid+mblname[0], koXmlCache,20,false);list2str = "";}else{KoXmlCache koXmlCache = new KoXmlCache();listAddValue.add(list2str);koXmlCache.setValue(listAddValue);cacheManager.putCacheInfo(gid+mblname[0], koXmlCache,20,false);list2str = "";}mapname.put(name, name);//手体品牌名称}return true;}public static KoXmlCache getCacheMblValues(String typeName){KoXmlSetNameToCache koXmlList = new KoXmlSetNameToCache();typeName = typeName.replace(",", "");KoXmlCache cache = cacheManager.getCacheInfo(typeName);KoXmlCache kxcache =  (KoXmlCache)cache.getValue();return kxcache;}public static StringBuffer getWritePage(String str,String gid){KoXmlSetNameToCache xmlTest = new KoXmlSetNameToCache();KoXmlCache koXmlCache = xmlTest.getOkShowGameMsg(gid);List list = (List)koXmlCache.getValue();System.out.println(list.get(0));StringBuffer buffer = new StringBuffer();str = str.replace("[", "").replace("[", "").replace("[", "").replace("[", "").replace("]", "");String[] s = str.split("");String[] a = null;String[] urls = null;String name = "";String type = "";String big = "";String url = "";//得游戏名称buffer.append("【"+list.get(0)+"】").append("<br/>");if(s.length>1){String[] names =s[0].split("=");buffer.append("【"+names[0]+"】").append("<br/>");}buffer.append("请选择适合您机型的下载点").append("<br/>");for (int i = 0; i < s.length; i++) {a =s[i].split("=");for (int j = 0; j < a.length; j++) {//System.out.println(a[i]);name = a[0];type = a[1];big = a[2];url = a[3];}urls = url.split(",");if(urls.length == 1){buffer.append("下载点"+(i+1)+":  ");buffer.append("<a href=\"").append(urls[0]).append("\">").append(FileType.getPicfileType(urls[0])+"("+big+")").append("</a><br/>");buffer.append(type).append("<br/>");}else{buffer.append("下载点"+(i+1)+":  ");buffer.append("<a href=\"").append(urls[0]).append("\">").append(FileType.getPicfileType(urls[0])+"jar  |  ").append("</a>");buffer.append("<a href=\"").append(urls[1]).append("\">").append(FileType.getPicfileType(urls[1])+"("+big+")").append("</a><br/>");buffer.append(type).append("<br/>");}}System.out.println(buffer);return buffer;}public static void main(String[] args) {      //写些测试}}

package xml2system;import java.util.ArrayList;import java.util.HashSet;import java.util.List;import java.util.Vector;import javax.xml.parsers.DocumentBuilder;import javax.xml.parsers.DocumentBuilderFactory;import org.w3c.dom.DOMException;import org.w3c.dom.Document;import org.w3c.dom.Element;import org.w3c.dom.NodeList;import org.w3c.dom.Text;public class KoXmlSetNameToCache {private static Vector xml_Vector;private List mblnamelist = new ArrayList();private List mblmsglist = new ArrayList();private List msglist = new ArrayList();static CacheManager cacheManager ;/** * 分析xml文件，遍历每个节点，放入MAP里 * @param id * @return * @throws Exception */public Boolean readXMLFile(String gid) throws Exception {//为解析XML作准备，创建DocumentBuilderFactory实例,指定DocumentBuilder　DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();DocumentBuilder db = null;try {db = dbf.newDocumentBuilder();} catch (Exception pce) {System.err.println(pce); //出异常时输出异常信息，然后退出，下同　System.exit(1);}Document doc = null;try {//doc = db.parse(inFile);doc = db.parse("http://***:8086/openface/down.e?rid="+gid);} catch (DOMException dom) {System.err.println(dom.getMessage());System.exit(1);}//下面是解析XML的全过程，比较简单，先取根元素"<results>"　Element root = doc.getDocumentElement();//取"<result>"元素列表NodeList resultxmlNode = root.getElementsByTagName("result");//依次取每个"<result>"元素　Element result = (Element) resultxmlNode.item(0);//创建一个<result>的Bean实例　XmlBean xmlBean = new XmlBean();//取"name"元素，下面类同　NodeList names = result.getElementsByTagName("name");if (names.getLength() == 1) {Element e = (Element) names.item(0);Text t = (Text) e.getFirstChild();xmlBean.setName(t.getNodeValue());}NodeList pic = result.getElementsByTagName("pic");if (pic.getLength() == 1) {Element e = (Element) pic.item(0);Text t = (Text) e.getFirstChild();xmlBean.setPic(t.getNodeValue());}NodeList evaluation = result.getElementsByTagName("evaluation");if (evaluation.getLength() == 1) {Element e = (Element) evaluation.item(0);Text t = (Text) e.getFirstChild();xmlBean.setEvaluation(Integer.parseInt(t.getNodeValue()));}NodeList downcount = result.getElementsByTagName("downcount");if (downcount.getLength() == 1) {Element e = (Element) downcount.item(0);Text t = (Text) e.getFirstChild();xmlBean.setDowncount(Integer.parseInt(t.getNodeValue()));}NodeList type = result.getElementsByTagName("type");if (type.getLength() == 1) {Element e = (Element) type.item(0);Text t = (Text) e.getFirstChild();xmlBean.setType(t.getNodeValue());}NodeList introduction = result.getElementsByTagName("introduction");if (introduction.getLength() == 1) {Element e = (Element) introduction.item(0);Text t = (Text) e.getFirstChild();xmlBean.setIntroduction(t.getNodeValue());}NodeList downurlsxmlNode = result.getElementsByTagName("downurls");for (int i = 0; i < downurlsxmlNode.getLength(); i++) {Element downurls = (Element) downurlsxmlNode.item(i);//内部的supportMobilesNodeList supportMobiles = downurls.getElementsByTagName("supportMobiles");Element supportMobiles_results = (Element) supportMobiles.item(0);NodeList brand = supportMobiles_results.getElementsByTagName("brand");if (brand.getLength() == 1) {Element e_brand = (Element) brand.item(0);Text t_brand = (Text) e_brand.getFirstChild();xmlBean.setBrand(t_brand.getNodeValue());}NodeList mobiles = supportMobiles_results.getElementsByTagName("mobiles");if (mobiles.getLength() == 1) {Element e_mobiles = (Element) mobiles.item(0);Text t_mobiles = (Text) e_mobiles.getFirstChild();xmlBean.setMobiles(t_mobiles.getNodeValue());}NodeList size = downurls.getElementsByTagName("size");if (size.getLength() == 1) {Element e_size = (Element) size.item(0);Text t_size = (Text) e_size.getFirstChild();xmlBean.setSize(t_size.getNodeValue());}NodeList downurl = downurls.getElementsByTagName("downurl");if (downurl.getLength()  == 1) {Element e_downurl = (Element) downurl.item(0);Text t_downurl = (Text) e_downurl.getFirstChild();xmlBean.setDownurl(t_downurl.getNodeValue());}xml_Vector.add(xmlBean);xmlBean = (XmlBean)xml_Vector.get(i);//把机型名称放入Listmblnamelist.add(xmlBean.getBrand());HashSet hs = new HashSet(mblnamelist);KoXmlCache koXmlCache = new KoXmlCache();koXmlCache.setValue(hs);cacheManager.putCacheInfo(gid, koXmlCache,20,false);KoXmlCache koXmlCacheContext = new KoXmlCache();mblmsglist.add(xmlBean.getBrand());mblmsglist.add("=");mblmsglist.add(xmlBean.getMobiles());mblmsglist.add("=");mblmsglist.add(xmlBean.getSize());mblmsglist.add("=");mblmsglist.add(xmlBean.getDownurl());mblmsglist.add("");koXmlCacheContext.setValue(mblmsglist);cacheManager.putCacheInfo(gid+"context", koXmlCacheContext,20,false);}KoXmlCache koXmlCacheContexts = getKoXmlCacheContextValue(gid);//KoXmlCache cache = (KoXmlCache) koXmlCache.getValue();String cacheString = koXmlCacheContexts.getValue().toString();KoXmlSetMapCache.setMapListValue(cacheString,gid);    System.out.println("---"+cacheString);       msglist.add(xmlBean.getName());msglist.add(xmlBean.getPic());msglist.add(xmlBean.getEvaluation());msglist.add(xmlBean.getDowncount());msglist.add(xmlBean.getType());KoXmlCache koXmlCacheMsg = new KoXmlCache();koXmlCacheMsg.setValue(msglist);cacheManager.putCacheInfo(gid+"msg", koXmlCacheMsg,20,false);   /*KoXmlSetMapCache ko = new KoXmlSetMapCache();ko.setMapListValue(str,gid);*/    return true;}public static KoXmlCache getOkShowGameMsg(String gid){int num = CacheManager.getCacheSize(gid+"msg");//System.out.println(CacheManager.getCacheAllkey());KoXmlCache cache = null;KoXmlCache koXmlCache = null;if (num>0) {koXmlCache = cacheManager.getCacheInfo(gid+"msg");cache = (KoXmlCache) koXmlCache.getValue();}else{KoXmlSetNameToCache xmlTest = new KoXmlSetNameToCache();String xmlString = xmlTest.getOkShowPhoneType(gid);koXmlCache = cacheManager.getCacheInfo(gid+"msg");cache = (KoXmlCache) koXmlCache.getValue();}return cache;}/** * 通过gid获得手机品牌名称 * @param gid * @return String */public static String getOkShowPhoneType(String gid){KoXmlSetNameToCache xmlTest = new KoXmlSetNameToCache();xmlTest.xml_Vector = new Vector();String mblmapstring = "";String str = "";try {if(xmlTest.readXMLFile(gid)){System.out.println("放入Cache成功!!!");}else{System.out.println("放入Cache失败!!!");}mblmapstring =  cacheManager.getCacheInfo(gid)+"";} catch (Exception e) {// TODO Auto-generated catch blocke.printStackTrace();}return mblmapstring;}public static KoXmlCache getKoXmlCacheValue(String gid){int num = CacheManager.getCacheSize(gid);//System.out.println(CacheManager.getCacheAllkey());KoXmlCache cache = null;KoXmlCache koXmlCache = null;if (num>0) {koXmlCache = cacheManager.getCacheInfo(gid);cache = (KoXmlCache) koXmlCache.getValue();}else{KoXmlSetNameToCache xmlTest = new KoXmlSetNameToCache();String xmlString = xmlTest.getOkShowPhoneType(gid);koXmlCache = cacheManager.getCacheInfo(gid);cache = (KoXmlCache) koXmlCache.getValue();}return cache;}public static KoXmlCache getKoXmlCacheContextValue(String gid){int num = CacheManager.getCacheSize(gid);//System.out.println(CacheManager.getCacheAllkey());KoXmlCache cache = null;KoXmlCache koXmlCache = null;if (num>0) {koXmlCache = cacheManager.getCacheInfo(gid+"context");cache = (KoXmlCache) koXmlCache.getValue();}else{KoXmlSetNameToCache xmlTest = new KoXmlSetNameToCache();String xmlString = xmlTest.getOkShowPhoneType(gid);koXmlCache = cacheManager.getCacheInfo(gid+"context");cache = (KoXmlCache) koXmlCache.getValue();}return cache;}public static void main(String[] args) {//查询XML遍历的类KoXmlSetNameToCache xmlTest = new KoXmlSetNameToCache();xmlTest.xml_Vector = new Vector();//传来的gidString gid = "54530";try {xmlTest.readXMLFile(gid);} catch (Exception e) {// TODO Auto-generated catch blocke.printStackTrace();}//通过GID获得初始化信息 可以先加入一个判断是否为空KoXmlCache koXmlCache = getOkShowGameMsg(gid);List list = (List)koXmlCache.getValue();System.out.println(list.get(0));String[] pics = list.get(1).toString().split(",");System.out.println(pics[0]);System.out.println(list.get(2));System.out.println(list.get(3));}}

package xml2system;/** * 下面为类分析格式如下： */public class XmlBean {    private String name;    private String pic;    private int evaluation;    private int downcount;    private String type;    private String introduction;    private String supportMobiles;    private String size;    private String downurl;    private String brand;    private String mobiles;        public String getName() {return name;}public void setName(String name) {this.name = name;}public String getPic() {return pic;}public void setPic(String pic) {this.pic = pic;} public int getEvaluation() {return evaluation;}public void setEvaluation(int evaluation) {this.evaluation = evaluation;}public int getDowncount() {return downcount;}public void setDowncount(int downcount) {this.downcount = downcount;}public String getType() {return type;}public void setType(String type) {this.type = type;}public String getIntroduction() {return introduction;}public void setIntroduction(String introduction) {this.introduction = introduction;}public String getSupportMobiles() {return supportMobiles;}public void setSupportMobiles(String supportMobiles) {this.supportMobiles = supportMobiles;}public String getSize() {return size;}public void setSize(String size) {this.size = size;}public String getDownurl() {return downurl;}public void setDownurl(String downurl) {this.downurl = downurl;}public String getBrand() {return brand;}public void setBrand(String brand) {this.brand = brand;}public String getMobiles() {return mobiles;}public void setMobiles(String mobiles) {this.mobiles = mobiles;}}




	
				
	
					   
抓取网页读写文件存入map
	  	   
Python抓取网页链接，存入mysql
	  	   
定时抓取网页连接,提取网页内容,存入数据库
	  	   
Flume 抓取日志文件存入MySQL中
	  	   
PHP抓取文件内容，存入数组
	  	   
HttpClient抓取网页文件方法
	  	   
flume-ng抓取日志文件存入mysql中
	  	   
python网络爬虫抓取动态网页并将数据存入数据库MySQL
	  	   
CURL 爬虫，抓取网页并写入文件
	  	   
网页抓取
	  	   
网页抓取
	  	   
抓取网页
	  	   
网页抓取
	  	   
抓取网页
	  	   
网页抓取
	  	   
网页抓取
	  	   
抓取网页
	  	   
网页抓取
	     		  
	  	   
Protel 99 小知识点
	  	   
proerties操作
	  	   
名人传记
	  	   
unity Mathf 数学运算(C#)
	  	   
前端优化技术
	  	   
抓取网页读写文件存入map
	  	   
QT 就少写了一个Q引发的巨大问题,
	  	   
EMRPad30电子病历控件的医院名称是否可以修改?
	  	   
六项精进——稻盛和夫
	  	   
FireFox 中form.submit() 不执行 解决方案
	  	   
Quartz_框架快速入门
	  	   
UML中的四种关系
	  	   
设置QTableView的默认行高
	  	   
2011年总结——玩