HttpClient对网页编码的精确识别

最近用Httpclient对网页进行采集，因为采集的网页编码不确定，主要是中文的网址，而httpclient对编码的识别也是靠response的head来识别的，但是有的服务器根本不返回这个头，httpclient默认就采用了ISO-8859-1的编码。上网搜索了下，有人写出了浏览器对页面编码的自动识别原理，还是个北京人呢。我找到了他的java实现算法，为了保证写好的代码不会被重新大修改，我就extends了httpclient中的GetMethod（因为我在用这个类）对里面的编码识别进行了修改，完整的算法如下（使用了chardet.jar这个类库）：

这个代码是对chardet.jar的使用，这个算法来自网上：

 

/** * */package com.baseframework.support;import java.io.BufferedInputStream;import java.io.IOException;import java.io.InputStream;import org.mozilla.intl.chardet.nsDetector;import org.mozilla.intl.chardet.nsICharsetDetectionObserver;import org.mozilla.intl.chardet.nsPSMDetector;/** * @author sunyanan 判断字节流的编码 * */public class CharsetDetector {private boolean found = false;private String result;private int lang;private static CharsetDetector c = new CharsetDetector();private CharsetDetector(){}public static CharsetDetector getInstance() {return c;}public String[] detectChineseCharset(InputStream in) throws IOException {lang = nsPSMDetector.CHINESE;String[] prob;// Initalize the nsDetector() ;nsDetector det = new nsDetector(lang);// Set an observer...// The Notify() will be called when a matching charset is found.det.Init(new nsICharsetDetectionObserver() {public void Notify(String charset) {found = true;result = charset;}});BufferedInputStream imp = new BufferedInputStream(in);byte[] buf = new byte[1024];int len;boolean isAscii = true;while ((len = imp.read(buf, 0, buf.length)) != -1) {// Check if the stream is only ascii.if (isAscii)isAscii = det.isAscii(buf, len);// DoIt if non-ascii and not done yet.if (!isAscii) {if (det.DoIt(buf, len, false))break;}}imp.close();in.close();det.DataEnd();if (isAscii) {found = true;prob = new String[] { "ASCII" };} else if (found) {prob = new String[] { result };} else {prob = det.getProbableCharsets();}return prob;}public String[] detectAllCharset(InputStream in) throws IOException {try {lang = nsPSMDetector.ALL;return detectChineseCharset(in);} catch (IOException e) {throw e;}}}

 

下面是对GetMethod的扩充

 

 /** * */package com.baseframework.httpcient;import java.io.IOException;import java.io.InputStream;import org.apache.commons.httpclient.methods.GetMethod;import org.apache.commons.logging.Log;import org.apache.commons.logging.LogFactory;import com.baseframework.support.CharsetDetector;/** * @author sunyanan * 对标准org.apache.commons.httpclient.methods.GetMethod的重写，主要是为了覆盖其父类对Charset的探测，这个探测时选用的开源的jar */public class GetMethodForCharset extends GetMethod {private Log log = LogFactory.getLog(GetMethodForCharset.class);public GetMethodForCharset() {super();}public GetMethodForCharset(String uri) {super(uri);}/** * 主要实现的是对这个方法的重写 */ public String getResponseCharSet() { String charset = getContentCharSet(getResponseHeader("Content-Type")); // 默认情况下选择的是 ISO-8859-1，那么就判断如果是这个字符编码的时候再来探测 if(charset.equalsIgnoreCase("ISO-8859-1")) { // 使用组件来判断 try {InputStream is = getResponseBodyAsStream();String cs[] = CharsetDetector.getInstance().detectAllCharset(is);if(cs != null && cs.length > 0) {charset = cs[0];}} catch (IOException e) {e.printStackTrace();} } log.debug("charset used: " + charset); return charset; }}