HTML Parser Jsoup - 网页抓取百度百科信息的例子

目标：获取百度百科基本信息、信息列表、人物图片（同名情况暂不考虑）。

重点：调用开源Jar包Jsoup对HTML解析。网页抓取、简单爬虫。

例子（部分类去掉，运行需改改code）：

import java.io.IOException;import java.lang.reflect.InvocationTargetException;import java.lang.reflect.Method;import java.util.Calendar;import java.util.Date;import java.util.HashMap;import java.util.List;import java.util.Map;import org.codehaus.jackson.JsonNode;import org.codehaus.jackson.JsonProcessingException;import org.codehaus.jackson.map.ObjectMapper;import org.jsoup.Jsoup;import org.jsoup.nodes.Document;import org.jsoup.nodes.Element;import org.jsoup.select.Elements;import org.slf4j.Logger;import org.slf4j.LoggerFactory;public class BaiduBaikeUtils{private static Logger logger = LoggerFactory.getLogger(BaiduBaikeUtils.class);final static String BAIKE_SEARCH = "http://baike.baidu.com/search/word?word=";final static String DOUBLE_QUOTE = "\"";final static String NON_WORD_CHAR = "[^\\p{L}\\p{Nd}]+";/* * 获取百科Url */private static String getBaikeSearchUrl(String name){String url = null;url = BAIKE_SEARCH + name;return url;}private static Document getDocument(String url){Document doc = null;try {doc = Jsoup.connect(url).timeout(5000).get();} catch (IOException e) {logger.error("error to connect to Baidu baike" + e.getClass().getName() + ": " + e.getMessage());}return doc;}/* * 获取具体信息列表 */private static WeiboPerson parseBaseInfoWrap(Document doc){final String BASE_INFO_WRAP = "baseInfoWrapDom";WeiboPerson weiboPerson = new WeiboPerson();Map<String, String> biTitleMap = setBiTitleMap();if (doc == null) return null;if (null != doc.getElementById(BASE_INFO_WRAP)) {Elements elements = doc.getElementById(BASE_INFO_WRAP).getElementsByClass("biItemInner");if (null != elements && elements.size() > 0) {for (Element element : elements) {setWeiboPerson(weiboPerson, element, biTitleMap);}}}return weiboPerson;}private static Map<String, String> setBiTitleMap(){final Map<String, String> biTitleMap = new HashMap<String, String>();biTitleMap.put("中文名", "Name");biTitleMap.put("本名", "Name");biTitleMap.put("外文名", "En_name");biTitleMap.put("性别", "Gender");biTitleMap.put("国籍", "Country");biTitleMap.put("民族", "Nation");biTitleMap.put("毕业院校", "Institution");biTitleMap.put("别名", "Alias");biTitleMap.put("身高", "Height");biTitleMap.put("体重", "Weight");biTitleMap.put("出生地", "BirthPlace");biTitleMap.put("出生日期", "Birthday");biTitleMap.put("出生时间", "Birthday");biTitleMap.put("逝世日期", "Passaway");biTitleMap.put("去世时间", "Passaway");biTitleMap.put("职业", "Career");biTitleMap.put("最高职务", "Career");biTitleMap.put("信仰", "Faith");biTitleMap.put("主要成就", "Achievement");biTitleMap.put("代表作品", "Opus");biTitleMap.put("所属政党", "Party");biTitleMap.put("血型", "BloodType");biTitleMap.put("妻子", "Spouse");biTitleMap.put("丈夫", "Spouse");biTitleMap.put("学位", "Degree");biTitleMap.put("籍贯", "NativePlace");biTitleMap.put("星座", "Constellation");return biTitleMap;}private static void setWeiboPerson(WeiboPerson weiboPerson, Element element, Map<String, String> biTitleMap){final String BI_TITLE = "biTitle";final String BI_CONTENT = "biContent";for (String biTitleMapKey : biTitleMap.keySet()) {if (null != element.getElementsByClass(BI_TITLE) && element.getElementsByClass(BI_TITLE).size() > 0) {String key = element.getElementsByClass(BI_TITLE).get(0).text().replaceAll(NON_WORD_CHAR, "");// 处理多行信息String textHtml = element.getElementsByClass(BI_CONTENT).get(0).toString().replaceAll("<br />",",");String text = Jsoup.parse(textHtml).text();// 设置具体信息if (biTitleMapKey.equals(key)) {try {Class clazz = weiboPerson.getClass(); Method method;method = clazz.getMethod("set" + biTitleMap.get(biTitleMapKey), String.class);method.invoke(weiboPerson, new Object[]{text});} catch (SecurityException e) {logger.error(e.getMessage(), e);} catch (NoSuchMethodException e) {logger.error(e.getMessage(), e);} catch (IllegalArgumentException e) {logger.error(e.getMessage(), e);} catch (IllegalAccessException e) {logger.error(e.getMessage(), e);} catch (InvocationTargetException e) {logger.error(e.getMessage(), e);}}}}}/* * 获取基本信息 */private static String parseBaseInfo(Document doc){final String LINE_SEPARATOR = "\r\n";String baseInfoStr = "";if (doc == null) return null;Elements elements = null;// 概况if (doc.getElementsByClass("card-summary-content").size() != 0) {elements = doc.getElementsByClass("card-summary-content").get(0).getElementsByClass("para");if (null != elements && elements.size() > 0) {for (Element element : elements) {baseInfoStr += element.text() + LINE_SEPARATOR;}return baseInfoStr;}}// 基本信息if (null != doc.getElementById("lemmaContent-0")) {elements = doc.getElementById("lemmaContent-0").getAllElements();if (null != elements && elements.size() > 0) {for (Element element : elements) {if ("headline-1".equals(element.className())) {if (null != element.getElementsByTag("a")) {// 标题为基本信息if ("baseinfo".equals(element.getElementsByTag("a").attr("name"))) continue;}// 其他标题break;}if ("para".equals(element.className())) {baseInfoStr += element.text() + LINE_SEPARATOR;}}return baseInfoStr;}}return null;}/* * 获取人物图片 */private static String parseRolePicUrl(Document doc, String name){Elements elements = null;String picSrc = null;// 侧边图片if (null != doc.getElementById("side")) {elements = doc.getElementById("side").getElementsByTag("img");picSrc = getPicSrc(elements, name);}// 其他位置图片if (!StringUtils.isEmptyString(picSrc)) return picSrc;elements = doc.getElementsByTag("img");picSrc = getPicSrc(elements, name);return picSrc;}private static String getPicSrc(Elements elements, String name){if (null != elements && elements.size() > 0) {for (Element element : elements) {if (name.equals(element.attr("alt"))) {if (!StringUtils.isEmptyString(element.attr("src"))) return element.attr("src");else return element.attr("data-src");}}}return null;}/* * 获取当前title */private static String getTitle(Document doc){Elements elements = doc.getElementsByClass("polysemeTitle");if (null != elements && elements.size() > 0) {return elements.get(0).text();}return null;}/* * 获取百度百科信息 */public static WeiboPerson getBaiduBaikeInfo(String name){String url = getBaikeSearchUrl(name);Document doc = getDocument(url);WeiboPerson weiboPerson = new WeiboPerson();weiboPerson = parseBaseInfoWrap(doc);String baseInfoStr = parseBaseInfo(doc);String rolePicUrl = parseRolePicUrl(doc, name);//String title = getTitle(doc);weiboPerson.setDescription(baseInfoStr);weiboPerson.setImageUrl(rolePicUrl);//weiboPerson.setTitle(title);return weiboPerson;}/* * 获取年份 */public static int getYear(Date date){Calendar calendar = Calendar.getInstance();calendar.setTime(date);return calendar.get(Calendar.YEAR);}/* * 测试 */public static void main(String[] args) {String name = "李敏镐";  // 李敏镐，张超，李刚，张灵甫，曹操，孔明，obama，马云，迈克尔·乔丹，李明// 测试：连接//String url = getBaikeSearchUrl(name);//System.out.println("url:" + url);//Document doc = getDocument(url);// 测试：具体信息列表//WeiboPerson weiboPerson = parseBaseInfoWrap(doc);//System.out.println("Name:" + weiboPerson.getName());// 测试：基本信息//String baseInfoStr = parseBaseInfo(doc);//System.out.println("baseInfoStr:" + baseInfoStr);// 测试：人物图片//String rolePicUrl = parseRolePicUrl(doc, name);//System.out.println("rolePicUrl:" + rolePicUrl);//getTitle(doc);// 总测试//WeiboPerson weiboPerson = getBaiduBaikeInfo(name);//System.out.println("Name:" + weiboPerson.getName());//System.out.println("Country:" + weiboPerson.getCountry());//System.out.println("Career:" + weiboPerson.getCareer());//System.out.println("baseInfoStr:" + weiboPerson.getDescription());//System.out.println("rolePicUrl:" + weiboPerson.getImageUrl());}}

补充：

基于已有HTML源文件分析。

private static Document getDocument(String filePath){Document doc = null;try {File file = new File(filePath);        doc = Jsoup.parse(file,"GBK","http://www.mk.org/");} catch (Exception e) {            System.out.println("error to parse" + e.getClass().getName() + ": " + e.getMessage());        }        return doc;    }

public static void main(String[] args) {String searchNo = "";String filePath = "";if (args.length >= 1)searchNo = args[0];if (args.length >= 2)filePath = args[1];elsefilePath = "src/bankResource/trans.html";if (searchNo == null || searchNo == "") {System.out.println("Search No is required.");} else {// get documentDocument doc = getDocument(filePath);...