htmlunit 学习笔记

htmlunit 是一款开源的java 页面分析工具，读取页面后，可以有效的使用htmlunit分析页面上的内容。项目可以模拟浏览器运行，被誉为java浏览器的开源实现。是一个没有界面的浏览器，运行速度迅速。是junit的扩展之一

采用的是Rhinojs引擎。模拟js运行

常规意义上，该项目可以用来进行页面的测试工作，实现网页自动化测试，（包括JS）

但是一般来说，在小型爬虫项目中，这种框架十分常用，可以有效的分析出 dom的标签，并且有效的运行页面上的js以便得到一些需要执行JS才能得到的值。

我们研究的爬虫应用中 仅仅用Httpclient+jsoup是不够的，因为有些页面数据是js加载出来的 httpclient解析不出来，比如百度云用户页面，淘宝页面，连开源中国博客的博客内容都是Js加载或者处理的，所以httpClient是解析不出来的，这里htmlunit就是一个很好的方案，内嵌js浏览器，模拟Js运行，把结果执行出来。这个就是我们想要的。所以搞爬虫,htmlunit必须要掌握好；
Htlmlunit我使用了最新版，最新版需要jdk1.8不然会报错jdk版本对应不上。

package com.myactiviti.HelloActiviti;import java.io.IOException;import java.net.MalformedURLException;import java.util.ArrayList;import java.util.HashMap;import java.util.List;import java.util.Map;import com.gargoylesoftware.htmlunit.BrowserVersion;import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;import com.gargoylesoftware.htmlunit.WebClient;import com.gargoylesoftware.htmlunit.html.DomElement;import com.gargoylesoftware.htmlunit.html.DomNodeList;import com.gargoylesoftware.htmlunit.html.HtmlDivision;import com.gargoylesoftware.htmlunit.html.HtmlPage;import com.gargoylesoftware.htmlunit.html.HtmlSpan;import com.gargoylesoftware.htmlunit.html.HtmlTable;/** * Hello world! * */public class HtmlUnit1 {    public static void main(String[] args) {        Map<String, Integer> map = getHighIP.getInstance().map;        getHtml(map);    }    public static void getHtml(Map<String, Integer> ips) {        Object[] array = ips.keySet().toArray();        String ip = array[0].toString();        System.out.println(ip+"ip and port"+ips.get(ip));        WebClient webclient = new WebClient(BrowserVersion.FIREFOX_52,ip,ips.get(ip));        webclient.getOptions().setTimeout(10000);        try {            HtmlPage page = webclient.getPage("http://www.java1234.com/");           //System.out.println("网页内容："+page.asXml());            HtmlDivision htmlDivision = page.getHtmlElementById("navMenu");            //System.out.println("网页内容txt："+htmlDivision.asText());            DomNodeList<DomElement> aList = page.getElementsByTagName("a");             for (DomElement domElement : aList) {                 System.out.println("通过标签获取："+domElement.asXml());            }            HtmlSpan item = (HtmlSpan) page.getByXPath("//div[@id='navMenu'][1]/ul/li[2]/a/span").get(0); // xpath方式            System.out.println("xpath方式获取某个元素值："+item.asText());        } catch (FailingHttpStatusCodeException e) {            // TODO Auto-generated catch block            System.out.println("+++++++++++++++++++++++++++++++++++++++=");            ips.remove(ip);            getHtml(ips);        } catch (MalformedURLException e) {            // TODO Auto-generated catch block            System.out.println("________________________");            ips.remove(ip);            getHtml(ips);        } catch (Exception e) {            ips.remove(ip);            System.out.println("有异常啊！");            getHtml(ips);            // TODO Auto-generated catch block        }finally {            webclient.close();        }    }}

通过单例模式爬取网站http://www.xicidaili.com/上的高匿IP放到Map中

package com.myactiviti.HelloActiviti;import java.io.IOException;import java.net.MalformedURLException;import java.util.ArrayList;import java.util.HashMap;import java.util.List;import java.util.Map;import com.gargoylesoftware.htmlunit.BrowserVersion;import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;import com.gargoylesoftware.htmlunit.WebClient;import com.gargoylesoftware.htmlunit.html.DomElement;import com.gargoylesoftware.htmlunit.html.DomNodeList;import com.gargoylesoftware.htmlunit.html.HtmlDivision;import com.gargoylesoftware.htmlunit.html.HtmlPage;import com.gargoylesoftware.htmlunit.html.HtmlSpan;import com.gargoylesoftware.htmlunit.html.HtmlTable;/** * Hello world! * */public class getHighIP {    Map<String, Integer> map = new HashMap<String, Integer>();    private static getHighIP instance=null;    public static synchronized getHighIP getInstance(){        if(instance==null){               instance=new getHighIP();        }        return instance;    }    private getHighIP(){        WebClient webclient = new WebClient(BrowserVersion.FIREFOX_52);        //webclient.getOptions().setJavaScriptEnabled(false);//关闭js        //webclient.getOptions().setCssEnabled(false);//关闭css        //获取高匿        try {            HtmlPage ipPage = webclient.getPage("http://www.xicidaili.com/");            HtmlTable htmlTable = ipPage.getHtmlElementById("ip_list");            for (int i = 2; i < 10; i++) {                String ip = htmlTable.getCellAt(i, 1).asText();                String port =htmlTable.getCellAt(i, 2).asText();                map.put(ip, Integer.parseInt(port));            }            //System.out.println(ips);        } catch (FailingHttpStatusCodeException e1) {            // TODO Auto-generated catch block            e1.printStackTrace();        } catch (MalformedURLException e1) {            // TODO Auto-generated catch block            e1.printStackTrace();        } catch (IOException e1) {            // TODO Auto-generated catch block            e1.printStackTrace();        }finally {            webclient.close();        }    }}