HtmlUnit入门

来源：互联网发布：windows 2000系统编辑：程序博客网时间：2024/06/07 03:46

HtmlUnit介绍

htmlunit 是一款开源的java 页面分析工具，读取页面后，可以有效的使用htmlunit分析页面上的内容。项目可以模拟浏览器运行，被誉为java浏览器的开源实现。是一个没有界面的浏览器，运行速度迅速。

入门案例

pom.xml加入依赖

    <dependency>        <groupId>net.sourceforge.htmlunit</groupId>        <artifactId>htmlunit</artifactId>        <version>2.23</version>    </dependency>

获取页面内容

        /**         * 获取页面内容         */        public class HtmlUnitDemo03 {            public static void main(String[] args) {                //实例化web客户端                WebClient webClient = new WebClient(BrowserVersion.FIREFOX_45,"124.206.107.125",3128);//使用代理IP                try {                    //解析获取页面                    HtmlPage page = webClient.getPage("http://www.java1234.com");                    HtmlElement div = page.getHtmlElementById("navMenu");                    System.out.println(div.asXml());                    System.out.println("---------------");                    //获取a标签                    DomNodeList<DomElement> dnList = page.getElementsByTagName("a");                    for (int i = 0; i < dnList.getLength(); i++) {                        DomElement element = dnList.get(i);                        System.out.println(element.asXml());                        System.out.println(element.asText());                    }                    System.out.println("------------------");                    List<HtmlListItem> itemList = (List<HtmlListItem>) page.getByXPath("//div[@id='navMenu']/ul/li");                    System.out.println(itemList.get(0).asXml());                } catch (FailingHttpStatusCodeException e) {                    e.printStackTrace();                } catch (MalformedURLException e) {                    e.printStackTrace();                } catch (ElementNotFoundException e) {                    e.printStackTrace();                } catch (IOException e) {                    e.printStackTrace();                } finally {                    //释放资源                    webClient.close();                }            }        }

取消css,js支持

        /**         * 取消css,js支持         */        public class HtmlUnitDemo04 {            public static void main(String[] args) {                //实例化web客户端                WebClient webClient = new WebClient(BrowserVersion.FIREFOX_45,"124.206.107.125",3128);//使用代理IP                //取消css支持                webClient.getOptions().setCssEnabled(false);                //取消js支持                webClient.getOptions().setJavaScriptEnabled(false);                try {                    //解析获取页面                    HtmlPage page = webClient.getPage("http://www.java1234.com");                    HtmlElement div = page.getHtmlElementById("navMenu");                    System.out.println(div.asXml());                    System.out.println("---------------");                    //获取a标签                    DomNodeList<DomElement> dnList = page.getElementsByTagName("a");                    for (int i = 0; i < dnList.getLength(); i++) {                        DomElement element = dnList.get(i);                        System.out.println(element.asXml());                    }                    System.out.println("------------------");                    List<HtmlListItem> itemList = (List<HtmlListItem>) page.getByXPath("//div[@id='navMenu']/ul/li");                    System.out.println(itemList.get(0).asXml());                } catch (FailingHttpStatusCodeException e) {                    e.printStackTrace();                } catch (MalformedURLException e) {                    e.printStackTrace();                } catch (ElementNotFoundException e) {                    e.printStackTrace();                } catch (IOException e) {                    e.printStackTrace();                } finally {                    //释放资源                    webClient.close();                }            }        }

模拟表单搜索

        /**         * 模拟表单搜索         */        public class HtmlUnitDemo05 {            public static void main(String[] args) {                //实例化web客户端                WebClient webClient = new WebClient(BrowserVersion.FIREFOX_45,"124.206.107.125",3128);//使用代理IP                try {                    //解析获取页面                    HtmlPage page = webClient.getPage("http://blog.java1234.com/index.html");                    //获取搜索表单                    HtmlForm form = page.getFormByName("myform");                    //获取查询文本框                    HtmlTextInput textInput = form.getInputByName("q");                    //获取提交按钮                    HtmlSubmitInput submitInput = form.getInputByName("submitButton");                    //输入文本框                    textInput.setValueAttribute("java");                    //模拟提交(点击)                    HtmlPage resultPage = submitInput.click();                    System.out.println(resultPage.asXml());                } catch (FailingHttpStatusCodeException e) {                    e.printStackTrace();                } catch (MalformedURLException e) {                    e.printStackTrace();                } catch (ElementNotFoundException e) {                    e.printStackTrace();                } catch (IOException e) {                    e.printStackTrace();                } finally {                    //释放资源                    webClient.close();                }            }        }

获取ajax加载的内容

        /**         * 获取ajax加载的内容         */        public class HtmlUnitDemo06 {            public static void main(String[] args) {                //实例化web客户端                WebClient webClient = new WebClient(BrowserVersion.FIREFOX_45);//使用代理IP                try {                    //解析获取页面                    HtmlPage page = webClient.getPage("https://pan.baidu.com/share/home?uk=305605848#category/type=0");                    //等待htmlunit执行js                    Thread.sleep(10000);                    System.out.println("网页html:"+page.asXml());                } catch (FailingHttpStatusCodeException e) {                    e.printStackTrace();                } catch (MalformedURLException e) {                    e.printStackTrace();                } catch (ElementNotFoundException e) {                    e.printStackTrace();                } catch (IOException e) {                    e.printStackTrace();                } catch (InterruptedException e) {                    e.printStackTrace();                } finally {                    //释放资源                    webClient.close();                }            }        }

阅读全文

0 0