Jsoup爬取全国行政区域信息并入库

1 数据来源

中华人民共和国国家统计局点击打开链接

2 Jsoup解析

//Jsoup.parse()似乎获取不了数据        Document document = Jsoup.connect("http://www.stats.gov.cn/tjsj/tjbz/xzqhdm/201703/t20170310_1471429.html").post();        //在浏览器在分析其DOM结构得到如下解析步骤        Elements select = document.select(".MsoNormal");        List<Area> areas = new ArrayList<>();        int size =0;        Area area =null;        Integer parentCode =null;        for (Element e:select) {            Elements span = e.select("span");             size = span.size();                area = new Area();                if(size==3){//是省级或直辖市                    ////比较坑,String s="110000    ";拿到的文本内容如s,直接用String.trim()出来的字符串还是无法解析成数字所以自己写了一个                    parentCode=Integer.parseInt(StringUtil.trim(span.get(0).text().trim()));                    System.out.println(span.get(0).text().trim()+"|");                    area.setCode(parentCode);                    area.setName(span.get(2).text().trim());                    System.out.println(span.get(2).text().trim()+"|");                }                else if(size==4){//是省级下面                    area.setParentCode(parentCode);                    area.setCode(Integer.parseInt(StringUtil.trim(span.get(1).text().trim())));                    System.out.println(span.get(1).text().trim()+"|");                    area.setName(span.get(3).text().trim());                    System.out.println(span.get(3).text().trim()+"|");                }                areas.add(area);                //System.out.println(span.toString());        }        System.out.println("总共解析到"+areas.size()+"个地区数据");        /**         * 入库,建表和数据库相关操作不再缀述         */        java.sql.Connection connection = MyTest.getConnection();        connection.setAutoCommit(false);        PreparedStatement preparedStatement = connection.prepareStatement("insert into area values(?,?,?)");        for (Area a:areas             ) {            preparedStatement.setInt(1,a.getCode());            preparedStatement.setString(2,a.getName());            preparedStatement.setObject(3,a.getParentCode());            preparedStatement.execute();        }        connection.commit();        connection.close();public static String trim(String s){        byte[] bytes = s.getBytes();        StringBuffer stringBuffer =new StringBuffer();        int cnt =0;        for (byte b:bytes){            if(b>=0){                cnt++;            }        }        byte [] newb = new byte[cnt];        cnt=0;        for (int i = 0; i <bytes.length ; i++) {            if(bytes[i]>=0){                newb[cnt++]=bytes[i];            }        }        return new String(newb);    }