HttpURLConnection抓取数据

来源：互联网发布：交友网站知乎编辑：程序博客网时间：2024/05/29 17:08

当页面中的数据为json格式时可以采取这种方法

public void yjStep(){
          URL url;
          int num=0;
          for (int i = 1; i <=page; i++) {
              try {
                   url = new URL("这里为你要访问的链接 ");

                    String data="action=010102&param={\"PageNo\":\""+i+"\",\"Sjbm\":\"430000\",\"Qymc\":\"\"}";


                    byte[] postDataBytes = data.getBytes("UTF-8");
                    int len = postDataBytes.length;

                    HttpURLConnection conn = (HttpURLConnection)url.openConnection();
                    conn.setRequestMethod("POST");
                    conn.setRequestProperty("Accept","application/json, text/javascript, */*; q=0.01");
                    conn.setRequestProperty("Accept-Encoding","gzip, deflate");
                    conn.setRequestProperty("Accept-Language","zh-CN,zh;q=0.8");
                    conn.setRequestProperty("Cache-Control","no-cache");
                    conn.setRequestProperty("Connection","keep-alive");
                    conn.setRequestProperty("Content-Length", String.valueOf(postDataBytes.length));
                    conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
                   conn.setRequestProperty("Host","jzsgl.coc.gov.cn");
                   conn.setRequestProperty("Origin","http://jzsgl.coc.gov.cn");
                   conn.setRequestProperty("Pragma","no-cache");

                   conn.setRequestProperty("Referer","http://jzsgl.coc.gov.cn/archisearch/cxyjjzs/qylist.aspx?sjbm=430000");
                   conn.setRequestProperty("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.134 Safari/537.36");
                   conn.setRequestProperty("X-Requested-With","XMLHttpRequest");
                   //conn.setRequestProperty("X_REQUESTED_WITH","XMLHttpRequest");

                    conn.setDoOutput(true);
                    conn.getOutputStream().write(data.getBytes());
                    //conn.getOutputStream().flush();

                    Reader in = new BufferedReader(new InputStreamReader(conn.getInputStream(), "UTF-8"));


                    BufferedReader r = new BufferedReader(in);
                    StringBuilder b = new StringBuilder();
                    String line;
                    while((line=r.readLine())!=null) {
                       b.append(line);
                       b.append("\r\n");
                    }

                    String st = b.toString();
                    ObjectMapper mapper = new ObjectMapper();
                   mapper.setPropertyNamingStrategy(PropertyNamingStrategy.CAMEL_CASE_TO_LOWER_CASE_WITH_UNDERSCORES);
                    JsonNode node=mapper.readValue(st,JsonNode.class);
                   JsonNode xx=node.findPath("Data");
                   page=Integer.parseInt(node.findPath("PagerInfo").findValue("pagecount").asText().toString());
                   for (int j = 0; j < xx.size(); j++) {
                       String sjbm = xx.get(j).findValue("sjbm").asText();
                       String qymc = xx.get(j).findValue("qymc").asText();
                       String zclb = "00";
                       int count00 = Integer.parseInt(xx.get(j).findValue("count00").asText());
                       if(count00 >0){
                           List<Map<String,Object>> list = show(sjbm, qymc, zclb);
                           num += list.size();
                           System.out.println(list.size()+"==================");
                       }
                   }
                   System.out.println("------------------第"+i+"页------------------------------");
                  } catch (Exception e) {
                      e.printStackTrace();
                      continue;
                  }
              System.out.println(num+"------------end-------------");
           }
   }

0 0