java爬取网页源代码，解析

来源：互联网发布：纸模软件apk 编辑：程序博客网时间：2024/06/04 18:18

1.搜索词的地址采用模拟地址方法（通过分析搜索引擎的参数得到，如百度），然后将搜索词加到模拟的地址中。

2.函数的输入参数是模拟地址。

String query = URLEncoder.encode("何青松", "UTF-8");String url="http://www.baidu.com/s?wd="+query+"&pn="+p*10+"&tn=baiduhome_pg&ie=utf-8"public void MakeQuery(String domain) {  try {   HttpClient httpClient = new HttpClient();   GetMethod getMethod = new GetMethod(domain);      //System.out.println("*************************************************************");   //System.out.println(getMethod);   try{    httpClient.executeMethod(getMethod);    }catch(Exception e){     System.out.println("网络问题");    }   getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER,     new DefaultHttpMethodRetryHandler());   int statusCode = httpClient.executeMethod(getMethod);   if (statusCode != HttpStatus.SC_OK) {    System.err.println("Method failed: "      + getMethod.getStatusLine());   }      byte[] responseBody = getMethod.getResponseBody();      //System.out.println("*************************************************************");   //System.out.println(responseBody);         String response = new String(responseBody, "UTF-8");      //System.out.println("*************************************************************");   //System.out.println(response);      //Jsoup解析html   Document doc=Jsoup.parse(response);   //System.out.println("*************************************************************");   //System.out.println(doc);     Elements contents=doc.getElementsByClass("f");   for(Element content:contents){    Element links = content.getElementsByTag("a").first();     String linkHref = links.attr("href");//链接     String linkText = links.text();//摘要     FoursearchZH.map.put(linkHref, linkText);          System.out.println("------------------");     System.out.println(linkHref);        System.out.println(linkText);    }  } catch (Exception e) {   System.err.println("Something went wrong...");   e.printStackTrace();  }  }