使用jsoup爬虫抓取页面

package com.chongdong.log.test;import java.io.IOException;import java.util.HashMap;import java.util.Map;import org.jsoup.Connection;import org.jsoup.Connection.Method;import org.jsoup.Jsoup;import org.jsoup.helper.HttpConnection.Response;import org.jsoup.nodes.Document;import org.jsoup.select.Elements;import org.junit.Test;/** * * 类名称：JsoupTest * 类描述： jsoup 抓取 mitsuku聊天 信息* 创建人：zk * 创建时间：2015-7-20 下午3:52:06 * 修改人：zk * 修改时间：2015-7-20 下午3:52:06 * 修改备注： * 开发进度：* @version 1.0 * */publicclassJsoupTest {public staticvoid postMitSuKu(){ Map<String, String> map =new HashMap<String, String>();/** 表单 提交的参数 * input:how old are you botid:9fa364f2fe345a10 custid:c04f62ad1e044059 faq : http://www.pandorabots.com/botmaster/en/faq#h1 * **/ map.put("input","hi"); map.put("botid","9fa364f2fe345a10");//开启 可在 跳转到聊天框界面 获取 标示 map.put("custid","c04f62ad1e044059");//当一个空请求 即可获取 相当于 标示 Connection conn = Jsoup.connect("http://fiddle.pandorabots.com/pandora/talk-xml");/* conn.header("(Request-Line)", "POST /cgi-bin/login?lang=zh_CN HTTP/1.1");*/ conn.header("Accept","*/*"); conn.header("Accept-Encoding","gzip,deflate,sdch"); conn.header("Accept-Language","zh-CN,zh;q=0.8");/* conn.header("Content-Length", "58");*/ conn.header("Origin","http://www.square-bear.co.uk"); conn.header("Pragma","no-cache"); conn.header("Connection","Keep-Alive"); //必须 填写 表单提交 conn.header("Content-Type","application/x-www-form-urlencoded"); conn.header("Host","fiddle.pandorabots.com"); conn.header("Referer","http://www.square-bear.co.uk/mitsuku/mitsy_retro.swf"); conn.header("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36");try { Response response = (Response) conn.ignoreContentType(true).method(Method.POST).data(map).execute();//response. String json=response.body(); System.out.println(json); }catch (IOException e) {// TODO Auto-generated catch block e.printStackTrace(); } }///////////////////////////////////////////////////////////////////////////////////////////方案2： 通过 开发抓包工具可知 表单的提交方式 应该为http post 此处为 get 方法 不合适/** * 请求英文对话的网页，抓取结果 * @param url * @return */private static StringprocessLogic(String url){ String result =""; try { Document document = Jsoup.connect(url).ignoreContentType(true).ignoreHttpErrors(true) .followRedirects(true).timeout(5000).userAgent("Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/4.0;)").get(); Elements elements = document.select("result that"); result = elements.text(); System.out.println(result); }catch (Exception e) { e.printStackTrace(); result ="ok"; } return result; }public staticvoid main(String[] args) {for (int i =0; i < 100; i++) {// Thread thread=new Thread();//thread.start(); postMitSuKu(); }/* 通过 开发抓包工具可知 表单的提交方式 应该为http post 此处为 get 方法 不合适 String url = "http://fiddle.pandorabots.com/pandora/talk-xml?input=%s&botid=9fa364f2fe345a10&custid=bbbb30debe1bc7f7"; processLogic(url); */ }}