开始学习——爬取微博页面的html

今天开始正式着手写爬取微博搜索的爬虫，上个星期，找了很多资料，也尝试了很多，想了很多，这个星期开始一点一点的写吧，新手入门，从最简单的开始，先爬取一个微博页面的Html。

package sina_weibo;//根据Demo4获取微博的Html文件import java.awt.Desktop;import java.io.IOException;import java.net.URI;import java.net.URISyntaxException;import org.apache.http.client.ClientProtocolException;import org.apache.http.client.config.CookieSpecs;import org.apache.http.client.config.RequestConfig;import org.apache.http.client.methods.CloseableHttpResponse;import org.apache.http.client.methods.HttpGet;import org.apache.http.config.Registry;import org.apache.http.config.RegistryBuilder;import org.apache.http.cookie.Cookie;import org.apache.http.cookie.CookieOrigin;import org.apache.http.cookie.CookieSpec;import org.apache.http.cookie.CookieSpecProvider;import org.apache.http.cookie.MalformedCookieException;import org.apache.http.impl.client.CloseableHttpClient;import org.apache.http.impl.client.HttpClients;import org.apache.http.impl.cookie.BestMatchSpecFactory;import org.apache.http.impl.cookie.BrowserCompatSpec;import org.apache.http.impl.cookie.BrowserCompatSpecFactory;import org.apache.http.protocol.HttpContext;import org.apache.http.util.EntityUtils;@SuppressWarnings("deprecation")public class getHtml {    /**     * 用默认浏览器打开指定网址     * @param url     * @throws URISyntaxException      * @throws IOException      */    public void runBroswer(String url) throws URISyntaxException, IOException {          Desktop desktop = Desktop.getDesktop();          if (Desktop.isDesktopSupported() && desktop.isSupported(Desktop.Action.BROWSE)) {              URI uri = new URI(url);              desktop.browse(uri);            }    }    /**     * 由url得到html     * @param url     * @return html     * @throws URISyntaxException     * @throws ClientProtocolException     * @throws IOException     */    public String getHTML(String url) throws URISyntaxException, ClientProtocolException, IOException {        CookieSpecProvider easySpecProvider = new CookieSpecProvider() {            @SuppressWarnings("deprecation")            public CookieSpec create(HttpContext context) {                return new BrowserCompatSpec() {                    @Override                    public void validate(Cookie cookie, CookieOrigin origin)                            throws MalformedCookieException {                    }                };            }        };        @SuppressWarnings("deprecation")        Registry<CookieSpecProvider> r = RegistryBuilder                .<CookieSpecProvider> create()                .register(CookieSpecs.BEST_MATCH, new BestMatchSpecFactory())                .register(CookieSpecs.BROWSER_COMPATIBILITY,                        new BrowserCompatSpecFactory())                .register("easy", easySpecProvider).build();        RequestConfig requestConfig = RequestConfig.custom()                .setCookieSpec("easy")                .setSocketTimeout(1000)//设置socket超时时间                .setConnectTimeout(1000)//设置connect超时时间                .build();        CloseableHttpClient httpClient = HttpClients.custom()                .setDefaultCookieSpecRegistry(r)                .setDefaultRequestConfig(requestConfig).build();        HttpGet httpGet = new HttpGet(url);        httpGet.setConfig(requestConfig);        String html = "html获取失败";//用于验证是否正常取到html        try{            CloseableHttpResponse response = httpClient.execute(httpGet);            html = EntityUtils.toString(response.getEntity());            //System.out.println(html);//打印返回的html        } catch(IOException e){            System.out.println("****连接超时，程序自动重连****");        }        return html;    }    public String get () throws IOException, URISyntaxException, IOException{        getHtml html = new getHtml();        String url = "http://weibo.com/1642088277/C8P1zpVDP";        String data = "";        html.runBroswer(url);        data = html.getHTML(url);        return data;    }    public static void main(String[] args) throws IOException, URISyntaxException{        getHtml test = new getHtml();        System.out.println(test.get());    }}

返回的即为一个微博页面的Html，之后对其用jsoup进行解析处理。