网络爬虫ip代理服务器【程序样例】

爬虫有的时候会遇到被禁ip的情况，这个时候你可以找一下代理网站，抓取一下ip，来进行动态的轮询就没问题了，也可以用别人做好的第三方ip代理平台，比如说crawlera，crawlera是一个利用代理IP地址池来做分布式下载的第三方平台。【具体介绍请看这篇博客：http://blog.csdn.net/djd1234567/article/details/51741557】

package daili;import java.io.BufferedReader;import java.io.IOException;import java.io.InputStream;import java.io.InputStreamReader;import java.io.UnsupportedEncodingException;import java.net.InetSocketAddress;import java.net.MalformedURLException;import java.net.Proxy;import java.net.URL;import java.net.URLConnection;/* * author:合肥工业大学 管院学院 钱洋  *[e-mail] *博客地址:http://blog.csdn.net/qy20115549/*/public class GetHtml {    public static void main(String[] args) throws UnsupportedEncodingException {        //输入代理ip，端口，及所要爬取的url        gethtml("183.136.217.74",8080,"http://club.autohome.com.cn/bbs/forum-c-2533-1.html?orderby=dateline&qaType=-1");    }    public static String gethtml(String ip,int port,String url) throws UnsupportedEncodingException{        URL url1 = null;        try {            url1 = new URL(url);        } catch (MalformedURLException e1) {            e1.printStackTrace();        }        InetSocketAddress addr = null;        //代理服务器的ip及端口        addr = new InetSocketAddress(ip, port);        Proxy proxy = new Proxy(Proxy.Type.HTTP, addr); // http proxy        InputStream in = null;        try {            URLConnection conn = url1.openConnection(proxy);            conn.setConnectTimeout(3000);            in = conn.getInputStream();        } catch (Exception e) {            System.out.println("ip " + " is not aviable");//异常IP        }        String s = convertStreamToString(in);        System.out.println(s);        return s;    }    public static String convertStreamToString(InputStream is) throws UnsupportedEncodingException {        if (is == null)            return "";        BufferedReader reader = new BufferedReader(new InputStreamReader(is,"gb2312"));        StringBuilder sb = new StringBuilder();        String line = null;        try {            while ((line = reader.readLine()) != null) {                sb.append(line + "/n");            }        } catch (IOException e) {            e.printStackTrace();        } finally {            try {                is.close();            } catch (IOException e) {                e.printStackTrace();            }        }        return sb.toString();    }}

如下图，便可以抓取到url对应的html内容。