httpclient

来源：互联网发布：ubuntu怎么安装python 编辑：程序博客网时间：2024/04/28 07:53

package xiaogang.enif.net;import java.io.IOException;import java.net.Socket;import java.net.UnknownHostException;import java.security.KeyManagementException;import java.security.KeyStore;import java.security.KeyStoreException;import java.security.NoSuchAlgorithmException;import java.security.UnrecoverableKeyException;import java.security.cert.CertificateException;import java.security.cert.X509Certificate;import javax.net.ssl.SSLContext;import javax.net.ssl.TrustManager;import javax.net.ssl.X509TrustManager;import org.apache.http.HttpHost;import org.apache.http.HttpResponse;import org.apache.http.HttpVersion;import org.apache.http.client.methods.HttpGet;import org.apache.http.client.methods.HttpHead;import org.apache.http.client.methods.HttpPost;import org.apache.http.client.methods.HttpUriRequest;import org.apache.http.client.params.HttpClientParams;import org.apache.http.conn.ClientConnectionManager;import org.apache.http.conn.params.ConnManagerParams;import org.apache.http.conn.params.ConnPerRouteBean;import org.apache.http.conn.params.ConnRouteParams;import org.apache.http.conn.scheme.PlainSocketFactory;import org.apache.http.conn.scheme.Scheme;import org.apache.http.conn.scheme.SchemeRegistry;import org.apache.http.conn.ssl.SSLSocketFactory;import org.apache.http.impl.client.DefaultHttpClient;import org.apache.http.impl.conn.tsccm.ThreadSafeClientConnManager;import org.apache.http.params.BasicHttpParams;import org.apache.http.params.HttpConnectionParams;import org.apache.http.params.HttpParams;import org.apache.http.params.HttpProtocolParams;import xiaogang.enif.utils.IOUtils;import android.content.Context;import android.net.Proxy;import android.text.TextUtils;public class HttpManager {    private static final int DEFAULT_MAX_CONNECTIONS = 30;    private static final int DEFAULT_SOCKET_TIMEOUT = 20 * 1000;    private static final int DEFAULT_SOCKET_BUFFER_SIZE = 8192;    private static DefaultHttpClient sHttpClient;    static {        final HttpParams httpParams = new BasicHttpParams();        ConnManagerParams.setTimeout(httpParams, 1000);        ConnManagerParams.setMaxConnectionsPerRoute(httpParams, new ConnPerRouteBean(10));        ConnManagerParams.setMaxTotalConnections(httpParams, DEFAULT_MAX_CONNECTIONS);        HttpProtocolParams.setVersion(httpParams, HttpVersion.HTTP_1_1);        HttpProtocolParams.setContentCharset(httpParams, "UTF-8");        HttpConnectionParams.setStaleCheckingEnabled(httpParams, false);        HttpClientParams.setRedirecting(httpParams, false);        HttpProtocolParams.setUserAgent(httpParams, "Android client");        HttpConnectionParams.setSoTimeout(httpParams, DEFAULT_SOCKET_TIMEOUT);        HttpConnectionParams.setConnectionTimeout(httpParams, DEFAULT_SOCKET_TIMEOUT);        HttpConnectionParams.setTcpNoDelay(httpParams, true);        HttpConnectionParams.setSocketBufferSize(httpParams, DEFAULT_SOCKET_BUFFER_SIZE);        SchemeRegistry schemeRegistry = new SchemeRegistry();        schemeRegistry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80));        try {            KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType());            trustStore.load(null, null);            SSLSocketFactory sf = new MySSLSocketFactory(trustStore);            sf.setHostnameVerifier(SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);            schemeRegistry.register(new Scheme("https", sf, 443));        } catch (Exception ex) {            // do nothing, just keep not crash        }        ClientConnectionManager manager = new ThreadSafeClientConnManager(httpParams, schemeRegistry);        sHttpClient = new DefaultHttpClient(manager, httpParams);    }    private HttpManager() {    }    public static HttpResponse execute(HttpHead head) throws IOException {        return sHttpClient.execute(head);    }    public static HttpResponse execute(HttpHost host, HttpGet get) throws IOException {        return sHttpClient.execute(host, get);    }    public static HttpResponse execute(Context context, HttpGet get) throws IOException {        if (!IOUtils.isWifiAvailable(context) && isWapNetwork()) {            setWapProxy();            return sHttpClient.execute(get);        }        final HttpHost host = (HttpHost)sHttpClient.getParams().getParameter(                ConnRouteParams.DEFAULT_PROXY);        if (host != null) {            sHttpClient.getParams().removeParameter(ConnRouteParams.DEFAULT_PROXY);        }        return sHttpClient.execute(get);    }    private static void setSinaWapProxy() {        final HttpHost para = (HttpHost)sHttpClient.getParams().getParameter(                ConnRouteParams.DEFAULT_PROXY);        if (para != null) {            sHttpClient.getParams().removeParameter(ConnRouteParams.DEFAULT_PROXY);        }        String host = Proxy.getDefaultHost();        int port = Proxy.getDefaultPort();        HttpHost httpHost = new HttpHost(host, port);        HttpParams httpParams = new BasicHttpParams();        httpParams.setParameter(ConnRouteParams.DEFAULT_PROXY, httpHost);    }    public static HttpResponse execute(Context context, HttpUriRequest post) throws IOException {        if (!IOUtils.isWifiAvailable(context) && isWapNetwork()) {            setSinaWapProxy();        }        return sHttpClient.execute(post);    }    public static HttpResponse execute(Context context, HttpPost post) throws IOException {        if (!IOUtils.isWifiAvailable(context) && isWapNetwork()) {            setWapProxy();            return sHttpClient.execute(post);        }        final HttpHost host = (HttpHost)sHttpClient.getParams().getParameter(                ConnRouteParams.DEFAULT_PROXY);        if (host != null) {            sHttpClient.getParams().removeParameter(ConnRouteParams.DEFAULT_PROXY);        }        return sHttpClient.execute(post);    }    private static boolean isWapNetwork() {        final String proxyHost = android.net.Proxy.getDefaultHost();        return !TextUtils.isEmpty(proxyHost);    }    private static void setWapProxy() {        final HttpHost host = (HttpHost)sHttpClient.getParams().getParameter(                ConnRouteParams.DEFAULT_PROXY);        if (host == null) {            final String host1 = Proxy.getDefaultHost();            int port = Proxy.getDefaultPort();            HttpHost httpHost = new HttpHost(host1, port);            sHttpClient.getParams().setParameter(ConnRouteParams.DEFAULT_PROXY, httpHost);        }    }    private static class MySSLSocketFactory extends SSLSocketFactory {        SSLContext sslContext = SSLContext.getInstance("TLS");        public MySSLSocketFactory(KeyStore truststore) throws NoSuchAlgorithmException,        KeyManagementException, KeyStoreException, UnrecoverableKeyException {            super(truststore);            TrustManager tm = new X509TrustManager() {                @Override                public void checkClientTrusted(X509Certificate[] chain, String authType)                        throws CertificateException {                }                @Override                public void checkServerTrusted(X509Certificate[] chain, String authType)                        throws CertificateException {                }                @Override                public X509Certificate[] getAcceptedIssuers() {                    return null;                }            };            sslContext.init(null, new TrustManager[] {                    tm            }, null);        }        @Override        public Socket createSocket(Socket socket, String host, int port, boolean autoClose)                throws IOException, UnknownHostException {            return sslContext.getSocketFactory().createSocket(socket, host, port, autoClose);        }        @Override        public Socket createSocket() throws IOException {            return sslContext.getSocketFactory().createSocket();        }    }}

在一次服务器异常的排查过程当中（服务器异常排查的过程我会另起文章），我们决定使用HttpClient4.X替代HttpClient3.X或者HttpConnection。
为什么使用HttpClient4？主要是HttpConnection没有连接池的概念，多少次请求就会建立多少个IO，在访问量巨大的情况下服务器的IO可能会耗尽。
HttpClient3也有连接池的东西在里头，使用MultiThreadedHttpConnectionManager，大致过程如下：
view plainprint?
MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager();
HttpClient client = new HttpClient(connectionManager);...// 在某个线程中。
GetMethod get = new GetMethod("http://jakarta.apache.org/");
try {
client.executeMethod(get);// print response to stdout
System.out.println(get.getResponseBodyAsStream());
} finally {
// be sure the connection is released back to the connection
managerget.releaseConnection();
}

可以看出来，它的方式与jdbc连接池的使用方式相近，我觉得比较不爽的就是需要手动调用releaseConnection去释放连接。对每一个HttpClient.executeMethod须有一个method.releaseConnection()与之匹配。

HttpClient4在这点上做了改进，使用我们常用的InputStream.close()来确认连接关闭（4.1版本之前使用entity.consumeContent()来确认内容已经被消耗关闭连接）。具体方式如下：
view plainprint?
...HttpClient client = null;InputStream in = null;
try{
client = HttpConnectionManager.getHttpClient();
HttpGet get = new HttpGet();
get.setURI(new URI(urlPath));
HttpResponse response = client.execute(get);
HttpEntity entity =response.getEntity();
if( entity != null ){
in = entity.getContent();
....
}catch (Exception e){
....
}finally{
if (in != null){
try{in.close ();}catch (IOException e){
e.printStackTrace ();
}
}
}

好说完了连接池的使用流程，现在来说一说连接池在使用时最重要的几个参数。我用4.1的版本实现了一个简单的HttpConnectionManager，代码如下：
view plainprint?
public class HttpConnectionManager {

private static HttpParams httpParams;
private static ClientConnectionManager connectionManager;

/**
* 最大连接数
*/
public final static int MAX_TOTAL_CONNECTIONS = 800;
/**
* 获取连接的最大等待时间
*/
public final static int WAIT_TIMEOUT = 60000;
/**
* 每个路由最大连接数
*/
public final static int MAX_ROUTE_CONNECTIONS = 400;
/**
* 连接超时时间
*/
public final static int CONNECT_TIMEOUT = 10000;
/**
* 读取超时时间
*/
public final static int READ_TIMEOUT = 10000;

static {
httpParams = new BasicHttpParams();
// 设置最大连接数
ConnManagerParams.setMaxTotalConnections(httpParams, MAX_TOTAL_CONNECTIONS);
// 设置获取连接的最大等待时间
ConnManagerParams.setTimeout(httpParams, WAIT_TIMEOUT);
// 设置每个路由最大连接数
ConnPerRouteBean connPerRoute = new ConnPerRouteBean(MAX_ROUTE_CONNECTIONS);
ConnManagerParams.setMaxConnectionsPerRoute(httpParams,connPerRoute);
// 设置连接超时时间
HttpConnectionParams.setConnectionTimeout(httpParams, CONNECT_TIMEOUT);
// 设置读取超时时间
HttpConnectionParams.setSoTimeout(httpParams, READ_TIMEOUT);

SchemeRegistry registry = new SchemeRegistry();
registry.register(new Scheme("http", PlainSocketFactory.getSocketFactory(), 80));
registry.register(new Scheme("https", SSLSocketFactory.getSocketFactory(), 443));

connectionManager = new ThreadSafeClientConnManager(httpParams, registry);
}

public static HttpClient getHttpClient() {
return new DefaultHttpClient(connectionManager, httpParams);
}

}

最大连接数、获取连接的最大等待时间、读取超时时间这些配置应该比较容易理解，一般的连接池都会有这些配置，比较特别的是每个路由(route)最大连接数。

什么是一个route？

这里route的概念可以理解为运行环境机器到目标机器的一条线路。举例来说，我们使用HttpClient的实现来分别请求 www.baidu.com 的资源和 www.bing.com 的资源那么他就会产生两个route。

这里为什么要特别提到route最大连接数这个参数呢，因为这个参数的默认值为2，如果不设置这个参数值默认情况下对于同一个目标机器的最大并发连接只有2 个！这意味着如果你正在执行一个针对某一台目标机器的抓取任务的时候，哪怕你设置连接池的最大连接数为200，但是实际上还是只有2个连接在工作，其他剩余的198个连接都在等待，都是为别的目标机器服务的。

怎么样蛋疼吧，我是已经有过血的教训了，在切换到HttpClient4.1的起初没有注意到这个配置，最后使得服务承受的压力反而不如从前了，所以在这里特别提醒大家注意。

HttpClient4.X 教程下载：
http://svn.apache.org/repos/asf/httpcomponents/httpclient/trunk/httpclient-contrib/docs/translated-tutorial/httpclient-tutorial-simplified-chinese.pdf

关于版本的补充：
网友w2449008821提醒之后我才发现在HttpClient4.1+的版本ConnManagerParams已经被Deprecated了。
我在写这篇日志的时候时候的httpclient 版本是4.0.3，从4.0版本之后ConnManagerParams被Deprecated，没想到一个小版本升级会有这么大变化。
官网教程举例了新的连接池设置：
view plainprint?
SchemeRegistry schemeRegistry = new SchemeRegistry();
schemeRegistry.register(
new Scheme("http", 80, PlainSocketFactory.getSocketFactory()));
schemeRegistry.register(
new Scheme("https", 443, SSLSocketFactory.getSocketFactory()));

ThreadSafeClientConnManager cm = new ThreadSafeClientConnManager(schemeRegistry);
// Increase max total connection to 200
cm.setMaxTotalConnections(200);
// Increase default max connection per route to 20
cm.setDefaultMaxPerRoute(20);
// Increase max connections for localhost:80 to 50
HttpHost localhost = new HttpHost("locahost", 80);
cm.setMaxForRoute(new HttpRoute(localhost), 50);

HttpClient httpClient = new DefaultHttpClient(cm);
ConnManagerParams的功能被挪到了 ThreadSafeClientConnManager 和 HttpConnectionParams两个类：
static ConnPerRoutegetMaxConnectionsPerRoute(HttpParams params)
Deprecated. use ThreadSafeClientConnManager.getMaxForRoute(org.apache.http.conn.routing.HttpRoute)
static intgetMaxTotalConnections(HttpParams params)
Deprecated. use ThreadSafeClientConnManager.getMaxTotal()
static longgetTimeout(HttpParams params)
Deprecated. use HttpConnectionParams.getConnectionTimeout(HttpParams)
static voidsetMaxConnectionsPerRoute(HttpParams params, ConnPerRoute connPerRoute)
Deprecated. use ThreadSafeClientConnManager.setMaxForRoute(org.apache.http.conn.routing.HttpRoute, int)
static voidsetMaxTotalConnections(HttpParams params, int maxTotalConnections)
Deprecated. use ThreadSafeClientConnManager.setMaxTotal(int)
static voidsetTimeout(HttpParams params, long timeout)
Deprecated. use HttpConnectionParams.setConnectionTimeout(HttpParams, int)

HttpClient 4：

连接超时：

?
httpclient.getParams().setParameter(CoreConnectionPNames.CONNECTION_TIMEOUT,60000);
// 或者
HttpConnectionParams.setConnectionTimeout(params,6000);

读取超时：

?
httpclient.getParams().setParameter(CoreConnectionPNames.SO_TIMEOUT,60000);
// 或者
HttpConnectionParams.setSoTimeout(params,60000);

HttpClient 3：

连接超时：

?
httpClient.getHttpConnectionManager().getParams().setConnectionTimeout(60000);

读取超时：

?
httpClient.getHttpConnectionManager().getParams().setSoTimeout(60000);