Apache HttpClient4.5(三)
来源:互联网 发布:淘宝如何做展现词 编辑:程序博客网 时间:2024/05/16 11:15
转载自:
http://blog.csdn.net/z69183787/article/details/78039601
http://blog.csdn.net/z69183787/article/details/77964021
HttpClient中的RequestConfig在使用时:
RequestConfig requestConfig = RequestConfig.custom() .setConnectionRequestTimeout(1000) .setSocketTimeout(1000) .setConnectTimeout(1000) .build();
有三个超时:connectionRequestTimeout
、connectTimeout
、socketTimeout
,它们各自含义: connectTimeOut
:指建立连接的超时时间; connectionRequestTimeOut
:指从连接池获取到连接的超时时间,如果是非连接池的话,该参数暂时没有发现有什么用处。 socketTimeOut
:指客户端和服务进行数据交互的时间,是指两者之间如果两个数据包之间的时间大于该时间则认为超时,而不是整个交互的整体时间,比如如果设置1秒超时,如果每隔0.8秒传输一次数据,传输10次,总共8秒,这样是不超时的。而如果任意两个数据包之间的时间超过了1秒,则超时。
更深理解可以参照:http://www.baeldung.com/httpclient-connection-management
RetryHandler重试方式ServiceUnavailableRetryStrategy
并且可以自定义retry时间的间隔:
public static CloseableHttpClient getHttpClient() { ServiceUnavailableRetryStrategy serviceUnavailableRetryStrategy = new ServiceUnavailableRetryStrategy() { /** * retry逻辑 */ @Override public boolean retryRequest(HttpResponse response, int executionCount, HttpContext context) { if (executionCount <= 3) return true; else return false; } /** * retry间隔时间 */ @Override public long getRetryInterval() { return 2000; } }; PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); cm.setMaxTotal(MAX_TOTAL); cm.setDefaultMaxPerRoute(MAX_PERROUTE); CloseableHttpClient httpClient = HttpClients.custom().setRetryHandler(new DefaultHttpRequestRetryHandler()) .setConnectionManager(cm).build(); return httpClient; }
除此之外,还有StandardHttpRequestRetryHandler
、DefaultHttpRequestRetryHandler
,或则自定义,实现HttpRequestRetryHandler
即可。
两个主机建立连接的过程是很复杂的一个过程,涉及到多个数据包的交换,并且也很耗时间。Http连接需要的三次握手开销很大,这一开销对于比较小的http消息来说更大。但是如果直接使用已经建立好的http连接,这样花费就比较小,吞吐率更大。
httpclient的连接池中只有一个httpclient的实例,可以看看CloseableHttpClient
和PoolingHttpClientConnectionManager
的源码,会发现httpclient实例通过execute执行get或post获取连接的时候,会通过实例关联的connectionManager的connect()来建立连接,这里的connectionManager是一个连接池的实现, 因此connect方法调用的是PoolingHttpClientConnectionManager重写的实现,这就是一个连接池建立连接的实现,具体看源码。
简单示例:
public class HttpClientUtil { static final int timeOut = 10 * 1000; private static CloseableHttpClient httpClient = null; private final static Object syncLock = new Object(); private static void config(HttpRequestBase httpRequestBase) { // 设置Header等 // httpRequestBase.setHeader("User-Agent", "Mozilla/5.0"); // httpRequestBase // .setHeader("Accept", // "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"); // httpRequestBase.setHeader("Accept-Language", // "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3");// "en-US,en;q=0.5"); // httpRequestBase.setHeader("Accept-Charset", // "ISO-8859-1,utf-8,gbk,gb2312;q=0.7,*;q=0.7"); // 配置请求的超时设置 RequestConfig requestConfig = RequestConfig.custom() .setConnectionRequestTimeout(timeOut) .setConnectTimeout(timeOut).setSocketTimeout(timeOut).build(); httpRequestBase.setConfig(requestConfig); } public static CloseableHttpClient getHttpClient(String url) { String hostname = url.split("/")[2]; int port = 80; if (hostname.contains(":")) { String[] arr = hostname.split(":"); hostname = arr[0]; port = Integer.parseInt(arr[1]); } if (httpClient == null) { synchronized (syncLock) { if (httpClient == null) { httpClient = createHttpClient(200, 40, 100, hostname, port); } } } return httpClient; } public static CloseableHttpClient createHttpClient(int maxTotal, int maxPerRoute, int maxRoute, String hostname, int port) { ConnectionSocketFactory plainsf = PlainConnectionSocketFactory .getSocketFactory(); LayeredConnectionSocketFactory sslsf = SSLConnectionSocketFactory .getSocketFactory(); Registry<ConnectionSocketFactory> registry = RegistryBuilder .<ConnectionSocketFactory> create().register("http", plainsf) .register("https", sslsf).build(); PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager( registry); // 将最大连接数增加 cm.setMaxTotal(maxTotal); // 将每个路由基础的连接增加 cm.setDefaultMaxPerRoute(maxPerRoute); HttpHost httpHost = new HttpHost(hostname, port); // 将目标主机的最大连接数增加 cm.setMaxPerRoute(new HttpRoute(httpHost), maxRoute); // 请求重试处理 HttpRequestRetryHandler httpRequestRetryHandler = new HttpRequestRetryHandler() { public boolean retryRequest(IOException exception, int executionCount, HttpContext context) { if (executionCount >= 5) {// 如果已经重试了5次,就放弃 return false; } if (exception instanceof NoHttpResponseException) {// 如果服务器丢掉了连接,那么就重试 return true; } if (exception instanceof SSLHandshakeException) {// 不要重试SSL握手异常 return false; } if (exception instanceof InterruptedIOException) {// 超时 return false; } if (exception instanceof UnknownHostException) {// 目标服务器不可达 return false; } if (exception instanceof ConnectTimeoutException) {// 连接被拒绝 return false; } if (exception instanceof SSLException) {// SSL握手异常 return false; } HttpClientContext clientContext = HttpClientContext .adapt(context); HttpRequest request = clientContext.getRequest(); // 如果请求是幂等的,就再次尝试 if (!(request instanceof HttpEntityEnclosingRequest)) { return true; } return false; } }; CloseableHttpClient httpClient = HttpClients.custom() .setConnectionManager(cm) .setRetryHandler(httpRequestRetryHandler).build(); return httpClient; } private static void setPostParams(HttpPost httpost, Map<String, Object> params) { List<NameValuePair> nvps = new ArrayList<NameValuePair>(); Set<String> keySet = params.keySet(); for (String key : keySet) { nvps.add(new BasicNameValuePair(key, params.get(key).toString())); } try { httpost.setEntity(new UrlEncodedFormEntity(nvps, "UTF-8")); } catch (UnsupportedEncodingException e) { e.printStackTrace(); } } public static String post(String url, Map<String, Object> params) throws IOException { HttpPost httppost = new HttpPost(url); config(httppost); setPostParams(httppost, params); CloseableHttpResponse response = null; try { response = getHttpClient(url).execute(httppost, HttpClientContext.create()); HttpEntity entity = response.getEntity(); String result = EntityUtils.toString(entity, "utf-8"); EntityUtils.consume(entity); return result; } catch (Exception e) {// e.printStackTrace(); throw e; } finally { try { if (response != null) response.close(); } catch (IOException e) { e.printStackTrace(); } } } public static String get(String url) { HttpGet httpget = new HttpGet(url); config(httpget); CloseableHttpResponse response = null; try { response = getHttpClient(url).execute(httpget, HttpClientContext.create()); HttpEntity entity = response.getEntity(); String result = EntityUtils.toString(entity, "utf-8"); EntityUtils.consume(entity); return result; } catch (IOException e) { e.printStackTrace(); } finally { try { if (response != null) response.close(); } catch (IOException e) { e.printStackTrace(); } } return null; } public static void main(String[] args) { // URL列表数组 String[] urisToGet = { "http://blog.csdn.net/catoop/article/details/38849497", "http://blog.csdn.net/catoop/article/details/38849497" }; long start = System.currentTimeMillis(); try { int pagecount = urisToGet.length; ExecutorService executors = Executors.newFixedThreadPool(pagecount); CountDownLatch countDownLatch = new CountDownLatch(pagecount); for (int i = 0; i < pagecount; i++) { HttpGet httpget = new HttpGet(urisToGet[i]); config(httpget); // 启动线程抓取 executors .execute(new GetRunnable(urisToGet[i], countDownLatch)); } countDownLatch.await(); executors.shutdown(); } catch (InterruptedException e) { e.printStackTrace(); } finally { System.out.println("线程" + Thread.currentThread().getName() + "," + System.currentTimeMillis() + ", 所有线程已完成,开始进入下一步!"); } long end = System.currentTimeMillis(); System.out.println("consume -> " + (end - start)); } static class GetRunnable implements Runnable { private CountDownLatch countDownLatch; private String url; public GetRunnable(String url, CountDownLatch countDownLatch) { this.url = url; this.countDownLatch = countDownLatch; } @Override public void run() { try { System.out.println(HttpClientUtil.get(url)); } finally { countDownLatch.countDown(); } } }}
- Apache HttpClient4.5(三)
- Apache HttpClient4.5(一)
- Apache HttpClient4.5(二)
- Apache Httpclient4.5学习笔记
- HttpClient4.5 Apache 教程 [记录]
- Apache HttpClient4.5 设置TLS协议
- HttpClient4.5 基础教程 执行请求<三>
- Apache HttpClient4.2入门
- JavaWeb开发-用Apache的HttpClient4.5完成HttpGet请求
- 【JavaWeb开发】用Apache的HttpClient4.5完成HttpGet请求
- 【JavaWeb开发】用Apache的HttpClient4.5完成HttpPost请求
- 【HttpClient4.5中文教程】【第一章 :基础】1.1执行请求(三)
- 【HttpClient4.5中文教程】【第一章 :基础】1.1执行请求(三)
- Httpclient4.5
- httpclient4.5
- HttpClient4.5
- httpClient4.5
- 【HttpClient4.5实训】一.HttpClient4.5模拟浏览器GET请求访问新浪网(非原文教程)
- python实现希尔排序
- Intellij添加Maven模块
- 3.2 Linux C程序解析
- java8之行为参数化(二)
- 习题6-13 电子表格计算器(Spreadsheet Calculator, ACM/ICPC World Finals 1992, UVa215)
- Apache HttpClient4.5(三)
- 第十天 PHP的首次讲课
- jedis连接池及redis封装(util)
- [二分+差分]BZOJ 4326——NOIP2015 运输计划
- sklearn Classification metrics
- 写一个函数返回参数二进制中 1 的个数,提示:使用位操作符:按位与&
- 读取文件乱码问题
- 数据结构栈的一系列操作
- linux 文件系统的建立