HttpClient4.x 多线程请求

基本过程：使用httpclient发起多线程的请求，将返回的数据临时放在ArrayBlockingQueue中，单独开启一个线程读取ArrayBlockingQueue中的数据并写入磁盘上的文件。

启动请求线程import java.util.concurrent.ArrayBlockingQueue;import java.util.concurrent.BlockingQueue;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;import org.apache.http.client.methods.HttpGet;import org.apache.http.impl.client.CloseableHttpClient;import org.apache.http.impl.client.HttpClients;import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;public class MultiReq {

//数据返回是否完毕标志位public static volatile int status =0;public static BlockingQueue<String> bqueue = new ArrayBlockingQueue<String>(1000);static ExecutorService exec = Executors.newCachedThreadPool();private static String[] urls = {};public void req() {PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();

//最大线程数cm.setMaxTotal(140);CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).build();for(int i=0,n=urls.length;i<n;i++){HttpGet httpget = new HttpGet(urls[i]);exec.execute(new GetThread(httpClient,httpget));}}public static void main(String[] args){MultiReq req = new MultiReq();MultiReq.exec.execute(new getData());req.req();}}

发起请求的线程

package newreq;import java.io.IOException;import java.util.concurrent.ArrayBlockingQueue;import java.util.concurrent.BlockingQueue;import org.apache.http.HttpEntity;import org.apache.http.HttpStatus;import org.apache.http.client.ClientProtocolException;import org.apache.http.client.methods.CloseableHttpResponse;import org.apache.http.client.methods.HttpGet;import org.apache.http.client.protocol.HttpClientContext;import org.apache.http.impl.client.CloseableHttpClient;import org.apache.http.protocol.HttpContext;import org.apache.http.util.EntityUtils;public class GetThread implements Runnable{    private final CloseableHttpClient httpClient;    private final HttpContext context;    private final HttpGet httpget;    public GetThread(CloseableHttpClient httpClient, HttpGet httpget) {        this.httpClient = httpClient;        this.context = HttpClientContext.create();        this.httpget = httpget;    }@Overridepublic void run(){try {            CloseableHttpResponse response = httpClient.execute(                    httpget, context);                        try {                            HttpEntity entity = response.getEntity();                String res = EntityUtils.toString(entity);                if(!"[]".equals(res)){                MultiReq.bqueue.add(res);                }                if(MultiReq.status==0){                if(response.getStatusLine().getStatusCode()!=HttpStatus.SC_OK || entity==null || "[]".equals(res)){                synchronized(httpClient){                MultiReq.status=1;                }                }                }                            } finally {                response.close();            }        } catch (ClientProtocolException ex) {            // Handle protocol errors        } catch (IOException ex) {            // Handle I/O errors        }}}

处理返回的数据

import java.io.BufferedWriter;import java.io.FileWriter;import java.io.IOException;import java.text.DateFormat;import java.text.SimpleDateFormat;import java.util.Date;import java.util.HashMap;import java.util.Map;import com.alibaba.fastjson.JSONArray;import com.alibaba.fastjson.JSONObject;public class getData implements Runnable{);private static BufferedWriter bw = null;static{try {bw =new BufferedWriter(new FileWriter("D:\\file\\0723.txt"));} catch (IOException e) {// TODO Auto-generated catch blocke.printStackTrace();}}@Overridepublic void run(){while(true){if (MultiReq.bqueue.size() > 0 && MultiReq.status == 0) {String readed = MultiReq.bqueue.poll();bw.write(readed);                                        bw.flush();}}}}