JAVA 中线程队列BlockingQueue的使用

这几天在做一个项目，涉及到大批量处理xml报文回执的操作，一开始用的单线程，结果处理了10000多个文件，用了半个多小时还没处理完，后来就考虑多线程，结果发现老发生文件解析冲突，之后就用到了线程队列的东东....

关键代码如下：

package XXXXXX.web.sd.receipt;


import java.io.File;
import java.io.FileInputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;


import jcifs.smb.SmbFile;
import jcifs.smb.SmbFileInputStream;


import org.apache.log4j.Logger;
import org.dom4j.Document;
import org.dom4j.Element;
import org.dom4j.io.SAXReader;


import com.routdata.gzdatachage.logic.common.ILoadDataLogic;
import com.routdata.gzdatachage.util.CommonUtils;
import com.routdata.gzdatachage.util.Const;


/**
 *  报关单回执信息入库
 * @author  
 *  @version Oct 18, 2015
 */

//程序线程入口类
public class DeclareSingleGjReceiptJob implements Runnable {
private ILoadDataLogic loadDataLogic;
public ILoadDataLogic getLoadDataLogic() {
return loadDataLogic;
}


public void setLoadDataLogic(ILoadDataLogic loadDataLogic) {
this.loadDataLogic = loadDataLogic;
}

private final static Logger LOG = Logger.getLogger(DeclareSingleGjReceiptJob.class);
@Override
public void run()  {// 电商企业订单信息申报
try {
while(true){
LOG.info("回执入库任务开始......");
Long downLoadType =Long.parseLong(CommonUtils.findUploadPath(Const.FILE_TPYE_HG, Const.DOWNLOAD_TYPE)); //获取下载文件类型(0 本地 1 远程)
if (downLoadType==0) {
getCustThreadsByDir();  //本地目录
}else {
//getShareCustThreadsByDir();  //共享目录
}
LOG.info("回执入库任务结束......");
Thread.sleep(20000);
}
} catch (Exception e) {
e.printStackTrace();
}
}


//获取客户目录分开线程个数
public void getCustThreadsByDir() throws Exception {
//获取客户发送目录路径
String sendDirPath = CommonUtils.findDownloadPath(Const.FILE_TPYE_HG, Const.DOWNLOAD_DIR);
LocalFile instance = new LocalFile();
File custSendDir = new File(sendDirPath);
instance.Quentstart(20, loadDataLogic);  //初始化20个线程
if (custSendDir != null && custSendDir.isDirectory()) {
if(custSendDir.listFiles() != null && custSendDir.list().length > 0) {
instance.addFilesIgnoreInterrupted(custSendDir.listFiles());
}else {
LOG.info("总署报关单 下载文件路径：" + custSendDir +"目录为空，没有文件可解析");
}
instance.shutdown();
instance.waiting();
}
}


//获取客户目录分开线程个数
public void getShareCustThreadsByDir() throws Exception {
//获取客户发送目录路径
String sendDirPath = "smb:"+CommonUtils.findDownloadPath(Const.FILE_TPYE_HG, Const.DOWNLOAD_DIR);
ShareFile instance = new ShareFile();
SmbFile custSendDir = new SmbFile(sendDirPath);
instance.Quentstart(30, loadDataLogic); //初始化10个线程
if (custSendDir != null && custSendDir.isDirectory()) {
if(custSendDir.listFiles() != null && custSendDir.list().length > 0) {
instance.addFilesIgnoreInterrupted(custSendDir.listFiles());
}else {
LOG.info("总署报关单 下载文件路径：" + custSendDir +"目录为空，没有文件可解析");
}
instance.shutdown();
instance.waiting();
}
}
}

//主要实现类

package com.routdata.gzdatachage.web.sd.receipt;


import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Queue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.LinkedBlockingQueue;


import org.apache.log4j.Logger;
import org.dom4j.Document;
import org.dom4j.Element;
import org.dom4j.io.SAXReader;
import com.routdata.gzdatachage.logic.common.ILoadDataLogic;
import com.routdata.gzdatachage.util.CommonUtils;
import com.routdata.gzdatachage.util.Const;


/**
 * Class description goes here.
 *
 * @author Administrator
 * @since 2016-4-24
 */
public class LocalFile {
private final static Logger LOG = Logger.getLogger(DeclareSingleGjReceiptJob.class);
private static int i = 0;
public static final int THREAD_POOL_SIZE = 2;
private final File POISON = new File("");
private final BlockingQueue<File> files = new LinkedBlockingQueue<File>(1000);
private final ConcurrentLinkedQueue<String> infos = new ConcurrentLinkedQueue<String>();
private Thread[] pool = null;
private volatile boolean running = false;




public void Quentstart(int poolSize, ILoadDataLogic loadDataLogic) {
//this.custDir = custDir;
//this.compCode = compCode;
this.loadDataLogic = loadDataLogic;


pool = new Thread[poolSize];
FileWorker worker = new FileWorker();
LOG.info("新启动风信子线程:"+poolSize);
for(int i=0;i<poolSize;i++){
pool[i] = new Thread(worker,"线程"+(i+1));
pool[i].start();
LOG.info(pool[i].getName()+"------启动");
}
running = true;
}


private class FileWorker implements Runnable {
@Override
public void run() {


File file = null;
try {
while((file=files.take())!= POISON){
LOG.debug(Thread.currentThread().getName()+"准备取数据...");
//while(true){
LOG.info(Thread.currentThread().getName()+"将数据"+file.getName()+"从队列中取出，目前数据总量为："+files.size());
try {
doWork(file);
} catch (Exception e) {
onException(e,file);
}
}
files.put(POISON);
} catch (InterruptedException e) {
}




}


private void onException(Exception e, File file) {
e.printStackTrace();
}


private int doWork(File readFile) {
//TODO 你要做的事情
}
} 




public void addFilesIgnoreInterrupted(File[] files) {
for(File file : files){
try {
LOG.debug(Thread.currentThread().getName()+"准备放数据!");
this.files.put(file); 
LOG.debug(Thread.currentThread().getName()+"已经放了数据!"+file.getName());
} catch (InterruptedException e) {
}
}
}


public void shutdown(){
try {
if(running){
running = false;
files.put(POISON);
}
} catch (Exception e) {
}
}


public void waiting(){
if(running || !files.contains(POISON)){
shutdown();
//throw new IllegalStateException("You must call shutdown() function before.");
} 
for(Thread t : pool){
try {
t.join();
} catch (InterruptedException e) {
}
}
}


public Queue<String> getInfos(){
return infos;
}


public static final Logger logger = Logger
.getLogger(LocalFile.class);


private ILoadDataLogic loadDataLogic;


/**
* @return the loadDataLogic
*/
public ILoadDataLogic getLoadDataLogic() {
return loadDataLogic;
}
/**
* @param loadDataLogic the loadDataLogic to set
*/
public void setLoadDataLogic(ILoadDataLogic loadDataLogic) {
this.loadDataLogic = loadDataLogic;
}
private String threadNo;
//private String custDir;//客户目录
// private String compCode;//客户备案号
/*public String getCustDir() {
return custDir;
}


public void setCustDir(String custDir) {
this.custDir = custDir;
}*/

public String getThreadNo() {
return threadNo;
}


public void setThreadNo(String threadNo) {
this.threadNo = threadNo;
}
}

===============================================以上看不懂没关系，下面有个例子，拷过去看一下就明白了 执行main方法===========================================================================

package test;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;


public class Quene {
    public static void main(String[] args) {
        final BlockingQueue queue = new ArrayBlockingQueue(100);
        for(int i=0;i<5;i++){
            new Thread("线程"+i){
                public void run(){
                 //   while(true){
                        try {
                        long time=(long)(Math.random()*1000);
                            Thread.sleep(time);
                            System.out.println(Thread.currentThread().getName() + "准备放数据!");                            
                            queue.put(time);
                            System.out.println(Thread.currentThread().getName() + "已经放了数据:"+time +                             
                                        "队列目前有" + queue.size() + "个数据");
                        } catch (InterruptedException e) {
                            e.printStackTrace();
                        }


                   // }
                }
                
            }.start();
        }
        for(int i=0;i<2;i++){
        new Thread("线程"+i){
            public void run(){
                while(true){
                    try {
                        //将此处的睡眠时间分别改为100和1000，观察运行结果
                        Thread.sleep(200);
                        System.out.println(Thread.currentThread().getName() + "准备取数据!");
                        Object ta=queue.take();
                        System.out.println(Thread.currentThread().getName() + "已经取走数据:"+ta +                             
                                "队列目前有" + queue.size() + "个数据");                    
                    } catch (InterruptedException e) {
                        e.printStackTrace();
                    }
                }
            }
            
        }.start();  
        }
    }
}