必须串行执行程序如何提高执行效率之队列方法

场景是meta与index，为1对多，index与user为1对多，程序需要先判断meta表中是否存在逻辑进行表的处理，然后index根据meta_id来进行是否表存在逻辑，然后user表再根据index_id进行表存储；这样逻辑处理的需要串行，这样操作处理数据库效率会非常低。针对这种情况，可以采用三个固定队列的方式，meta队列，index队列，user队列，队列按照先进先出的策略执行，这样就会使一个串行线程，增加缓冲时间。例如meta线程处理完一个后，index线程进行处理，这时又有线程进入了meta队列进行处理了，增加队列错开时间。如下图：

线程图

线程执行流程，以及效率提示执行缩略图

中间index处理的线程，处理metaQueue，增加到indexQueue

public class IndexProducer implements Runnable {    private BlockingQueue<IndexForQueue>  metaqueue ;    private BlockingQueue<VbIndex> indexqueue ;    private VbIndexRepository vbIndexRepository ;    private  int chat_time ;    private int liveStartTime ;    private int vacuateTime ;    private int videoId ;    private long time;    public IndexProducer(BlockingQueue<IndexForQueue> mq,BlockingQueue<VbIndex> iq,VbIndexRepository vbIndexRepository){        this.metaqueue=mq;        this.indexqueue=iq ;        this.vbIndexRepository =vbIndexRepository ;    }    @Override    public void run(){        while (true){            log.info("index producer ,metaqueue size:"+metaqueue.size()+";indexqueue size:"+indexqueue.size());            try{                IndexForQueue ifq = metaqueue.take() ;                VbMeta meta = ifq.getVbMeta() ;                chat_time = ifq.getChat_time() ;                liveStartTime = ifq.getLiveStartTime() ;                vacuateTime = ifq.getVacuateTime() ;                videoId = ifq.getVideoId() ;                time = ifq.getTime() ;                long metaId = meta.getMetaId() ;                int timestamp = chat_time - liveStartTime ;                int offset =  timestamp <0 ? 0:timestamp;                int indextime = (offset/vacuateTime)*vacuateTime;                VbIndex index = vbIndexRepository.queryIndexByVideoTimeMeta(videoId,Long.valueOf(String.valueOf(indextime)),metaId) ;//            boolean idxexist = false ;                if(null==index){                    index = new VbIndex();                    index.setCreateTime(time);                    index.setMetaId(metaId);                    index.setTimestamp(Long.valueOf(String.valueOf(indextime)));                    index.setVideoId(videoId);                    index = vbIndexRepository.save(index) ;                }                indexqueue.put(index);            } catch (InterruptedException e) {                log.error("indexproducer run error:"+e);                e.printStackTrace();            }        }    }}

user入库的线程，处理indexQueue队列

public class IndexConsumer implements Runnable {    private BlockingQueue<VbIndex> indexQueue ;    private VbUserRepository vbUserRepository ;    private VbConfigRepository vbConfigRepository ;    private long time ;    private  int chat_time ;    private int liveStartTime ;    private int uid ;    private int videoId ;    @Value("${lexue.client}")    private String client ;    public IndexConsumer(BlockingQueue<VbIndex> iq,VbUserRepository vbUserRepository,VbConfigRepository vbConfigRepository,long time,int chat_time,int liveStartTime ,int uid,int videoId){        this.indexQueue =iq ;        this.vbUserRepository = vbUserRepository ;        this.vbConfigRepository= vbConfigRepository ;        this.time=time ;        this.chat_time =chat_time ;        this.liveStartTime = liveStartTime ;        this.uid =uid ;        this.videoId = videoId ;    }    @Override    public void run(){        log.info("indexConsumer run,indexQueue size:"+indexQueue.size());        try{            int timestamp = (chat_time - liveStartTime) <0?0:(chat_time - liveStartTime) ;            VbIndex index = indexQueue.take() ;            VbUser user = new VbUser();            user.setCreateTime(time);            user.setIndexId(index.getIndexId());            user.setTimestamp(timestamp);            user.setUid(uid);            user.setVideoId(videoId);            vbUserRepository.save(user) ;            VbConfig vc = vbConfigRepository.findOne(videoId) ;            if(vc==null || "0".equals(vc.getUpdateStatus())){                updateConfig(videoId) ;            }        } catch (InterruptedException e) {            log.error("indexConsumer error:"+e);            e.printStackTrace();        }    }

}

主流程处理：meta为主流程处理完后，放入metaQueue中，index线程是一直循环处理，user线程是固定线程池进行执行

private static Thread indexpro=null ;    ExecutorService userservice = Executors.newFixedThreadPool(10);    /**     * 定时监控线程是否挂掉     */    @Scheduled(cron="0/30 * *  * * ? " )    private void checkThread(){        if(indexpro ==null || !indexpro.isAlive()){            indexpro = new Thread(new IndexProducer(metaQueue,indexQueue,vbIndexRepository)) ;            indexpro.start();            log.warn("indexThread new start");        }    }    /**     * 直播弹幕入库     * @param uid     * @param liveroom     * @param content     * @param chat_time     * @param msg_type     */    public void addLiveBullets(int uid , int liveroom , String content ,int chat_time ,int msg_type ){        log.info("addLiveBullets start:"+new Date());        if(!("".equals(content) || liveroom <1)){            try{                Live live = cacheService.queryLiveByRoom(liveroom) ;                if(live !=null){                    VbMeta meta = vbMetaRepository.queryByContent(content) ;                    long time = System.currentTimeMillis()/1000 ;                    if(null==meta){                        meta = new VbMeta() ;                        meta.setContent(content);                        meta.setContentType(Short.valueOf(String.valueOf(msg_type)));                        meta.setCount(Long.valueOf("1"));                        meta.setCreateTime(time);                        meta.setDisplay(Short.valueOf("0"));                    }else{                        meta.setCount(meta.getCount()+1);                        meta.setUpdateTime(time);                    }                    meta = vbMetaRepository.save(meta) ;                    IndexForQueue ifq = new IndexForQueue() ;                    ifq.setVbMeta(meta);                    ifq.setVideoId(live.getVideoId());                    ifq.setChat_time(chat_time);                    ifq.setLiveStartTime(live.getStartTime());                    ifq.setTime(time);                    ifq.setVacuateTime(vacuateTime);                    metaQueue.put(ifq);                    log.info("meta deal end:"+new Date());                    Thread indexcons = new Thread(new IndexConsumer(indexQueue,vbUserRepository,vbConfigRepository,time,chat_time,live.getStartTime(),uid,live.getVideoId())) ;                    userservice.execute(indexcons);                }else{                    log.info("not query live by liveroom:"+liveroom);                }            }catch (Exception e){                log.error("addLiveBullets error:"+e);            }        }else{            log.info("信息不完整，忽略该信息content："+content);        }        log.info("addLiveBullets end:"+new Date());    }

最后测试的结果，提高效率差不多是3倍的效果，类似于切分了三次操作库。每分钟插入数据3张表，每张表各2000多条数据。后续可能需要再设置结构，进行相对提升。

另一种方案：就是不进行通过db的方式，不进行数据逻辑校验，等数据全部入库后，再进行数据的合并逻辑处理。