多线程的PipeLine实现实例

多生产者多消费者的Blocking Queue

/** * A blocking queue which wraps {@code ArrayBlockingQueue}, and with the * following features.  * <li>Use null object as end of queue. * <li>Supports muti-producers for the queue. Only if all the producers put a position-pill to * the queue, the queue will reach end of life. *  * @param <T> *            element type */public class Queue<T> {        private ArrayBlockingQueue<Element<T>> queue = null;        private AtomicInteger lifeValue ;           private static int DEFAULT_LIFE_VALUE = 1;        /**     *      * @param capacity The queue size     * @param lifeValue It is set as the number of producer of the queue as a rule     */    public Queue(int capacity, int lifeValue) {        this.queue = new ArrayBlockingQueue<Element<T>>(capacity);        this.lifeValue = new AtomicInteger(lifeValue);    }           public Queue(int capacity) {        this( capacity, DEFAULT_LIFE_VALUE );    }        /**     * Put an position-pill to the queue.       * @throws InterruptedException     */    @SuppressWarnings("unchecked")    public void putEnd() throws InterruptedException{        int life = this.lifeValue.decrementAndGet();        if(life <= 0 ){            this.queue.put(Element.NULL);        }    }        /**        * Put an element to the queue     * @param element     * @throws InterruptedException     */    public void put(T element ) throws InterruptedException{        try {            queue.put(new Element<T>(element));        } catch (InterruptedException e) {            this.queue.clear();            throw e;        }    }        /**     * Take an element from the head of queue.      * If queue is empty, the operation will be blocked.     *      * @return a non-nullable object if queue if not empty, or null if the queue is ended     * @throws InterruptedException      */    public T take() throws InterruptedException{        Element<T> element = this.queue.take();        if( Element.NULL == element){            this.queue.put(Element.NULL);        }        return element.getObject();            }         /**     * Take elements in batch from the queue. The operation will be blocked for a non-ended queue.     * The fetched list size should be equal to count unless the queue is ended     * @param count the desired element size to fetch     * @return a List which contains the batch fetched element, or null if the queue is ended and empty     * @throws InterruptedException     */    public List<T> batchTake(int count) throws InterruptedException{                List<T> result = new ArrayList<T>(count);        int size = 0;        while(size < count){            T object = this.take();            if(object == null){                break;            }else{                                     result.add(object);                size ++;            }        }        if(result.isEmpty()){            return null;        }        return result;    }             /**     * Wrapper of object, which supports null      *     * @param <T>     */    private static class Element<T> {        @SuppressWarnings({"unchecked","rawtypes"})        static Element NULL = new Element(null);        private T object;        Element(T o) {            this.object = o;        }        T getObject() {            return this.object;        }    }}

PipeLine调度框架

/** *  * An executor that uses multiple threads to run a job which involve muti-steps pipeline like source > intermediate steps > output. *  * <li> Steps can be executed concurrently, and they communicates with each other by {@code Queue} * <li> A step can be executed with muti-thread concurrently, and the thread number can be configured *  * @author yuliang * * @param <S> source type * @param <O> output type */@SuppressWarnings("rawtypes")public class ParallelPipeLineExecutor<S, O> {    protected static final Logger LOGGER = Logger.getLogger(FourStepPiplineExecutor.class);        private String pipeLineName;    private ISource<S> source;    private Queue<S> sourceQueue;    private List<Queue> queues = new LinkedList<Queue>();    private List<PipeLineStep> processorSteps = new LinkedList<PipeLineStep>();            public ParallelPipeLineExecutor() {    }        public void setName(String pipeLineName) {        this.pipeLineName = pipeLineName;    }    public void setSource(ISource<S> source, int sourceQueueSize) {        this.source = source;        this.sourceQueue = new Queue<S>(sourceQueueSize);    }        private String generateStepProcessorName(String stepName, int stepSequence){        return this.pipeLineName+"."+stepName+"#"+stepSequence;    }    private <T1, T2> void addNextStep(String name, ProcessorProvider<T1, T2> provider, int threadNum, int outQueueSize) {        String stepName = (name == null) ? "Step" + (processorSteps.size()+1) : name;        PipeLineStep step = new PipeLineStep(stepName, provider, threadNum);        this.processorSteps.add(step);        if (outQueueSize > 0) {            Queue<T1> outQueue = new Queue<T1>(outQueueSize, threadNum);            queues.add(outQueue);        }    }    /**     * Add next step for the pipeline     *      * @param provider     *            a {@code Processor} provider     * @param threadNum     *            the thread number that run the step     * @param outQueueSize     *            the size of queue where the step output to     */    public <T1, T2> void addNextStep(ProcessorProvider<T1, T2> provider, int threadNum, int outQueueSize) {        Preconditions.checkArgument(threadNum > 0, "threadNum: " + threadNum + "should be positive");        Preconditions.checkArgument(outQueueSize >= threadNum, "outQueueSize: " + outQueueSize                + "should be greater than threadNum:" + threadNum);        if (provider instanceof IdentityProcessor.Provider) {            LOGGER.info("Skip a step because it use Indentity.Provider");        } else {                        this.addNextStep(null, provider, threadNum, outQueueSize);        }    }    /**     * Add sink step for the pipeline     *      * @param provider     *            provider a {@code Processor} provider     * @param threadNum     *            the thread number that run the step     */    public void addSinks(ProcessorProvider<O, Void> provider, int threadNum) {        Preconditions.checkArgument(threadNum > 0, "threadNum: " + threadNum + "should be positive");        this.addNextStep("Output", provider, threadNum, 0);    }    /**     * Launch the execution of the pipeline and wait until it is finished or exception occurs     */    public void launchAndWait() {        int workersCount = 0;        final ExecutorService executorService = Executors.newCachedThreadPool();        ExecutorCompletionService<Worker> workerCompletionService = new ExecutorCompletionService<Worker>(                executorService);        try {            //submit source worker            Worker sourceWorker = new SourceWorker<S>(this.source, this.sourceQueue);                        sourceWorker.setName(this.generateStepProcessorName("sourceWorker", workersCount));            workerCompletionService.submit(sourceWorker);            workersCount++;            //submit processor workers including sink workers            Queue inQueue = sourceQueue;            Iterator<Queue> queuesIter = queues.iterator();            for (PipeLineStep step : processorSteps) {                final Queue outQueue = queuesIter.hasNext() ? queuesIter.next() : null;                int stepSequence = 0;                for (IProcessor processor : step.generateProcessors()) {                    stepSequence ++;                     ProcessWorker worker = new ProcessWorker(inQueue, processor, outQueue);                    worker.setName(this.generateStepProcessorName(step.getName(), stepSequence));                    workerCompletionService.submit(worker);                    workersCount++;                }                inQueue = outQueue;            }                         //wait for completion or exception            while (workersCount > 0) {                Worker w = workerCompletionService.take().get();                if (w.getException() != null) {                    LOGGER.error(w + " finished with exception");                    throw w.getException();                } else {                    LOGGER.info(workersCount + " finished ");                }                workersCount--;                LOGGER.info(workersCount + " threads still running...");            }        } catch (Exception e) {            //Just throw exceptions so that job failure cause be be directly shown in DJS/HAM             throw new RuntimeException(e);        } finally {            executorService.shutdown();        }    }        /**     * A process step in the pipe line     *      */    private static class PipeLineStep {        private String name;        private ProcessorProvider processorProvider;        private int concurrentThreadNum;        public PipeLineStep(String name, ProcessorProvider provider, int threadNum) {            this.name = name;            this.processorProvider = provider;            this.concurrentThreadNum = threadNum;        }        public String getName() {            return name;        }        public List<IProcessor> generateProcessors() throws Exception {            List<IProcessor> processors = new ArrayList<IProcessor>();            for (int i = 0; i < concurrentThreadNum; i++) {                processors.add(processorProvider.get());            }            return processors;        }    }}

Worker类

/** * A {@code Worker} which can pull inputs from {@code inQueue}, process inputs  * and output the results to {@code outQueue} continuously * @author yuliang *  * @param <I> process input type * @param <O> process output type */public class ProcessWorker<I, O> extends Worker {    private Queue<I> inQueue;    private IProcessor<I, O> processor;    private Queue<O> outQueue;    private int batchSize = 1;    public ProcessWorker(Queue<I> inQueue, IProcessor<I, O> processor, Queue<O> outQueue) {        super();        Preconditions.checkArgument(batchSize > 0, "batchSize should be greater than 0");        this.inQueue = inQueue;        this.processor = processor;        this.outQueue = outQueue;        this.batchSize = processor.getBatchSize();    }    @Override    public void work() throws Exception {        try {            if (this.batchSize > 1 && this.processor.isBatchSupported()) {                this.processByBatch();            } else {                this.processOneByOne();            }        } finally {            if (outQueue != null) {                outQueue.putEnd();            }        }    }    /**     * Fetch inputs from {@code inQueue}, and process inputs one by one and put     * results to {@code outQueue}     */    private void processOneByOne() throws Exception {        while (true) {            I in = inQueue.take();            if (in == null) {                break;            }            O o = processor.process(in);            if (o != null && outQueue != null) {                outQueue.put(o);            }        }    }    /**     * Fetch inputs from {@code inQueue}, and process inputs in batch and put     * results to {@code outQueue}     */    private void processByBatch() throws Exception {        while (true) {            List<I> in = inQueue.batchTake(this.batchSize);            if (in == null) {                break;            }            List<O> outs = processor.batchProcess(in);            if (outs != null && outQueue != null) {                for (O o : outs) {                    if (o != null) {                        outQueue.put(o);                    }                }            }        }    }}

/** *  * A source worker is a {@code Worker} which produce source elements  * @author yuliang * * @param <S> source element type */public class SourceWorker <S> extends Worker{    private Queue<S> outQueue;         private ISource<S> source;                public SourceWorker(ISource<S> source, Queue<S> sourceQueue) {        super();        this.source = source;        this.outQueue = sourceQueue;        }        @Override    public void work() throws Exception {        try{            while(true){                                S o = source.nextSource();                if(o == null){                    break;                }                this.outQueue.put(o);            }                    }finally{            this.outQueue.putEnd();                   }    }}