zookeeper的barriers和queue简单案例

1、介绍

应该都用过jdk自带的countdownlatch、cyclicbarrier和queue，都知道前者是一个可循环使用的多线程同步栅栏，后者是一个队列，用于常用于异步操作。但在分布式环境下如何做到用barrier进行同步多台机器实例的运行？如何进行队列消费？也许想到了消息队列jmx，或者用redis等分布式nosql缓存做同步，但其实zookeeper也可以，只要好好使用好getChildren这个方法。

2、barrier

barrier同步栅栏，可以对多个机器或线程进行步调协同，类似于赛马游戏，只有所有赛道上的马都准备好了，比赛才会开始，只有赛马都过了终点才算比赛结束，barrier就是为了解决这个问题。

如上图所示，类似与countdownlatch，我们可以在指定节点（如barrier1）下，每一个线程或机器都创建一个属于自己的“临时有序节点”（如192.168.1.10000000001），然后监听节点（如barrier1）的变更通知，每次收到barrier1的变更通知（使用watch），则会调用getChildren，获取当前barrier1下的子节点个数，一旦满足要求，比如3，则视为“比赛开始”；

如何判断“比赛结束”？只要满足getChildren().size() == 0即可视为比赛结束。

3、queue

queue的案例是一个producer-consumer的案例，即producer将消息发送给队列（实际上是在/queue1节点下创建了一个有序节点），而consumer则轮询方式不断从/queue1节点调用getChildren列表，从列表中获取序号最小的节点进行消费，消费完毕即删除该节点。

基本思路和barrier类似，就是充分利用getChildren及有序节点，getChildren获取所有子节点列表（也就是队列），而有序节点则保证了队列是有序。同时，zk.delete的操作也是原子的，保证了消息只能被消费一次。

有几个缺点：

• 单个消息不能太大（kb级别），因为是用节点存储一个队列消息，zk本身不是存储系统，每个节点存储量有限
• 队列长度也有限，且队列过长，还需要从队列中获取序号最小的节点，本身也有耗时

4、代码

• 基类 SyncPrimitive

package com.xxxxx.zookeeper;import org.apache.zookeeper.WatchedEvent;import org.apache.zookeeper.Watcher;import org.apache.zookeeper.ZooKeeper;import java.io.IOException;/** * Created by wushiweijun on 2017/11/22. */public class SyncPrimitive implements Watcher{    static ZooKeeper zk = null;    /*互斥锁*/    static Integer mutex;    String root;    public SyncPrimitive(String address) {        if(zk == null){            try {                System.out.println("Starting ZK:");                zk = new ZooKeeper(address, 3000, this);                mutex = new Integer(-1);                System.out.println("Finished starting ZK: " + zk);            } catch (IOException e) {                System.out.println(e.toString());                zk = null;            }        }    }    @Override    synchronized public void process(WatchedEvent event) {        synchronized (mutex) {            /*有变化，你醒来看下*/            mutex.notify();        }    }}

• barrier

package com.xxxxx.zookeeper;import org.apache.zookeeper.CreateMode;import org.apache.zookeeper.KeeperException;import org.apache.zookeeper.ZooDefs;import org.apache.zookeeper.data.Stat;import java.net.InetAddress;import java.net.UnknownHostException;import java.util.List;import java.util.Random;/** * Created by wushiweijun on 2017/11/22. */public class Barrier extends SyncPrimitive {    private int size;    private String name;    /**     * 创建栅栏根节点，后续所有的处理应用，都是在这个节点下创建子节点     *     * @param address     * @param root     * @param size     */    Barrier(String address, String root, int size) {        super(address);        this.root = root;        this.size = size;        // Create barrier node        if (zk != null) {            try {                /*同步调用，判断节点是否存在*/                Stat s = zk.exists(root, false);                if (s == null) {                    /*创建一个znode节点*/                    zk.create(root, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE,                            CreateMode.PERSISTENT);                }            } catch (KeeperException e) {                System.out                        .println("Keeper exception when instantiating queue: "                                + e.toString());            } catch (InterruptedException e) {                System.out.println("Interrupted exception");            }        }        // My node name        try {            name = new String(InetAddress.getLocalHost().getCanonicalHostName().toString());        } catch (UnknownHostException e) {            System.out.println(e.toString());        }    }    /**     * Join barrier     *     * @return     * @throws KeeperException     * @throws InterruptedException     */    boolean enter() throws KeeperException, InterruptedException{        /*创建临时有序节点*/        zk.create(root + "/" + name, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE,                CreateMode.EPHEMERAL_SEQUENTIAL);        while (true) {            synchronized (mutex) {                List<String> list = zk.getChildren(root, true);                if (list.size() < size) {                    /*没到，你睡吧*/                    mutex.wait();                } else {                    return true;                }            }        }    }    /**     * Wait until all reach barrier     *     * @return     * @throws KeeperException     * @throws InterruptedException     */    boolean leave() throws KeeperException, InterruptedException{        /*删除临时有序节点*/        zk.delete(root + "/" + name, 0);        while (true) {            synchronized (mutex) {                List<String> list = zk.getChildren(root, true);                if (list.size() > 0) {                    mutex.wait();                } else {                    return true;                }            }        }    }    public static void main(String[] args) {        args = "localhost:2181,localhost:2182,localhost:2183 2".split(" ");        Barrier b = new Barrier(args[0], "/b1", new Integer(args[1]));        try{            boolean flag = b.enter();            System.out.println("Entered barrier: " + args[1]);            if(!flag) System.out.println("Error when entering the barrier");        } catch (KeeperException e){        } catch (InterruptedException e){        }        // Generate random integer        Random rand = new Random();        int r = rand.nextInt(100);        // Loop for rand iterations        for (int i = 0; i < r; i++) {            try {                Thread.sleep(100);            } catch (InterruptedException e) {            }        }        try{            b.leave();        } catch (KeeperException e){        } catch (InterruptedException e){        }        System.out.println("Left barrier");    }}

• queue

package com.xxxxxx.zookeeper;import org.apache.zookeeper.CreateMode;import org.apache.zookeeper.KeeperException;import org.apache.zookeeper.ZooDefs;import org.apache.zookeeper.data.Stat;import java.nio.ByteBuffer;import java.util.List;/** * 这个简单案例，producer和consumer使用的是一个整数 * * Created by wushiweijun on 2017/11/22. */public class Queue extends SyncPrimitive{    /**     * Constructor of producer-consumer queue     *     * @param address     * @param name     */    Queue(String address, String name) {        super(address);        this.root = name;        // Create ZK node name        if (zk != null) {            try {                Stat s = zk.exists(root, false);                if (s == null) {                    /*创建znode节点*/                    zk.create(root, new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE,                            CreateMode.PERSISTENT);                }            } catch (KeeperException e) {                System.out                        .println("Keeper exception when instantiating queue: "                                + e.toString());            } catch (InterruptedException e) {                System.out.println("Interrupted exception");            }        }    }    /**     * Add element to the queue.     *     * @param i     * @return     */    boolean produce(int i) throws KeeperException, InterruptedException{        ByteBuffer b = ByteBuffer.allocate(4);        byte[] value;        // Add child with value i        b.putInt(i);        value = b.array();        /*创建持久有序节点*/        zk.create(root + "/element", value, ZooDefs.Ids.OPEN_ACL_UNSAFE,                CreateMode.PERSISTENT_SEQUENTIAL);        return true;    }    /**     * Remove first element from the queue.     *     * @return     * @throws KeeperException     * @throws InterruptedException     */    int consume() throws KeeperException, InterruptedException{        int retvalue = -1;        Stat stat = null;        String path = null;        // Get the first element available        while (true) {            synchronized (mutex) {                List<String> list = zk.getChildren(root, true);                if (list.size() == 0) {                    System.out.println("Going to wait");                    /*没到，你睡吧*/                    mutex.wait();                } else {                    Integer min = new Integer(list.get(0).substring(7));                    for(String s : list){                        /*去除掉element前缀，获取后面的序号*/                        Integer tempValue = new Integer(s.substring(7));                        //System.out.println("Temporary value: " + tempValue);                        if(tempValue < min) min = tempValue;                    }                    path = String.format("/element%010d" ,min);                    System.out.println("Temporary value: " + root + path);                    byte[] b = zk.getData(root + path,                            false, stat);                    /*使用了zk的原子性，如果节点被其他消费了删除了，则这个操作将报KeeperException.NoNode*/                    zk.delete(root + path, 0);                    ByteBuffer buffer = ByteBuffer.wrap(b);                    retvalue = buffer.getInt();                    return retvalue;                }            }        }    }    public static void main(String[] args) {//        args = "localhost:2181,localhost:2182,localhost:2183 10 p".split(" ");        args = "localhost:2181,localhost:2182,localhost:2183 10 c".split(" ");        Queue q = new Queue(args[0], "/app1");        System.out.println("Input: " + args[0]);        int i;        Integer max = new Integer(args[1]);        if (args[2].equals("p")) {            System.out.println("Producer");            for (i = 0; i < max; i++)                try{                    q.produce(10 + i);                } catch (KeeperException e){                } catch (InterruptedException e){                }        } else {            System.out.println("Consumer");            for (i = 0; i < max; i++) {                try{                    int r = q.consume();                    System.out.println("Item: " + r);                } catch (KeeperException e){                    i--;                } catch (InterruptedException e){                }            }        }    }}

5、参考

[1] http://zookeeper.apache.org/doc/trunk/zookeeperTutorial.html