Jedis客户端分片--构建高可用客户端分片

记录是一种精神，是加深理解最好的方式之一。

最近深入研究了Jedis的源码，对Jedis的使用进行深入理解，提笔记录。
曹金桂 [e-mail](如有遗漏之处还请指教)
时间：2016年12月06日15:00

概述

分片允许数据存放在不同的物理机器上，以适应数据量过大的场景，克服单台机器内存或者磁盘空间的限制。Redis在3.0之前是没有集群的，就算现在3.0推出3.0集群版本，也没有几家公司用于生产。Jedis作为Java的Redis客户端工具，其内部也提供了客户的分片的实现。具体可以参考我的文章《Jedis使用教程完整版》，讲解了Jedis客户端使用，包括了Jedis的客户端分片的使用方法。

Jedis客户端分片存在问题

Jedis的客户端分片采用一致性hash算法，实现数据的分配存储。但当某一个分片服务挂了之后，客户端需要修改服务的指向才能动态的切换到当前分片的从服务去。其实完全可以采用和JedisSentinel一样的方法，利用Jedis的Sentinel构建高可用的服务。

实现

个人参照JedisSentinelPool的实现，完成了对构建Jedis分片的高可用的修改。源码如下：

package redis.clients.jedis;import org.apache.commons.pool2.PooledObject;import org.apache.commons.pool2.PooledObjectFactory;import org.apache.commons.pool2.impl.DefaultPooledObject;import org.apache.commons.pool2.impl.GenericObjectPoolConfig;import redis.clients.jedis.exceptions.JedisConnectionException;import redis.clients.jedis.exceptions.JedisException;import redis.clients.util.Hashing;import redis.clients.util.Pool;import java.util.*;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.atomic.AtomicBoolean;import java.util.logging.Level;import java.util.logging.Logger;import java.util.regex.Pattern;public class SentinelShardedJedisPool extends Pool<SentinelShardedJedis> {    protected Logger log = Logger.getLogger(getClass().getName());    private final Map<String, JedisShardInfo> shardInfoMap = new ConcurrentHashMap<>(); // key=masterName    private final Map<String, String> sentinels = new ConcurrentHashMap<>(); // key=host+port value=masterName    private SentinelShardedJedisFactory factory;    private Set<MasterListener> masterListeners = new HashSet<>();    public SentinelShardedJedisPool(Set<SentinelInfo> sentinelInfos, final GenericObjectPoolConfig poolConfig, Hashing algo, Pattern keyTagPattern) {        initShards(sentinelInfos);        factory = new SentinelShardedJedisFactory(shardInfoMap.values(), algo, keyTagPattern);        super.initPool(poolConfig, factory);    }    private void initShards(Set<SentinelInfo> sentinelInfos) {        for (SentinelInfo info : sentinelInfos) {            JedisShardInfo shardInfo = getJedisSharedInfoFromSentinelInfo(info);            sentinels.put(shardInfo.getHost() + ":" + shardInfo.getPort(), info.getMasterName());            shardInfoMap.put(info.getMasterName(), shardInfo);            for (String sentinel : info.getSentinels()) {                HostAndPort hap = HostAndPort.parseString(sentinel);                MasterListener masterListener = new MasterListener(info.getMasterName(), hap.getHost(), hap.getPort());                masterListeners.add(masterListener);                masterListener.setDaemon(true);                masterListener.start();            }        }    }    private JedisShardInfo getJedisSharedInfoFromSentinelInfo(SentinelInfo info) {        log.info("Trying to find master from available Sentinels...");        for (String sentinel : info.getSentinels()) {            final HostAndPort hap = HostAndPort.parseString(sentinel);            log.fine("Connecting to Sentinel " + hap);            Jedis jedis = null;            try {                jedis = new Jedis(hap.getHost(), hap.getPort());                List<String> masterAddr = jedis.sentinelGetMasterAddrByName(info.getMasterName());                if (masterAddr == null || masterAddr.size() != 2) {                    log.warning("Can not get master addr, master name: " + info.getMasterName() + ". Sentinel: " + hap + ".");                    continue;                }                String host = masterAddr.get(0);                int port = Integer.parseInt(masterAddr.get(1));                JedisShardInfo jedisShardInfo = new JedisShardInfo(host, port);                log.fine("Found Redis master at " + masterAddr);                return jedisShardInfo;            } catch (JedisException e) {                log.warning("Cannot get master address from sentinel running @ " + hap + ". Reason: " + e + ". Trying next one.");            } finally {                if (jedis != null) {                    jedis.close();                }            }        }        throw new JedisConnectionException("All sentinels down, cannot determine where is " + info.getMasterName() + " master is running...");    }    private void reInitPool() {        internalPool.clear();    }    @Override    public void destroy() {        for (MasterListener m : masterListeners) {            m.shutdown();        }        super.destroy();    }    @Override    public SentinelShardedJedis getResource() {        while (true) {            SentinelShardedJedis jedis = super.getResource();            for (Jedis shard : jedis.getAllShards()) {                String host = shard.getClient().getHost();                int port = shard.getClient().getPort();                if (sentinels.containsKey(host + ":" + port)) {                    jedis.setDataSource(this);                    return jedis;                }            }            returnBrokenResource(jedis);        }    }    @Override    protected void returnBrokenResource(final SentinelShardedJedis resource) {        if (resource != null) {            returnBrokenResourceObject(resource);        }    }    @Override    protected void returnResource(final SentinelShardedJedis resource) {        if (resource != null) {            resource.resetState();            returnResourceObject(resource);        }    }    private class MasterListener extends Thread {        private String masterName;        private String sentinelHost;        private int sentinelPort;        private volatile Jedis j;        private AtomicBoolean running = new AtomicBoolean(false);        public MasterListener(String masterName, String host, int port) {            this.masterName = masterName;            this.sentinelHost = host;            this.sentinelPort = port;            this.setName(String.format("MasterListener-%s-[%s:%d]", masterName, host, port));        }        @Override        public void run() {            running.set(true);            while (running.get()) {                j = new Jedis(sentinelHost, sentinelPort);                try {                    if (!running.get()) {                        break;                    }                    j.subscribe(new JedisPubSub() {                        @Override                        public void onMessage(String channel, String message) {                            log.fine("Sentinel " + sentinelHost + ":" + sentinelPort + " published: " + message + ".");                            String[] switchMasterMsg = message.split(" ");                            if (switchMasterMsg.length > 3) {                                if (masterName.equals(switchMasterMsg[0])) {                                    JedisShardInfo oldInfo = shardInfoMap.get(masterName);                                    sentinels.remove(oldInfo.getHost() + ":" + oldInfo.getPort());                                    JedisShardInfo newInfo = new JedisShardInfo(switchMasterMsg[3], switchMasterMsg[4]);                                    newInfo.setConnectionTimeout(oldInfo.getConnectionTimeout());                                    newInfo.setPassword(oldInfo.getPassword());                                    newInfo.setSoTimeout(oldInfo.getSoTimeout());                                    shardInfoMap.put(masterName, newInfo);                                    sentinels.put(newInfo.getHost() + ":" + newInfo.getPort(), masterName);                                    reInitPool();                                } else {                                    log.fine("Ignoring message on +switch-master for master name " + switchMasterMsg[0] + ", our master name is " + masterName);                                }                            } else {                                log.severe("Invalid message received on Sentinel " + sentinelHost + ":" + sentinelPort + " on channel +switch-master: " + message);                            }                        }                    }, "+switch-master");                } catch (JedisConnectionException e) {                    if (running.get()) {                        log.log(Level.SEVERE, "Lost connection to Sentinel at " + sentinelHost + ":" + sentinelPort + ". Sleeping 5000ms and retrying.", e);                        try {                            Thread.sleep(5000);                        } catch (InterruptedException e1) {                            log.log(Level.SEVERE, "Sleep interrupted: ", e1);                        }                    } else {                        log.fine("Unsubscribing from Sentinel at " + sentinelHost + ":" + sentinelPort);                    }                } finally {                    j.close();                }            }        }        public void shutdown() {            try {                log.fine("Shutting down listener on " + sentinelHost + ":" + sentinelPort);                running.set(false);                if (j != null) {                    j.disconnect();                }            } catch (Exception e) {                log.log(Level.SEVERE, "Caught exception while shutting down: ", e);            }        }    }    /**     * PoolableObjectFactory custom impl.     */    private class SentinelShardedJedisFactory implements PooledObjectFactory<SentinelShardedJedis> {        private Collection<JedisShardInfo> shards;        private Hashing algo;        private Pattern keyTagPattern;        public SentinelShardedJedisFactory(Collection<JedisShardInfo> shards, Hashing algo, Pattern keyTagPattern) {            this.shards = shards;            this.algo = algo;            this.keyTagPattern = keyTagPattern;        }        @Override        public PooledObject<SentinelShardedJedis> makeObject() throws Exception {            SentinelShardedJedis jedis = new SentinelShardedJedis(new ArrayList<>(shards), algo, keyTagPattern);            return new DefaultPooledObject<SentinelShardedJedis>(jedis);        }        @Override        public void destroyObject(PooledObject<SentinelShardedJedis> pooledShardedJedis) throws Exception {            final SentinelShardedJedis shardedJedis = pooledShardedJedis.getObject();            for (Jedis jedis : shardedJedis.getAllShards()) {                try {                    try {                        jedis.quit();                    } catch (Exception e) {                    }                    jedis.disconnect();                } catch (Exception e) {                }            }        }        @Override        public boolean validateObject(PooledObject<SentinelShardedJedis> pooledShardedJedis) {            try {                SentinelShardedJedis jedis = pooledShardedJedis.getObject();                for (Jedis shard : jedis.getAllShards()) {                    String host = shard.getClient().getHost();                    int port = shard.getClient().getPort();                    if (!shardInfoMap.containsKey(host + port)) {                        return false;                    }                    if (!shard.ping().equals("PONG")) {                        return false;                    }                }                return true;            } catch (Exception ex) {                return false;            }        }        @Override        public void activateObject(PooledObject<SentinelShardedJedis> p) throws Exception {        }        @Override        public void passivateObject(PooledObject<SentinelShardedJedis> p) throws Exception {        }    }    public static class SentinelInfo {        private String masterName;        private String clientName;        private Set<String> sentinels;        public SentinelInfo(String masterName, String clientName, Set<String> sentinels) {            this.masterName = masterName;            this.clientName = clientName;            this.sentinels = sentinels;        }        public String getMasterName() {            return masterName;        }        public String getClientName() {            return clientName;        }        public Set<String> getSentinels() {            return sentinels;        }    }}

package redis.clients.jedis;import redis.clients.util.Hashing;import java.util.List;import java.util.regex.Pattern;public class SentinelShardedJedis extends ShardedJedis {    protected SentinelShardedJedisPool dataSource = null;    public SentinelShardedJedis(List<JedisShardInfo> shards) {        super(shards);    }    public SentinelShardedJedis(List<JedisShardInfo> shards, Hashing algo) {        super(shards, algo);    }    public SentinelShardedJedis(List<JedisShardInfo> shards, Pattern keyTagPattern) {        super(shards, keyTagPattern);    }    public SentinelShardedJedis(List<JedisShardInfo> shards, Hashing algo, Pattern keyTagPattern) {        super(shards, algo, keyTagPattern);    }    public void setDataSource(SentinelShardedJedisPool sentinelShardedJedisPool) {        this.dataSource = sentinelShardedJedisPool;    }}

使用方法

JedisPoolConfig poolConfig = new JedisPoolConfig();poolConfig.setMinIdle(4);poolConfig.setMaxTotal(100);poolConfig.setMaxWaitMillis(30000);Set<String> sentinels = new HashSet<>();sentinels.add("192.168.2.122:26379");sentinels.add("192.168.2.128:26379");Set<SentinelShardedJedisPool.SentinelInfo> sentinelInfos = new HashSet<>();sentinelInfos.add(new SentinelShardedJedisPool.SentinelInfo("mymaster1", "m1", sentinels));sentinelInfos.add(new SentinelShardedJedisPool.SentinelInfo("mymaster2", "m2", sentinels));SentinelShardedJedisPool pool = new SentinelShardedJedisPool(sentinelInfos, poolConfig, Hashing.MURMUR_HASH, null);SentinelShardedJedis jedis = pool.getResource();jedis.set("key", "value");jedis.close();pool.close();

小结

本实现个人已经测试通过。但若采用本方案必须要经过一定的性能测试。如果问题欢迎一起讨论。