01 storm 源码阅读 storm的进程间消息通信实现netty server实现

/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements.  See the NOTICE file * distributed with this work for additional information * regarding copyright ownership.  The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License.  You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */package backtype.storm.messaging.netty;import backtype.storm.Config;import backtype.storm.messaging.IConnection;import backtype.storm.messaging.TaskMessage;import backtype.storm.utils.Utils;import org.jboss.netty.bootstrap.ServerBootstrap;import org.jboss.netty.channel.Channel;import org.jboss.netty.channel.ChannelFactory;import org.jboss.netty.channel.group.ChannelGroup;import org.jboss.netty.channel.group.DefaultChannelGroup;import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory;import org.slf4j.Logger;import org.slf4j.LoggerFactory;import java.net.InetSocketAddress;import java.util.ArrayList;import java.util.Arrays;import java.util.HashMap;import java.util.Iterator;import java.util.List;import java.util.Map;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.Executors;import java.util.concurrent.LinkedBlockingQueue;import java.util.concurrent.ThreadFactory;import java.util.concurrent.atomic.AtomicInteger;import org.jboss.netty.bootstrap.ServerBootstrap;import org.jboss.netty.channel.Channel;import org.jboss.netty.channel.ChannelFactory;import org.jboss.netty.channel.group.ChannelGroup;import org.jboss.netty.channel.group.DefaultChannelGroup;import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory;import org.slf4j.Logger;import org.slf4j.LoggerFactory;import backtype.storm.Config;import backtype.storm.messaging.ConnectionWithStatus;import backtype.storm.messaging.IConnection;import backtype.storm.messaging.TaskMessage;import backtype.storm.metric.api.IStatefulObject;import backtype.storm.utils.Utils;/** * netty服务端，做进程之间通信用。传递tutle。 <br> * 服务端主要负责计数为主，按照一定得数据结构存储收到的消息，和转发收到的消息。 *  * @author mosi.li *  */class Server extends ConnectionWithStatus implements IStatefulObject {private static final Logger LOG = LoggerFactory.getLogger(Server.class);@SuppressWarnings("rawtypes")Map storm_conf;int port;// 这个messagesEnqueued，只是提供统计用，记录进来的消息数量。private final ConcurrentHashMap<String, AtomicInteger> messagesEnqueued = new ConcurrentHashMap<String, AtomicInteger>();// 这个messagesDequeued，只是提供统计用，记录出去的消息数量。private final AtomicInteger messagesDequeued = new AtomicInteger(0);// pendingMessages提供统计用，表示处理中的消息数量。private final AtomicInteger[] pendingMessages;// Create multiple queues for incoming messages. The size equals the number// of receiver threads.// For message which is sent to same task, it will be stored in the same// queue to preserve the message order.// 表示。。进来的消息的详细情况，目的地是相同task的消息放在同一个队列里，这是个数组队列。incoming messages的消息private LinkedBlockingQueue<ArrayList<TaskMessage>>[] message_queue;// 下面3个类是netty的实现，实现进程间的消息通信volatile ChannelGroup allChannels = new DefaultChannelGroup("storm-server");final ChannelFactory factory;//使用factory来设置netty的通道类型final ServerBootstrap bootstrap;//netty服务端的核心类// message_queue的队列数组的长度。private int queueCount;// 任务和任务下的数据流通的队列的对应关系。private volatile HashMap<Integer, Integer> taskToQueueId = null;int roundRobinQueueId;private volatile boolean closing = false;List<TaskMessage> closeMessage = Arrays.asList(new TaskMessage(-1, null));@SuppressWarnings("rawtypes")Server(Map storm_conf, int port) {this.storm_conf = storm_conf;this.port = port;queueCount = Utils.getInt(storm_conf.get(Config.WORKER_RECEIVER_THREAD_COUNT), 1);roundRobinQueueId = 0;taskToQueueId = new HashMap<Integer, Integer>();message_queue = new LinkedBlockingQueue[queueCount];pendingMessages = new AtomicInteger[queueCount];for (int i = 0; i < queueCount; i++) {message_queue[i] = new LinkedBlockingQueue<ArrayList<TaskMessage>>();pendingMessages[i] = new AtomicInteger(0);}// Configure the server.int buffer_size = Utils.getInt(storm_conf.get(Config.STORM_MESSAGING_NETTY_BUFFER_SIZE));int maxWorkers = Utils.getInt(storm_conf.get(Config.STORM_MESSAGING_NETTY_SERVER_WORKER_THREADS));ThreadFactory bossFactory = new NettyRenameThreadFactory(name()+ "-boss");ThreadFactory workerFactory = new NettyRenameThreadFactory(name()+ "-worker");if (maxWorkers > 0) {factory = new NioServerSocketChannelFactory(Executors.newCachedThreadPool(bossFactory),Executors.newCachedThreadPool(workerFactory), maxWorkers);} else {factory = new NioServerSocketChannelFactory(Executors.newCachedThreadPool(bossFactory),Executors.newCachedThreadPool(workerFactory));}LOG.info("Create Netty Server " + name() + ", buffer_size: "+ buffer_size + ", maxWorkers: " + maxWorkers);bootstrap = new ServerBootstrap(factory);//设置channel类型bootstrap.setOption("child.tcpNoDelay", true);bootstrap.setOption("child.receiveBufferSize", buffer_size);bootstrap.setOption("child.keepAlive", true);// Set up the pipeline factory.bootstrap.setPipelineFactory(new StormServerPipelineFactory(this));// Bind and start to accept incoming connections.Channel channel = bootstrap.bind(new InetSocketAddress(port));allChannels.add(channel);}/** * 将数据塞入对应的task里的队列。 *  * @param msgs * @return */private ArrayList<TaskMessage>[] groupMessages(List<TaskMessage> msgs) {ArrayList<TaskMessage> messageGroups[] = new ArrayList[queueCount];for (int i = 0; i < msgs.size(); i++) {TaskMessage message = msgs.get(i);int task = message.task();if (task == -1) {closing = true;return null;}Integer queueId = getMessageQueueId(task);if (null == messageGroups[queueId]) {messageGroups[queueId] = new ArrayList<TaskMessage>();}messageGroups[queueId].add(message);}return messageGroups;}/** * 获取某个任务，对应的消息队列id *  * @param task * @return */private Integer getMessageQueueId(int task) {// try to construct the map from taskId -> queueId in round robin// manner.Integer queueId = taskToQueueId.get(task);if (null == queueId) {synchronized (this) {queueId = taskToQueueId.get(task);if (queueId == null) {queueId = roundRobinQueueId++;if (roundRobinQueueId == queueCount) {roundRobinQueueId = 0;}HashMap<Integer, Integer> newRef = new HashMap<Integer, Integer>(taskToQueueId);newRef.put(task, queueId);taskToQueueId = newRef;}}}return queueId;}/** * 添加消息时，改变进入消息队列的数量 *  * @param from * @param amount */private void addReceiveCount(String from, int amount) {// This is possibly lossy in the case where a value is deleted// because it has received no messages over the metrics collection// period and new messages are starting to come in. This is// because I don't want the overhead of a synchronize just to have// the metric be absolutely perfect.AtomicInteger i = messagesEnqueued.get(from);if (i == null) {i = new AtomicInteger(amount);AtomicInteger prev = messagesEnqueued.putIfAbsent(from, i);if (prev != null) {prev.addAndGet(amount);}} else {i.addAndGet(amount);}}/** * 收到一个消息的时候，做队列变化 enqueue a received message *  * @throws InterruptedException */protected void enqueue(List<TaskMessage> msgs) throws InterruptedException {if (null == msgs || msgs.size() == 0 || closing) {return;}ArrayList<TaskMessage> messageGroups[] = groupMessages(msgs);if (null == messageGroups || closing) {return;}for (int receiverId = 0; receiverId < messageGroups.length; receiverId++) {ArrayList<TaskMessage> msgGroup = messageGroups[receiverId];if (null != msgGroup) {// 所有的消息队列，加上消息message_queue[receiverId].put(msgGroup);// 处理中的消息队列数量pendingMessages[receiverId].addAndGet(msgGroup.size());}}}/** * 从队列里消费消息,返回即将消费的消息迭代器，用于处理 */public Iterator<TaskMessage> recv(int flags, int receiverId) {if (closing) {return closeMessage.iterator();}ArrayList<TaskMessage> ret = null;int queueId = receiverId % queueCount;if ((flags & 0x01) == 0x01) {// non-blockingret = message_queue[queueId].poll();} else {try {ArrayList<TaskMessage> request = message_queue[queueId].take();LOG.debug("request to be processed: {}", request);ret = request;} catch (InterruptedException e) {LOG.info("exception within msg receiving", e);ret = null;}}if (null != ret) {messagesDequeued.addAndGet(ret.size());pendingMessages[queueId].addAndGet(0 - ret.size());return ret.iterator();}return null;}/** * register a newly created channel *  * @param channel */protected void addChannel(Channel channel) {allChannels.add(channel);}/** * close a channel *  * @param channel */protected void closeChannel(Channel channel) {channel.close().awaitUninterruptibly();allChannels.remove(channel);}/** * close all channels, and release resources */public synchronized void close() {if (allChannels != null) {allChannels.close().awaitUninterruptibly();factory.releaseExternalResources();allChannels = null;}}public void send(int task, byte[] message) {throw new UnsupportedOperationException("Server connection should not send any messages");}public void send(Iterator<TaskMessage> msgs) {throw new UnsupportedOperationException("Server connection should not send any messages");}public String name() {return "Netty-server-localhost-" + port;}/** * 一个状态的标示。标示netty的连接状态. */@Overridepublic Status status() {if (closing) {return Status.Closed;} else if (!connectionEstablished(allChannels)) {return Status.Connecting;} else {return Status.Ready;}}private boolean connectionEstablished(Channel channel) {return channel != null && channel.isBound();}private boolean connectionEstablished(ChannelGroup allChannels) {boolean allEstablished = true;for (Channel channel : allChannels) {if (!(connectionEstablished(channel))) {allEstablished = false;break;}}return allEstablished;}/** * 为统计消息提供信息.为了效率不加锁。是不准确的， * 提供dequeuedMessages（出去的消息数），（pending）处理中的,enqueued (进来的) */public Object getState() {LOG.info("Getting metrics for server on port {}", port);HashMap<String, Object> ret = new HashMap<String, Object>();ret.put("dequeuedMessages", messagesDequeued.getAndSet(0));ArrayList<Integer> pending = new ArrayList<Integer>(pendingMessages.length);for (AtomicInteger p : pendingMessages) {pending.add(p.get());}ret.put("pending", pending);HashMap<String, Integer> enqueued = new HashMap<String, Integer>();Iterator<Map.Entry<String, AtomicInteger>> it = messagesEnqueued.entrySet().iterator();while (it.hasNext()) {Map.Entry<String, AtomicInteger> ent = it.next();// Yes we can delete something that is not 0 because of races, but// that is OK for metricsAtomicInteger i = ent.getValue();if (i.get() == 0) {it.remove();} else {enqueued.put(ent.getKey(), i.getAndSet(0));}}ret.put("enqueued", enqueued);return ret;}@Overridepublic String toString() {return String.format("Netty server listening on port %s", port);}}