02 storm 源码阅读 storm的进程间消息通信实现netty client实现

/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements.  See the NOTICE file * distributed with this work for additional information * regarding copyright ownership.  The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License.  You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */package backtype.storm.messaging.netty;import backtype.storm.Config;import backtype.storm.messaging.IConnection;import backtype.storm.messaging.TaskMessage;import backtype.storm.utils.StormBoundedExponentialBackoffRetry;import backtype.storm.utils.Utils;import org.jboss.netty.bootstrap.ClientBootstrap;import org.jboss.netty.channel.Channel;import org.jboss.netty.channel.ChannelFactory;import org.jboss.netty.channel.ChannelFuture;import org.jboss.netty.channel.ChannelFutureListener;import org.slf4j.Logger;import org.slf4j.LoggerFactory;import java.net.InetSocketAddress;import java.net.SocketAddress;import java.util.ArrayList;import java.util.Iterator;import java.util.List;import java.util.Map;import java.util.HashMap;import java.util.concurrent.Callable;import java.util.concurrent.ScheduledExecutorService;import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicBoolean;import java.util.concurrent.atomic.AtomicInteger;import java.util.concurrent.atomic.AtomicLong;import java.util.concurrent.atomic.AtomicReference;import com.google.common.util.concurrent.*;import org.jboss.netty.bootstrap.ClientBootstrap;import org.jboss.netty.channel.Channel;import org.jboss.netty.channel.ChannelFactory;import org.jboss.netty.channel.ChannelFuture;import org.jboss.netty.channel.ChannelFutureListener;import org.slf4j.Logger;import org.slf4j.LoggerFactory;import backtype.storm.Config;import backtype.storm.messaging.ConnectionWithStatus;import backtype.storm.metric.api.IStatefulObject;import backtype.storm.messaging.TaskMessage;import backtype.storm.utils.StormBoundedExponentialBackoffRetry;import backtype.storm.utils.Utils;/** * A Netty client for sending task messages to a remote destination (Netty * server). *  * Implementation details: *  * - Sending messages, i.e. writing to the channel, is performed asynchronously. * - Messages are sent in batches to optimize for network throughput at the * expense of network latency. The message batch size is configurable. - * Connecting and reconnecting are performed asynchronously. - Note: The current * implementation drops any messages that are being enqueued for sending if the * connection to the remote destination is currently unavailable. - A background * flusher thread is run in the background. It will, at fixed intervals, check * for any pending messages (i.e. messages buffered in memory) and flush them to * the remote destination iff background flushing is currently enabled. */public class Client extends ConnectionWithStatus implements IStatefulObject {private static final Logger LOG = LoggerFactory.getLogger(Client.class);private static final String PREFIX = "Netty-Client-";private static final long NO_DELAY_MS = 0L;// 0秒，隔天处理private static final long MINIMUM_INITIAL_DELAY_MS = 30000L;// 30秒延迟private static final long PENDING_MESSAGES_FLUSH_TIMEOUT_MS = 600000L;// 5分钟结束private static final long PENDING_MESSAGES_FLUSH_INTERVAL_MS = 1000L;// 一秒刷一次消息到服务端private static final long DISTANT_FUTURE_TIME_MS = Long.MAX_VALUE;//private final StormBoundedExponentialBackoffRetry retryPolicy;private final ClientBootstrap bootstrap;// netty，客户端private final InetSocketAddress dstAddress;// 服务端地址protected final String dstAddressPrefixedName;// 服务端前缀/** * The channel used for all write operations from this client to the remote * destination. */private final AtomicReference<Channel> channelRef = new AtomicReference<Channel>(null);/** * Maximum number of reconnection attempts we will perform after a * disconnect before giving up. */private final int maxReconnectionAttempts;/** * Total number of connection attempts. */private final AtomicInteger totalConnectionAttempts = new AtomicInteger(0);/** * Number of connection attempts since the last disconnect. */private final AtomicInteger connectionAttempts = new AtomicInteger(0);/** * Number of messages successfully sent to the remote destination. */private final AtomicInteger messagesSent = new AtomicInteger(0);/** * Number of messages that could not be sent to the remote destination. */private final AtomicInteger messagesLost = new AtomicInteger(0);/** * Number of messages buffered in memory. */private final AtomicLong pendingMessages = new AtomicLong(0);/** * This flag is set to true if and only if a client instance is being * closed. */private volatile boolean closing = false;/** * When set to true, then the background flusher thread will flush any * pending messages on its next run. */private final AtomicBoolean backgroundFlushingEnabled = new AtomicBoolean(false);/** * The absolute time (in ms) when the next background flush should be * performed. *  * Note: The flush operation will only be performed if * backgroundFlushingEnabled is true, too. */private final AtomicLong nextBackgroundFlushTimeMs = new AtomicLong(DISTANT_FUTURE_TIME_MS);/** * The time interval (in ms) at which the background flusher thread will be * run to check for any pending messages to be flushed. */private final int flushCheckIntervalMs;/** * How many messages should be batched together before sending them to the * remote destination. *  * Messages are batched to optimize network throughput at the expense of * latency. */private final int messageBatchSize;private MessageBatch messageBatch = null;private final ListeningScheduledExecutorService scheduler;protected final Map stormConf;@SuppressWarnings("rawtypes")Client(Map stormConf, ChannelFactory factory,ScheduledExecutorService scheduler, String host, int port) {closing = false;this.stormConf = stormConf;this.scheduler = MoreExecutors.listeningDecorator(scheduler);int bufferSize = Utils.getInt(stormConf.get(Config.STORM_MESSAGING_NETTY_BUFFER_SIZE));LOG.info("creating Netty Client, connecting to {}:{}, bufferSize: {}",host, port, bufferSize);messageBatchSize = Utils.getInt(stormConf.get(Config.STORM_NETTY_MESSAGE_BATCH_SIZE), 262144);flushCheckIntervalMs = Utils.getInt(stormConf.get(Config.STORM_NETTY_FLUSH_CHECK_INTERVAL_MS), 10);maxReconnectionAttempts = Utils.getInt(stormConf.get(Config.STORM_MESSAGING_NETTY_MAX_RETRIES));int minWaitMs = Utils.getInt(stormConf.get(Config.STORM_MESSAGING_NETTY_MIN_SLEEP_MS));int maxWaitMs = Utils.getInt(stormConf.get(Config.STORM_MESSAGING_NETTY_MAX_SLEEP_MS));retryPolicy = new StormBoundedExponentialBackoffRetry(minWaitMs,maxWaitMs, maxReconnectionAttempts);// Initiate connection to remote destinationbootstrap = createClientBootstrap(factory, bufferSize);dstAddress = new InetSocketAddress(host, port);dstAddressPrefixedName = prefixedName(dstAddress);connect(NO_DELAY_MS);// Launch background flushing threadpauseBackgroundFlushing();long initialDelayMs = Math.min(MINIMUM_INITIAL_DELAY_MS, maxWaitMs* maxReconnectionAttempts);scheduler.scheduleWithFixedDelay(createBackgroundFlusher(),initialDelayMs, flushCheckIntervalMs, TimeUnit.MILLISECONDS);}/** * 创建netty客户端 *  * @param factory * @param bufferSize * @return */private ClientBootstrap createClientBootstrap(ChannelFactory factory,int bufferSize) {ClientBootstrap bootstrap = new ClientBootstrap(factory);bootstrap.setOption("tcpNoDelay", true);bootstrap.setOption("sendBufferSize", bufferSize);bootstrap.setOption("keepAlive", true);bootstrap.setPipelineFactory(new StormClientPipelineF9actory(this));return bootstrap;}private String prefixedName(InetSocketAddress dstAddress) {if (null != dstAddress) {return PREFIX + dstAddress.toString();}return "";}/** * 由google 的guava的同步包维护了一个线程池，在后台定时的刷新，pending中的msg。 *  * @return */private Runnable createBackgroundFlusher() {return new Runnable() {@Overridepublic void run() {if (!closing && backgroundFlushingEnabled.get()&& nowMillis() > nextBackgroundFlushTimeMs.get()) {LOG.debug("flushing {} pending messages to {} in background",messageBatch.size(), dstAddressPrefixedName);flushPendingMessages();}}};}/** * 用来控制后台的往服务端发射pending消息的定时任务。暂停，当主动发射的时候，是暂停的。当isFull。批量到了的时候。会触发 */private void pauseBackgroundFlushing() {backgroundFlushingEnabled.set(false);}/** * 启动发射消息的定时任务。 */private void resumeBackgroundFlushing() {backgroundFlushingEnabled.set(true);}/** * 定时任务执行的方法，针对pending队列，发射消息的方法。 */private synchronized void flushPendingMessages() {Channel channel = channelRef.get();if (containsMessages(messageBatch)) {if (connectionEstablished(channel)) {if (channel.isWritable()) {pauseBackgroundFlushing();MessageBatch toBeFlushed = messageBatch;flushMessages(channel, toBeFlushed);messageBatch = null;} else if (closing) {// Ensure background flushing is enabled so that we// definitely have a chance to re-try the flush// operation in case the client is being gracefully closed// (where we have a brief time window where// the client will wait for pending messages to be sent).resumeBackgroundFlushing();}} else {closeChannelAndReconnect(channel);}}}private long nowMillis() {return System.currentTimeMillis();}/** * 和服务端，建立连接 We will retry connection with exponential back-off policy */private synchronized void connect(long delayMs) {try {if (closing) {return;}if (connectionEstablished(channelRef.get())) {return;}connectionAttempts.getAndIncrement();if (reconnectingAllowed()) {totalConnectionAttempts.getAndIncrement();LOG.info("connection attempt {} to {} scheduled to run in {} ms",connectionAttempts.get(), dstAddressPrefixedName,delayMs);ListenableFuture<Channel> channelFuture = scheduler.schedule(new Connector(dstAddress, connectionAttempts.get()),delayMs, TimeUnit.MILLISECONDS);Futures.addCallback(channelFuture,new FutureCallback<Channel>() {@Overridepublic void onSuccess(Channel result) {if (connectionEstablished(result)) {setChannel(result);LOG.info("connection established to {}",dstAddressPrefixedName);connectionAttempts.set(0);} else {reconnectAgain(new RuntimeException("Returned channel was actually not established"));}}@Overridepublic void onFailure(Throwable t) {reconnectAgain(t);}private void reconnectAgain(Throwable t) {String baseMsg = String.format("connection attempt %s to %s failed",connectionAttempts,dstAddressPrefixedName);String failureMsg = (t == null) ? baseMsg: baseMsg + ": " + t.toString();LOG.error(failureMsg);long nextDelayMs = retryPolicy.getSleepTimeMs(connectionAttempts.get(), 0);connect(nextDelayMs);}});} else {close();throw new RuntimeException("Giving up to connect to "+ dstAddressPrefixedName + " after "+ connectionAttempts + " failed attempts");}} catch (Exception e) {throw new RuntimeException("Failed to connect to "+ dstAddressPrefixedName, e);}}private void setChannel(Channel channel) {channelRef.set(channel);}private boolean reconnectingAllowed() {return !closing&& connectionAttempts.get() <= (maxReconnectionAttempts + 1);}private boolean connectionEstablished(Channel channel) {// Because we are using TCP (which is a connection-oriented transport// unlike UDP), a connection is only fully// established iff the channel is connected. That is, a TCP-based// channel must be in the CONNECTED state before// anything can be read or written to the channel.//// See:// - http://netty.io/3.9/api/org/jboss/netty/channel/ChannelEvent.html// -// http://stackoverflow.com/questions/13356622/what-are-the-netty-channel-state-transitionsreturn channel != null && channel.isConnected();}/** * 表示连接状态的标示 Note: Storm will check via this method whether a worker can be * activated safely during the initial startup of a topology. The worker * will only be activated once all of the its connections are ready. */@Overridepublic Status status() {if (closing) {return Status.Closed;} else if (!connectionEstablished(channelRef.get())) {return Status.Connecting;} else {return Status.Ready;}}/** * Receiving messages is not supported by a client. *  * @throws java.lang.UnsupportedOperationException *             whenever this method is being called. */@Overridepublic Iterator<TaskMessage> recv(int flags, int clientId) {throw new UnsupportedOperationException("Client connection should not receive any messages");}/** * 发射单个消息。都是批量发送的，这个得看看在哪用了。 *  * @param taskId * @param payload */@Overridepublic void send(int taskId, byte[] payload) {TaskMessage msg = new TaskMessage(taskId, payload);List<TaskMessage> wrapper = new ArrayList<TaskMessage>(1);wrapper.add(msg);send(wrapper.iterator());}/** * 批量发射消息，追加一个isfull的批量。到的时候flush到服务端，属于延迟发射，提高网络利用率 Enqueue task messages * to be sent to the remote destination (cf. `host` and `port`). */@Overridepublic synchronized void send(Iterator<TaskMessage> msgs) {if (closing) {int numMessages = iteratorSize(msgs);LOG.error("discarding {} messages because the Netty client to {} is being closed",numMessages, dstAddressPrefixedName);return;}if (!hasMessages(msgs)) {return;}Channel channel = channelRef.get();if (!connectionEstablished(channel)) {// Closing the channel and reconnecting should be done before// handling the messages.closeChannelAndReconnect(channel);handleMessagesWhenConnectionIsUnavailable(msgs);return;}// Collect messages into batches (to optimize network throughput), then// flush them.while (msgs.hasNext()) {TaskMessage message = msgs.next();if (messageBatch == null) {messageBatch = new MessageBatch(messageBatchSize);}messageBatch.add(message);if (messageBatch.isFull()) {MessageBatch toBeFlushed = messageBatch;flushMessages(channel, toBeFlushed);messageBatch = null;}}// Handle any remaining messages in case the "last" batch was not full.if (containsMessages(messageBatch)) {if (connectionEstablished(channel) && channel.isWritable()) {// We can write to the channel, so we flush the remaining// messages immediately to minimize latency.pauseBackgroundFlushing();MessageBatch toBeFlushed = messageBatch;messageBatch = null;flushMessages(channel, toBeFlushed);} else {// We cannot write to the channel, which means Netty's internal// write buffer is full.// In this case, we buffer the remaining messages and wait for// the next messages to arrive.//// Background:// Netty 3.x maintains an internal write buffer with a high// water mark for each channel (default: 64K).// This represents the amount of data waiting to be flushed to// operating system buffers. If the// outstanding data exceeds this value then the channel is set// to non-writable. When this happens, a// INTEREST_CHANGED channel event is triggered. Netty sets the// channel to writable again once the data// has been flushed to the system buffers.//// See http://stackoverflow.com/questions/14049260resumeBackgroundFlushing();nextBackgroundFlushTimeMs.set(nowMillis()+ flushCheckIntervalMs);}}}private boolean hasMessages(Iterator<TaskMessage> msgs) {return msgs != null && msgs.hasNext();}/** * 当连接不可用是，丢掉消息。消息,通过send过来. We will drop pending messages and let * at-least-once message replay kick in. *  * Another option would be to buffer the messages in memory. But this option * has the risk of causing OOM errors, especially for topologies that * disable message acking because we don't know whether the connection * recovery will succeed or not, and how long the recovery will take. */private void handleMessagesWhenConnectionIsUnavailable(Iterator<TaskMessage> msgs) {LOG.error("connection to {} is unavailable", dstAddressPrefixedName);dropMessages(msgs);}/** * 追加丢掉消息的计数器. *  * @param msgs */private void dropMessages(Iterator<TaskMessage> msgs) {// We consume the iterator by traversing and thus "emptying" it.int msgCount = iteratorSize(msgs);messagesLost.getAndAdd(msgCount);LOG.error("dropping {} message(s) destined for {}", msgCount,dstAddressPrefixedName);}private int iteratorSize(Iterator<TaskMessage> msgs) {int size = 0;if (msgs != null) {while (msgs.hasNext()) {size++;msgs.next();}}return size;}/** * 定时任务执行的方法，同时也是当批量到达的时候执行的，发射消息的方法。整个类基本就是为了做这件事 Asynchronously writes the * message batch to the channel. *  * If the write operation fails, then we will close the channel and trigger * a reconnect. */private synchronized void flushMessages(Channel channel,final MessageBatch batch) {if (!containsMessages(batch)) {return;}final int numMessages = batch.size();pendingMessages.getAndAdd(numMessages);LOG.debug("writing {} messages to channel {}", batch.size(),channel.toString());ChannelFuture future = channel.write(batch);future.addListener(new ChannelFutureListener() {public void operationComplete(ChannelFuture future)throws Exception {pendingMessages.getAndAdd(0 - numMessages);if (future.isSuccess()) {LOG.debug("sent {} messages to {}", numMessages,dstAddressPrefixedName);messagesSent.getAndAdd(batch.size());} else {LOG.error("failed to send {} messages to {}: {}",numMessages, dstAddressPrefixedName,future.getCause());closeChannelAndReconnect(future.getChannel());messagesLost.getAndAdd(numMessages);}}});}/** * 关netty连接.,并重连 *  * @param channel */private synchronized void closeChannelAndReconnect(Channel channel) {if (channel != null) {channel.close();if (channelRef.compareAndSet(channel, null)) {connect(NO_DELAY_MS);}}}private boolean containsMessages(MessageBatch batch) {return batch != null && !batch.isEmpty();}/** * Gracefully close this client. *  * We will attempt to send any pending messages (i.e. messages currently * buffered in memory) before closing the client. */@Overridepublic void close() {if (!closing) {LOG.info("closing Netty Client {}", dstAddressPrefixedName);// Set closing to true to prevent any further reconnection attempts.closing = true;flushPendingMessages();waitForPendingMessagesToBeSent();closeChannel();}}private synchronized void waitForPendingMessagesToBeSent() {LOG.info("waiting up to {} ms to send {} pending messages to {}",PENDING_MESSAGES_FLUSH_TIMEOUT_MS, pendingMessages.get(),dstAddressPrefixedName);long totalPendingMsgs = pendingMessages.get();long startMs = nowMillis();while (pendingMessages.get() != 0) {try {long deltaMs = nowMillis() - startMs;if (deltaMs > PENDING_MESSAGES_FLUSH_TIMEOUT_MS) {LOG.error("failed to send all pending messages to {} within timeout, {} of {} messages were not "+ "sent", dstAddressPrefixedName,pendingMessages.get(), totalPendingMsgs);break;}Thread.sleep(PENDING_MESSAGES_FLUSH_INTERVAL_MS);} catch (InterruptedException e) {break;}}}private synchronized void closeChannel() {if (channelRef.get() != null) {channelRef.get().close();LOG.debug("channel to {} closed", dstAddressPrefixedName);}}/** * 为metric统计框架，搜集信息 *  * @return */@Overridepublic Object getState() {LOG.info("Getting metrics for client connection to {}",dstAddressPrefixedName);HashMap<String, Object> ret = new HashMap<String, Object>();ret.put("reconnects", totalConnectionAttempts.getAndSet(0));ret.put("sent", messagesSent.getAndSet(0));ret.put("pending", pendingMessages.get());ret.put("lostOnSend", messagesLost.getAndSet(0));ret.put("dest", dstAddress.toString());String src = srcAddressName();if (src != null) {ret.put("src", src);}return ret;}private String srcAddressName() {String name = null;Channel c = channelRef.get();if (c != null) {SocketAddress address = c.getLocalAddress();if (address != null) {name = address.toString();}}return name;}@Overridepublic String toString() {return String.format("Netty client for connecting to %s",dstAddressPrefixedName);}/** * netty框架的回调 * Asynchronously establishes a Netty connection to the remote address, * returning a Netty Channel on success. */private class Connector implements Callable<Channel> {private final InetSocketAddress address;private final int connectionAttempt;public Connector(InetSocketAddress address, int connectionAttempt) {this.address = address;if (connectionAttempt < 1) {throw new IllegalArgumentException("connection attempt must be >= 1 (you provided "+ connectionAttempt + ")");}this.connectionAttempt = connectionAttempt;}@Overridepublic Channel call() throws Exception {LOG.debug("connecting to {} [attempt {}]", address.toString(),connectionAttempt);Channel channel = null;ChannelFuture future = bootstrap.connect(address);future.awaitUninterruptibly();Channel current = future.getChannel();if (future.isSuccess() && connectionEstablished(current)) {channel = current;LOG.debug("successfully connected to {}, {} [attempt {}]",address.toString(), channel.toString(),connectionAttempt);} else {LOG.debug("failed to connect to {} [attempt {}]",address.toString(), connectionAttempt);if (current != null) {current.close();}}return channel;}}}