Eclipse Paho Java Client分析——整体架构分析

前言

Eclipse Paho Java Client是IBM实现的MQTT客户端，源码地址。
本系列文章主要分析该库的整体设计，编程思想，希望能从中学到一些东西。
下面是引用官方的github上的说明：

The Paho Java Client is an MQTT client library written in Java for developing applications that run on the JVM or other Java compatible platforms such as Android
The Paho Java Client provides two APIs: MqttAsyncClient provides a fully asychronous API where completion of activities is notified via registered callbacks. MqttClient is a synchronous wrapper around MqttAsyncClient where functions appear synchronous to the application.

可以看到，该库可以在JVM和Android平台运行。库提供同步和异步两种类型的API。

常用类说明

MqttConnectOptions：连接选项设置
MqttAsyncClient：异步client
ClientComms：处理客户端与服务器间的通信，发送和接收消息。
NetworkModule：定义了网络模块的接口
TCPNetworkModule：对NetworkModule的实现，不加密。
SSLNetworkModule：对NetworkModule的实现，加密。
ConnectActionListener：用于监听连接的状态变化
MqttToken：提供用于跟踪异步操作完成的机制。
MqttWireMessage：消息类型的定义。
CommsReceiver：从MQTT服务器接收消息。
CommsSender：处理消息发送。

连接过程

new MqttAsyncClient(创建对象)

public IMqttToken connect(MqttConnectOptions options, Object userContext, IMqttActionListener callback)            throws MqttException, MqttSecurityException {    final String methodName = "connect";    // 判断当前连接状态    if (comms.isConnected()) {        throw ExceptionHelper.createMqttException(MqttException.REASON_CODE_CLIENT_CONNECTED);    }    if (comms.isConnecting()) {        throw new MqttException(MqttException.REASON_CODE_CONNECT_IN_PROGRESS);    }    if (comms.isDisconnecting()) {        throw new MqttException(MqttException.REASON_CODE_CLIENT_DISCONNECTING);    }    if (comms.isClosed()) {        throw new MqttException(MqttException.REASON_CODE_CLIENT_CLOSED);    }    this.connOpts = options;    this.userContext = userContext;    final boolean automaticReconnect = options.isAutomaticReconnect();    // 创建网络模块    comms.setNetworkModules(createNetworkModules(serverURI, options));    comms.setReconnectCallback(new MqttCallbackExtended() {        public void messageArrived(String topic, MqttMessage message) throws Exception {        }        public void deliveryComplete(IMqttDeliveryToken token) {        }        public void connectComplete(boolean reconnect, String serverURI) {        }        public void connectionLost(Throwable cause) {            // 如果设置了自动重连，在连接丢失时开始自动重连            if (automaticReconnect) {                    // Automatic reconnect is set so make sure comms is in resting state                    comms.setRestingState(true);                    reconnecting = true;                    startReconnectCycle();            }        }    });    // Insert our own callback to iterate through the URIs till the connect succeeds    MqttToken userToken = new MqttToken(getClientId());    ConnectActionListener connectActionListener = new ConnectActionListener(this, persistence, comms, options, userToken, userContext, callback, reconnecting);    userToken.setActionCallback(connectActionListener);    userToken.setUserContext(this);    // If we are using the MqttCallbackExtended, set it on the connectActionListener    if(this.mqttCallback instanceof MqttCallbackExtended){        connectActionListener.setMqttCallbackExtended((MqttCallbackExtended)this.mqttCallback);    }    comms.setNetworkModuleIndex(0);    // 连接服务器    connectActionListener.connect();    return userToken;}

ConnectActionListener.connect

1. new MqttToken，创建一个token并设置回调函数
2. comms.connect，调用ClientComms的连接函数

public void connect() throws MqttPersistenceException {    MqttToken token = new MqttToken(client.getClientId());    token.setActionCallback(this);    token.setUserContext(this);    persistence.open(client.getClientId(), client.getServerURI());    if (options.isCleanSession()) {      persistence.clear();    }    if (options.getMqttVersion() == MqttConnectOptions.MQTT_VERSION_DEFAULT) {      options.setMqttVersion(MqttConnectOptions.MQTT_VERSION_3_1_1);    }    try {      // 继续连接      comms.connect(options, token);    } catch (MqttException e) {      onFailure(token, e);    }}

ClientComms.connect

1. 创建ClientComms$ConnectBG
2. ClientComms$ConnectBG.start，开始连接服务器

public void connect(MqttConnectOptions options, MqttToken token) throws MqttException {    final String methodName = "connect";    synchronized (conLock) {        if (isDisconnected() && !closePending) {            // 修改当前连接状态为正在连接            conState = CONNECTING;            conOptions = options;            MqttConnect connect = new MqttConnect(client.getClientId(),                    conOptions.getMqttVersion(),                    conOptions.isCleanSession(),                    conOptions.getKeepAliveInterval(),                    conOptions.getUserName(),                    conOptions.getPassword(),                    conOptions.getWillMessage(),                    conOptions.getWillDestination());            this.clientState.setKeepAliveSecs(conOptions.getKeepAliveInterval());            this.clientState.setCleanSession(conOptions.isCleanSession());            this.clientState.setMaxInflight(conOptions.getMaxInflight());            tokenStore.open();            // 开启一个线程执行连接过程            ConnectBG conbg = new ConnectBG(this, token, connect);            conbg.start();        }        else {            // @TRACE 207=connect failed: not disconnected {0}            log.fine(CLASS_NAME,methodName,"207", new Object[] {new Byte(conState)});            if (isClosed() || closePending) {                throw new MqttException(MqttException.REASON_CODE_CLIENT_CLOSED);            } else if (isConnecting()) {                throw new MqttException(MqttException.REASON_CODE_CONNECT_IN_PROGRESS);            } else if (isDisconnecting()) {                throw new MqttException(MqttException.REASON_CODE_CLIENT_DISCONNECTING);            } else {                throw ExceptionHelper.createMqttException(MqttException.REASON_CODE_CLIENT_CONNECTED);            }        }    }}

ClientComms$ConnectBG.run

1. networkModule.start()，连接服务器
2. new CommsReceiver，新建消息接收类
3. new CommsSender，新建消息发送类
4. ClientComms.internalSend，发送消息

private class ConnectBG implements Runnable {    ClientComms     clientComms = null;    Thread          cBg = null;    MqttToken       conToken;    MqttConnect     conPacket;    ......    public void run() {        final String methodName = "connectBG:run";        MqttException mqttEx = null;        try {            // Reset an exception on existing delivery tokens.            // This will have been set if disconnect occured before delivery was            // fully processed.            MqttDeliveryToken[] toks = tokenStore.getOutstandingDelTokens();            for (int i=0; i<toks.length; i++) {                toks[i].internalTok.setException(null);            }            // Save the connect token in tokenStore as failure can occur before send            tokenStore.saveToken(conToken,conPacket);            // 连接网络            NetworkModule networkModule = networkModules[networkModuleIndex];            networkModule.start();            // 创建接收线程            receiver = new CommsReceiver(clientComms, clientState, tokenStore, networkModule.getInputStream());            receiver.start("MQTT Rec: "+getClient().getClientId());            // 创建发送线程            sender = new CommsSender(clientComms, clientState, tokenStore, networkModule.getOutputStream());            sender.start("MQTT Snd: "+getClient().getClientId());            callback.start("MQTT Call: "+getClient().getClientId());            // 发送连接数据包                          internalSend(conPacket, conToken);        } catch (MqttException ex) {            mqttEx = ex;        } catch (Exception ex) {            mqttEx =  ExceptionHelper.createMqttException(ex);        }        if (mqttEx != null) {            shutdownConnection(conToken, mqttEx);        }    }}

ClientComms.internalSend

1. 将客户端和token联系起来
2. this.clientState.send

void internalSend(MqttWireMessage message, MqttToken token) throws MqttException {    if (token.getClient() == null ) {        // Associate the client with the token - also marks it as in use.        token.internalTok.setClient(getClient());    } else {        // Token is already in use - cannot reuse        throw new MqttException(MqttException.REASON_CODE_TOKEN_INUSE);    }    try {        // Persist if needed and send the message        this.clientState.send(message, token);    } catch(MqttException e) {        if (message instanceof MqttPublish) {            this.clientState.undo((MqttPublish)message);        }        throw e;    }}

ClientComms.clientState.send

1. 设置messageId
2. 将消息加入消息队列，然后通知发送线程

if (message instanceof MqttConnect) {    synchronized (queueLock) {        // Add the connect action at the head of the pending queue ensuring it jumps        // ahead of any of other pending actions.        tokenStore.saveToken(token, message);        // 将连接数据包放到队列头部        pendingFlows.insertElementAt(message,0);        // 通知发送线程        queueLock.notifyAll();    }}

发送数据线程

CommsSender.run

1. 从队列中获取消息
2. 通过socket发送消息

public void run() {    final String methodName = "run";    MqttWireMessage message = null;    while (running && (out != null)) {        try {            // 从队列中获取消息，等待信号            message = clientState.get();            if (message != null) {                if (message instanceof MqttAck) {                    // 如果是ack消息，直接发送                    out.write(message);                    out.flush();                } else {                    MqttToken token = tokenStore.getToken(message);                    // While quiescing the tokenstore can be cleared so need                     // to check for null for the case where clear occurs                    // while trying to send a message.                    if (token != null) {                        synchronized (token) {                            // 发送消息                            out.write(message);                            try {                                out.flush();                            } catch (IOException ex) {                                if (!(message instanceof MqttDisconnect)) {                                    throw ex;                                }                            }                            // 发送完成通知发送等待线程                            clientState.notifySent(message);                        }                    }                }            } else {                log.fine(CLASS_NAME,methodName,"803");                running = false;            }        } catch (MqttException me) {            handleRunException(message, me);        } catch (Exception ex) {                    handleRunException(message, ex);            }    } }

接收数据线程

CommsReceiver

public void run() {    MqttToken token = null;    while (running && (in != null)) {        try {            // 读socket            receiving = in.available() > 0;            MqttWireMessage message = in.readMqttWireMessage();            receiving = false;            // 如果是ack消息，则通知发送线程接收到了ack消息            if (message instanceof MqttAck) {                token = tokenStore.getToken(message);                if (token!=null) {                    synchronized (token) {                        // Ensure the notify processing is done under a lock on the token                        // This ensures that the send processing can complete  before the                         // receive processing starts! ( request and ack and ack processing                        // can occur before request processing is complete if not!                        clientState.notifyReceivedAck((MqttAck)message);                    }                } else {                    // It its an ack and there is no token then something is not right.                    // An ack should always have a token assoicated with it.                    throw new MqttException(MqttException.REASON_CODE_UNEXPECTED_ERROR);                }            } else {                // 新消息到达                clientState.notifyReceivedMsg(message);            }        }        catch (MqttException ex) {            running = false;            // Token maybe null but that is handled in shutdown            clientComms.shutdownConnection(token, ex);        }         catch (IOException ioe) {            running = false;            // An EOFException could be raised if the broker processes the             // DISCONNECT and ends the socket before we complete. As such,            // only shutdown the connection if we're not already shutting down.            if (!clientComms.isDisconnecting()) {                clientComms.shutdownConnection(token, new MqttException(MqttException.REASON_CODE_CONNECTION_LOST, ioe));            }        }        finally {            receiving = false;        }    }}

异步API变同步API分析

发送消息后会返回一个token，可以使用这个token来等待消息发送完成，很神奇。
client.publish本身是异步的，但是调用waitForCompletion后就变成同步的了。

client.publish(topic, message).waitForCompletion(remainTime * 1000);

看一下这个等待机制是如何实现的。

publish

public IMqttDeliveryToken publish(String topic, MqttMessage message, Object userContext, IMqttActionListener callback) throws MqttException,            MqttPersistenceException {        //Checks if a topic is valid when publishing a message.        MqttTopic.validate(topic, false/*wildcards NOT allowed*/);        // 创建一个token        MqttDeliveryToken token = new MqttDeliveryToken(getClientId());        token.setActionCallback(callback);        token.setUserContext(userContext);        token.setMessage(message);        token.internalTok.setTopics(new String[] {topic});        MqttPublish pubMsg = new MqttPublish(topic, message);        // 发送消息        comms.sendNoWait(pubMsg, token);        // 这里返回token        return token;    }

comms.sendNoWait

public void sendNoWait(MqttWireMessage message, MqttToken token) throws MqttException {        final String methodName = "sendNoWait";        if (isConnected() ||                (!isConnected() && message instanceof MqttConnect) ||                (isDisconnecting() && message instanceof MqttDisconnect)) {            if(disconnectedMessageBuffer != null && disconnectedMessageBuffer.getMessageCount() != 0){                this.clientState.persistBufferedMessage(message);                disconnectedMessageBuffer.putMessage(message, token);            } else {                // 发送消息                this.internalSend(message, token);            }        } else if(disconnectedMessageBuffer != null && isResting()){            this.clientState.persistBufferedMessage(message);            disconnectedMessageBuffer.putMessage(message, token);        } else {            throw ExceptionHelper.createMqttException(MqttException.REASON_CODE_CLIENT_NOT_CONNECTED);        }    }

internalSend

void internalSend(MqttWireMessage message, MqttToken token) throws MqttException {    final String methodName = "internalSend";    if (token.getClient() == null ) {        // Associate the client with the token - also marks it as in use.        token.internalTok.setClient(getClient());    } else {        // Token is already in use - cannot reuse        throw new MqttException(MqttException.REASON_CODE_TOKEN_INUSE);    }    try {        // 通过clientState发送消息到服务器，不再往下看        // Persist if needed and send the message        this.clientState.send(message, token);    } catch(MqttException e) {        if (message instanceof MqttPublish) {            this.clientState.undo((MqttPublish)message);        }        throw e;    }}

Token.waitForCompletion

发送消息得到token后，调用waitForCompletion开始等待服务器的回答。

public void waitForCompletion(long timeout) throws MqttException {    final String methodName = "waitForCompletion";    // 开始等待    MqttWireMessage resp = waitForResponse(timeout);    if (resp == null && !completed) {        exception = new MqttException(MqttException.REASON_CODE_CLIENT_TIMEOUT);        throw exception;    }    checkResult();}

waitForResponse

protected MqttWireMessage waitForResponse(long timeout) throws MqttException {    final String methodName = "waitForResponse";    synchronized (responseLock) {        while (!this.completed) {            if (this.exception == null) {                try {                    if (timeout <= 0) {                        // 无限等待                        responseLock.wait();                    } else {                        // 带超时的等待                        responseLock.wait(timeout);                    }                } catch (InterruptedException e) {                    exception = new MqttException(e);                }            }            if (!this.completed) {                if (this.exception != null) {                    throw exception;                }                if (timeout > 0) {                    // time up and still not completed                    break;                }            }        }    }    return this.response;}

CommsReceiver.run

在调用token.waitForComplete后，调用者最终在responseLock.wait()处等待。
所以查看接收线程，应该会在收到消息后，通知调用者。

• clientState.notifyReceivedAck((MqttAck)message)，接收到服务器ack
• notifyReceivedAck
• notifyResult
• notifyComplete

protected void notifyResult(MqttWireMessage ack, MqttToken token, MqttException ex) {    final String methodName = "notifyResult";    // unblock any threads waiting on the token      token.internalTok.markComplete(ack, ex);    // 就是这里了，通知发送完成    token.internalTok.notifyComplete();    // Let the user know an async operation has completed and then remove the token    if (ack != null && ack instanceof MqttAck && !(ack instanceof MqttPubRec)) {        callback.asyncOperationComplete(token);    }    // There are cases where there is no ack as the operation failed before     // an ack was received     if (ack == null ) {        callback.asyncOperationComplete(token);    }}

notifyComplete实现

protected void notifyComplete() {        final String methodName = "notifyComplete";        synchronized (responseLock) {            // If pending complete is set then normally the token can be marked            // as complete and users notified. An abnormal error may have             // caused the client to shutdown beween pending complete being set            // and notifying the user.  In this case - the action must be failed.            if (exception == null && pendingComplete) {                completed = true;                pendingComplete = false;            } else {                pendingComplete = false;            }            // 这里通知等待的调用者            responseLock.notifyAll();        }        synchronized (sentLock) {            sent=true;              sentLock.notifyAll();        }    }

数据包到底层网络

以publish一条消息做为例子，在publish消息时，会创建一个MqttPublish的消息。看看这个消息是怎么到最后变成mqtt数据包格式的二进制流发生出去的。

public IMqttDeliveryToken publish(String topic, MqttMessage message, Object userContext, IMqttActionListener callback) throws MqttException,            MqttPersistenceException {    final String methodName = "publish";    //@TRACE 111=< topic={0} message={1}userContext={1} callback={2}    log.fine(CLASS_NAME,methodName,"111", new Object[] {topic, userContext, callback});    //Checks if a topic is valid when publishing a message.    MqttTopic.validate(topic, false/*wildcards NOT allowed*/);    MqttDeliveryToken token = new MqttDeliveryToken(getClientId());    token.setActionCallback(callback);    token.setUserContext(userContext);    token.setMessage(message);    token.internalTok.setTopics(new String[] {topic});    // 创建一个MqttPublish消息    MqttPublish pubMsg = new MqttPublish(topic, message);    comms.sendNoWait(pubMsg, token);    //@TRACE 112=<    log.fine(CLASS_NAME,methodName,"112");    return token;}

MqttPublish定义

public class MqttPublish extends MqttPersistableWireMessage {    private MqttMessage message;    private String topicName;    private byte[] encodedPayload = null;    public MqttPublish(String name, MqttMessage message) {        super(MqttWireMessage.MESSAGE_TYPE_PUBLISH);        topicName = name;        this.message = message;    }

MqttMessage定义

public class MqttMessage {    private boolean mutable = true;    private byte[] payload;    private int qos = 1;    private boolean retained = false;    private boolean dup = false;    private int messageId;

MqttOutputStream

public void write(MqttWireMessage message) throws IOException, MqttException {    final String methodName = "write";    byte[] bytes = message.getHeader();    byte[] pl = message.getPayload();//      out.write(message.getHeader());//      out.write(message.getPayload());        // 发现数据包头    out.write(bytes,0,bytes.length);    clientState.notifySentBytes(bytes.length);    int offset = 0;    int chunckSize = 1024;    // 发送数据内容    while (offset < pl.length) {        int length = Math.min(chunckSize, pl.length - offset);        out.write(pl, offset, length);        offset += chunckSize;        clientState.notifySentBytes(length);    }}

MqttWireMessage

这个类实现了将数据包头部和payload变成二进制流。

public byte[] getHeader() throws MqttException {    try {        int first = ((getType() & 0x0f) << 4) ^ (getMessageInfo() & 0x0f);        byte[] varHeader = getVariableHeader();        int remLen = varHeader.length + getPayload().length;        ByteArrayOutputStream baos = new ByteArrayOutputStream();        DataOutputStream dos = new DataOutputStream(baos);        dos.writeByte(first);        dos.write(encodeMBI(remLen));        dos.write(varHeader);        dos.flush();        return baos.toByteArray();    } catch(IOException ioe) {        throw new MqttException(ioe);    }}