ConsumerNetworkClient 分析

来源：互联网发布：泉州java培训机构编辑：程序博客网时间：2024/05/17 20:29

ConsumerNetworkClient 是对NetworkClient的一个封装，提供了额外的一些功能，首先我们看一下它有哪些属性：

# NetworkClient

private final KafkaClient client;

# 等待被发送的请求队列

private final Map<Node,List<ClientRequest>> unsent = new HashMap<>();

# 元数据信息

private final Metadata metadata;

# 重试时间

private final long retryBackoffMs;

# ClientRequest在unsent列表的超时时间

private final long unsentExpiryMs;

# KafkaConsumer是否正在执行不可中断的方法的次数，每次进入一个不可中断方法就加1，退出减1，它只会被KafkaConsumer线程修改，其他线程不能修改

private int wakeupDisabledCount = 0;

# 当请求完成时，它们会在调用之前被转移到这个队列中。这样做的目的是为了避免在持有该对象的监视器时调用它们，从而为死锁打开大门。

private finalConcurrentLinkedQueue<RequestFutureCompletionHandler> pendingCompletion =new ConcurrentLinkedQueue<>();

接着我们看一些它的核心方法：

# poll(long timeout, long now, PollConditionpollCondition) 主要用于发送请求，检测连接状态，处理超时请求

public void poll(longtimeout, long now, PollCondition pollCondition) {
    // 触发pendingCompleted里的请求
    firePendingCompletedRequests();

    synchronized (this) {
        // 发送现在我们能发送的所有请求
        trySend(now);

        // 检测我们是否还需要poll
        if (pollCondition== null || pollCondition.shouldBlock()) {
            // if thereare no requests in flight, do not block longer than the retrybackoff
            if (client.inFlightRequestCount() ==0)
                timeout = Math.min(timeout,retryBackoffMs);
            client.poll(Math.min(MAX_POLL_TIMEOUT_MS,timeout), now);
            now = time.milliseconds();
        } else {
            client.poll(0,now);
        }
        // 检测连接状态，检测消费者与每个Node之间的连接状态，当检测到连接断开的Node时，会将其在unsent列表中对应的全部Client
        // Request对象清除掉，之后调用这些ClientRequest函数，并且设置disconnect标记为true
        checkDisconnects(now);

        // 检测wakeup和wakeupDisabledCount,查看是否有其他线程中断，如果有中断请求，则抛出WakeupExeception
        maybeTriggerWakeup();
        // 再次试图发送，可能现在内存已经清理了或者Node可以连接上了
        trySend(now);

        // 处理unsent中的超时请求，它会遍历整个unsent集合，检测，每一个ClientRequest是否超时，调用超时ClientRequest
        // 的回调函数，并将其从unsent列表删除
        failExpiredRequests(now);
    }

    // 触发pendingCompleted里的请求
    firePendingCompletedRequests();
}

# poll(RequestFuture<?> future, longtimeout): 是一个实现阻塞发送请求的功能

public boolean poll(RequestFuture<?> future, long timeout) {    long begin = time.milliseconds();    long remaining = timeout;    long now = begin;    do {        poll(remaining, now, future);// 请求未完成则调用poll方法        now = time.milliseconds();        long elapsed = now - begin;        remaining = timeout - elapsed;    } while (!future.isDone() && remaining > 0); //循环检测future的完成情况    return future.isDone();}

# send(Node, ApiKeys api, shortversion,AbstractRequest request):

会将等待发送的请求封装成ClinetRequest

private RequestFuture<ClientResponse> send(Node node, ApiKeys api, short version, AbstractRequest request) {    long now = time.milliseconds();    RequestFutureCompletionHandler completionHandler = new RequestFutureCompletionHandler();    RequestHeader header = client.nextRequestHeader(api, version);    RequestSend send = new RequestSend(node.idString(), header, request.toStruct());    // 将这个ClinetRequest放到unsent列表    put(node, new ClientRequest(now, true, send, completionHandler));    // 如果NetworkCLient被poll阻塞，则唤醒它    client.wakeup();    // 返回RequestFuture对象    return completionHandler.future;}

# put(Node node, ClientRequest request): 往unsent列表添加ClientRequest 这些请求等待被发送

private void put(Node node, ClientRequest request) {    synchronized (this) {        List<ClientRequest> nodeUnsent = unsent.get(node);        if (nodeUnsent == null) {            nodeUnsent = new ArrayList<>();            unsent.put(node, nodeUnsent);        }        nodeUnsent.add(request);    }}

# leastLoadedNode: 获取集群中负载最少的节点

public Node leastLoadedNode() {    synchronized (this) {        return client.leastLoadedNode(time.milliseconds());    }}

# awaitMetadataUpdate(): 除非元数据已经更新，否则一直阻塞

public void awaitMetadataUpdate() {    awaitMetadataUpdate(Long.MAX_VALUE);}

# awaitMetadataUpdate(long timeout): 除非元数据已经更新，否则在指定的时间内一直阻塞

public boolean awaitMetadataUpdate(long timeout) {    long startMs = time.milliseconds();    int version = this.metadata.requestUpdate();    do {        poll(timeout);    } while (this.metadata.version() == version && time.milliseconds() - startMs < timeout);    return this.metadata.version() > version;}

# ensureFreshMetadata(): 确保元数据应经刷新

public void ensureFreshMetadata() {    if (this.metadata.updateRequested() || this.metadata.timeToNextUpdate(time.milliseconds()) == 0)        awaitMetadataUpdate();}

# awaitPendingRequests(Node node): 等待指定节点的pedning的请求已经完成，否则一直阻塞

public void awaitPendingRequests(Node node) {    while (pendingRequestCount(node) > 0)        poll(retryBackoffMs);}

# pendingRequestCount(Node node)：获得指定节点pending的request数量

public int pendingRequestCount(Node node) {    synchronized (this) {        List<ClientRequest> pending = unsent.get(node);        int unsentCount = pending == null ? 0 : pending.size();        return unsentCount + client.inFlightRequestCount(node.idString());    }}

# pendingRequestCount()：获得所有节点pending的request数量

public int pendingRequestCount() {    synchronized (this) {        int total = 0;        for (List<ClientRequest> requests: unsent.values())            total += requests.size();        return total + client.inFlightRequestCount();    }}

# firePendingCompletedRequests()：触发pendingCompleted里的请求

private void firePendingCompletedRequests() {    boolean completedRequestsFired = false;    for (;;) {        // 获取RequestFutureCompletionHandler        RequestFutureCompletionHandler completionHandler = pendingCompletion.poll();        // 如果为空，跳出循环        if (completionHandler == null)            break;        completionHandler.fireCompletion();        completedRequestsFired = true;    }    // 为了防止NetworkClient在poll方法中被阻塞，唤醒这个NetworkClient    if (completedRequestsFired)        client.wakeup();}

# checkDisconnects(long now)：检测连接状态，检测消费者与每个Node之间的连接状态，当检测到连接断开的Node时，会将其在unsent列表中对应的全部Client

private void checkDisconnects(long now) {    Iterator<Map.Entry<Node, List<ClientRequest>>> iterator = unsent.entrySet().iterator();    while (iterator.hasNext()) {        Map.Entry<Node, List<ClientRequest>> requestEntry = iterator.next();        Node= requestEntry.getKey();        if (client.connectionFailed(node)) {            // Remove entry before invoking request callback to avoid callbacks handling            // coordinator failures traversing the unsent list again.            iterator.remove();            for (ClientRequest request : requestEntry.getValue()) {                RequestFutureCompletionHandler handler =                        (RequestFutureCompletionHandler) request.callback();                handler.onComplete(new ClientResponse(request, now, true, null));            }        }    }}

# failExpiredRequests(long now)： unsent中的超时请求，它会遍历整个unsent集合，检测，每一个ClientRequest是否超时，调用超时ClientRequest的回调函数，并将其从unsent列表删除

private void failExpiredRequests(long now) {    // clear all expired unsent requests and fail their corresponding futures    Iterator<Map.Entry<Node, List<ClientRequest>>> iterator = unsent.entrySet().iterator();    while (iterator.hasNext()) {        Map.Entry<Node, List<ClientRequest>> requestEntry = iterator.next();        Iterator<ClientRequest> requestIterator = requestEntry.getValue().iterator();        while (requestIterator.hasNext()) {            ClientRequest request = requestIterator.next();            if (request.createdTimeMs() < now - unsentExpiryMs) {                RequestFutureCompletionHandler handler = (RequestFutureCompletionHandler) request.callback();                handler.onFailure(new TimeoutException("Failed to send request after " + unsentExpiryMs + " ms."));                requestIterator.remove();            } else                break;        }        if (requestEntry.getValue().isEmpty())            iterator.remove();    }}

# trySend(long now)：尝试发送请求

private boolean trySend(long now) {    // send any requests that can be sent now    boolean requestsSent = false;    // 遍历unsent列表里的ClientRequest，判断NetworkClient是否已准备好，如果准备好了就开始发送请求    for (Map.Entry<Node, List<ClientRequest>> requestEntry: unsent.entrySet()) {        Node= requestEntry.getKey();        Iterator<ClientRequest> iterator = requestEntry.getValue().iterator();        while (iterator.hasNext()) {            ClientRequest request = iterator.next();            if (client.ready(node, now)) {                client.send(request, now);                iterator.remove();// 从unsent集合列表删除该请求                requestsSent = true;            }        }    }    return requestsSent;}

# maybeTriggerWakeup()：检测wakeup 和 wakeupDisabledCount,查看是否有其他线程中断，如果有中断请求，则抛出WakeupExeception

private void maybeTriggerWakeup() {    if (wakeupDisabledCount == 0 && wakeup.get()) {        wakeup.set(false);        throw new WakeupException();    }}

内部类

RequestFutureCompletionHandler：一个回调接口的实现，用于在请求完成时根据相应执行该回调方法。如果在处理请求时存在断开连接，那么这个处理程序也会被调用。

// 请求结果，可以判断请求是否完成，请求成功还是失败，请求是否可重试private final RequestFuture<ClientResponse> future;// 响应结果private ClientResponse response;

// 当请求完成的时候，触发某些动作public void fireCompletion() {    if (e != null) {        future.raise(e);    } else if (response.wasDisconnected()) {        ClientRequest request = response.request();        RequestSend send = request.request();        ApiKeys api = ApiKeys.forId(send.header().apiKey());        int correlation = send.header().correlationId();        log.debug("Cancelled {} request {} with correlation id {} due to node {} being disconnected",                api, request, correlation, send.destination());        future.raise(DisconnectException.INSTANCE);    } else {        future.complete(response);    }}

阅读全文

0 0