sender分析之创建请求
来源:互联网 发布:淘宝如何避免售假违规 编辑:程序博客网 时间:2024/06/06 01:05
一 Sender run方法调用流程
# 从Metadata获取集群元数据
# 调用RecordAccumulator.ready方法,根据RecordAccumulator的缓存情况,选出可以向哪些Node发送消息,返回ReadyCheckResult对象
# 如果ReadyCheckResult存在某些分区没有leader副本,则调用Metadata的requestUpdate方法,标记需要更新kafka的集群信息
# 针对ReadyCheckResult的readyNodes集合,循环调用NetworkClient的ready方法,目的是检测网络I/O方面是否符合发送条件,不符合发送条件的Node将会从readyNodes集合中删除
# 调用RecordAccumulator的drain方法获取待发送的消息集合
# 调用RecordAccumulator的abortExpiredBatchers方法处理RecordAccumulator中超时的消息
# 调用Sender的createProduceRequests方法,将发送的消息封装成ClientRequest请求
# 调用NetworkClient.send方法,将ClientRequest写入KafkaChannel的send字段
# 调用NetworkClient的poll方法,将KafkaChannel中send字段保存的ClientRequest发送出去,同时还会处理服务端发回的响应处理超时请求,调用用户自定义的函数等
voidrun(long now) {
// 从Metadata获取集群元数据
Cluster cluster= metadata.fetch();
// 根据RecordAccumulator的缓存情况,选出可以向哪些Node发送消息,返回ReadyCheckResult对象
RecordAccumulator.ReadyCheckResultresult= this.accumulator.ready(cluster,now);
// 如果ReadyCheckResult存在某些分区没有leader副本,则调用Metadata的requestUpdate方法,标记需要更新kafka的集群信息
if (!result.unknownLeaderTopics.isEmpty()) {
for (Stringtopic : result.unknownLeaderTopics)
this.metadata.add(topic);
this.metadata.requestUpdate();
}
// 针对ReadyCheckResult的readyNodes集合,循环调用NetworkClient的ready方法,
// 目的是检测网络I/O方面是否符合发送条件,不符合发送条件的Node将会从readyNodes集合中删除
Iterator<Node>iter = result.readyNodes.iterator();
long notReadyTimeout= Long.MAX_VALUE;
while (iter.hasNext()) {
Node node = iter.next();
if (!this.client.ready(node,now)) {
iter.remove();
notReadyTimeout= Math.min(notReadyTimeout,this.client.connectionDelay(node,now));
}
}
// 调用RecordAccumulator的drain方法获取待发送的消息集合
Map<Integer,List<RecordBatch>>batches = this.accumulator.drain(cluster,result.readyNodes,
this.maxRequestSize,now);
// 是否需要保证消息的顺序
if (guaranteeMessageOrder) {
// 遍历record batch
for (List<RecordBatch>batchList : batches.values()) {
for (RecordBatchbatch : batchList)
this.accumulator.mutePartition(batch.topicPartition);
}
}
// 调用RecordAccumulator的abortExpiredBatchers方法处理RecordAccumulator中超时的消息
List<RecordBatch>expiredBatches = this.accumulator.abortExpiredBatches(this.requestTimeout,now);
for (RecordBatchexpiredBatch : expiredBatches)
this.sensors.recordErrors(expiredBatch.topicPartition.topic(),expiredBatch.recordCount);
sensors.updateProduceRequestMetrics(batches);
// 创建生产者请求
List<ClientRequest>requests = createProduceRequests(batches,now);
long pollTimeout= Math.min(result.nextReadyCheckDelayMs,notReadyTimeout);
if (result.readyNodes.size() >0) {
log.trace("Nodes with data ready tosend: {}",result.readyNodes);
log.trace("Created {} producerequests: {}",requests.size(),requests);
pollTimeout = 0;
}
// 将ClientRequest写入KafkaChannel的send字段
for (ClientRequestrequest : requests)
client.send(request,now);
// 调用NetworkClient的poll方法,将KafkaChannel中send字段保存的ClientRequest发送出去,
// 同时还会处理服务端发回的响应处理超时请求,调用用户自定义的函数等
this.client.poll(pollTimeout,now);
}
二 创建请求
我们先分析ProduceRequest和ProduceResponse消息体格式:
api_key: API标识
api_version: API版本号
correaltion_id: 一个单调递增序号
client_id: 客户端id
acks: 确认机制,0 不需要确认,1 只需要leader确认,-1所有副本都需要确认
timeout: 超时时间
topic: topic名称
partition: partition编号
record_set: 消息
correaltion_id: 一个单调递增序号
topic: topic名称
partition: partition编号
error_code: 错误码
base_offset: 服务端为消息生成的一个offset
timestamp: 瞬间戳
throttle_time_ms: 延长时间
private List<ClientRequest> createProduceRequests(Map<Integer, List<RecordBatch>> collated, long now) { // 保存创建的ClientRequest列表 List<ClientRequest> requests = new ArrayList<ClientRequest>(collated.size()); for (Map.Entry<Integer, List<RecordBatch>> entry : collated.entrySet()) // 将发往同一个Node的RecordBatch封装成ClientRequest requests.add(produceRequest(now, entry.getKey(), acks, requestTimeout, entry.getValue())); return requests;}
private ClientRequest produceRequest(long now, int destination, short acks, int timeout, List<RecordBatch> batches) { Map<TopicPartition, ByteBuffer> produceRecordsByPartition = new HashMap<TopicPartition, ByteBuffer>(batches.size()); final Map<TopicPartition, RecordBatch> recordsByPartition = new HashMap<TopicPartition, RecordBatch>(batches.size()); // 将RecordBatch按照partiiton分类,同时构建集合 for (RecordBatch batch : batches) { TopicPartition tp = batch.topicPartition; produceRecordsByPartition.put(tp, batch.records.buffer()); recordsByPartition.put(tp, batch); } // 创建ProduceRequest和RequestSend ProduceRequest request = new ProduceRequest(acks, timeout, produceRecordsByPartition); RequestSend send = new RequestSend(Integer.toString(destination), this.client.nextRequestHeader(ApiKeys.PRODUCE), request.toStruct()); // 创建RequestCompletionHandler作为回调对象 RequestCompletionHandler callback = new RequestCompletionHandler() { public void onComplete(ClientResponse response) { handleProduceResponse(response, recordsByPartition, time.milliseconds()); } }; // 创建ClientRequest对象 return new ClientRequest(now, acks != 0, send, callback);}
- sender分析之创建请求
- sender分析之Selector
- Kafka源码分析之Sender
- Kafka源码分析之Sender
- storyboard之 prepareForSegue:sender:
- storyboard之 prepareForSegue:sender:
- storyboard之 prepareForSegue:sender:
- storyboard之 prepareForSegue:sender:
- storyboard之 prepareForSegue:sender:
- qt学习之sender
- sender
- Sender
- QT分析之HTTP请求
- QT分析之HTTP请求
- ddpush 学习之路 12 Sender.java
- iOS学习之——prepareForSegue:sender:
- http请求之GET、POST对比分析
- Struts2源码分析之请求处理
- csdn如何转载别人的文章
- hdu 2112 HDU Today
- HBuilder更改为自定义的背景颜色
- SSH框架搭建和整合(struts2、spring4、hibernate5)
- python:syntaxerror :missing parenteses in call to print
- sender分析之创建请求
- CocoStudio UIButton setPressedActionEnabled 子控件缩放解决方案
- RAM简单介绍
- Excel转Plist
- PHP操作Excel – PHPExcel 基本用法详解
- KingEditor结合SpringMVC上传图片
- [python基础理论]python学习笔记1
- javaWeb服务详解(含源代码,测试通过,注释)
- handler中Looper的用法