RocketMQ源码解析-PushConsumer(3)
来源:互联网 发布:淘宝网雷丝高领打底衫 编辑:程序博客网 时间:2024/06/05 21:51
public boolean putMessage(final List<MessageExt> msgs) { boolean dispatchToConsume = false; try { this.lockTreeMap.writeLock().lockInterruptibly(); try { int validMsgCnt = 0; for (MessageExt msg : msgs) { MessageExt old = msgTreeMap.put(msg.getQueueOffset(), msg); if (null == old) { validMsgCnt++; this.queueOffsetMax = msg.getQueueOffset(); } } msgCount.addAndGet(validMsgCnt); if (!msgTreeMap.isEmpty() && !this.consuming) { dispatchToConsume = true; this.consuming = true; } if (!msgs.isEmpty()) { MessageExt messageExt = msgs.get(msgs.size() - 1); String property = messageExt.getProperty(MessageConst.PROPERTY_MAX_OFFSET); if (property != null) { long accTotal = Long.parseLong(property) - messageExt.getQueueOffset(); if (accTotal > 0) { this.msgAccCnt = accTotal; } } } } finally { this.lockTreeMap.writeLock().unlock(); } } catch (InterruptedException e) { log.error("putMessage exception", e); } return dispatchToConsume;}
取回来的消息将会在proceeQueue当中存放在其中的treeMap中(整个操作为了保证线程安全,全程加锁),并且在之后统计消费的数量统计。
在存放消息之后,将是消息的消费。
调用defaultMQPushConsumerImpl下的ConsumeMessageService的submitConsumeRequest()方法来消费消息。
@Overridepublic void submitConsumeRequest(// final List<MessageExt> msgs, // final ProcessQueue processQueue, // final MessageQueue messageQueue, // final boolean dispatchToConsume) { final int consumeBatchSize = this.defaultMQPushConsumer.getConsumeMessageBatchMaxSize(); if (msgs.size() <= consumeBatchSize) { ConsumeRequest consumeRequest = new ConsumeRequest(msgs, processQueue, messageQueue); this.consumeExecutor.submit(consumeRequest); } else { for (int total = 0; total < msgs.size();) { List<MessageExt> msgThis = new ArrayList<MessageExt>(consumeBatchSize); for (int i = 0; i < consumeBatchSize; i++, total++) { if (total < msgs.size()) { msgThis.add(msgs.get(total)); } else { break; } } ConsumeRequest consumeRequest = new ConsumeRequest(msgThis, processQueue, messageQueue); this.consumeExecutor.submit(consumeRequest); } }}
在消费消息的一开始会判断消息个数与一次最高允许消费的消息条数,如果小于,那直接回封装成ConsumeRequest消费请求丢入线程池中进行消费,而一旦大于,将会按照一次最大批量处理条数进行分次处理。
@Overridepublic void run() { if (this.processQueue.isDropped()) { log.info("the message queue not be able to consume, because it's dropped {}", this.messageQueue); return; } MessageListenerConcurrently listener = ConsumeMessageConcurrentlyService.this.messageListener; ConsumeConcurrentlyContext context = new ConsumeConcurrentlyContext(messageQueue); ConsumeConcurrentlyStatus status = null; ConsumeMessageContext consumeMessageContext = null; if (ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.hasHook()) { consumeMessageContext = new ConsumeMessageContext(); consumeMessageContext .setConsumerGroup(ConsumeMessageConcurrentlyService.this.defaultMQPushConsumer .getConsumerGroup()); consumeMessageContext.setMq(messageQueue); consumeMessageContext.setMsgList(msgs); consumeMessageContext.setSuccess(false); ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl .executeHookBefore(consumeMessageContext); } long beginTimestamp = System.currentTimeMillis(); try { ConsumeMessageConcurrentlyService.this.resetRetryTopic(msgs); status = listener.consumeMessage(Collections.unmodifiableList(msgs), context); } catch (Throwable e) { log.warn("consumeMessage exception: {} Group: {} Msgs: {} MQ: {}",// RemotingHelper.exceptionSimpleDesc(e),// ConsumeMessageConcurrentlyService.this.consumerGroup,// msgs,// messageQueue); } long consumeRT = System.currentTimeMillis() - beginTimestamp; if (null == status) { log.warn("consumeMessage return null, Group: {} Msgs: {} MQ: {}",// ConsumeMessageConcurrentlyService.this.consumerGroup,// msgs,// messageQueue); status = ConsumeConcurrentlyStatus.RECONSUME_LATER; } if (ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl.hasHook()) { consumeMessageContext.setStatus(status.toString()); consumeMessageContext.setSuccess(ConsumeConcurrentlyStatus.CONSUME_SUCCESS == status); ConsumeMessageConcurrentlyService.this.defaultMQPushConsumerImpl .executeHookAfter(consumeMessageContext); } ConsumeMessageConcurrentlyService.this.getConsumerStatsManager().incConsumeRT( ConsumeMessageConcurrentlyService.this.consumerGroup, messageQueue.getTopic(), consumeRT); if (!processQueue.isDropped()) { ConsumeMessageConcurrentlyService.this.processConsumeResult(status, context, this); } else { log.warn("processQueue is dropped without process consume result. messageQueue={}, msgs={}", messageQueue, msgs); }}
在这里将会在线程里调用用户配置的linstener对消息进行消费处理,在消费完毕之后,将会调用processConsumeResult()方法对消费结果进行处理。
public void processConsumeResult(// final ConsumeConcurrentlyStatus status, // final ConsumeConcurrentlyContext context, // final ConsumeRequest consumeRequest//) { int ackIndex = context.getAckIndex(); if (consumeRequest.getMsgs().isEmpty()) return; switch (status) { case CONSUME_SUCCESS: if (ackIndex >= consumeRequest.getMsgs().size()) { ackIndex = consumeRequest.getMsgs().size() - 1; } int ok = ackIndex + 1; int failed = consumeRequest.getMsgs().size() - ok; this.getConsumerStatsManager().incConsumeOKTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(), ok); this.getConsumerStatsManager().incConsumeFailedTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(), failed); break; case RECONSUME_LATER: ackIndex = -1; this.getConsumerStatsManager().incConsumeFailedTPS(consumerGroup, consumeRequest.getMessageQueue().getTopic(), consumeRequest.getMsgs().size()); break; default: break; } switch (this.defaultMQPushConsumer.getMessageModel()) { case BROADCASTING: for (int i = ackIndex + 1; i < consumeRequest.getMsgs().size(); i++) { MessageExt msg = consumeRequest.getMsgs().get(i); log.warn("BROADCASTING, the message consume failed, drop it, {}", msg.toString()); } break; case CLUSTERING: List<MessageExt> msgBackFailed = new ArrayList<MessageExt>(consumeRequest.getMsgs().size()); for (int i = ackIndex + 1; i < consumeRequest.getMsgs().size(); i++) { MessageExt msg = consumeRequest.getMsgs().get(i); boolean result = this.sendMessageBack(msg, context); if (!result) { msg.setReconsumeTimes(msg.getReconsumeTimes() + 1); msgBackFailed.add(msg); } } if (!msgBackFailed.isEmpty()) { consumeRequest.getMsgs().removeAll(msgBackFailed); this.submitConsumeRequestLater(msgBackFailed, consumeRequest.getProcessQueue(), consumeRequest.getMessageQueue()); } break; default: break; } long offset = consumeRequest.getProcessQueue().removeMessage(consumeRequest.getMsgs()); if (offset >= 0) { this.defaultMQPushConsumerImpl.getOffsetStore().updateOffset(consumeRequest.getMessageQueue(), offset, true); }}
在这里,如果消息消费成功,则将在这里进行对ack也就是消费成功的消息数量进行赋值。
在广播模式下,即使消息消费失败,也只是进行日志记录,并没有别的操作。
在集群模式下,如果消息消费失败,将会先尝试将消息发送回broker,如果发送回给broker也失败了,那么将会将这条消息在一定时间后重新尝试消费,否则就从消息失败数组中移除这个消息。
最后,更新消息队列的消费进度。PushConsumer的消息消费结束。阅读全文
0 0
- RocketMQ源码解析-PushConsumer(3)
- RocketMQ源码解析-PushConsumer(1)
- RocketMQ源码解析-PushConsumer(2)
- RocketMQ源码解析:Filtersrv
- rocketmq源码解析系列-开头
- RocketMQ源码解析:Message存储
- RocketMQ源码解析:高可用
- RocketMQ源码解析:事务消息
- RocketMQ源码解析-Producer启动
- RocketMQ原理以及源码解析目录介绍
- RocketMQ源码解析:Message发送&接收
- RocketMQ源码解析-Producer消息发送
- RocketMQ源码解析-Consumer启动(1)
- RocketMQ源码解析-Consumer启动(2)
- RocketMQ源码解析-PullConsumer取消息(1)
- RocketMQ源码解析-PullConsumer取消息(2)
- RocketMQ源码解析-Broker的HA实现
- RocketMQ源码解析-Broker的消息存储
- Qt5+vs2015无法打开源文件moc_XXX.h/cpp的问题
- Struts2 概述
- 简析XML和HTML的区别
- 11.2
- java常见面试题之单例模式
- RocketMQ源码解析-PushConsumer(3)
- Android 组件学习笔记(九宫格游戏,imageview设置图片满屏技巧)
- string类方法
- 键盘输入数组
- struts2中关于package的namespace和访问路径嵌套的处理关系
- 【Java基础】equals和==的区别以及Integer(-128~127)值的==和equals比较
- python中的input方法
- Nginx负载均衡服务器的双机高可用
- 函数(方法)