java 获取kafka offsets(偏移量)
来源:互联网 发布:微信小程序服务器php 编辑:程序博客网 时间:2024/05/17 02:57
public KafkaInfoClient(){ init(); } public Map<Integer,Long> getEarliestOffset(String topic) { //kafka.api.OffsetRequest.EarliestTime() = -2 return getTopicOffset(topic,kafka.api.OffsetRequest.EarliestTime()); } /*** * 获取指定 topic 的所有分区 offset * @param topic * @param whichTime 要获取offset的时间,-1 最新,-2 最早 * @return */ public Map<Integer,Long> getTopicOffset(String topic, long whichTime) { HashMap<Integer, Long> offsets = new HashMap<Integer, Long>(); TreeMap<Integer, PartitionMetadata> leaders = this.findLeader(hosts, topic); for( int part:leaders.keySet()){ PartitionMetadata metadata = leaders.get(part); String leadBroker = metadata.leader().host(); int leadPort = metadata.leader().port(); SimpleConsumer consumer = new SimpleConsumer(leadBroker, leadPort, timeOut, bufferSize, clientID); long partitionOffset = this.getPartitionOffset(consumer, topic, part, whichTime); offsets.put(part,partitionOffset); } return offsets; } /*** * 获取 offset * @param consumer SimpleConsumer * @param topic topic * @param partition partition * @param whichTime 要获取offset的时间,-1 最新,-2 最早 * @return */ private long getPartitionOffset(SimpleConsumer consumer, String topic, int partition, long whichTime) { TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition); Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>(); requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1)); //PartitionOffsetRequestInfo(long time, int maxNumOffsets) 中的第二个参数maxNumOffsets,没弄明白是什么意思,但是测试后发现传入1 时返回whichTime 对应的offset,传入2 返回一个包含最大和最小offset的元组 kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo, OffsetRequest.CurrentVersion(), consumer.clientId()); OffsetResponse response = consumer.getOffsetsBefore(request); if (response.hasError()) { logger.error("Error fetching data Offset Data the Broker. Reason:{}", response.errorCode(topic, partition)); return 0; } long[] offsets = response.offsets(topic, partition); return offsets[0]; } /*** * 获取每个 partition 元数据信息 * @param bootstraps (host,port) * @param topic topic * @return */ private TreeMap<Integer, PartitionMetadata> findLeader(Map<String, Integer> bootstraps, String topic) { TreeMap<Integer, PartitionMetadata> map = new TreeMap<Integer, PartitionMetadata>(); loop: for (Map.Entry<String, Integer> bootstrap : bootstraps.entrySet()) { SimpleConsumer consumer = null; try { consumer = new SimpleConsumer(bootstrap.getKey(), bootstrap.getValue(), timeOut, bufferSize, clientID); List<String> topics = Collections.singletonList(topic); TopicMetadataRequest req = new TopicMetadataRequest(topics); kafka.javaapi.TopicMetadataResponse resp = consumer.send(req); List<TopicMetadata> metaData = resp.topicsMetadata(); for (TopicMetadata item : metaData) { for (PartitionMetadata part : item.partitionsMetadata()) { map.put(part.partitionId(), part); } } } catch (Exception e) { logger.error("Error communicating with Broker [{}] to find Leader for [{}] Reason: ",bootstrap,topic,e); } finally { if (consumer != null) ``` consumer.close(); } } return map; }
阅读全文
0 0
- java 获取kafka offsets(偏移量)
- 如何导出zk中的偏移量(offsets)
- kafka偏移量offset--java
- Kafka 消息偏移量
- kafka控制offset偏移量
- 获取两点的偏移量(角度)
- kafka同步zookeeper前移偏移量
- Kafka 消息偏移量的维护
- Spark Streaming管理Kafka偏移量
- 获取类成员偏移量
- 如何管理Spark Streaming消费Kafka的偏移量(一)
- 如何管理Spark Streaming消费Kafka的偏移量(二)
- 如何管理Spark Streaming消费Kafka的偏移量(三)
- 获取struct成员偏移量的方法
- C++:获取类成员的偏移量
- C++:获取类成员的偏移量
- activiti获取偏移量minx与miny
- 获取类成员的偏移量
- Netty4.0学习笔记系列之四:混合使用coder和handler
- GIT-子模块
- Mybatis association与collection关联查询
- **a[3][4],(**a)[3][4], *(*a)[3][4], *(*a[3])[4]等变量占用的内存
- html,css,js代码加载顺序问题
- java 获取kafka offsets(偏移量)
- Netty4.0学习笔记系列之五:自定义通讯协议
- Java中的装箱和拆箱
- 导航栏,利用bootstrap做导航栏
- 利用Miniflow创建一个深度神经网络
- Python CGI 初体验
- UVA11584
- bzoj1503: [NOI2004]郁闷的出纳员
- Netty4.0学习笔记系列之六:多种通讯协议支持