kafka consumer group的删除和offset恢复

运维kafka时，会有时需要删除无效或已下线的consumer group，若设置的offset保存位置是zookeepr，则数据都在zookeeper中，可自行删除，方式就是删除zookeeper相应节点。

kafka的offset若想按照生产时间进行恢复，事实严重不准，具体可参见：关于kafka中的timestamp与offset的对应关系 

恢复步骤：

1. 获取该consumer group消费topic的各partition该时间的offset

2. 在zookeeper中修改这些offset

进行操作期间需要暂停该group对topic的消费，恢复offset后再重启，否则修改不生效。

上代码！

ResetOffsetOperator.resetOffset(topic, group, null, whichTime);ZookeeperOperator zookeeperOperator = new ZookeeperOperator();zookeeperOperator.deleteUselessConsumer("test1");

public class ZookeeperOperator implements Watcher {  private ZooKeeper zooKeeper = null;  private static final Logger log = LoggerFactory      .getLogger(ZookeeperOperator.class);  public ZookeeperOperator() throws IOException {    this(KafkaConf.ZOOKEEPERHOST);  }  public ZookeeperOperator(String zookeeperHost) throws IOException {    zooKeeper = new ZooKeeper(zookeeperHost, 5000, ZookeeperOperator.this);  }  public void close() {    try {      zooKeeper.close();    } catch (InterruptedException e) {      log.error("Failed to close zookeeper:", e);    }  }  /**   * 读取指定节点下孩子节点数目   *   * @param path 节点path   * @return   */  private int readChildren(String path) {    try {      return zooKeeper.getChildren(path, false, null).size();    } catch (KeeperException e) {      log.error("Read error,KeeperException:" + path, e);      return 0;    } catch (InterruptedException e) {      log.error("Read error,InterruptedException:" + path, e);      return 0;    }  }  private List<String> getChildrenList(String path) {    try {      return zooKeeper.getChildren(path, false, null);    } catch (KeeperException e) {      log.error("Read error,KeeperException:" + path, e);      return null;    } catch (InterruptedException e) {      log.error("Read error,InterruptedException:" + path, e);      return null;    }  }  private boolean setData(String path, String data) {    try {      zooKeeper.setData(path, data.getBytes(), -1);    } catch (KeeperException e) {      log.error("Set error,KeeperException:" + path + " data:" + data, e);      return false;    } catch (InterruptedException e) {      log.error("Set error,InterruptedException:" + path + " data:" + data, e);      return false;    }    return true;  }  private boolean deleteData(String path) {    try {      zooKeeper.delete(path, -1);    } catch (InterruptedException e) {      log.error("delete error,InterruptedException:" + path, e);      return false;    } catch (KeeperException e) {      log.error("delete error,KeeperException:" + path, e);      return false;    }    return true;  }  private boolean recursivelyDeleteData(String path) {    List<String> childList = getChildrenList(path);    if (childList == null) {      return false;    } else if (childList.isEmpty()) {      deleteData(path);    } else {      for (String childName : childList) {        String childPath = path + "/" + childName;        List<String> grandChildList = getChildrenList(childPath);        if (grandChildList == null) {          return false;        } else if (grandChildList.isEmpty()) {          deleteData(childPath);        } else {          recursivelyDeleteData(childPath);        }      }      deleteData(path);    }    return true;  }  private String getData(String path) {    try {      return new String(zooKeeper.getData(path, false, null));    } catch (KeeperException e) {      log.error("Read error,KeeperException:" + path, e);      return "";    } catch (InterruptedException e) {      log.error("Read error,InterruptedException:" + path, e);      return "";    }  }  /**   * 读取指定节点下孩子节点数目   *   * @param topic kafka topic 名称   * @return   */  public int readTopicChildren(String topic) {    StringBuilder sb = new StringBuilder().append("/brokers/topics/")        .append(topic).append("/partitions");    return readChildren(sb.toString());  }  public boolean setTopicGroupOffset(String topic, String group,      String partition, String data) {    StringBuilder sb = new StringBuilder().append("/consumers/").append(group)        .append("/offsets/").append(topic).append("/").append(partition);    return setData(sb.toString(), data);  }  public String getTopicGroupOffset(String topic, String group,      String partition) {    StringBuilder sb = new StringBuilder().append("/consumers/").append(group)        .append("/offsets/").append(topic).append("/").append(partition);    System.out.println(sb.toString());    return getData(sb.toString());  }  public boolean deleteUselessConsumer(String topic, String group) {    if (topic.endsWith("-1")) {      StringBuilder sb = new StringBuilder().append("/consumers/")          .append(group);      return recursivelyDeleteData(sb.toString());    } else {      StringBuilder sb = new StringBuilder().append("/consumers/").append(group)          .append("/offsets/").append(topic);      return recursivelyDeleteData(sb.toString());    }  }  public boolean deleteUselessLikeConsumer(String topic, String group) {    String path = "/consumers";    List<String> childList = getChildrenList(path);    int success = 0;    int count = 0;    for (String child : childList) {      if (child.startsWith(group)) {        count++;        if (deleteUselessConsumer(topic, child)) {          success++;        }      }    }    if (success == count) {      return true;    } else {      return false;    }  }  public boolean deleteUselessLikeConsumer(String group) {    return deleteUselessLikeConsumer("-1", group);  }  public boolean deleteUselessConsumer(String group) {    return deleteUselessConsumer("-1", group);  }  @Override  public void process(WatchedEvent event) {    log.info("Receive Event：" + event.getState());  }}

public class ResetOffsetOperator {  public static boolean resetOffset(final String topic, String group,      Properties properties, long whichTime) throws IOException {    if (StringUtils.isBlank(topic) || StringUtils.isBlank(group)) {      System.err.println("topic or group can not be null or empty!");      System.exit(2);    }    if (properties == null) {      properties = new Properties();      properties.setProperty("zookeeper.connect", KafkaConf.ZOOKEEPERHOST);      properties.setProperty("group.id", group);      // zookeeper最大超时时间，就是心跳的间隔，若是没有反映，那么认为已经死了，不易过大      properties.setProperty("zookeeper.session.timeout.ms", "10000");      // ZooKeeper集群中leader和follower之间的同步时间      properties.setProperty("zookeeper.sync.time.ms", "2000");      // 自动提交offset到zookeeper的时间间隔      properties.setProperty("auto.commit.interval.ms", "3000");      // 当zookeeper中没有初始的offset时候的处理方式 。smallest ：重置为最小值 largest:重置为最大值 anything else：抛出异常      properties.setProperty("auto.offset.reset", "largest");    }    ZookeeperOperator zookeeper = new ZookeeperOperator(        properties.getProperty("zookeeper.connect"));    GroupOperator groupOperator = new GroupOperator(topic, group, whichTime,        zookeeper);    return groupOperator.retryResetGroupOffset();  }}

public class GroupOperator {  private static final Logger LOG = LoggerFactory      .getLogger(GroupOperator.class);  private List<String> m_replicaBrokers;  private ZookeeperOperator zookeeper;  private String topic;  private List<String> seeds;  private int port;  private long whichTime;  private int partitionNum;  private String group;  private int retryNum = 5;  /**   * 初始化   *   * @param topic   * @param whichTime timestamp(13位)/-1(latest)/-2(earliest)   * @throws java.io.IOException   */  public GroupOperator(String topic, String group, long whichTime,  ZookeeperOperator zookeeper)      throws IOException {    this.topic = topic;    this.group = group;    this.whichTime = whichTime;    m_replicaBrokers = new ArrayList<String>();    seeds = KafkaConf.getBrokerHost();    port = Integer.parseInt(KafkaConf.BROKERPORT);    this.zookeeper = zookeeper;    partitionNum = zookeeper.readTopicChildren(topic);  }  /**   * 将zookeeper中该group对应该topic下的所有分区的offset恢复为所希望的时间   */  public boolean resetGroupOffset() {    List<Long> offsetList = new ArrayList<Long>();    for (int partition = 0; partition < partitionNum; partition++) {      long offset = getOffset(partition);      if (offset == -1) {        LOG.error("Failed to get offset of " + group + " with partition:"            + partition);        return false;      } else {        offsetList.add(offset);      }    }    for (int partition = 0; partition < partitionNum; partition++) {      boolean isSuccess = zookeeper          .setTopicGroupOffset(topic, group, String.valueOf(partition),              String.valueOf(offsetList.get(partition)));      if (!isSuccess) {        LOG.error("Failed to reset offset of topic:" + topic + " group:" + group            + " partition" + partition + " value:" + offsetList.get(partition));        return false;      }    }    return true;  }  public boolean retryResetGroupOffset() {    for (int retry = 0; retry < retryNum; retry++) {      if (resetGroupOffset()) {        return true;      }    }    return false;  }  /**   * 获取该partition要恢复时间的offset   *   * @param partition   * @return   */  public long getOffset(int partition) {    // find the meta data about the topic and partition we are interested in    PartitionMetadata metadata = findLeader(partition);    if (metadata == null) {      LOG.error("Can't find metadata for Topic and Partition. Exiting");      return -1;    }    if (metadata.leader() == null) {      LOG.error("Can't find Leader for Topic and Partition. Exiting");      return -1;    }    String leadBroker = metadata.leader().host();    String clientName = "Client_" + topic + "_" + partition;    SimpleConsumer consumer = new SimpleConsumer(leadBroker, port, 100000,        64 * 1024, clientName);    long readOffset = getAssignedOffset(consumer, partition, clientName);    if (consumer != null) {      consumer.close();    }    return readOffset;  }  /**   * Finding the Lead Broker for a Topic and Partition   *   * @param a_partition 分区id，从0开始   * @return   */  private PartitionMetadata findLeader(int a_partition) {    PartitionMetadata returnMetaData = null;    loop:    for (String seed : seeds) {      SimpleConsumer consumer = null;      try {        consumer = new SimpleConsumer(seed, port, 100000, 64 * 1024,            "leaderLookup");        List<String> topics = Collections.singletonList(topic);        TopicMetadataRequest req = new TopicMetadataRequest(topics);        kafka.javaapi.TopicMetadataResponse resp = consumer.send(req);        List<TopicMetadata> metaData = resp.topicsMetadata();        for (TopicMetadata item : metaData) {          for (PartitionMetadata part : item.partitionsMetadata()) {            if (part.partitionId() == a_partition) {              returnMetaData = part;              break loop;            }          }        }      } catch (Exception e) {        LOG.error("Error communicating with Broker [" + seed            + "] to find Leader for [" + topic + ", " + a_partition            + "] Reason: " + e);      } finally {        if (consumer != null)          consumer.close();      }    }    if (returnMetaData != null) {      m_replicaBrokers.clear();      for (kafka.cluster.Broker replica : returnMetaData.replicas()) {        m_replicaBrokers.add(replica.host());      }    }    return returnMetaData;  }  public long getAssignedOffset(SimpleConsumer consumer, int partition,      String clientName) {    TopicAndPartition topicAndPartition = new TopicAndPartition(topic,        partition);    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();    requestInfo        .put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));    kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(        requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);    OffsetResponse response = consumer.getOffsetsBefore(request);    if (response.hasError()) {      System.out.println(          "Error fetching data Offset Data the Broker. Reason: " + response              .errorCode(topic, partition));      return -1;    }    long[] offsets = response.offsets(topic, partition);    return offsets[0];  }}