Spark取到Kafka，出现ZK和Kafka offset不一致

在项目中用到Spark Streaming读取Kafka，应用的是Kafka的low level的API因此手动的把Offset存储到ZK（每次执行成功后，才更新zk中的offset信息）当中，但是如果出现Kafka出现网络问题或者ZK没有写入到情况就会出现ZK的offset和Kafka的offset不一致。此时就要对比Kafka和ZK中的Offset

PS：另外spark中也可以做checkpoint来保存state

• Using checkpoints
• Keeping track of the offsets that have been processed.
  另外it takes time for Spark to prepare them and store them
  checkpoint比较耗时（平均时间3S做checkpoint)
  墙裂推荐：http://aseigneurin.github.io/2016/05/07/spark-kafka-achieving-zero-data-loss.html

逻辑：
如果ZK中的offset小于 EarliestOffset 大于LastestOffset说明ZK中的offset已经失效，把ZK中的offset更新为EarliestOffset；如果ZK的offset在 EarliestOffset 大于LastestOffset之间那么以ZK的offset为准

KafkaUtil （SimpleConsumer从Kafka读取offset）

public class KafkaUtil implements Serializable {    private static final long serialVersionUID = -7708717328840L;    private static KafkaUtil kafkaUtil = null;    private KafkaUtil() {    }    public static KafkaUtil getInstance() {        if (kafkaUtil == null) {            synchronized (KafkaUtil.class) {                if (kafkaUtil == null) {                    kafkaUtil = new KafkaUtil();                }            }        }        return kafkaUtil;    }    /**     * 从brokerList中获取host     *     * @param brokerList     * @return     */    public String[] getHostFromBrokerList(String brokerList) {        String[] brokers = brokerList.split(",");        for (int i = 0; i < brokers.length; i++) {            brokers[i] = brokers[i].split(":")[0];        }        return brokers;    }    /**     * 从brokerList中获取port     *     * @param brokerList     * @return     */    public Map<String, Integer> getPortFromBrokerList(String brokerList) {        Map<String, Integer> portMap = new HashMap<String, Integer>();        String[] brokers = brokerList.split(",");        for (int i = 0; i < brokers.length; i++) {            String host = brokers[i].split(":")[0];            Integer port = Integer.valueOf(brokers[i].split(":")[1]);            portMap.put(host, port);        }        return portMap;    }    public KafkaTopicOffset topicAndMetadataRequest(String brokerList, String topic) {        List<String> topics = Collections.singletonList(topic);        TopicMetadataRequest topicMetadataRequest = new TopicMetadataRequest(topics);        KafkaTopicOffset kafkaTopicOffset = new KafkaTopicOffset(topic);        String[] hosts = getHostFromBrokerList(brokerList);        Map<String, Integer> portMap = getPortFromBrokerList(brokerList);        for (String host : hosts) {            SimpleConsumer simpleConsumer = null;            try {                simpleConsumer = new SimpleConsumer(host, portMap.get(host), Constant.TIME_OUT, Constant.BUFFERSIZE, Constant.groupId);                kafka.javaapi.TopicMetadataResponse response = simpleConsumer.send(topicMetadataRequest);                List<TopicMetadata> topicMetadatas = response.topicsMetadata();                for (TopicMetadata metadata : topicMetadatas) {                    for (PartitionMetadata partitionMetadata : metadata.partitionsMetadata()) {                        kafkaTopicOffset.getLeaderList().put(partitionMetadata.partitionId(), partitionMetadata.leader().host());                        kafkaTopicOffset.getOffsetList().put(partitionMetadata.partitionId(), 0L);                    }                }            } catch (Exception e) {                e.printStackTrace();            } finally {                if (simpleConsumer != null) {                    simpleConsumer.close();                }            }        }        return kafkaTopicOffset;    }    /**     * 从Kafka取出某个topic中某个partition的最小或者最大offset     *     * @param brokerList     * @param topic     * @return     */    public KafkaTopicOffset getOffset(String brokerList, String topic, String flag) {        KafkaTopicOffset kafkaTopicOffset = topicAndMetadataRequest(brokerList, topic);        String[] hosts = getHostFromBrokerList(brokerList);        Map<String, Integer> portMap = getPortFromBrokerList(brokerList);        for (String host : hosts) {            Iterator iterator = kafkaTopicOffset.getOffsetList().entrySet().iterator();            SimpleConsumer simpleConsumer = null;            try {                simpleConsumer = new SimpleConsumer(host, portMap.get(host), Constant.TIME_OUT, Constant.BUFFERSIZE, Constant.groupId);                while (iterator.hasNext()) {                    Map.Entry<Integer, Long> entry = (Map.Entry<Integer, Long>) iterator.next();                    int partitionId = entry.getKey();                    //判断当前的host是否为leader                    if (!kafkaTopicOffset.getLeaderList().get(partitionId).equals(partitionId)) {                        continue;                    }                    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partitionId);                    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfoMap = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();                    if (flag.equals(Constant.EARLIEST_OFFSET)) {                        requestInfoMap.put(topicAndPartition, new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.EarliestTime(), 1));                    } else if (flag.equals(Constant.LATEST_OFFSET)) {                        requestInfoMap.put(topicAndPartition, new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1));                    }                    OffsetRequest offsetRequest = new OffsetRequest(requestInfoMap, kafka.api.OffsetRequest.CurrentVersion(), Constant.groupId);                    OffsetResponse offsetResponse = simpleConsumer.getOffsetsBefore(offsetRequest);                    long[] offset = offsetResponse.offsets(topic, partitionId);                    if (offset.length > 0) {                        kafkaTopicOffset.getOffsetList().put(partitionId, offset[0]);                    }                }            } catch (Exception e) {                e.printStackTrace();            } finally {                if (simpleConsumer != null) {                    simpleConsumer.close();                }            }        }        return kafkaTopicOffset;    }}