Kafka low level API8 Consumer

import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
 
import org.apache.log4j.Logger;
 
import com.jd.Seckill.UdpRecv.Utils.Config;
 
import kafka.api.FetchRequest;
import kafka.api.FetchRequestBuilder;
import kafka.api.PartitionOffsetRequestInfo;
import kafka.cluster.Broker;
import kafka.common.ErrorMapping;
import kafka.common.TopicAndPartition;
import kafka.javaapi.FetchResponse;
import kafka.javaapi.OffsetRequest;
import kafka.javaapi.OffsetResponse;
import kafka.javaapi.PartitionMetadata;
import kafka.javaapi.TopicMetadata;
import kafka.javaapi.TopicMetadataRequest;
import kafka.javaapi.consumer.SimpleConsumer;
import kafka.message.MessageAndOffset;
import kafka.javaapi.TopicMetadataResponse;
 
publicclassKafkaConsume_LowAPI8
{
privatestaticLogger log =Logger.getLogger(KafkaConsume_LowAPI8.class);
privateList<String> m_replicaBrokers =newArrayList<String>();
 
publicKafkaConsume_LowAPI8()
{
m_replicaBrokers =newArrayList<String>();
}
 
publicvoid run(long a_maxReads,String a_topic,int a_partition,List<String> a_seedBrokers,int a_port)throwsException
{
// 找出目的topic与分区的leader broker（负责数据的读取与写入）
PartitionMetadata metadata = findLeader(a_seedBrokers, a_port, a_topic, a_partition);
if(null== metadata)
{
log.info("Can't find metadata for Topic and Partition. Exiting");
return;
}
if(null== metadata.leader())
{
log.info("Can't find Leader for Topic and Partition. Exiting");
return;
}
String leadBroker = metadata.leader().host();
String clientName ="Client_"+ a_topic +"_"+ a_partition;
 
SimpleConsumer consumer =newSimpleConsumer(leadBroker, a_port,100000,64*1024, clientName);
// 获取要读取的offset数值
long readOffset = getLastOffset(consumer, a_topic, a_partition, kafka.api.OffsetRequest.LatestTime(), clientName);
 
int numErrors =0;
while(a_maxReads >0)
{
if(consumer ==null)
consumer =newSimpleConsumer(leadBroker, a_port,100000,64*1024, clientName);
 
// Reading the Data,获取数据
FetchRequest req =newFetchRequestBuilder().clientId(clientName).addFetch(a_topic, a_partition, readOffset,100000).build();
FetchResponse fetchResponse = consumer.fetch(req);
 
if(fetchResponse.hasError())
{
numErrors++;
// Something went wrong!
short code = fetchResponse.errorCode(a_topic, a_partition);
log.error("Error fetching data from the Broker:"+ leadBroker +" Reason: "+ code);
if(numErrors >5)
break;
if(code ==ErrorMapping.OffsetOutOfRangeCode())
{
// We asked for an invalid offset. For simple case ask for
// the last element to reset
readOffset = getLastOffset(consumer, a_topic, a_partition, kafka.api.OffsetRequest.LatestTime(), clientName);
continue;
}
consumer.close();
consumer =null;
leadBroker = findNewLeader(leadBroker, a_topic, a_partition, a_port);
continue;
}
numErrors =0;
 
long numRead =0;
for(MessageAndOffset messageAndOffset : fetchResponse.messageSet(a_topic, a_partition))
{
long currentOffset = messageAndOffset.offset();
if(currentOffset < readOffset)
{
log.error("Found an old offset: "+ currentOffset +" Expecting: "+ readOffset);
continue;
}
readOffset = messageAndOffset.nextOffset();
ByteBuffer payload = messageAndOffset.message().payload();
byte[] bytes =newbyte[payload.limit()];
payload.get(bytes);
log.info(String.valueOf(messageAndOffset.offset())+": "+newString(bytes,"UTF-8"));
numRead++;
a_maxReads--;
}
 
if(numRead ==0)
{
try
{
Thread.sleep(1000);
}
catch(InterruptedException ie)
{
}
}
}
if(consumer !=null)
consumer.close();
}
 
// Now define where to start reading data. Kafka includes two constants to
// help, kafka.api.OffsetRequest.EarliestTime() finds the beginning of the
// data in the logs and starts streaming from there,
// kafka.api.OffsetRequest.LatestTime() will only stream new messages. Don’t
// assume that offset 0 is the beginning offset, since messages age out of
// the log over time
publicstaticlong getLastOffset(SimpleConsumer consumer,String topic,int partition,long whichTime,String clientName)
{
log.info("getLastOffset,topic:"+ topic +",partition:"+ partition +",whichTime:"+ whichTime +",clientName:"+ clientName);
TopicAndPartition topicAndPartition =newTopicAndPartition(topic, partition);
Map<TopicAndPartition,PartitionOffsetRequestInfo> requestInfo =newHashMap<TopicAndPartition,PartitionOffsetRequestInfo>();
requestInfo.put(topicAndPartition,newPartitionOffsetRequestInfo(whichTime,1));
OffsetRequest request =newOffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
OffsetResponse response = consumer.getOffsetsBefore(request);
if(response.hasError())
{
System.out.println("Error fetching data Offset Data the Broker. Reason: "+ response.errorCode(topic, partition));
return0;
}
long[] offsets = response.offsets(topic, partition);
return offsets[0];
}
 
privateString findNewLeader(String a_oldLeader,String a_topic,int a_partition,int a_port)throwsException
{
for(int i =0; i <3; i++)
{
boolean goToSleep =false;
PartitionMetadata metadata = findLeader(m_replicaBrokers, a_port, a_topic, a_partition);
if(metadata ==null)
{
goToSleep =true;
}
elseif(metadata.leader()==null)
{
goToSleep =true;
}
elseif(a_oldLeader.equalsIgnoreCase(metadata.leader().host())&& i ==0)
{
goToSleep =true;
}
else
{
return metadata.leader().host();
}
if(goToSleep)
{
try
{
Thread.sleep(1000);
}
catch(InterruptedException e)
{
log.error(e);
}
}
}
log.error("Unable to find new leader after Broker failure. Exiting");
thrownewException("Unable to find new leader after Broker failure. Exiting");
}
 
// 获取指定topic与分区的broker leader
privatePartitionMetadata findLeader(List<String> a_seedBrokers,int a_port,String a_topic,int a_partition)
{
PartitionMetadata returnMetaData =null;
 
loop:for(String seed : a_seedBrokers)
{
SimpleConsumer consumer =null;
try
{
// soTimeout与buffersize是什么意思？
consumer =newSimpleConsumer(seed, a_port,100000,64*1024,"leaderLookup");
// singletonlist返回一个只包含指定对象的不可变列表
List<String> topics =Collections.singletonList(a_topic);
TopicMetadataRequest req =newTopicMetadataRequest(topics);
TopicMetadataResponse resp = consumer.send(req);
 
List<TopicMetadata> metaData = resp.topicsMetadata();
for(TopicMetadata item : metaData)
{
for(PartitionMetadata part : item.partitionsMetadata())
{
if(part.partitionId()== a_partition)
{
returnMetaData = part;
// 跳出外循环；如果只break那么只会跳出内循环
break loop;
}
}
}
}
catch(Exception e)
{
System.out.println(
"Error communicating with Broker ["+ seed +"] to find Leader for ["+ a_topic +", "+ a_partition +"] Reason: "+ e);
}
finally
{
if(consumer !=null)
consumer.close();
}
}
if(returnMetaData !=null)
{
m_replicaBrokers.clear();
for(Broker replica : returnMetaData.replicas())
{
m_replicaBrokers.add(replica.host());
}
}
return returnMetaData;
}
 
publicvoid mainHandle()
{
KafkaConsume_LowAPI8 example =newKafkaConsume_LowAPI8();
long maxReads =25;
String topic =Config.getM_strConsume8Topic();
int partition =0;
List<String> seeds =newArrayList<String>();
seeds.add("10.189.122.213");
seeds.add("10.189.122.207");
seeds.add("10.189.122.208");
int port =9092;
try
{
example.run(maxReads, topic, partition, seeds, port);
}
catch(Exception e)
{
System.out.println("Oops:"+ e);
e.printStackTrace();
}
}
}

Downsides of using SimpleConsumer

The SimpleConsumer does require a significant amount of work not needed in the Consumer Groups:

1.           You must keep track of the offsets in your application to know where you left off consuming.
2.      You must figure out which Broker is the lead Broker for a topic and partition
3.      You  must handle Broker leader changes

Steps for using a SimpleConsumer

1. .    Find an active Broker and find out which Broker is the leader for your topic and partition
2.      Determine who the replica Brokers are for your topic and partition
3.     Build the request defining what data you are interested in
4.     .Fetch the data
5.      Identify and recover from leader changes