kafka生产者消费者

public class KafkaConsumer {private static final Logger logger = Logger.getLogger(KafkaAdmin.class);private Properties consumerConfig;private ConsumerConnector connector;private Map<String , Integer> topicCountMap;private Map<String , List<KafkaStream<String, String>>> streamMap;public KafkaConsumer(Map<String , Integer> topicCountMap , Properties consumerConfig){this.topicCountMap = topicCountMap;this.consumerConfig = consumerConfig;init() ;}private void init() {        Decoder<String> keyDecoder = new StringDecoder(new VerifiableProperties());        Decoder<String> valueDecoder = new StringDecoder(new VerifiableProperties());this.connector = kafka.consumer.Consumer.createJavaConsumerConnector(new ConsumerConfig(consumerConfig));streamMap = connector.createMessageStreams(topicCountMap , keyDecoder , valueDecoder) ;}public Map<String , List<KafkaStream<String, String>>> getStreamMap() {return streamMap;}}

public class KafkaProducer{protected Log logger = LogFactory.getLog(this.getClass());private Map<Integer,Producer<String,String>> producerMap=new HashMap<Integer,Producer<String,String>>();private String topic;private int producerNum;public KafkaProducer(String topic,int producerNum,Properties config){this.topic=topic;this.producerNum=producerNum;Producer<String,String> producer;for(int i=0;i<producerNum;i++){producer=new Producer<String,String>(new ProducerConfig(config));producerMap.put(Integer.valueOf(i), producer);}}public void sendMessage(String msgKey,String message){int position=TopicUtil.calcPartition(msgKey, producerNum);Producer<String,String> producer=producerMap.get(Integer.valueOf(position));producer.send(new KeyedMessage<String,String>(topic,msgKey,message));}}

public class ZKStringSerializer implements ZkSerializer {@Overridepublic Object deserialize(byte[] bytes) throws ZkMarshallingError {if(bytes==null){return null;} try {return new String(bytes, "UTF-8");} catch (UnsupportedEncodingException e) {throw new ZkMarshallingError(e);}}@Overridepublic byte[] serialize(Object data) throws ZkMarshallingError {try {return ((String)data).getBytes("UTF-8");} catch (UnsupportedEncodingException e) {throw new ZkMarshallingError(e);}}}

public class OrderFetcher {private Log logger = LogFactory.getLog(OrderFetcher.class) ;private KafkaStream<String , String> kafkaStream;private String topic = null;private int partition =-1;private long offset =-1l;public OrderFetcher(KafkaStream<String, String> kafkaStream){this.kafkaStream = kafkaStream;}public synchronized OrderMessageDTO fetchMessage()throws NoMessageException{OrderMessageDTO msgDTO = new OrderMessageDTO() ;JSONObject message = getMessage() ; //获取消息msgDTO.setReqOrderDTO(message) ;  //解释消息msgDTO.setOffset(offset);msgDTO.setTopic(topic);msgDTO.setPartition(partition);msgDTO.setDateCreated(new Date()) ;return msgDTO ;}private JSONObject getMessage() throws NoMessageException {JSONObject message = null ;try{ConsumerIterator<String , String> it = kafkaStream.iterator();MessageAndMetadata<String , String> mam = null ;if(it.hasNext()){mam=it.next();}message= JSONObject.parseObject(mam.message()) ;topic = mam.topic();partition = mam.partition();offset = mam.offset();}catch (ConsumerTimeoutException e) {throw new NoMessageException();}catch (Exception e) {throw new NoMessageException();}return message;}}

public class OrderFetcherPool {private ArrayBlockingQueue<OrderFetcher>  orderQueue = null ;private KafkaConsumer kafkaConsumer;public OrderFetcherPool(OrderDeliveryConfig config,PaesbCfgHolder paesbCfgHolder){Map<String ,Integer> topicCountMap = new HashMap<String,Integer>() ;topicCountMap.put(config.getOrderTopic() , config.getPartitionsOfOrderConsum()) ;topicCountMap.put(config.getFailOrderTopic() , config.getPartitionsOfFailOrderConsum()) ;kafkaConsumer = new KafkaConsumer(topicCountMap , paesbCfgHolder.getKafkaConsumerProps()) ;orderQueue = new ArrayBlockingQueue<OrderFetcher>(config.getPartitionsOfOrderConsum()) ;List<KafkaStream<String, String>> orderList = kafkaConsumer.getStreamMap().get(config.getOrderTopic());if(orderList!=null){for(KafkaStream<String , String> kafkaStream : orderList){orderQueue.offer(new OrderFetcher(kafkaStream));}}}public OrderFetcher getOrderFetcher(){try {return orderQueue.take();} catch (InterruptedException e) {return null;}}public void releaseOrderFetcher(OrderFetcher fetcher){orderQueue.offer(fetcher);}}

public class OrderDelivery  implements Runnable {protected Log logger = LogFactory.getLog(this.getClass());private OrderHandler handler;private OrderFetcher fetcher ;private String threadId="";public OrderDelivery(OrderFetcherPool fetchPool , OrderHandler handler , String threadId){this.threadId=threadId;this.fetcher = fetchPool.getOrderFetcher()  ;this.handler = handler ;}@Overridepublic void run() {while(true){logger.info(threadId+" is alive!");doWork();}}private void doWork() {OrderMessageDTO msgDTO=null;try{msgDTO = fetchMessage() ;if(msgDTO != null){   processOrder(msgDTO) ; //----------处理订单}}catch(Exception e){logger.info("handle order error "+e.getMessage(),e);}}private OrderMessageDTO fetchMessage(){OrderMessageDTO msgDTO = null;try {msgDTO = fetcher.fetchMessage();} catch (NoMessageException e) {}return msgDTO;}private void processOrder(OrderMessageDTO msgDTO){OrderRequest request = setRequestData(msgDTO);OrderResponse response = new OrderResponse();try{    handler.processOrder(request, response); }catch(Exception e){}}private OrderRequest setRequestData(OrderMessageDTO msgDTO) {OrderRequest request=new OrderRequest();request.setAcceptNo(msgDTO.getId());request.setOrderDTO(msgDTO.getReqOrderDTO());return request;}}

public class OrderDeliveryContext{private  Log logger = LogFactory.getLog(OrderDeliveryContext.class);//--线程池private ExecutorService deliveryOrderthreadPool;//正常的订单交付线程池//--workerprivate List<OrderDelivery> orderDelivererList;//交付者列表private OrderFetcherPool orderFetcherPool;private OrderRecord orderRecord ;private OrderDeliveryConfig config;//交付配置private OrderHandler handler;//订单处理器private KafkaProducer failOrderProducer;//消息生产者private PaesbCfgHolder paesbCfgHolder;public OrderDeliveryContext(){}@PostConstructpublic void init() throws Exception {logger.info("OrderDeliveryContext initail.........");    /*公共配置*/config = new OrderDeliveryConfig(paesbCfgHolder.getPaesbProps()) ;      /*创建topic*/    KafkaAdmin.createTopic(config.getOrderTopic(), config.getPartitionsOfOrder(), config.getReplicaOfOrder(), config.getZkHost()) ; /*kafka配置*/orderFetcherPool = new OrderFetcherPool(config , paesbCfgHolder) ;deliveryOrderthreadPool = Executors.newFixedThreadPool(config.getOrderHandlerThreadSize());orderDelivererList = new ArrayList<OrderDelivery>();OrderDelivery deliverer = null ;for(int i = 0 ; i < config.getOrderHandlerThreadSize() ; i++){deliverer = new OrderDelivery(orderFetcherPool , handler , "Order-"+i) ;orderDelivererList.add(deliverer) ;deliveryOrderthreadPool.execute(deliverer) ;}deliveryOrderthreadPool.shutdown() ;logger.info("OrderDeliveryContext initail finished.........");}public OrderHandler getHandler() {return handler;}public void setHandler(OrderHandler handler) {this.handler = handler;}public OrderRecord getOrderRecord() {return orderRecord;}public void setOrderRecord(OrderRecord orderRecord) {this.orderRecord = orderRecord;}public KafkaProducer getProducer() {return failOrderProducer;}public void setProducer(KafkaProducer producer) {this.failOrderProducer = producer;}public ExecutorService getDeliveryOrderthreadPool() {return deliveryOrderthreadPool;}public void setDeliveryOrderthreadPool(ExecutorService deliveryOrderthreadPool) {this.deliveryOrderthreadPool = deliveryOrderthreadPool;}public OrderDeliveryConfig getConfig() {return config;}public void setConfig(OrderDeliveryConfig config) {this.config = config;}public KafkaProducer getFailOrderProducer() {return failOrderProducer;}public void setFailOrderProducer(KafkaProducer failOrderProducer) {this.failOrderProducer = failOrderProducer;}public PaesbCfgHolder getPaesbCfgHolder() {return paesbCfgHolder;}public void setPaesbCfgHolder(PaesbCfgHolder paesbCfgHolder) {this.paesbCfgHolder = paesbCfgHolder;}public OrderFetcherPool getOrderFetcherPool() {return orderFetcherPool;}public void setOrderFetcherPool(OrderFetcherPool orderFetcherPool) {this.orderFetcherPool = orderFetcherPool;}}