canal与spring整合

由于之前项目需要，了解到了阿里优秀框架canal，使用起来真是很不错很方便，分库分表情况下用来缓存或是ES数据的同步很便捷。在尝试过程中，走了点弯路，总结此文，希望能够对观者有所帮助。

Canal官方下载地址

https://github.com/alibaba/canal/releases

下载的canal.deployer启动canal服务端，然后编写客户端程序消费数据，此方式对于应用部署和维护显然不是很方便。为此结合canal.deployer代码编写与spring整合方式，使web应用启动时即能启动服务端，又能消费数据。

canal消费方采用策略模式，定义BaseProcess抽象类，包含processInsert、processUpdate、processDelete三个抽象方法，分别用于处理三种类型的数据操作。封装processConvert方法将RowChage中的数据反射为javaBean对象。

1.    Canal服务端代码CanalServer

 

package com.scy.canal.server;import java.io.FileInputStream;import java.util.List;import java.util.Properties;import org.apache.commons.lang.StringUtils;import org.apache.commons.logging.Log;import org.apache.commons.logging.LogFactory;import com.alibaba.otter.canal.deployer.CanalController;/** * canal服务端代码 * @author suicy * */public class CanalServer {private static final Log logger = LogFactory.getLog(CanalServer.class);private static final String CLASSPATH_URL_PREFIX = "classpath:";private CanalController controller;private List<String> configs;public void startup() {logger.debug("CanalServer startup 准备启动canal服务端...");try {Properties properties = new Properties();for (String conf : configs) {if (conf.startsWith(CLASSPATH_URL_PREFIX)) {conf = StringUtils.substringAfter(conf, CLASSPATH_URL_PREFIX);properties.load(CanalServer.class.getClassLoader().getResourceAsStream(conf));} else {properties.load(new FileInputStream(conf));}}logger.debug("CanalController创建开始...");controller = new CanalController(properties);logger.debug("CanalController创建结束...");controller.start();logger.debug("CanalServer startup 启动canal服务端成功！");Runtime.getRuntime().addShutdownHook(new Thread() {public void run() {try {logger.info("## stop the canal server");controller.stop();} catch (Throwable e) {logger.warn("##something goes wrong when stopping canal Server:\n{}", e);} finally {logger.info("## canal server is down.");}}});} catch (Throwable e) {logger.error("CanalServer startup 启动canal服务端失败，", e);System.exit(0);}}public void shutdown() {try {controller.stop();} catch (Throwable e) {logger.error("CanalServer shutdown canal服务端异常，", e);}}public void setConfigs(List<String> configs) {this.configs = configs;}}

2.    服务端配置文件spring-config-canal-server.xml

 

<?xml version="1.0" encoding="utf-8"?><beans xmlns="http://www.springframework.org/schema/beans"xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd"><bean class="com.scy.canal.server.CanalServer" init-method="startup" destroy-method="shutdown"><property name="configs"><list><value>classpath:props/canal.properties</value></list></property></bean> </beans>

3.    Canal数据变更封装类CanalRowChange

 

package com.scy.canal.entity;import java.io.Serializable;import java.util.List;import com.alibaba.otter.canal.protocol.CanalEntry.EventType;import com.alibaba.otter.canal.protocol.CanalEntry.RowData;public class CanalRowChange implements Serializable{private static final long serialVersionUID = -90027012566550680L;private String schemaName;private String tableName;private List<RowData> rowData;private EventType eventType;public String getSchemaName() {return schemaName;}public void setSchemaName(String schemaName) {this.schemaName = schemaName;}public String getTableName() {return tableName;}public void setTableName(String tableName) {this.tableName = tableName;}public List<RowData> getRowData() {return rowData;}public void setRowData(List<RowData> rowData) {this.rowData = rowData;}public EventType getEventType() {return eventType;}public void setEventType(EventType eventType) {this.eventType = eventType;}}

4.    Canal消费端代码CanalConsumer

 

package com.scy.canal.client;import java.util.List;import java.util.Map;import org.slf4j.Logger;import org.slf4j.LoggerFactory;import com.alibaba.otter.canal.client.CanalConnector;import com.alibaba.otter.canal.client.CanalConnectors;import com.alibaba.otter.canal.protocol.Message;import com.alibaba.otter.canal.protocol.CanalEntry.Column;import com.alibaba.otter.canal.protocol.CanalEntry.Entry;import com.alibaba.otter.canal.protocol.CanalEntry.EntryType;import com.alibaba.otter.canal.protocol.CanalEntry.EventType;import com.alibaba.otter.canal.protocol.CanalEntry.RowChange;import com.alibaba.otter.canal.protocol.CanalEntry.RowData;import com.scy.canal.entity.CanalRowChange;import com.scy.canal.process.BaseProcess;/** * 数据消费 * @author suicy * */public class CanalConsumer {    private static final Logger logger = LoggerFactory.getLogger(CanalConsumer.class);    //数据处理类key为表名，value为对应的处理类    private Map<String, BaseProcess> processor;         public static volatile boolean running = true;        private String destination;        private String zkServers;            public void init(){        // 创建链接    logger.error("--------CanalConsumer destination:"+destination+" start------------");Thread thread = new Thread(new Runnable() {public void run() {CanalConnector connector = CanalConnectors.newClusterConnector(zkServers, destination, null, null);int batchSize = 1000;try {connector.connect();connector.subscribe();while ( running ) {Message message = connector.getWithoutAck(batchSize); // 获取指定数量的数据long batchId = message.getId();int size = message.getEntries().size();if (batchId == -1 || size == 0) {try {Thread.sleep(1000);} catch (InterruptedException e) {e.printStackTrace();}} else {boolean res = parseData(message.getEntries(),destination);if(res){connector.ack(batchId);}else{connector.rollback(batchId);}}}} finally {connector.disconnect();}}});thread.start();}    /**     * 数据处理     * @param entrys     * @param destination2     * @return     */private boolean parseData(List<Entry> entrys, String destination2) {for (Entry entry : entrys) {if ( EntryType.TRANSACTIONBEGIN.equals(entry.getEntryType()) || EntryType.TRANSACTIONEND.equals(entry.getEntryType()) ) {continue;}RowChange rowChage = null;try {rowChage = RowChange.parseFrom(entry.getStoreValue());} catch (Exception e) {throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(),e);}EventType eventType = rowChage.getEventType();String schemaName = entry.getHeader().getSchemaName();String tableName = entry.getHeader().getTableName();/*1. 输出数据变更日志*/if( logger.isErrorEnabled() ){if(eventType == EventType.INSERT || eventType == EventType.DELETE || eventType == EventType.UPDATE ){logger.error("================> binlog["+entry.getHeader().getLogfileName()+":"+ entry.getHeader().getLogfileOffset()+"] , "+ "name["+schemaName+":"+tableName+"] , eventType : "+eventType);for (RowData rowData : rowChage.getRowDatasList()) {if (eventType == EventType.DELETE) {printColumn(rowData.getBeforeColumnsList());} else if (eventType == EventType.INSERT) {printColumn(rowData.getAfterColumnsList());} else if (eventType == EventType.UPDATE){logger.error("-------> before");printColumn(rowData.getBeforeColumnsList());logger.error("-------> after");printColumn(rowData.getAfterColumnsList());}}}}/*2. 数据处理*/if( EventType.INSERT.equals(eventType)||EventType.DELETE.equals(eventType)||EventType.UPDATE.equals(eventType)){//构造CanalRowChange对象CanalRowChange rowChange = bulidCanalRowChange(schemaName, tableName, eventType, rowChage.getRowDatasList());/*根据表明获取数据处理类*/if(processor.containsKey(tableName.toLowerCase())){boolean res = false;//根据事件类型调用相应的数据处理方法if (EventType.UPDATE.equals(eventType)) {res = processor.get(tableName).processUpdate(rowChange);} else if (EventType.INSERT.equals(eventType)) {res = processor.get(tableName).processInsert(rowChange);} else if (EventType.DELETE.equals(eventType)) {res = processor.get(tableName).processDelete(rowChange);}if(!res){logger.error("================> binlog["+entry.getHeader().getLogfileName()+":"+ entry.getHeader().getLogfileOffset()+"] , "+ "name["+schemaName+":"+tableName+"] , eventType : "+eventType);return res;}}}}return true;}private CanalRowChange bulidCanalRowChange(String schemaName, String tableName, EventType eventType, List<RowData> rowData){CanalRowChange change = new CanalRowChange();change.setSchemaName(schemaName);change.setEventType(eventType);change.setRowData(rowData);change.setTableName(tableName);return change;}private void printColumn(List<Column> columns) {for (Column column : columns) {logger.error(column.getName() + " : " + column.getValue() + "    update=" + column.getUpdated());}}    protected void stop() {        if (!running) {            return;        }        running = false;    }    public Map<String, BaseProcess> getProcessor() {return processor;}public void setProcessor(Map<String, BaseProcess> processor) {this.processor = processor;}public String getDestination() {return destination;}public void setDestination(String destination) {this.destination = destination;}public String getZkServers() {return zkServers;}public void setZkServers(String zkServers) {this.zkServers = zkServers;}}

5.    消费端配置文件spring-config-canal-client.xml

 

<?xml version="1.0" encoding="utf-8"?><beans xmlns="http://www.springframework.org/schema/beans"xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd"default-autowire="byName"><bean id="canalConsumer_A1" class="com.scy.canal.client.CanalConsumer" init-method="init" destroy-method="stop" ><property name="destination" value="A1" /><property name="zkServers" value="${canal.zkServers}" /><property name="processor"><!-- 配置数据处理类key为表明,value为处理类 为BaseProcess子类--><map><entry key="goods" value-ref="goodsProcess"></entry></map></property></bean><bean id="goodsProcess" class="com.scy.canal.process.GoodsProcess"></bean></beans>

6.    数据消费父类BaseProcess

 

package com.scy.canal.process;import java.lang.reflect.Field;import java.util.ArrayList;import java.util.HashMap;import java.util.List;import java.util.Map;import org.apache.commons.lang.StringUtils;import org.springframework.core.convert.support.DefaultConversionService;import com.alibaba.otter.canal.protocol.CanalEntry.Column;import com.alibaba.otter.canal.protocol.CanalEntry.RowData;import com.scy.canal.entity.CanalRowChange;import com.scy.canal.entity.DateConverter;/** * 数据消费基类 *  * @author suicy * */public abstract class BaseProcess {private DefaultConversionService convertor = new DefaultConversionService() {{addConverter(new DateConverter());}};private Map<String, Map<String, Field>> clzFieldsCached;/** * 处理数据添加方法 * @param rowChange * @return */public abstract boolean processInsert(CanalRowChange rowChange);/** * 处理修改数据方法 * @param rowChange * @return */public abstract boolean processUpdate(CanalRowChange rowChange);/** * 处理删除数据方法 * @param rowChange * @return */public abstract boolean processDelete(CanalRowChange rowChange);/** * 数据转换方法 *  * @param rowChange * @param clz * @param isAfter * @return * @throws IllegalAccessException * @throws InstantiationException */public <T> List<T> processConvert(CanalRowChange rowChange, Class<T> clz, boolean isAfter)throws InstantiationException, IllegalAccessException {if (rowChange == null || clz == null) {return null;}if (clzFieldsCached == null) {clzFieldsCached = new HashMap<String, Map<String, Field>>();}Map<String, Field> fieldscached = clzFieldsCached.get(clz.getName());if (fieldscached == null || fieldscached.size() <= 0) {fieldscached = new HashMap<String, Field>();for (Field field : clz.getDeclaredFields()) {field.setAccessible(true);fieldscached.put(field.getName().toLowerCase(), field);}clzFieldsCached.put(clz.getName(), fieldscached);}List<RowData> rowDatas = rowChange.getRowData();if (rowDatas == null || rowDatas.size() <= 0) {return null;}List<T> beans = new ArrayList<T>();for (RowData rowData : rowDatas) {T bean = clz.newInstance();List<Column> cols = isAfter ? rowData.getAfterColumnsList() : rowData.getBeforeColumnsList();if (cols == null || cols.size() <= 0) {return null;}int count = 0;for (Column col : cols) {String name = col.getName();String value = col.getValue();Field field = fieldscached.get(name.toLowerCase());if (field == null) {continue;}if (StringUtils.isNotBlank(value)) {Object nvalue = convertor.convert(value, field.getType());field.set(bean, nvalue);}count++;}if (count != fieldscached.size()) {return null;}beans.add(bean);}if (beans.size() != rowDatas.size()) {return null;}return beans;}}

7.    goods消费代码实例GoodsProcess

package com.scy.canal.process;import java.util.List;import org.apache.commons.logging.Log;import org.apache.commons.logging.LogFactory;import com.scy.canal.entity.CanalRowChange;import com.scy.canal.entity.Goods;public class GoodsProcess extends BaseProcess{private static final Log logger = LogFactory.getLog(GoodsProcess.class);@Overridepublic boolean processInsert(CanalRowChange rowChange) {try {List<Goods> data = super.processConvert(rowChange, Goods.class, true);logger.info("添加前"+data);} catch (InstantiationException e) {e.printStackTrace();} catch (IllegalAccessException e) {e.printStackTrace();}return true;}@Overridepublic boolean processUpdate(CanalRowChange rowChange) {try {List<Goods> data = super.processConvert(rowChange, Goods.class, false);List<Goods> data2 = super.processConvert(rowChange, Goods.class, true);logger.info("修改前："+data);logger.info("修改后："+data2);return true;} catch (InstantiationException e) {e.printStackTrace();} catch (IllegalAccessException e) {e.printStackTrace();}return false;}@Overridepublic boolean processDelete(CanalRowChange rowChange) {try {List<Goods>data = super.processConvert(rowChange, Goods.class, false);logger.info("删除前："+data);return true;} catch (InstantiationException e) {e.printStackTrace();} catch (IllegalAccessException e) {e.printStackTrace();}return false;}}

8.    Pom文件

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"><modelVersion>4.0.0</modelVersion><groupId>com.scy</groupId><artifactId>canalDemo</artifactId><packaging>war</packaging><version>0.0.1-SNAPSHOT</version><name>canalDemo Maven Webapp</name><url>http://maven.apache.org</url>              <properties>       <jdk.version>1.6</jdk.version>     <project.build.sourceEncoding>GBK</project.build.sourceEncoding>     <org.springframework.version>3.1.1.RELEASE</org.springframework.version>     <mybatis.version>3.2.0</mybatis.version>     <jdbc.mysql.version>5.1.8</jdbc.mysql.version>     <junit.version>4.10</junit.version>     <velocity.version>1.7</velocity.version>     <velocity.tools.version>2.0</velocity.tools.version>     <canal.version>1.0.24</canal.version>       </properties>              <dependencies>              <!-- spring -->     <dependency>     <groupId>org.springframework</groupId>     <artifactId>spring-beans</artifactId>     <version>${org.springframework.version}</version>     </dependency>     <dependency>     <groupId>org.springframework</groupId>     <artifactId>spring-jdbc</artifactId>     <version>${org.springframework.version}</version>     </dependency>     <dependency>     <groupId>org.springframework</groupId>     <artifactId>spring-web</artifactId>     <version>${org.springframework.version}</version>     </dependency>     <dependency>     <groupId>org.springframework</groupId>     <artifactId>spring-webmvc</artifactId>     <version>${org.springframework.version}</version>     </dependency>     <dependency>     <groupId>org.springframework</groupId>     <artifactId>spring-orm</artifactId>     <version>${org.springframework.version}</version>     </dependency>     <dependency>     <groupId>org.springframework</groupId>     <artifactId>spring-tx</artifactId>     <version>${org.springframework.version}</version>     </dependency>      <dependency>     <groupId>org.quartz-scheduler</groupId>     <artifactId>quartz</artifactId>     <version>1.8.4</version>     </dependency>       <!-- mybatis -->     <dependency>     <groupId>org.mybatis</groupId>     <artifactId>mybatis</artifactId>     <version>${mybatis.version}</version>     </dependency>     <dependency>     <groupId>org.mybatis</groupId>     <artifactId>mybatis-spring</artifactId>     <version>1.1.0</version>     </dependency>              <!-- mySql -->       <dependency>     <groupId>mysql</groupId>     <artifactId>mysql-connector-java</artifactId>     <version>${jdbc.mysql.version}</version>     <type>jar</type>     <scope>runtime</scope>     </dependency>               <!-- log -->     <dependency>     <groupId>log4j</groupId>     <artifactId>log4j</artifactId>     <version>1.2.14</version>     </dependency>     <dependency>     <groupId>org.slf4j</groupId>     <artifactId>slf4j-api</artifactId>     <version>1.6.0</version>     <scope>compile</scope>     </dependency>     <dependency>     <groupId>org.slf4j</groupId>     <artifactId>slf4j-log4j12</artifactId>     <version>1.6.0</version>     <scope>compile</scope>     </dependency>     <dependency>     <groupId>javax.servlet</groupId>     <artifactId>servlet-api</artifactId>     <version>2.5</version>     </dependency>               <!-- ump 依赖包 begin -->     <dependency>     <groupId>org.aspectj</groupId>     <artifactId>aspectjweaver</artifactId>     <version>1.6.6</version>     </dependency>     <dependency>     <groupId>cglib</groupId>     <artifactId>cglib-nodep</artifactId>     <version>2.2</version>     </dependency>     <dependency>     <groupId>ant</groupId>     <artifactId>ant</artifactId>     <version>1.6.2</version>     </dependency>          <!-- net client -->         <dependency>     <groupId>commons-net</groupId>     <artifactId>commons-net</artifactId>     <version>3.3</version>         </dependency>                  <!-- chain -->     <dependency>     <groupId>commons-chain</groupId>     <artifactId>commons-chain</artifactId>     <version>1.2</version>     </dependency> <dependency>    <groupId>com.alibaba</groupId>    <artifactId>druid</artifactId>    <version>0.2.10</version></dependency>     <!-- test begin -->     <dependency>     <groupId>junit</groupId>     <artifactId>junit</artifactId>     <version>${junit.version}</version>     <scope>provided</scope>     </dependency>     <dependency>     <groupId>org.springframework</groupId>     <artifactId>spring-test</artifactId>     <version>${org.springframework.version}</version>     <scope>provided</scope>     </dependency><dependency><groupId>com.alibaba.otter</groupId><artifactId>canal.deployer</artifactId><version>${canal.version}</version><exclusions><exclusion><groupId>org.springframework</groupId><artifactId>spring</artifactId></exclusion><exclusion><groupId>commons-io</groupId><artifactId>commons-io</artifactId></exclusion><exclusion><groupId>ch.qos.logback</groupId><artifactId>logback-core</artifactId></exclusion><exclusion><groupId>ch.qos.logback</groupId><artifactId>logback-classic</artifactId></exclusion><exclusion><groupId>org.slf4j</groupId><artifactId>jcl-over-slf4j</artifactId></exclusion><exclusion><groupId>org.slf4j</groupId><artifactId>slf4j-api</artifactId></exclusion></exclusions></dependency><dependency><groupId>com.alibaba.otter</groupId><artifactId>canal.client</artifactId><version>${canal.version}</version><exclusions><exclusion><groupId>org.springframework</groupId><artifactId>spring</artifactId></exclusion><exclusion><groupId>commons-io</groupId><artifactId>commons-io</artifactId></exclusion><exclusion><groupId>ch.qos.logback</groupId><artifactId>logback-core</artifactId></exclusion><exclusion><groupId>ch.qos.logback</groupId><artifactId>logback-classic</artifactId></exclusion><exclusion><groupId>org.slf4j</groupId><artifactId>jcl-over-slf4j</artifactId></exclusion><exclusion><groupId>org.slf4j</groupId><artifactId>slf4j-api</artifactId></exclusion><exclusion><groupId>org.apache.zookeeper</groupId><artifactId>zookeeper</artifactId></exclusion></exclusions></dependency><dependency>    <groupId>org.apache.zookeeper</groupId>    <artifactId>zookeeper</artifactId>    <version>3.4.9</version></dependency></dependencies><build><finalName>canalDemo</finalName><plugins><plugin><groupId>org.apache.maven.plugins</groupId><artifactId>maven-compiler-plugin</artifactId><configuration><source>1.8</source><target>1.8</target><encoding>UTF-8</encoding><!-- 指定编码格式，否则在DOS下运行mvn compile命令时会出现莫名的错误，因为系统默认使用GBK编码 --></configuration></plugin><plugin><groupId>org.apache.maven.plugins</groupId><artifactId>maven-resources-plugin</artifactId><version>2.6</version><configuration><encoding>UTF-8</encoding><!-- 指定编码格式，否则在DOS下运行mvn命令时当发生文件资源copy时将使用系统默认使用GBK编码 --></configuration></plugin></plugins></build></project>

实例代码下载链接

http://download.csdn.net/download/suijiarui/9988995