关于游戏中cache的后写入回写机制

来源：互联网发布：linux 部分中文乱码编辑：程序博客网时间：2024/04/30 15:03

这几天把引擎中关于cache回写部分的操作进行一次整理，并把原来分散到由个模块自已负责的cache回写操作集中到一起来进行回写操作，我这里暂时称他为后写入管理器。

TEWriterBehindsManager.java

package com.te.server.engine.writerbehind;import java.util.Collection;import java.util.HashMap;import java.util.Iterator;import java.util.List;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.ScheduledExecutorService;import org.jdom.Element;/** * * 后写入方式管理器，负责管理所有角色对应的后写入方式 * * 后写入方式，不负责管理cache，但他负责所有注册cache类型数据的IO操作，从而把cache操作与io操作分离 * * 在游戏中有一些cache是全局的，比如说帮会，拍买行的交易，等，这些在后写入管理器都有对应的全局后写入的映射表 * 我们大部分是每个角色有一个后写入器，后写入器中有针对在游戏中的所有Cache后写入队例，每个角色的后写入器都会 * 启动一个定时器，按给定时间的间隔进行后写入，后写入中双分为insert,update,delete等操作，这些操作在定时器到 * 达时会按insert,update,delete等次序进行操作，为更有效的节省开销，在每个操作放入到后写入器时都会检查，并且 * 合并已有的操作，比如放入一个update操作，后写入器会检查是否有相同对象的insert操作，如果有就不把update操作 * 放入队列 * * 注意: * 1.由于后写入器，并不知道各自cache中的数据对象的结构，但为了区分是否为同一个数据，所以一般情况下都是用Object中的 * equals方法，所以我们最好是让需要认为是相同对象的实例实现hashCode,与 equals方法 * * 2.对于要操作的cache中的数据对象，最好是同一对象，比如我们插入一个数据对象，之后再更新一个数据对象，我们不仅要做到 * 他们在数据库中的key是相同的，而且最好是同一对象，如果万不得以不能是同一对象，最好是在相同键不同对象的情况下，重 * 新实现hashCode()与equals()方法，让他同的判断是正确的。 * * * * * @author 胡玮 * */public class TEWriterBehindsManager {/** * 全局Behind，给游戏中一些不在角色之下管理的Cache使用的 */private final HashMap<Integer, TEWriterBehind> globalWriterBehindMap = new HashMap<Integer, TEWriterBehind>();/** * 角色对应的Behind */private ConcurrentHashMap<Long, TEWriterBehind> writerBehindMap = null;/**定时器*/ScheduledExecutorService timer;/**系统CacheWriter工厂*/TECacheWriterFactory systemFactory = null;/**自定义CacheWriter工厂*/TECacheWriterFactory customFactory = null;/**cache类型映射表*/HashMap<Integer, TECacheType> cacheTypeMap = new HashMap<Integer, TECacheType>(); /**写间隔时间*/long writeDelayMs = 0;/**每次进行插入批量处理的数量*/int insertBathingSize = 0;/**每次进行更新批量处理的数量*/int updateBathingSize = 0;/**每次进行删除批量处理的数量*/int deleteBathingSize = 0;/**初次回写失败后，失败重写的次数*/int retryAttempts = 0;/**失败重写的间隔时间*/long retryAttemptDelay = 1;public TEWriterBehindsManager(ScheduledExecutorService timer){this.timer = timer;}/** * * 启动管理器 * * * @param configElement * @throws Exception */public void start(Element configElement) throws Exception {fillCacheTypeMap();int initialCapacity = Integer.parseInt(configElement.getChild("ConcurrentMap_structure").getChildTextTrim("initialCapacity"));float loadFactor = Float.parseFloat(configElement.getChild("ConcurrentMap_structure").getChildTextTrim("loadFactor"));int concurrencyLevel = Integer.parseInt(configElement.getChild("ConcurrentMap_structure").getChildTextTrim("concurrencyLevel"));writerBehindMap = new ConcurrentHashMap<Long, TEWriterBehind>(initialCapacity, loadFactor, concurrencyLevel);systemFactory = (TECacheWriterFactory) Class.forName(configElement.getChild("systemWriterFactory").getAttributeValue("class")).newInstance();writeDelayMs = Long.parseLong(configElement.getChild("config").getAttributeValue("writeDelayMs"));insertBathingSize = Integer.parseInt(configElement.getChild("config").getAttributeValue("insertBathingSize"));updateBathingSize = Integer.parseInt(configElement.getChild("config").getAttributeValue("updateBathingSize"));deleteBathingSize = Integer.parseInt(configElement.getChild("config").getAttributeValue("deleteBathingSize"));retryAttempts = Integer.parseInt(configElement.getChild("config").getAttributeValue("retryAttempts"));retryAttemptDelay = Integer.parseInt(configElement.getChild("config").getAttributeValue("retryAttemptDelay"));Element customCacheTypes = configElement.getChild("customCacheTypes");if(customCacheTypes != null) {customFactory = (TECacheWriterFactory) Class.forName(customCacheTypes.getAttributeValue("writerFactory")).newInstance();List<Element> elist = customCacheTypes.getChildren("cacheType");for(Iterator<Element> iter = elist.iterator(); iter.hasNext();) {Element e = iter.next();boolean isGlobal = Boolean.parseBoolean(e.getAttributeValue("global"));int id = Integer.parseInt(e.getAttributeValue("id"));String name = e.getAttributeValue("name");if(cacheTypeMap.containsKey(id)) {throw new Exception("Cache类型有重复的，不能声名重复的");}TECacheType ct = new TECacheType(id, isGlobal, name);cacheTypeMap.put(ct.getId(), ct);}}createGlobalBehind(); //创建全局的Behind}/** * 以系统的cache类型填充cacheTypeMap; */private void fillCacheTypeMap() {cacheTypeMap.put(TECacheType.CACHE_GANG.getId(), TECacheType.CACHE_GANG);cacheTypeMap.put(TECacheType.CACHE_ITEM.getId(), TECacheType.CACHE_ITEM);cacheTypeMap.put(TECacheType.CACHE_MAIL.getId(), TECacheType.CACHE_MAIL);cacheTypeMap.put(TECacheType.CACHE_MISSION.getId(), TECacheType.CACHE_MISSION);cacheTypeMap.put(TECacheType.CACHE_ROLE.getId(), TECacheType.CACHE_ROLE);cacheTypeMap.put(TECacheType.CACHE_SKILL.getId(), TECacheType.CACHE_SKILL);cacheTypeMap.put(TECacheType.CACHE_TRANSACTION.getId(), TECacheType.CACHE_TRANSACTION);}/** * 创建全局的WriterBehind */private void createGlobalBehind() {Collection<TECacheType> values = cacheTypeMap.values();for(Iterator<TECacheType> iter = values.iterator(); iter.hasNext();) {TECacheType ct = iter.next();if(!ct.isGlobal()) {continue;}TEWriterBehind behind = new TEWriterBehind();behind.start(this, ct);globalWriterBehindMap.put(ct.getId(), behind);}}/** * * 放入一个角色的WriterBehind，并启动 * * * @param roleId * @param behind */public void putBehind(long roleId, TEWriterBehind behind) {if(behind != null) {behind.start(this);writerBehindMap.put(roleId, behind);}}/** * * 移除一个角色的WriterBehind，并停止 * * * @param roleId */public void removeBehind(long roleId) {TEWriterBehind behind = writerBehindMap.remove(roleId);if(behind != null) {behind.stop();}}/** * * 获取一个角色的WriterBehind * * * @param roleId * @return */public TEWriterBehind getBehind(long roleId) {return writerBehindMap.get(roleId);}/** * * 获取一个全局的WriterBehind * * * @param cacheType * @return */public TEWriterBehind getGlobalBehind(TECacheType cacheType) {if(!cacheType.isGlobal()) {return null;}return globalWriterBehindMap.get(cacheType.getId());}public void shutdown() {//TODO:停止一切活动，并把所有需要回写的cache全部进行回写Collection<TEWriterBehind> values = writerBehindMap.values();for(Iterator<TEWriterBehind> iter = values.iterator(); iter.hasNext();) {TEWriterBehind behind = iter.next();behind.stop();}values = globalWriterBehindMap.values();for(Iterator<TEWriterBehind> iter = values.iterator(); iter.hasNext();) {TEWriterBehind behind = iter.next();behind.stop();}}}

TEWriterBehind.java

package com.te.server.engine.writerbehind;import java.util.Collection;import java.util.HashMap;import java.util.Iterator;import java.util.concurrent.ScheduledFuture;import java.util.concurrent.TimeUnit;/** * * 后写入器，每个角色与全局cache会对应一个后写入器，每个后写器将会启动一个定时器，定时器的间隔时间由配置文件决定 * 后写器提供api负责操作的写入，一个后写入器中可能对应N多个后写入队列，每个队列实际上对应一个cache类型。 * * * @author 胡玮 * */public class TEWriterBehind implements Runnable{private final HashMap<Integer, TEWriterBehindQueue> queueMap = new HashMap<Integer, TEWriterBehindQueue>();TEWriterBehindQueue[] queues;private ScheduledFuture future;private boolean isGlobal;private TEWriterBehindsManager manager;protected final void start(TEWriterBehindsManager manager) {this.manager = manager;isGlobal = false;Collection<TECacheType> values = manager.cacheTypeMap.values();for(Iterator<TECacheType> iter = values.iterator(); iter.hasNext();) {TECacheType ct = iter.next();if(ct.isGlobal()) {continue;}TEWriterBehindQueue queue = new TEWriterBehindQueue();TECacheWriter cacheWriter = null;if(manager.systemFactory != null) {cacheWriter = manager.systemFactory.createCacheWriter(ct);}if(cacheWriter == null && manager.customFactory != null) {cacheWriter = manager.customFactory.createCacheWriter(ct);}queue.cacheWriter = cacheWriter;queue.queueId = ct.getId();queue.cacheName = ct.getName();queueMap.put(ct.getId(), queue);}queues = new TEWriterBehindQueue[queueMap.size()];int index = 0;Collection<TEWriterBehindQueue> qValues = queueMap.values();for(Iterator<TEWriterBehindQueue> iter = qValues.iterator(); iter.hasNext();) {queues[index++] = iter.next();}future = manager.timer.scheduleAtFixedRate(this, manager.writeDelayMs, manager.writeDelayMs, TimeUnit.SECONDS);}/** * 启动全局Behind * @param manager * @param type */protected final void start(TEWriterBehindsManager manager, TECacheType type) {this.manager = manager;this.isGlobal = true;TEWriterBehindQueue queue = new TEWriterBehindQueue();TECacheWriter cacheWriter = null;if(manager.systemFactory != null) {cacheWriter = manager.systemFactory.createCacheWriter(type);}if(cacheWriter == null && manager.customFactory != null) {cacheWriter = manager.customFactory.createCacheWriter(type);}queue.cacheWriter = cacheWriter;queue.queueId = type.getId();queue.cacheName = type.getName();queueMap.put(type.getId(), queue);queues = new TEWriterBehindQueue[1];queues[0] = queue;future = manager.timer.scheduleAtFixedRate(this, manager.writeDelayMs, manager.writeDelayMs, TimeUnit.SECONDS);}protected final void stop() {//TODO:把所有存在的操作全部回写到数据库中future.cancel(true);for(int i=0; i<queues.length; i++) {TEWriterBehindQueue behind = queues[i];behind.operation(0, 0, 0,manager.retryAttempts, manager.retryAttemptDelay);}}public void write(Object cacheObj, TECacheType cacheType, TEOperationType operationType) {if(cacheType.isGlobal() != isGlobal) {throw new RuntimeException("提交的cacheType类型不正确");}TEWriterBehindQueue queue = queueMap.get(cacheType.getId());queue.putOperation(cacheObj, operationType);}@Overridepublic void run() {for(int i=0; i<queues.length; i++) {TEWriterBehindQueue behind = queues[i];behind.operation(manager.insertBathingSize, manager.updateBathingSize, manager.deleteBathingSize,manager.retryAttempts, manager.retryAttemptDelay);}}}

TEWriterBehindQueue.java

package com.te.server.engine.writerbehind;import java.util.ArrayList;import java.util.HashSet;import java.util.Iterator;import java.util.Set;import java.util.concurrent.atomic.AtomicBoolean;import java.util.concurrent.locks.ReentrantReadWriteLock;import org.apache.log4j.Logger;/** * * 后写入队列，负责将对应cahce类型的操作保存，等待定时器时间到达时调用提供的API进行相应的IO操作 * * * @author 胡玮 * */public class TEWriterBehindQueue {private final static Logger LOGGER = Logger.getRootLogger(); //private static final int MS_IN_SEC = 1000;int queueId; String cacheName=""; private final ReentrantReadWriteLock queueLock = new ReentrantReadWriteLock(); private final ReentrantReadWriteLock.ReadLock queueReadLock = queueLock.readLock(); private final ReentrantReadWriteLock.WriteLock queueWriteLock = queueLock.writeLock(); //private final Condition queueIsEmpty = queueWriteLock.newCondition(); //private final Condition queueIsStopped = queueWriteLock.newCondition(); //private final AtomicLong lastProcessing = new AtomicLong(System.currentTimeMillis()); //private final AtomicLong lastWorkDone = new AtomicLong(System.currentTimeMillis()); private final AtomicBoolean busyProcessing = new AtomicBoolean(false); Set<Object> insertWaiting = new HashSet<Object>(); Set<Object> updateWaiting = new HashSet<Object>(); Set<Object> deleteWaiting = new HashSet<Object>(); TECacheWriter cacheWriter; public void putOperation(Object cacheObj, TEOperationType type) { queueWriteLock.lock(); try { if(type.equals(TEOperationType.INSERT)) { //插入 insertWaiting.add(cacheObj); } else if(type.equals(TEOperationType.UPDATE)) { //更新 if(!insertWaiting.contains(cacheObj)) { updateWaiting.add(cacheObj); } } else { //删除 insertWaiting.remove(cacheObj); updateWaiting.remove(cacheObj); deleteWaiting.add(cacheObj); } } finally { queueWriteLock.unlock(); } } void operation(int insertBathingSize, int updateBathingSize, int deleteBathingSize, int retryAttempts, long retryAttemptDelay) { if(cacheWriter == null) { LOGGER.error(cacheName+"[cacheId="+queueId+"] cacheWriter is not exist"); return; } if(busyProcessing.get()) { return; } else { busyProcessing.set(true); } ArrayList<Object> tempInserts = new ArrayList<Object>(); ArrayList<Object> tempUpdates = new ArrayList<Object>(); ArrayList<Object> tempDeletes = new ArrayList<Object>(); queueReadLock.lock(); try { _operation(tempInserts, insertWaiting, insertBathingSize); _operation(tempUpdates, updateWaiting, updateBathingSize); _operation(tempDeletes, deleteWaiting, deleteBathingSize); } finally { queueReadLock.unlock(); } int num = retryAttempts + 1; if(tempInserts.size() > 0) { //插入 for(int i=0; i<num; i++) { try { cacheWriter.insertAll(tempInserts); break; } catch (Exception e) { LOGGER.error(e.getMessage(), e); try { Thread.sleep(retryAttemptDelay * 1000); } catch (InterruptedException e1) { e1.printStackTrace(); } } } } if(tempUpdates.size() > 0) { //更新 for(int i=0; i<num; i++) { try { cacheWriter.updateAll(tempUpdates); break; } catch (Exception e) { LOGGER.error(e.getMessage(), e); try { Thread.sleep(retryAttemptDelay * 1000); } catch (InterruptedException e1) { e1.printStackTrace(); } } } } if(tempDeletes.size() > 0) { //删除 for(int i=0; i<num; i++) { try { cacheWriter.deleteAll(tempDeletes); break; } catch (Exception e) { LOGGER.error(e.getMessage(), e); try { Thread.sleep(retryAttemptDelay * 1000); } catch (InterruptedException e1) { e1.printStackTrace(); } } } }busyProcessing.set(false); } private void _operation(ArrayList<Object> _list, Set<Object> _waiting, int _bathingSize) { int index = 0; for(Iterator<Object> iter = _waiting.iterator(); iter.hasNext();) {if(_bathingSize >0 && index >= _bathingSize) {break;}Object cacheobj = iter.next();iter.remove();_list.add(cacheobj);index ++;} }}

TECacheType.java

package com.te.server.engine.writerbehind;/** * * cache类型，标识游戏中cache数据的类型，通过配置文件注册的cache类型分为全局与非全局的，非全局的则为每个角色中存在的 * cache类型，全局的则是在整个游戏系统中唯一存在的 * * * * @author 胡玮 * */public class TECacheType {public static final TECacheType CACHE_ITEM = new TECacheType(1, false, "item");public static final TECacheType CACHE_MISSION = new TECacheType(2, false, "mission");public static final TECacheType CACHE_ROLE = new TECacheType(3, false, "role");public static final TECacheType CACHE_SKILL = new TECacheType(4, false, "skill");public static final TECacheType CACHE_MAIL = new TECacheType(5, false, "mail");public static final TECacheType CACHE_GANG = new TECacheType(6, true, "gang");public static final TECacheType CACHE_TRANSACTION = new TECacheType(7, true, "transaction");private int id;private boolean isGlobal;private String name;public TECacheType(int id, boolean isGlobal, String name) {this.id = id;this.isGlobal = isGlobal;this.name = name;}/** * @return the id */public int getId() {return id;}/** * @return the isGlobal */public boolean isGlobal() {return isGlobal;}/** * @return the name */public String getName() {return name;}}

TECacheWriter.java

package com.te.server.engine.writerbehind;import java.sql.SQLException;import java.util.List;/** * * cache写操作器，这个接口由使用者实现，因为系统并不知道使用者要怎么样操作IO，操作器分为三个方法，分别对应数据库中的 * 插入，更新，删除 * * 注意： * 每个方法传入的是一个cache数据列表，传入列表的原因是希望使用者在实现时尽量一次进行IO操作，不要分散进行，这样能最大限度的 * 节省开销 * * * @author 胡玮 * */public interface TECacheWriter {public void insertAll(List<Object> elements) throws SQLException;public void updateAll(List<Object> elements) throws SQLException;public void deleteAll(List<Object> elements) throws SQLException;}

TECacheWriterFactory.java

package com.te.server.engine.writerbehind;/** * * CacheWriter的工厂接口，负责创建CacheWriter,原则上每一个cache类型对应一个CacheWriter，工厂分为系统工厂与自定义工厂 * 这二者都可以由配置文件定义工厂的具体实现，但是系统工厂是必须实现的，否则不能启动后写入管理器，系统工厂所做的事情与自 * 定义工厂是一样的，区别在于，我们知道游戏中一些我们耳熟能详的cache类型，比如道具，任务等等，这些是属于游戏引擎必然存 * 在，所以这里为他们定义了一个系统工厂，为这些cache类型创建对应的CacheWriter，而自定义的工厂，我们是为了方便扩展未知的 * cache类型而准备的。 * * * @author 胡玮 * */public interface TECacheWriterFactory {public TECacheWriter createCacheWriter(TECacheType cacheType);}

TEOperationType.java

package com.te.server.engine.writerbehind;/** * * 操作类型，需要通知后写入器进行怎样的操作，它们分别对应数据库的插入，更新，删除操作 * * @author 胡玮 * */public enum TEOperationType {INSERT,UPDATE,DELETE}

config.xml

<writer-behind>
 <ConcurrentMap_structure>
 <initialCapacity>100</initialCapacity>
 <loadFactor>0.8</loadFactor>
 <concurrencyLevel>20</concurrencyLevel>
 </ConcurrentMap_structure>

 <systemWriterFactory class="com.te.server.engine.writerbehind.SystemCacheWriterFactory"/>

 <config writeDelayMs="20"
 insertBathingSize="10"
 updateBathingSize="10"
 deleteBathingSize="10"
 retryAttempts="2"
 retryAttemptDelay="3"/>
 
</writer-behind>