C#开源磁盘/内存缓存引擎
来源:互联网 发布:速达软件功能 编辑:程序博客网 时间:2024/06/04 17:58
前言
昨天写了个 《基于STSdb和fastJson的磁盘/内存缓存》,大家可以先看看。下午用到业务系统时候,觉得可以改进一下,昨晚想了一个晚上,刚才重新实现一下。
更新
1. 增加了对批量处理的支持,写操作速度提升5倍,读操作提升100倍
2. 增加了一个存储provider,可以选择不用STSdb做存储,而用物理文件/Dictionary。
3. 增加了对并发的支持
需求
业务系统用的是数据库,数据量大,部分只读或相对稳定业务查询复杂,每次页面加载都要花耗不少时间(不讨论异步),觉得可以做一下高速缓存,譬如用nosql那种key/value快速存取结果
目的
提供一个简单易用的解决缓存方案,可以根据数据的大小缓存到内存或者磁盘。
实现
存取
方法1. 基于STSdb,提供高效的Key/Value存取,支持磁盘/内存,对Key无限制
方法2. 基于直接物理文件/Dictionary。Key必须是基本类型,譬如int/long/uint/ulong/DateTime/string等。
代码
代码比较简单,花了2个小时写的,很多情况没考虑,譬如磁盘空间/内存不足,自动回收过期缓存等,这些留给大家做家庭作业吧。另外,为了发布方便,STSdb和fastJson的代码都合并到一个项目里。
BaseCahce.cs
这是一个抽象基类,提供存取接口。
using System;using System.Collections.Generic;using System.Linq;using System.Text;namespace Com.SuperCache.Engine{ public abstract class BaseCache { protected internal const string KeyExpiration = "Expiration"; public abstract void Add<K>(string Category, K Key, object Data); public abstract void Add<K, V>(string Category, IEnumerable<KeyValuePair<K, V>> Items, DateTime? ExpirationDate); public abstract void Add<K>(string Category, K Key, object Data, DateTime? ExpirationDate); public abstract List<KeyValuePair<K, V>> Get<K, V>(string Category, IEnumerable<K> Keys); public abstract V Get<K, V>(string Category, K Key); }}
CahceEngine.cs
主要调用缓存引擎
using System;using System.Collections.Generic;using System.Linq;using System.Text;using STSdb4.Database;using fastJSON;using System.IO;namespace Com.SuperCache.Engine{ public enum CacheProviders { Default = 1, Raw = 2 } public class CacheEngine { private BaseCache cacheProvider = null; public CacheEngine(string DataPath): this(CacheProviders.Default, DataPath) { } public CacheEngine(CacheProviders Provider, string DataPath) { switch (Provider) { case CacheProviders.Default: cacheProvider = new STSdbCache(DataPath); break; case CacheProviders.Raw: cacheProvider = new RawCache(DataPath); break; default: break; } } public void Add<K>(string Category, K Key, object Data) { cacheProvider.Add<K>(Category, Key, Data); } public void Add<K, V>(string Category, IEnumerable<KeyValuePair<K, V>> Items, DateTime? ExpirationDate) { cacheProvider.Add<K, V>(Category, Items, ExpirationDate); } public void Add<K>(string Category, K Key, object Data, DateTime? ExpirationDate) { cacheProvider.Add<K>(Category, Key, Data, ExpirationDate); } public List<KeyValuePair<K, V>> Get<K, V>(string Category, IEnumerable<K> Keys) { return cacheProvider.Get<K, V>(Category, Keys); } public V Get<K, V>(string Category, K Key) { return cacheProvider.Get<K, V>(Category, Key); } }}
STSdbCache.cs
STSdb存储引擎
using System;using System.Collections.Generic;using System.Linq;using System.Text;using STSdb4.Database;using fastJSON;using System.IO;namespace Com.SuperCache.Engine{ public class STSdbCache : BaseCache { private string dataPath; private static IStorageEngine memoryInstance = null; private static object syncRoot = new object(); private bool isMemory = false; public STSdbCache(string DataPath) { dataPath = DataPath; if (!dataPath.EndsWith(Path.DirectorySeparatorChar.ToString())) dataPath += Path.DirectorySeparatorChar; isMemory = string.IsNullOrEmpty(DataPath); } public override void Add<K>(string Category, K Key, object Data) { Add(Category, Key, Data, null); } private IStorageEngine Engine { get { if (isMemory) { lock (syncRoot) { if (memoryInstance == null) memoryInstance = STSdb.FromMemory(); } return memoryInstance; } else return STSdb.FromFile(GetFile(false), GetFile(true)); } } private string GetExpirationTable(string Category) { return KeyExpiration + "_" + Category; } public override void Add<K, V>(string Category, IEnumerable<KeyValuePair<K, V>> Items, DateTime? ExpirationDate) { lock (syncRoot) { var engine = Engine; var table = engine.OpenXIndex<K, string>(Category); Items.ForEach(i => { var key = i.Key; var data = i.Value; //will only serialize object other than string var result = typeof(V) == typeof(string) ? data as string : JSON.Instance.ToJSON(data); table[key] = result; table.Flush(); //specify expiration var expiration = engine.OpenXIndex<K, DateTime>(GetExpirationTable(Category)); //default 30 mins to expire from now var expirationDate = ExpirationDate == null || ExpirationDate <= DateTime.Now ? DateTime.Now.AddMinutes(30) : (DateTime)ExpirationDate; expiration[key] = expirationDate; expiration.Flush(); }); engine.Commit(); //only dispose disk-based engine if (!isMemory) engine.Dispose(); } } public override void Add<K>(string Category, K Key, object Data, DateTime? ExpirationDate) { Add<K, object>(Category, new List<KeyValuePair<K, object>> { new KeyValuePair<K, object>(Key, Data) }, ExpirationDate); } private string GetFile(bool IsData) { if (!Directory.Exists(dataPath)) Directory.CreateDirectory(dataPath); return dataPath + "SuperCache." + (IsData ? "dat" : "sys"); } public override List<KeyValuePair<K, V>> Get<K, V>(string Category, IEnumerable<K> Keys) { var result = new List<KeyValuePair<K, V>>(); lock (syncRoot) { var engine = Engine; var table = engine.OpenXIndex<K, string>(Category); var expiration = engine.OpenXIndex<K, DateTime>(GetExpirationTable(Category)); var isCommitRequired = false; Keys.ForEach(key => { string buffer; V value; if (table.TryGet(key, out buffer)) { //will only deserialize object other than string value = typeof(V) == typeof(string) ? (V)(object)buffer : JSON.Instance.ToObject<V>(buffer); DateTime expirationDate; //get expiration date if (expiration.TryGet(key, out expirationDate)) { //expired if (expirationDate < DateTime.Now) { value = default(V); table.Delete(key); table.Flush(); expiration.Delete(key); expiration.Flush(); isCommitRequired = true; } } } else value = default(V); result.Add(new KeyValuePair<K, V>(key, value)); }); //only need to commit write actions if (isCommitRequired) engine.Commit(); //only dispose disk-based engine if (!isMemory) engine.Dispose(); } return result; } public override V Get<K, V>(string Category, K Key) { var buffer = Get<K, V>(Category, new K[] { Key }); var result = buffer.FirstOrDefault(); return result.Value; } }}
RawCache.cs
物理文件/Dictionary引擎
using System;using System.Collections.Generic;using System.Linq;using System.Text;using STSdb4.Database;using fastJSON;using System.IO;namespace Com.SuperCache.Engine{ public class RawCache : BaseCache { private string dataPath; private static Dictionary<string, object> memoryData = new Dictionary<string, object>(); private static Dictionary<string, DateTime?> memoryExpiration = new Dictionary<string, DateTime?>(); private static object syncRoot = new object(); private bool isMemory = false; public RawCache(string DataPath) { dataPath = DataPath; if (!dataPath.EndsWith(Path.DirectorySeparatorChar.ToString())) dataPath += Path.DirectorySeparatorChar; isMemory = string.IsNullOrEmpty(DataPath); } public override void Add<K>(string Category, K Key, object Data) { Add(Category, Key, Data, null); } private string GetExpirationTable(string Category) { return KeyExpiration + "_" + Category; } public override void Add<K, V>(string Category, IEnumerable<KeyValuePair<K, V>> Items, DateTime? ExpirationDate) { lock (syncRoot) { Items.ForEach(i => { var key = i.Key; var data = i.Value; if (isMemory) { var memKey = GetKey(Category, key.ToString()); memoryData[memKey] = data; memoryExpiration[memKey] = ExpirationDate; } else { //will only serialize object other than string var result = typeof(V) == typeof(string) ? data as string : JSON.Instance.ToJSON(data); File.WriteAllText(GetFile(Category, key.ToString(), true), result); //specify expiration //default 30 mins to expire from now var expirationDate = ExpirationDate == null || ExpirationDate <= DateTime.Now ? DateTime.Now.AddMinutes(30) : (DateTime)ExpirationDate; File.WriteAllText(GetFile(Category, key.ToString(), false), expirationDate.ToString()); } }); } } public override void Add<K>(string Category, K Key, object Data, DateTime? ExpirationDate) { Add<K, object>(Category, new List<KeyValuePair<K, object>> { new KeyValuePair<K, object>(Key, Data) }, ExpirationDate); } private string GetFile(string Category, string FileName, bool IsData) { var path = dataPath + Category.NormalizeFileName() + @"\"; if (!Directory.Exists(path)) Directory.CreateDirectory(path); return path + FileName.NormalizeFileName() + "." + (IsData ? "dat" : "exp"); } private string GetKey(string Category, string Key) { return Category + "_" + Key; } public override List<KeyValuePair<K, V>> Get<K, V>(string Category, IEnumerable<K> Keys) { var result = new List<KeyValuePair<K, V>>(); lock (syncRoot) { Keys.ForEach(key => { string buffer; V value; if (isMemory) { var memKey = GetKey(Category, key.ToString()); object memBuffer; if (memoryData.TryGetValue(memKey, out memBuffer)) { value = (V)memBuffer; DateTime? expirationDate; if (memoryExpiration.TryGetValue(memKey, out expirationDate)) { //expired if (expirationDate != null && (DateTime)expirationDate < DateTime.Now) { value = default(V); memoryData.Remove(memKey); memoryExpiration.Remove(memKey); } } } else value = default(V); } else { var dataFilePath = GetFile(Category, key.ToString(), true); if (File.Exists(dataFilePath)) { buffer = File.ReadAllText(dataFilePath); //will only deserialize object other than string value = typeof(V) == typeof(string) ? (V)(object)buffer : JSON.Instance.ToObject<V>(buffer); DateTime expirationDate; var expirationFilePath = GetFile(Category, key.ToString(), false); if (File.Exists(expirationFilePath)) { buffer = File.ReadAllText(expirationFilePath); expirationDate = Convert.ToDateTime(buffer); //expired if (expirationDate < DateTime.Now) { value = default(V); File.Delete(dataFilePath); File.Delete(expirationFilePath); } } } else value = default(V); } result.Add(new KeyValuePair<K, V>(key, value)); }); } return result; } public override V Get<K, V>(string Category, K Key) { var buffer = Get<K, V>(Category, new K[] { Key }); var result = buffer.FirstOrDefault(); return result.Value; } }}
Extensions.cs
扩展函数
using System;using System.Collections.Generic;using System.Linq;using System.Text;using System.IO;namespace Com.SuperCache.Engine{ public static class Extensions { public static void ForEach<T>(this IEnumerable<T> source, Action<T> action) { if (source != null) { foreach (var item in source) { action(item); } } } public static string NormalizeFileName(this string FileName) { var result = FileName; Path.GetInvalidFileNameChars().ForEach(c => { result = result.Replace(c.ToString(), string.Empty); }); return result; } }}
新建
构造CacheEngine需要传递缓存保存到哪个文件夹。
基于内存
如果你不喜欢基于磁盘的缓存,可以使用基于内存,构造函数传递空字符串便可。
增加/更新
同一个方法:Add。用户可以指定类型(Category),譬如User,Employee等。键(Key)支持泛型,值(Data)是object。有一个overload是过期日期(ExpirationDate),默认当前时间30分钟后
获取
Get方法需要指定类型(Category)和键(Key)。
例子
using System;using System.Collections.Generic;using System.Linq;using System.Text;using System.Threading;using System.Threading.Tasks;using System.Diagnostics;using Com.SuperCache.Engine;namespace Com.SuperCache.Test{ public class Foo { public string Name { get; set; } public int Age { get; set; } public double? Some { get; set; } public DateTime? Birthday { get; set; } } class Program { static void Main(string[] args) { //TestAddGet(); //Thread.Sleep(4000); //TestExpiration(); TestDefaultDiskPerformance(); TestRawMemoryPerformance(); TestRawDiskPerformance(); TestRawMemoryPerformance(); //TestConcurrent(); Console.Read(); } private static void TestConcurrent() { var w = new Stopwatch(); w.Start(); Parallel.For(1, 3, (a) => { var employees = Enumerable.Range((a - 1) * 1000, a * 1000).Select(i => new KeyValuePair<int, string>(i, "Wilson " + i + " Chen")); var engine = new CacheEngine(@"..\..\data"); engine.Add<int, string>("Employee", employees, DateTime.Now.AddMinutes(1)); }); w.Stop(); Console.WriteLine("add:" + w.Elapsed); var engine2 = new CacheEngine(@"..\..\data"); var o = engine2.Get<int, string>("Employee", 1005); Console.WriteLine(o); } private static void TestDefaultDiskPerformance() { TestPerformance(CacheProviders.Default, @"..\..\data"); } private static void TestDefaultMemoryPerformance() { TestPerformance(CacheProviders.Default, string.Empty); } private static void TestRawDiskPerformance() { TestPerformance(CacheProviders.Raw, @"..\..\data"); } private static void TestRawMemoryPerformance() { TestPerformance(CacheProviders.Raw, string.Empty); } private static void TestPerformance(CacheProviders Provider, string DataPath) { Console.WriteLine("Performance Test: " + Provider.ToString() + ", " + (string.IsNullOrEmpty(DataPath) ? "Memory" : DataPath)); var engine = new CacheEngine(Provider, DataPath); var w = new Stopwatch(); w.Start(); var employees = Enumerable.Range(0, 1000).Select(i => new KeyValuePair<int, string>(i, "Wilson " + i + " Chen")); engine.Add<int, string>("Employee", employees, DateTime.Now.AddMinutes(1)); w.Stop(); Console.WriteLine("add:" + w.Elapsed); /*w.Restart(); employees.ForEach(key => { var o1 = engine.Get<int, string>("Employee", key.Key); }); w.Stop(); Console.WriteLine("individual get:" + w.Elapsed);*/ w.Restart(); var keys = employees.Select(i => i.Key); var o = engine.Get<int, string>("Employee", keys); w.Stop(); Debug.Assert(o.Count == keys.Count()); Console.WriteLine("get:" + w.Elapsed); Console.WriteLine(); } private static void TestExpiration() { var engine = new CacheEngine(@"..\..\data"); var o = engine.Get<string, Foo>("User", "wchen"); Console.WriteLine(o != null ? o.Name : "wchen does not exist or expired"); } private static void TestAddGet() { var engine = new CacheEngine(@"..\..\data"); var f = new Foo { Name = "Wilson Chen", Age = 30, Birthday = DateTime.Now, Some = 123.456 }; engine.Add("User", "wchen", f, DateTime.Now.AddSeconds(5)); var o = engine.Get<string, Foo>("User", "wchen"); Console.WriteLine(o.Name); var o4 = engine.Get<string, Foo>("User", "foo"); Console.WriteLine(o4 != null ? o4.Name : "foo does not exist"); var o3 = engine.Get<string, string>("PlainText", "A"); Console.WriteLine(o3 ?? "A does not exist"); } }}
性能
通过上述性能测试例子,你会发现STSdb的磁盘存取速度要比一个记录对应一个物理文件快。想了解更多,请访问官方网站。
测试条件:1000条记录,7200RPM磁盘,i7。
引擎 介质 写入 读取 STSdb 磁盘 1.3s 0.06s 内存 0.01s 0.008s Raw 磁盘 1.0s 0.16s 内存 0.0004s 0.0008s
说明
项目中引用了System.Management是因为STSdb支持内存数据库,需要判断最大物理内存。如果不喜欢,大家可以移除引用,并且去掉STSdb4.Database.STSdb.FromMemory方法便可。
下载
点击这里下载
<script type="text/javascript"><!--google_ad_client = "ca-pub-1944176156128447";/* cnblogs 首页横幅 */google_ad_slot = "5419468456";google_ad_width = 728;google_ad_height = 90;//--></script><script type="text/javascript" src="http://pagead2.googlesyndication.com/pagead/show_ads.js"></script>
- C#开源磁盘/内存缓存引擎
- 内存缓存+磁盘缓存
- Banner 内存 磁盘 缓存
- SDWebImage内存缓存 磁盘缓存
- 磁盘缓存和内存缓存的区别
- Glide-内存缓存与磁盘缓存
- iOS 内存缓存和磁盘缓存
- 磁盘缓存和内存缓存的区别
- Android缓存机制Lrucache内存缓存和DiskLruCache磁盘缓存
- Glide 缓存策略 内存缓存和磁盘缓存
- Glide 缓存策略 内存缓存和磁盘缓存
- Glide 缓存策略 内存缓存和磁盘缓存
- Glide 缓存策略 内存缓存和磁盘缓存
- Bitmaps与优化|磁盘缓存|内存缓存|双缓存
- Linux清空内存和磁盘缓存
- Linux清空内存和磁盘缓存
- Linux清空内存和磁盘缓存
- Linux清空内存和磁盘缓存
- OSG入门即osgEarth建立一个地球的详细步骤
- B. Strings of Power
- 【图论05】并查集 1006 Zjnu Stadium
- TopCoder SRM 558 DIV2 Level 3:CatAndRabbit
- QWebView二次链接的问题
- C#开源磁盘/内存缓存引擎
- HTTP1.0与HTTP1.1的区别
- (2)WCF服务元数据交换、配置及编程开发的学习
- C++双向循环链表
- Ajax实现xml文件数据插入数据库(三)---javabean实现数据库插入
- 小程序(七)C语言 输入字符 检测后每个单词逆序输出
- ZOJ 3498 Javabeans
- PhotoShop算法原理解析系列
- Yelp处理1.02亿/月绝对造访人次及3900万评论的省钱秘诀