C# 分词帮助类
来源：互联网 发布：怎么做网络覆盖 编辑：程序博客网 时间：2024/05/16 11:31
using System;using System.Collections;using System.IO;using System.Text.RegularExpressions;namespace DotNet.Utilities{ /// <summary> /// 分词辅助类 /// </summary> public class SegList { public int MaxLength; private ArrayList m_seg; public int Count { get { return m_seg.Count; } } public SegList() { m_seg = new ArrayList(); MaxLength = 0; } public void Add(object obj) { m_seg.Add(obj); if (MaxLength < obj.ToString().Length) { MaxLength = obj.ToString().Length; } } public object GetElem(int i) { if (i < this.Count) return m_seg[i]; else return null; } public void SetElem(int i, object obj) { m_seg[i] = obj; } public bool Contains(object obj) { return m_seg.Contains(obj); } /// <summary> /// 按长度排序 /// </summary> public void Sort() { Sort(this); } /// <summary> /// 按长度排序 /// </summary> public void Sort(SegList list) { int max = 0; for (int i = 0; i < list.Count - 1; ++i) { max = i; for (int j = i + 1; j < list.Count; ++j) { string str1 = list.GetElem(j).ToString(); string str2 = list.GetElem(max).ToString(); int l1; int l2; if (str1 == "null") l1 = 0; else l1 = str1.Length; if (str2 == "null") l2 = 0; else l2 = str2.Length; if (l1 > l2) max = j; } object o = list.GetElem(max); list.SetElem(max, list.GetElem(i)); list.SetElem(i, o); } } } /// <summary> /// 分词类 /// </summary> //----------------调用---------------------- //Segment seg = new Segment(); //seg.InitWordDics(); //seg.EnablePrefix = true; //seg.Separator =" "; //seg.SegmentText("字符串", false).Trim(); //------------------------------------------- public class Segment { #region 私有字段 private string m_DicPath = System.Web.HttpContext.Current.Server.MapPath("bin/ShootSeg/sDict.dic"); private string m_NoisePath = System.Web.HttpContext.Current.Server.MapPath("bin/ShootSeg/sNoise.dic"); private string m_NumberPath = System.Web.HttpContext.Current.Server.MapPath("bin/ShootSeg/sNumber.dic"); private string m_WordPath = System.Web.HttpContext.Current.Server.MapPath("bin/ShootSeg/sWord.dic"); private string m_PrefixPath = System.Web.HttpContext.Current.Server.MapPath("bin/ShootSeg/sPrefix.dic"); private Hashtable htWords; private ArrayList alNoise; private ArrayList alNumber; private ArrayList alWord; private ArrayList alPrefix; private double m_EventTime = 0; /// <summary> /// 分隔符 /// </summary> private string m_Separator = " "; /// <summary> /// 用于验证汉字的正则表达式 /// </summary> private string strChinese = "[\u4e00-\u9fa5]"; #endregion #region 公有属性 /// <summary> /// 基本词典路径 /// </summary> public string DicPath { get { return m_DicPath; } set { m_DicPath = value; } } /// <summary> /// 数据缓存函数 /// </summary> /// <param name="key">索引键</param> /// <param name="val">缓存的数据</param> private static void SetCache(string key, object val) { if (val == null) val = " "; System.Web.HttpContext.Current.Application.Lock(); System.Web.HttpContext.Current.Application.Set(key, val); System.Web.HttpContext.Current.Application.UnLock(); } /// <summary> /// 读取缓存 /// </summary> private static object GetCache(string key) { return System.Web.HttpContext.Current.Application.Get(key); } /// <summary> /// 暂时无用 /// </summary> public string NoisePath { get { return m_NoisePath; } set { m_NoisePath = value; } } /// <summary> /// 数字词典路径 /// </summary> public string NumberPath { get { return m_NumberPath; } set { m_NumberPath = value; } } /// <summary> /// 字母词典路径 /// </summary> public string WordPath { get { return m_WordPath; } set { m_WordPath = value; } } /// <summary> /// 姓名前缀字典 用于纠错姓名 /// </summary> public string PrefixPath { get { return m_PrefixPath; } set { m_PrefixPath = value; } } /// <summary> /// 是否开启姓名纠错功能 /// </summary> public bool EnablePrefix { get { if (alPrefix.Count == 0) return false; else return true; } set { if (value) alPrefix = LoadWords(PrefixPath, alPrefix); else alPrefix = new ArrayList(); } } /// <summary> /// 用时每次进行加载或分词动作后改属性表示为上一次动作所用时间 /// 已精确到毫秒但分词操作在字符串较短时可能为0 /// </summary> public double EventTime { get { return m_EventTime; } } /// <summary> /// 分隔符,默认为空格 /// </summary> public string Separator { get { return m_Separator; } set { if (value != "" && value != null) m_Separator = value; } } #endregion #region 构造方法 /// <summary> /// 构造方法 /// </summary> public Segment() { } /// <summary> /// 构造方法 /// </summary> public Segment(string p_DicPath, string p_NoisePath, string p_NumberPath, string p_WordPath) { m_WordPath = p_DicPath; m_WordPath = p_NoisePath; m_WordPath = p_NumberPath; m_WordPath = p_WordPath; this.InitWordDics(); } #endregion #region 公有方法 /// <summary> /// 加载词列表 /// </summary> public void InitWordDics() { DateTime start = DateTime.Now; if (GetCache("jcms_dict") == null) { htWords = new Hashtable(); Hashtable father = htWords; Hashtable forfather = htWords; string strChar1; string strChar2; StreamReader reader = new StreamReader(DicPath, System.Text.Encoding.UTF8); string strline = reader.ReadLine(); SegList list; Hashtable child = new Hashtable(); long i = 0; while (strline != null && strline.Trim() != "") { i++; strChar1 = strline.Substring(0, 1); strChar2 = strline.Substring(1, 1); if (!htWords.ContainsKey(strChar1)) { father = new Hashtable(); htWords.Add(strChar1, father); } else { father = (Hashtable)htWords[strChar1]; } if (!father.ContainsKey(strChar2)) { list = new SegList(); if (strline.Length > 2) list.Add(strline.Substring(2)); else list.Add("null"); father.Add(strChar2, list); } else { list = (SegList)father[strChar2]; if (strline.Length > 2) { list.Add(strline.Substring(2)); } else { list.Add("null"); } father[strChar2] = list; } htWords[strChar1] = father; strline = reader.ReadLine(); } try { reader.Close(); } catch { } SetCache("jcms_dict", htWords); } htWords = (Hashtable)GetCache("jcms_dict"); alNoise = LoadWords(NoisePath, alNoise); alNumber = LoadWords(NumberPath, alNumber); alWord = LoadWords(WordPath, alWord); alPrefix = LoadWords(PrefixPath, alPrefix); TimeSpan duration = DateTime.Now - start; m_EventTime = duration.TotalMilliseconds; } /// <summary> /// 加载文本词组到ArrayList /// </summary> public ArrayList LoadWords(string strPath, ArrayList list) { StreamReader reader = new StreamReader(strPath, System.Text.Encoding.UTF8); list = new ArrayList(); string strline = reader.ReadLine(); while (strline != null) { list.Add(strline); strline = reader.ReadLine(); } try { reader.Close(); } catch { } return list; } /// <summary> /// 输出词列表 /// </summary> public void OutWords() { IDictionaryEnumerator idEnumerator1 = htWords.GetEnumerator(); while (idEnumerator1.MoveNext()) { IDictionaryEnumerator idEnumerator2 = ((Hashtable)idEnumerator1.Value).GetEnumerator(); while (idEnumerator2.MoveNext()) { SegList aa = (SegList)idEnumerator2.Value; for (int i = 0; i < aa.Count; i++) { Console.WriteLine(idEnumerator1.Key.ToString() + idEnumerator2.Key.ToString() + aa.GetElem(i).ToString()); } } } } /// <summary> /// 输出ArrayList /// </summary> public void OutArrayList(ArrayList list) { if (list == null) return; for (int i = 0; i < list.Count; i++) { Console.WriteLine(list[i].ToString()); } } /// <summary> /// 分词过程,不支持回车 /// </summary> /// <param name="strText">要分词的文本</param> /// <returns>分词后的文本</returns> public string SegmentText(string strText) { strText = (strText + "$").Trim(); if (htWords == null) return strText; if (strText.Length < 3) return strText; DateTime start = DateTime.Now; int length = 0; int preFix = 0; bool word = false; bool number = false; string reText = ""; string strPrefix = ""; string strLastChar = ""; string strLastWords = Separator; for (int i = 0; i < strText.Length - 1; i++) { #region 对于每一个字的处理过程 string strChar1 = strText.Substring(i, 1); string strChar2 = strText.Substring(i + 1, 1).Trim(); bool yes; SegList l; Hashtable h; if (reText.Length > 0) strLastChar = reText.Substring(reText.Length - 1); if (strChar1 == " ") { if ((number || word) && strLastChar != Separator) reText += this.Separator; yes = true; } else yes = false; int CharType = GetCharType(strChar1); switch (CharType) { case 1: #region 如果是数字，如果数字的上一位是字母要和后面的数字分开 if (word) { reText += Separator; } word = false; number = true; strLastWords = ""; break; #endregion case 2: case 5: #region 如果是字母 if (number) strLastWords = Separator; else strLastWords = ""; word = true; number = false; break; #endregion case 3: case 4: #region 第一级哈希表是否包含关键字，假如包含处理第二级哈希表 //上一个字是否为字母 if (word) reText += Separator; #region 检测上一个是否是数字，这个过程是用于修正数字后的量词的 if (number && CharType != 4) { h = (Hashtable)htWords["n"]; if (h.ContainsKey(strChar1)) { l = (SegList)h[strChar1]; if (l.Contains(strChar2)) { reText += strChar1 + strChar2 + Separator; yes = true; i++; } else if (l.Contains("null")) { reText += strChar1 + Separator; yes = true; } } else reText += Separator; } #endregion //非汉字数字的汉字 if (CharType == 3) { word = false; number = false; strLastWords = Separator; } else { word = false; number = true; strLastWords = ""; } //第二级哈希表取出 h = (Hashtable)htWords[strChar1]; //第二级哈希表是否包含关键字 if (h.ContainsKey(strChar2)) { #region 第二级包含关键字 //取出ArrayList对象 l = (SegList)h[strChar2]; //遍历每一个对象 看是否能组合成词 for (int j = 0; j < l.Count; j++) { bool have = false; string strChar3 = l.GetElem(j).ToString(); //对于每一个取出的词进行检测,看是否匹配，长度保护 if ((strChar3.Length + i + 2) < strText.Length) { //向i+2后取出m长度的字 string strChar = strText.Substring(i + 2, strChar3.Length).Trim(); if (strChar3 == strChar && !yes) { if (strPrefix != "") { reText += strPrefix + Separator; strPrefix = ""; preFix = 0; } reText += strChar1 + strChar2 + strChar; i += strChar3.Length + 1; have = true; yes = true; break; } } else if ((strChar3.Length + i + 2) == strText.Length) { string strChar = strText.Substring(i + 2).Trim(); if (strChar3 == strChar && !yes) { if (strPrefix != "") { reText += strPrefix + Separator; strPrefix = ""; preFix = 0; } reText += strChar1 + strChar2 + strChar; i += strChar3.Length + 1; have = true; yes = true; break; } } if (!have && j == l.Count - 1 && l.Contains("null") && !yes) { if (preFix == 1) { reText += strPrefix + strChar1 + strChar2; strPrefix = ""; preFix = 0; } else if (preFix > 1) { reText += strPrefix + strLastWords + strChar1 + strChar2; strPrefix = ""; preFix = 0; } else { if (CharType == 4) reText += strChar1 + strChar2; else reText += strChar1 + strChar2; strLastWords = this.Separator; number = false; } i++; yes = true; break; } else if (have) { break; } } #endregion //如果没有匹配还可能有一种情况，这个词语只有两个字，以这两个字开头的词语不存在 if (!yes && l.Contains("null")) { if (preFix == 1) { reText += strPrefix + strChar1 + strChar2; strPrefix = ""; preFix = 0; } else if (preFix > 1) { reText += strPrefix + strLastWords + strChar1 + strChar2; strPrefix = ""; preFix = 0; } else { if (CharType == 4) reText += strChar1 + strChar2; else reText += strChar1 + strChar2; strLastWords = this.Separator; number = false; } i++; yes = true; } if (reText.Length > 0) strLastChar = reText.Substring(reText.Length - 1); if (CharType == 4 && GetCharType(strLastChar) == 4) { number = true; } else if (strLastChar != this.Separator) reText += this.Separator; } #endregion break; default: #region 未知字符,可能是生僻字,也可能是标点符合之类 if (word && !yes) { reText += Separator; } else if (number && !yes) { reText += Separator; } number = false; word = false; strLastWords = this.Separator; break; #endregion } if (!yes && number || !yes && word) { reText += strChar1; yes = true; } if (!yes) { #region 处理姓名问题 if (preFix == 0) { if (alPrefix.Contains(strChar1 + strChar2)) { i++; strPrefix = strChar1 + strChar2; preFix++; } else if (alPrefix.Contains(strChar1)) { if (!number) { strPrefix = strChar1; preFix++; } else { reText += strChar1 + strLastWords; number = false; word = false; } } else { if (preFix == 3) { reText += strPrefix + Separator + strChar1 + Separator; strPrefix = ""; preFix = 0; } else if (preFix > 0) { if (Regex.IsMatch(strChar1, strChinese)) { strPrefix += strChar1; preFix++; } else { reText += strPrefix + Separator + strChar1 + Separator; strPrefix = ""; preFix = 0; } } else { reText += strChar1 + strLastWords; number = false; word = false; } } } else { if (preFix == 3) { reText += strPrefix + Separator + strChar1 + Separator; strPrefix = ""; preFix = 0; } else if (preFix > 0) { if (Regex.IsMatch(strChar1, strChinese)) { strPrefix += strChar1; preFix++; } else { reText += strPrefix + Separator + strChar1 + Separator; strPrefix = ""; preFix = 0; } } else { reText += strChar1 + strLastWords; number = false; } } #endregion } length = i; #endregion } #region 最后防止最后一个字的丢失 if (length < strText.Length - 1) { string strLastChar1 = strText.Substring(strText.Length - 1).Trim(); string strLastChar2 = strText.Substring(strText.Length - 2).Trim(); if (reText.Length > 0) strLastChar = reText.Substring(reText.Length - 1); if (preFix != 0) { reText += strPrefix + strLastChar1; } else { switch (GetCharType(strLastChar1)) { case 1: if (strLastChar1 != "." && strLastChar1 != "．") reText += strLastChar1; else reText += Separator + strLastChar1; break; case 2: case 5: if (alWord.Contains(strLastChar2)) reText += strLastChar1; break; case 3: case 4: if ((number || word) && strLastChar != Separator) reText += Separator + strLastChar1; else reText += strLastChar1; break; default: if (strLastChar != Separator) reText += Separator + strLastChar1; else reText += strLastChar1; break; } } if (reText.Length > 0) strLastChar = (reText.Substring(reText.Length - 1)); if (strLastChar != this.Separator) reText += this.Separator; } #endregion TimeSpan duration = DateTime.Now - start; m_EventTime = duration.TotalMilliseconds; return reText.Replace(" $", ""); //这里包含一个字的，则去掉 } /// <summary> /// 重载分词过程,支持回车 /// </summary> public string SegmentText(string strText, bool Enter) { if (Enter) { DateTime start = DateTime.Now; string[] strArr = strText.Split('\n'); string reText = ""; for (int i = 0; i < strArr.Length; i++) { reText += SegmentText(strArr[i]) + "\r\n"; } TimeSpan duration = DateTime.Now - start; m_EventTime = duration.TotalMilliseconds; return reText; } else { return SegmentText(strText); } } #region 判断字符类型 /// <summary> /// 判断字符类型,0为未知,1为数字,2为字母,3为汉字,4为汉字数字 /// </summary> private int GetCharType(string p_Char) { int CharType = 0; if (alNumber.Contains(p_Char)) CharType = 1; if (alWord.Contains(p_Char)) CharType = 2; if (htWords.ContainsKey(p_Char)) CharType += 3; return CharType; } #endregion #region 对加载的词典排序并重新写入 /// <summary> /// 对加载的词典排序并重新写入 /// </summary> public void SortDic() { SortDic(false); } /// <summary> /// 对加载的词典排序并重新写入 /// </summary> /// <param name="Reload">是否重新加载</param> public void SortDic(bool Reload) { DateTime start = DateTime.Now; StreamWriter sw = new StreamWriter(DicPath, false, System.Text.Encoding.UTF8); IDictionaryEnumerator idEnumerator1 = htWords.GetEnumerator(); while (idEnumerator1.MoveNext()) { IDictionaryEnumerator idEnumerator2 = ((Hashtable)idEnumerator1.Value).GetEnumerator(); while (idEnumerator2.MoveNext()) { SegList aa = (SegList)idEnumerator2.Value; aa.Sort(); for (int i = 0; i < aa.Count; i++) { if (aa.GetElem(i).ToString() == "null") sw.WriteLine(idEnumerator1.Key.ToString() + idEnumerator2.Key.ToString()); else sw.WriteLine(idEnumerator1.Key.ToString() + idEnumerator2.Key.ToString() + aa.GetElem(i).ToString()); } } } sw.Close(); if (Reload) InitWordDics(); TimeSpan duration = DateTime.Now - start; m_EventTime = duration.TotalMilliseconds; } #endregion /// <summary> /// 删除两行完全相同的词,暂时无用! /// </summary> /// <returns>相同词条个数</returns> public int Optimize() { int l = 0; DateTime start = DateTime.Now; Hashtable htOptimize = new Hashtable(); StreamReader reader = new StreamReader(DicPath, System.Text.Encoding.UTF8); string strline = reader.ReadLine(); while (strline != null && strline.Trim() != "") { if (!htOptimize.ContainsKey(strline)) htOptimize.Add(strline, null); else l++; } Console.WriteLine("ready"); try { reader.Close(); } catch { } StreamWriter sw = new StreamWriter(DicPath, false, System.Text.Encoding.UTF8); IDictionaryEnumerator ide = htOptimize.GetEnumerator(); while (ide.MoveNext()) sw.WriteLine(ide.Key.ToString()); try { sw.Close(); } catch { } TimeSpan duration = DateTime.Now - start; m_EventTime = duration.TotalMilliseconds; return l; } #endregion }} 0 0
- C# 分词帮助类
- C#时间帮助类
- c# JsonHelper 帮助类
- C# GridView帮助类
- C# Cookie帮助类
- C# 时间帮助类
- C# js帮助类
- FireBird C# 帮助类
- C# CookieHelper帮助类
- c# base64帮助类
- C# http帮助类
- C#数据查询帮助类
- C# MySQL DBHelper帮助类
- C#文件帮助类FoderHelper
- c#数据库帮助类SqlHelper
- c#通用json帮助类
- C# ADO.NET帮助类
- C# FTP简单帮助类
- <绝对干货>2017年ICLR视频地址
- RHEL-RH134-UNIT7-访问网络文件共享服务
- hi
- 深入理解JS—instanceof和原型链
- 快速进入陌生领域： HistCite 使用教程
- C# 分词帮助类
- @MappedSuperclass
- 深度学习目标检测算法性能分析
- 《JAVA与模式》之策略模式
- ubuntu 运行android studio出错unable to run mksdcard sdk
- 抽象类的内部类
- RHEL-RH134-UNIT8-LDAP 网络用户账户
- Java 剑指offer_面试题20_顺时针打印矩阵（思路在一步步分解之中）
- 水果展示案例
