python正向最大匹配分词和逆向最大匹配分词

正向最大匹配

# -*- coding:utf-8 -*-CODEC='utf-8'def u(s, encoding):    'converted other encoding to unicode encoding'    if isinstance(s, unicode):        return s    else:        return unicode(s, encoding)def fwd_mm_seg(wordDict, maxLen, str):    'forward max match segment'    wordList = []    segStr = str    segStrLen = len(segStr)    for word in wordDict:        print 'word: ', word    print "\n"    while segStrLen > 0:        if segStrLen > maxLen:            wordLen = maxLen        else:            wordLen = segStrLen        subStr = segStr[0:wordLen]        print "subStr: ", subStr        while wordLen > 1:            if subStr in wordDict:                print "subStr1: %r" % subStr                break            else:                print "subStr2: %r" % subStr                wordLen = wordLen - 1                subStr = subStr[0:wordLen]#            print "subStr3: ", subStr        wordList.append(subStr)        segStr = segStr[wordLen:]        segStrLen = segStrLen - wordLen    for wordstr in wordList:        print "wordstr: ", wordstr    return wordList                    def main():    fp_dict = open('words.dic')    wordDict = {}    for eachWord in fp_dict:        wordDict[u(eachWord.strip(), 'utf-8')] = 1    segStr = u'你好世界hello world'    print segStr    wordList = fwd_mm_seg(wordDict, 10, segStr)    print "==".join(wordList)    if __name__ == '__main__':    main()        

逆向最大匹配

# -*- coding:utf-8 -*-def u(s, encoding):    'converted other encoding to unicode encoding'    if isinstance(s, unicode):        return s    else:        return unicode(s, encoding)CODEC='utf-8'def bwd_mm_seg(wordDict, maxLen, str):    'forward max match segment'    wordList = []    segStr = str    segStrLen = len(segStr)    for word in wordDict:        print 'word: ', word    print "\n"    while segStrLen > 0:        if segStrLen > maxLen:            wordLen = maxLen        else:            wordLen = segStrLen        subStr = segStr[-wordLen:None]        print "subStr: ", subStr        while wordLen > 1:            if subStr in wordDict:                print "subStr1: %r" % subStr                break            else:                print "subStr2: %r" % subStr                wordLen = wordLen - 1                subStr = subStr[-wordLen:None]#            print "subStr3: ", subStr        wordList.append(subStr)        segStr = segStr[0: -wordLen]        segStrLen = segStrLen - wordLen    wordList.reverse()    for wordstr in wordList:        print "wordstr: ", wordstr    return wordList                    def main():    fp_dict = open('words.dic')    wordDict = {}    for eachWord in fp_dict:        wordDict[u(eachWord.strip(), 'utf-8')] = 1    segStr = ur'你好世界hello world'    print segStr    wordList = bwd_mm_seg(wordDict, 10, segStr)    print "==".join(wordList)if __name__ == '__main__':    main()