Python词法分析器实现

简单Python词法分析器简单实现:

词法分析器状态转换图：

词法分析器总流程图：

预处理程序：

词法分析器：

词法分析器程序详细设计

详细代码实现:

#!/usr/bin/env python3.4# coding=utf-8import sysimport stringkeywards = {}# 关键字部分keywards['False'] = 101keywards['class'] = 102keywards['finally'] = 103keywards['is'] = 104keywards['return'] = 105keywards['None'] = 106keywards['continue'] = 107keywards['for'] = 108keywards['lambda'] = 109keywards['try'] = 110keywards['True'] = 111keywards['def'] = 112keywards['from'] = 113keywards['nonlocal'] = 114keywards['while'] = 115keywards['and'] = 116keywards['del'] = 117keywards['global'] = 118keywards['not'] = 119keywards['with'] = 120keywards['as'] = 121keywards['elif'] = 122keywards['if'] = 123keywards['or'] = 124keywards['yield'] = 125keywards['assert'] = 126keywards['else'] = 127keywards['import'] = 128keywards['pass'] = 129keywards['break'] = 130keywards['except'] = 131keywards['in'] = 132keywards['raise'] = 133# 符号keywards['+'] = 201keywards['-'] = 202keywards['*'] = 203keywards['/'] = 204keywards['='] = 205keywards[':'] = 206keywards['<'] = 207keywards['>'] = 208keywards['%'] = 209keywards['&'] = 210keywards['!'] = 211keywards['('] = 212keywards[')'] = 213keywards['['] = 214keywards[']'] = 215keywards['{'] = 216keywards['}'] = 217keywards['#'] = 218keywards['|'] = 219keywards[','] = 220# 变量# keywards['var'] = 301# 常量# keywards['const'] = 401# Error# keywards['const'] = 501signlist = {}# 预处理函数，将文件中的空格，换行等无关字符处理掉def pretreatment(file_name):    try:        fp_read = open(file_name, 'r')        fp_write = open('file.tmp', 'w')        sign = 0        while True:            read = fp_read.readline()            if not read:                break            length = len(read)            i = -1            while i < length - 1:                i += 1                if sign == 0:                    if read[i] == ' ':                        continue                if read[i] == '#':                    break                elif read[i] == ' ':                    if sign == 1:                        continue                    else:                        sign = 1                        fp_write.write(' ')                elif read[i] == '\t':                    if sign == 1:                        continue                    else:                        sign = 1                        fp_write.write(' ')                elif read[i] == '\n':                    if sign == 1:                        continue                    else:                        fp_write.write(' ')                        sign = 1                elif read[i] == '"':                    fp_write.write(read[i])                    i += 1                    while i < length and read[i] != '"':                        fp_write.write(read[i])                        i += 1                    if i >= length:                        break                    fp_write.write(read[i])                elif read[i] == "'":                    fp_write.write(read[i])                    i += 1                    while i < length and read[i] != "'":                        fp_write.write(read[i])                        i += 1                    if i >= length:                        break                    fp_write.write(read[i])                else:                    sign = 3                    fp_write.write(read[i])    except Exception:        print(file_name, ': This FileName Not Found!')def save(string):    if string in keywards.keys():        if string not in signlist.keys():            signlist[string] = keywards[string]    else:        try:            float(string)            save_const(string)        except ValueError:            save_var(string)def save_var(string):    if string not in signlist.keys():        if len(string.strip()) < 1:            pass        else:            if is_signal(string) == 1:                signlist[string] = 301            else:                signlist[string] = 501def save_const(string):    if string not in signlist.keys():        signlist[string] = 401def save_error(string):    if string not in signlist.keys():        signlist[string] = 501def is_signal(s):    if s[0] == '_' or s[0] in string.ascii_letters:        for i in s:            if i in string.ascii_letters or i == '_' or i in string.digits:                pass            else:                return 0        return 1    else:        return 0def recognition(filename):    try:        fp_read = open(filename, 'r')        string = ""        sign = 0        while True:            read = fp_read.read(1)            if not read:                break            if read == ' ':                if len(string.strip()) < 1:                    sign = 0                    pass                else:                    if sign == 1 or sign == 2:                        string += read                    else:                        save(string)                        string = ""                        sign = 0            elif read == '(':                if sign == 1 or sign == 2:                    string += read                else:                    save(string)                    string = ""                    save('(')            elif read == ')':                if sign == 1 or sign == 2:                    string += read                else:                    save(string)                    string = ""                    save(')')            elif read == '[':                if sign == 1 or sign == 2:                    string += read                else:                    save(string)                    string = ""                    save('[')            elif read == ']':                if sign == 1 or sign == 2:                    string += read                else:                    save(string)                    string = ""                    save(']')            elif read == '{':                if sign == 1 or sign == 2:                    string += read                else:                    save(string)                    string = ""                    save('{')            elif read == '}':                if sign == 1 or sign == 2:                    string += read                else:                    save(string)                    string = ""                    save('}')            elif read == '<':                save(string)                string = ""                save('<')            elif read == '>':                save(string)                string = ""                save('>')            elif read == ',':                save(string)                string = ""                save(',')            elif read == "'":                string += read                if sign == 1:                    sign = 0                    save_const(string)                    string = ""                else:                    if sign != 2:                        sign = 1            elif read == '"':                string += read                if sign == 2:                    sign = 0                    save_const(string)                    string = ""                else:                    if sign != 1:                        sign = 2            elif read == ':':                if sign == 1 or sign == 2:                    string += read                else:                    save(string)                    string = ""                    save(':')            elif read == '+':                save(string)                string = ""                save('+')            elif read == '=':                save(string)                string = ""                save('=')            else:                string += read    except Exception as e:        print(e)def main():    if len(sys.argv) < 2:        print("Please Input FileName")    else:        pretreatment(sys.argv[1])    recognition('file.tmp')    for i in signlist.keys():        print("(", signlist[i], ",", i, ")")if __name__ == '__main__':    main()

本文固定链接:http://blog.dreamchasinger.cn/?p=625
欢迎访问我的自建博客:http://blog.dreamchasinger.cn/