自然语言处理1---文本预处理（删除非文本，非数字字符）

# -*- encoding:utf-8 -*-import re,HTMLParserimport sysreload(sys)import ossys.setdefaultencoding('utf-8')#输出的内容（爬下来的）是utf-8的形式def file_fill(file_dir):  # 得到文本.txt的路径    for root, dirs, files in os.walk(file_dir):        for f in files:            tmp_dir = '.\Reduced_sport' + '\\' + f  # 加上标签后的文本            text_init_dir = file_dir + '\\' + f  # 原始文本            # print text_init_dir            file_source = open(text_init_dir, 'r')            # ok_file = open(tmp_dir, 'a+')            ok_file = open(tmp_dir, 'a+')            line_content = file_source.readlines()            for lines in line_content:                text = lines.replace(' ', '').replace('>','')                # text=l.decode('ascii').encode('utf-8')                ok_file.write(text)            file_source.close()            ok_file.close()            print 'ok'if __name__ == "__main__":    folder = file_fill(".\Reduced\sport")