自然语言处理2---jieba分词

# -*- encoding:utf-8 -*-import re,HTMLParserimport sysreload(sys)import osimport jiebafrom sklearn.feature_extraction.text import TfidfVectorizer#sys.setdefaultencoding('utf-8')#输出的内容（爬下来的）是utf-8的形式def file_jieba(file_dir):  # 得到文本.txt的路径.\Fractal\Interne    for root, dirs, files in os.walk(file_dir):        for f in files:            tmp_dir = '.\Reduced_internet_preprocess' + '\\' + f  # 加上标签后的文本            text_init_dir = file_dir + '\\' + f  # 原始文本            # print text_init_dir            file_source = open(text_init_dir, 'r')            ok_file = open(tmp_dir, 'w+')            line_content = file_source.readlines()            for lines in line_content:                #file_source_decode = lines.decode('GBK')                file_source_cut = jieba.cut(lines, cut_all=False)                result = ' '.join(file_source_cut)                result = result.encode('utf-8')                ok_file.write(result)            file_source.close()            ok_file.close()            print 'ok'