tensorflow 实践(一)使用神经网络做中文情感分析
来源:互联网 发布:bj单身日记 知乎 编辑:程序博客网 时间:2024/06/06 05:56
本文使用哈工大做文本预处理; 两层隐层神经网络;
后注:不是标准的ann,做了去停用词和词性筛选,没有端到端。
# -*- coding: utf-8 -*-# @bref :使用tensorflow做中文情感分析import numpy as npimport tensorflow as tfimport randomfrom sklearn.feature_extraction.text import CountVectorizerimport osimport tracebackreal_dir_path = os.path.split(os.path.realpath(__file__))[0]pos_file = os.path.join(real_dir_path, 'data/pos_bak.txt')neg_file = os.path.join(real_dir_path, 'data/neg_bak.txt')#使用哈工大分词和词性标注from pyltp import Segmentor, Postaggerseg = Segmentor()seg.load('/root/git/ltp_data/cws.model')poser = Postagger()poser.load('/root/git/ltp_data/pos.model')real_dir_path = os.path.split(os.path.realpath(__file__))[0] #文件所在路径stop_words_file = os.path.join(real_dir_path, '../util/stopwords.txt')#定义允许的词性allow_pos_ltp = ('a', 'i', 'j', 'n', 'nh', 'ni', 'nl', 'ns', 'nt', 'nz', 'v', 'ws')#分词、去除停用词、词性筛选def cut_stopword_pos(s): words = seg.segment(''.join(s.split())) poses = poser.postag(words) stopwords = {}.fromkeys([line.rstrip() for line in open(stop_words_file)]) sentence = [] for i, pos in enumerate(poses): if (pos in allow_pos_ltp) and (words[i] not in stopwords): sentence.append(words[i]) return sentencedef create_vocab(pos_file, neg_file): def process_file(file_path): with open(file_path, 'r') as f: v = [] lines = f.readlines() for line in lines: sentence = cut_stopword_pos(line) v.append(' '.join(sentence)) return v sent = process_file(pos_file) sent += process_file(neg_file) tf_v = CountVectorizer(max_df=0.9, min_df=1) tf = tf_v.fit_transform(sent) #print tf_v.vocabulary_ return tf_v.vocabulary_.keys()#获取词汇vocab = create_vocab(pos_file, neg_file)#依据词汇将评论转化为向量def normalize_dataset(vocab): dataset = [] # vocab:词汇表; review:评论; clf:评论对应的分类, [0, 1]表示负面评论,[1, 0]表示正面 def string_to_vector(vocab, review, clf): words = cut_stopword_pos(review) # list of str features = np.zeros(len(vocab)) for w in words: if w.decode('utf-8') in vocab: features[vocab.index(w.decode('utf-8'))] = 1 return [features, clf] with open(pos_file, 'r') as f: lines = f.readlines() for line in lines: one_sample = string_to_vector(vocab, line, [1, 0]) dataset.append(one_sample) with open(neg_file, 'r') as f: lines = f.readlines() for line in lines: one_sample = string_to_vector(vocab, line, [0, 1]) dataset.append(one_sample) return datasetdataset = normalize_dataset(vocab)random.shuffle(dataset) #打乱顺序#取样本的10%作为测试数据test_size = int(len(dataset) * 0.1)dataset = np.array(dataset)train_dataset = dataset[:-test_size]test_dataset = dataset[-test_size:]print 'test_size = {}'.format(test_size)#print 'size of train_dataset is {}'.format(train_dataset)#Feed-forward nueral network#定义每个层有多少个神经元n_input_layer = len(vocab) #输入层每个神经元代表一个termn_layer_1 = 1000 #hiden layern_layer_2 = 1000 # hiden layern_output_layer = 2#定义待训练的神经网络def neural_netword(data): #定义第一层神经元的w和b, random_normal定义服从正态分布的随机变量 layer_1_w_b = {'w_':tf.Variable(tf.random_normal([n_input_layer, n_layer_1])), 'b_':tf.Variable(tf.random_normal([n_layer_1]))} layer_2_w_b = {'w_':tf.Variable(tf.random_normal([n_layer_1, n_layer_2])), 'b_':tf.Variable(tf.random_normal([n_layer_2]))} layer_output_w_b = {'w_':tf.Variable(tf.random_normal([n_layer_2, n_output_layer])), 'b_':tf.Variable(tf.random_normal([n_output_layer]))} layer_1 = tf.add(tf.matmul(data, layer_1_w_b['w_']), layer_1_w_b['b_']) layer_1 = tf.nn.relu(layer_1) #relu做激活函数 layer_2 = tf.add(tf.matmul(layer_1, layer_2_w_b['w_']), layer_2_w_b['b_']) layer_2 = tf.nn.relu(layer_2) layer_output = tf.add(tf.matmul(layer_2, layer_output_w_b['w_']), layer_output_w_b['b_']) return layer_outputbatch_size = 50X = tf.placeholder('float', [None, n_input_layer]) #None表示样本数量任意; 每个样本纬度是term数量Y = tf.placeholder('float')#使用数据训练神经网络def train_neural_network(X, Y): predict = neural_netword(X) #cost func是输出层softmax的cross entropy的平均值。 将softmax 放在此处而非nn中是为了效率. cost_func = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=predict, labels=Y)) #设置优化器 optimizer = tf.train.AdamOptimizer().minimize(cost_func) epochs = 13 #epoch本意是时代、纪, 这里是迭代周期 with tf.Session() as session: session.run(tf.initialize_all_variables()) #初始化所有变量,包括w,b random.shuffle(train_dataset) train_x = train_dataset[:, 0] #每一行的features; train_y = train_dataset[:, 1] #每一行的label print 'size of train_x is {}'.format(len(train_x)) for epoch in range(epochs): epoch_loss = 0 #每个周期的loss i = 0 while i < len(train_x): start = i end = i + batch_size batch_x = train_x[start:end] batch_y = train_y[start:end] #run的第一个参数fetches可以是单个,也可以是多个。 返回值是fetches的返回值。 #此处因为要打印cost,所以cost_func也在fetches中 _, c = session.run([optimizer, cost_func], feed_dict={X:list(batch_x), Y:list(batch_y)}) epoch_loss += c i = end print(epoch, ' : ', epoch_loss) #评估模型 test_x = test_dataset[:, 0] test_y = test_dataset[:, 1] #argmax能给出某个tensor对象在某一维上的其数据最大值所在的索引值, 这里是索引值的list。tf.equal用于检测匹配,返回bool型的list correct = tf.equal(tf.argmax(predict, 1), tf.argmax(Y, 1)) #tf.cast 可以将[True, False, True] 转化为[1, 0, 1] #reduce_mean用于在某一维上计算平均值, 未指定纬度则计算所有元素 accurqcy = tf.reduce_mean(tf.cast(correct, 'float')) print('准确率: {}'.format(accurqcy.eval({X:list(test_x), Y:list(test_y)}))) #等价: print session.run(accuracy, feed_dict={X:list(test_x), Y:list(test_y)})train_neural_network(X, Y)
最终的执行显示:
size of train_x is 31612(0, ' : ', 105508.38228607178)(1, ' : ', 11773.463727131188)(2, ' : ', 4551.4978754326503)(3, ' : ', 3576.6907950473492)(4, ' : ', 3144.6771814899175)(5, ' : ', 2911.1803286887775)(6, ' : ', 2691.8284285693276)(7, ' : ', 2651.9982114042473)(8, ' : ', 2882.4479921576026)(9, ' : ', 2665.3818837262743)(10, ' : ', 2551.3030235993206)(11, ' : ', 2838.3546982686303)(12, ' : ', 2770.5539811982608)准确率: 0.828587830067
0 0
- tensorflow 实践(一)使用神经网络做中文情感分析
- 神经网络之文本情感分析(一)
- 如何使用Stanford CoreNlp做中文情感分析
- 使用循环神经网络训练情感分析
- 神经网络之文本情感分析(二)
- 神经网络之文本情感分析(三)
- 神经网络之文本情感分析(四)
- 文本情感分析(一)
- Sentiment Analysis情感分析工具的使用(一):SentiStrength
- 怎样做情感分析
- 怎样做情感分析
- 基于回归神经网络的中文语句模型实践(Python+Tensorflow+阿里云)
- 使用tensorflow:LSTM神经网络预测股票(一)
- 中文情感倾向性分析(IEEE2010)
- 中文情感分析
- 中文情感分析语料库
- 中文情感分析语料库
- 中文情感分析语料库
- validate.js使用方法
- What is “origin” in Git?
- C语言判断素数
- 【linux】centOS firewall添加开放端口
- CSDN日报20170505 ——《创业时该不该用新手程序员》
- tensorflow 实践(一)使用神经网络做中文情感分析
- linux命令指南之四
- Unity自定义创建脚本模板菜单
- How Many Answers Are Wrong HDU
- 内核module加载顺序
- 个人模板 最长公共子序列
- 规则和统计(下)
- mysql的默认端口、索引类型
- FEP