程序博客网 > 简明python教程豆瓣

Simple Recurrent Unit For Sentence Classification tensorflow实现

来源：互联网 发布：简明python教程豆瓣 编辑：程序博客网 时间：2024/06/13 03:51

网上看到一篇文章关于SRU的原理以及实现，不过是基于pytorch，地址如下：

https://mp.weixin.qq.com/s/2I9_cF3F5MO3l5rE1VY2ng


在这里用tensorflow来实现,先看下SRUCell实现:

  1. import math
  2. import numpy as np
  3. import tensorflow as tf
  4. from tensorflow.python.ops.rnn_cell import RNNCell
  7. class SRUCell(RNNCell):
  8.     def __init__(self, num_units):
  9.         self.num_units = num_units
  11.     @property
  12.     def state_size(self):
  13.         return (self.num_units, self.num_units)
  15.     @property
  16.     def output_size(self):
  17.         return self.num_units
  19.     def __call__(self, x, state, scope=None):
  20.         with tf.variable_scope(scope or type(self).__name__):
  21.             c, _ = state
  23.             x_size = x.get_shape().as_list()[1] # x: [1, x_size], x_size == num_units
  25.             W_u = tf.get_variable('W_u',
  26.                 [x_size, 3 * self.num_units])
  28.             xh = tf.matmul(x, W_u) # [1, 3 * self.num_units]
  30.             tx, f, r = tf.split(xh, 3, 1) # [1, self.num_units] * 3
  31.             b_f = tf.get_variable('b_f', [self.num_units])
  32.             b_r = tf.get_variable('b_r', [self.num_units])
  34.             f = tf.sigmoid(f + b_f)
  35.             r = tf.sigmoid(r + b_r)
  37.             new_c = f * c + (1 - f) * tx # element wise - multiply
  38.             new_h = r * tf.tanh(new_c) + (1 - r) * x
  40.             return new_h, (new_c, new_h)


提供两种方式来实现，看下面:

  1. import tensorflow as tf
  2. from SRU_tensorflow import SRUCell
  3. class  TextRnn(object):
  4.     def __init__(self,config):
  5.         self.config=config
  6.         self.input_x=tf.placeholder(tf.int32,[None,self.config.seq_length],name='input_x')
  7.         self.input_y=tf.placeholder(tf.float32,[None,self.config.num_classes],name='input_y')
  8.         self.keep_prob=tf.placeholder(tf.float32,name='keep_prob')
  9.         self.isbirnn=self.config.isbirnn
  10.         # if  self.isbirnn:
  11.         #     self.birnn()
  12.         # else:
  13.         #     self.rnn()
  14.         if  self.isbirnn:
  15.              self.bisru()
  16.         else:
  17.              self.sru()
  18.     def input_embedding(self):
  19.         """词嵌套
  20.          这里先把指定gpu的程序去掉，线上用cpu部署，指定gpu模型会报错
  21.         """
  22.         #with tf.device('/gpu:0'):
  23.         embedding=tf.get_variable('embedding',[self.config.vocab_size,self.config.embedding_dim])
  24.         _inputs=tf.nn.embedding_lookup(embedding,self.input_x)
  25.         return  _inputs
  26.     def rnn(self):
  27.         def lstm_cell():
  28.             """lstm核"""
  29.             return tf.nn.rnn_cell.BasicLSTMCell(self.config.hidden_dim,state_is_tuple=True)
  30.         def gru_cell():
  31.             """gru核"""
  32.             return tf.nn.rnn_cell.GRUCell(self.config.hidden_dim)
  33.         def dropout():
  34.             """为每一个rnn后面加一个dropout层"""
  35.             if (self.config.rnn=='lstm'):
  36.                 cell=lstm_cell()
  37.             else:
  38.                 cell=gru_cell()
  39.             return tf.contrib.rnn.DropoutWrapper(cell,
  40.                 output_keep_prob=self.keep_prob)
  41.         embedding_inputs=self.input_embedding()
  42.         with tf.name_scope("rnn"):
  43.             cells=[dropout() for _ in range(self.config.num_layers)]
  44.             """
  45.             定义多层rnn
  46.             """
  47.             rnn_cell=tf.contrib.rnn.MultiRNNCell(cells,state_is_tuple=True)
  48.             _outputs,_=tf.nn.dynamic_rnn(cell=rnn_cell,inputs=embedding_inputs,dtype=tf.float32)
  49.             """
  50.             这里例如 _outputs size is[30,100,128]
  51.             则last的size变为  [30,128]
  52.             """
  53.             last= _outputs[:, -1, :]  # 取最后一个时序输出作为结果
  54.         with tf.name_scope("score"):
  55.             # 全连接层，后面接dropout以及relu激活
  56.             fc=tf.layers.dense(last,self.config.hidden_dim,name="fc1")
  57.             fc = tf.contrib.layers.dropout(fc, self.keep_prob)
  58.             fc = tf.nn.relu(fc)
  59.             #分类器
  60.             self.logits=tf.layers.dense(fc,self.config.num_classes,name="fc2")
  61.             self.pred_y=tf.nn.softmax(self.logits,name="pred_y")
  62.             tf.add_to_collection('pred_network', self.pred_y)
  63.         with tf.name_scope("loss"):
  64.             cross_entropy=tf.nn.softmax_cross_entropy_with_logits(logits=self.logits,labels=self.input_y)
  65.             self.loss=tf.reduce_mean(cross_entropy)
  66.         with tf.name_scope("optimize"):
  67.             # 优化器
  68.             optimizer = tf.train.AdamOptimizer(
  69.                 learning_rate=self.config.learning_rate)
  70.             self.optim = optimizer.minimize(self.loss)
  72.         with tf.name_scope("accuracy"):
  73.             # 准确率
  74.             correct_pred = tf.equal(tf.argmax(self.input_y, 1),
  75.                                     tf.argmax(self.pred_y, 1))
  76.             self.acc = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
  81.     def birnn(self):
  82.         def lstm_cell():
  83.             """lstm核"""
  84.             cell_fw=tf.nn.rnn_cell.BasicLSTMCell(self.config.hidden_dim,state_is_tuple=True)
  85.             cell_bw=tf.nn.rnn_cell.BasicLSTMCell(self.config.hidden_dim,state_is_tuple=True)
  86.             return cell_fw,cell_bw
  87.         def gru_cell():
  88.             """gru核"""
  89.             cell_fw= tf.nn.rnn_cell.GRUCell(self.config.hidden_dim)
  90.             cell_bw = tf.nn.rnn_cell.GRUCell(self.config.hidden_dim)
  91.             return cell_fw,cell_bw
  93.         embedding_inputs=self.input_embedding()
  94.         with tf.name_scope("birnn"):
  95.             if (self.config.rnn == 'lstm'):
  96.                 cell_fw, cell_bw=lstm_cell()
  97.             else:
  98.                 cell_fw, cell_fw=gru_cell()
  99.             outputs, output_states = tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_fw, embedding_inputs, dtype=tf.float32)
  100.             outputs = tf.concat(outputs, 2)  # 取最后一个时序输出作为结果
  101.             last = tf.transpose(outputs, [1, 0, 2])[-1]
  103.         with tf.name_scope("score"):
  104.             # 全连接层，后面接dropout以及relu激活
  105.             fc=tf.layers.dense(last,self.config.hidden_dim,name="fc1",
  106.                                kernel_regularizer=tf.contrib.layers.l2_regularizer(self.config.l2_reg_lambda)
  107.                                 ,bias_regularizer=tf.contrib.layers.l2_regularizer(self.config.l2_reg_lambda))
  108.             fc = tf.contrib.layers.dropout(fc, self.keep_prob)
  109.             fc = tf.nn.relu(fc)
  110.             #分类器
  111.             self.logits=tf.layers.dense(fc,self.config.num_classes,kernel_regularizer=tf.contrib.layers.l2_regularizer(self.config.l2_reg_lambda)
  112.                                 ,bias_regularizer=tf.contrib.layers.l2_regularizer(self.config.l2_reg_lambda),name="fc2")
  113.             self.pred_y=tf.nn.softmax(self.logits,name="pred_y")
  114.             tf.add_to_collection('pred_network', self.pred_y)
  115.         with tf.name_scope("loss"):
  116.             cross_entropy=tf.nn.softmax_cross_entropy_with_logits(logits=self.logits,labels=self.input_y)
  117.             self.loss=tf.reduce_mean(cross_entropy)
  118.         with tf.name_scope("optimize"):
  119.             # 优化器
  120.             optimizer = tf.train.AdamOptimizer(
  121.                 learning_rate=self.config.learning_rate)
  122.             self.optim = optimizer.minimize(self.loss)
  124.         with tf.name_scope("accuracy"):
  125.             # 准确率
  126.             correct_pred = tf.equal(tf.argmax(self.input_y, 1),
  127.                                     tf.argmax(self.pred_y, 1))
  128.             self.acc = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
  130.     def sru(self):
  131.         def sru_cell():
  132.             """lstm核"""
  133.             return SRUCell(self.config.hidden_dim//2)
  135.         def dropout():
  136.             """为每一个rnn后面加一个dropout层"""
  137.             cell=sru_cell()
  138.             return tf.contrib.rnn.DropoutWrapper(cell,
  139.                 output_keep_prob=self.keep_prob)
  140.         embedding_inputs=self.input_embedding()
  141.         with tf.name_scope("rnn"):
  142.             cells=[dropout() for _ in range(self.config.num_layers)]
  143.             """
  144.             定义多层rnn
  145.             """
  146.             rnn_cell=tf.contrib.rnn.MultiRNNCell(cells,state_is_tuple=True)
  147.             _outputs,_=tf.nn.dynamic_rnn(cell=rnn_cell,inputs=embedding_inputs,dtype=tf.float32)
  148.             """
  149.             这里例如 _outputs size is[30,100,128]
  150.             则last的size变为  [30,128]
  151.             """
  152.             last= _outputs[:, -1, :]  # 取最后一个时序输出作为结果
  153.         with tf.name_scope("score"):
  154.             # 全连接层，后面接dropout以及relu激活
  155.             fc=tf.layers.dense(last,self.config.hidden_dim,name="fc1")
  156.             fc = tf.contrib.layers.dropout(fc, self.keep_prob)
  157.             fc = tf.nn.relu(fc)
  158.             #分类器
  159.             self.logits=tf.layers.dense(fc,self.config.num_classes,name="fc2")
  160.             self.pred_y=tf.nn.softmax(self.logits,name="pred_y")
  161.             tf.add_to_collection('pred_network', self.pred_y)
  162.         with tf.name_scope("loss"):
  163.             cross_entropy=tf.nn.softmax_cross_entropy_with_logits(logits=self.logits,labels=self.input_y)
  164.             self.loss=tf.reduce_mean(cross_entropy)
  165.         with tf.name_scope("optimize"):
  166.             # 优化器
  167.             optimizer = tf.train.AdamOptimizer(
  168.                 learning_rate=self.config.learning_rate)
  169.             self.optim = optimizer.minimize(self.loss)
  171.         with tf.name_scope("accuracy"):
  172.             # 准确率
  173.             correct_pred = tf.equal(tf.argmax(self.input_y, 1),
  174.                                     tf.argmax(self.pred_y, 1))
  175.             self.acc = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
  177.     def bisru(self):
  178.         def sru():
  179.             """lstm核"""
  180.             cell_fw=SRUCell(self.config.hidden_dim//2)
  181.             cell_bw=SRUCell(self.config.hidden_dim//2)
  182.             return cell_fw,cell_bw
  184.         embedding_inputs=self.input_embedding()
  185.         with tf.name_scope("birnn"):
  186.             cell_fw, cell_bw=sru()
  187.             outputs, output_states = tf.nn.bidirectional_dynamic_rnn(cell_fw, cell_fw, embedding_inputs, dtype=tf.float32)
  188.             outputs = tf.concat(outputs, 2)  # 取最后一个时序输出作为结果
  189.             last = tf.transpose(outputs, [1, 0, 2])[-1]
  191.         with tf.name_scope("score"):
  192.             # 全连接层，后面接dropout以及relu激活
  193.             fc=tf.layers.dense(last,self.config.hidden_dim,name="fc1",
  194.                                kernel_regularizer=tf.contrib.layers.l2_regularizer(self.config.l2_reg_lambda)
  195.                                 ,bias_regularizer=tf.contrib.layers.l2_regularizer(self.config.l2_reg_lambda))
  196.             fc = tf.contrib.layers.dropout(fc, self.keep_prob)
  197.             fc = tf.nn.relu(fc)
  198.             #分类器
  199.             self.logits=tf.layers.dense(fc,self.config.num_classes,kernel_regularizer=tf.contrib.layers.l2_regularizer(self.config.l2_reg_lambda)
  200.                                 ,bias_regularizer=tf.contrib.layers.l2_regularizer(self.config.l2_reg_lambda),name="fc2")
  201.             self.pred_y=tf.nn.softmax(self.logits,name="pred_y")
  202.             tf.add_to_collection('pred_network', self.pred_y)
  203.         with tf.name_scope("loss"):
  204.             cross_entropy=tf.nn.softmax_cross_entropy_with_logits(logits=self.logits,labels=self.input_y)
  205.             self.loss=tf.reduce_mean(cross_entropy)
  206.         with tf.name_scope("optimize"):
  207.             # 优化器
  208.             optimizer = tf.train.AdamOptimizer(
  209.                 learning_rate=self.config.learning_rate)
  210.             self.optim = optimizer.minimize(self.loss)
  212.         with tf.name_scope("accuracy"):
  213.             # 准确率
  214.             correct_pred = tf.equal(tf.argmax(self.input_y, 1),
  215.                                     tf.argmax(self.pred_y, 1))
  216.             self.acc = tf.reduce_mean(tf.cast(correct_pred, tf.float32))



run sru:

  1. from  rnn_model import TextRnn
  2. from  configuration import TRNNConfig,Path,TCNNConfig
  3. from  data_utils_cut  import preocess_file,batch_iter
  4. import time
  5. import tensorflow as tf
  6. import os
  7. from  datetime  import timedelta
  8. #basepath="/Users/shuubiasahi/Documents/python"
  9. #noexperience
  10. #business
  11. #together
  12. basepath="/home/zhoumeixu"
  13. data_path=basepath+"/credit-tftextclassify/tensorflow/sex/cnn.txt"
  14. vocapath=basepath+"/credit-tftextclassify/tensorflow/sex/vocab.txt"
  15. modelpath=basepath+"/credit-tftextclassify/tensorflow/sex/"
  17. print(modelpath,"开始训练")
  18. def run_epoch(rnn=True):
  19.     # 载入数据
  20.     print('Loading data...')
  21.     start_time = time.time()
  23.     x_train, y_train, words = preocess_file(data_path,
  24.                                             vocapath)
  26.     if rnn:
  27.         print('Using RNN model...')
  28.         config = TRNNConfig()
  29.         config.vocab_size = len(words)
  30.         print("vocab_size is:", config.vocab_size)
  31.         model = TextRnn(config)
  32.         tensorboard_dir = basepath+'/boardlog'
  35.     end_time = time.time()
  36.     time_dif = end_time - start_time
  37.     time_dif = timedelta(seconds=int(round(time_dif)))
  38.     print('Time usage:', time_dif)
  40.     print('Constructing TensorFlow Graph...')
  41.     session = tf.Session()
  42.     session.run(tf.global_variables_initializer())
  43.     saver = tf.train.Saver()
  45.     # 配置 tensorboard
  46.     tf.summary.scalar("loss", model.loss)
  47.     tf.summary.scalar("accuracy", model.acc)
  49.     if not os.path.exists(tensorboard_dir):
  50.         os.makedirs(tensorboard_dir)
  52.     merged_summary = tf.summary.merge_all()
  53.     writer = tf.summary.FileWriter(tensorboard_dir)
  54.     writer.add_graph(session.graph)
  56.     # 生成批次数据
  57.     print('Generating batch...')
  58.     batch_train = batch_iter(list(zip(x_train, y_train)),
  59.         config.batch_size, config.num_epochs)
  61.     def feed_data(batch):
  62.         """准备需要喂入模型的数据"""
  63.         x_batch, y_batch = zip(*batch)
  64.         feed_dict = {
  65.             model.input_x: x_batch,
  66.             model.input_y: y_batch
  67.         }
  68.         return feed_dict, len(x_batch)
  70.     def evaluate(x_, y_):
  71.         """
  72.         模型评估
  73.         一次运行所有的数据会OOM，所以需要分批和汇总
  74.         """
  75.         batch_eval = batch_iter(list(zip(x_, y_)), 128, 1)
  77.         total_loss = 0.0
  78.         total_acc = 0.0
  79.         cnt = 0
  80.         for batch in batch_eval:
  81.             feed_dict, cur_batch_len = feed_data(batch)
  82.             feed_dict[model.keep_prob] = 1.0
  83.             loss, acc = session.run([model.loss, model.acc],
  84.                 feed_dict=feed_dict)
  85.             total_loss += loss * cur_batch_len
  86.             total_acc += acc * cur_batch_len
  87.             cnt += cur_batch_len
  89.         return total_loss / cnt, total_acc / cnt
  91.     # 训练与验证
  92.     print('Training and evaluating...')
  93.     start_time = time.time()
  94.     print_per_batch = config.print_per_batch
  95.     for i, batch in enumerate(batch_train):
  96.         feed_dict, _ = feed_data(batch)
  97.         feed_dict[model.keep_prob] = config.dropout_keep_prob
  99.         if i % 5 == 0:  # 每5次将训练结果写入tensorboard scalar
  100.             s = session.run(merged_summary, feed_dict=feed_dict)
  101.             writer.add_summary(s, i)
  103.         if i % print_per_batch == print_per_batch - 1:  # 每200次输出在训练集和验证集上的性能
  104.             loss_train, acc_train = session.run([model.loss, model.acc],
  105.                 feed_dict=feed_dict)
  106.             #loss, acc = evaluate(x_val, y_val)   验证机暂时不需要
  108.             # 时间
  109.             end_time = time.time()
  110.             time_dif = end_time - start_time
  111.             time_dif = timedelta(seconds=int(round(time_dif)))
  113.             msg = 'Iter: {0:>6}, Train Loss: {1:>6.2}, Train Acc: {2:>7.2%},'\
  114.                 + '  Time: {3}'
  115.             print(msg.format(i + 1, loss_train, acc_train, time_dif))
  116.         # if  i%10==0 and i>0:
  117.         #     graph=tf.graph_util.convert_variables_to_constants(session,session.graph_def,["keep_prob","input_x","score/pred_y"])
  118.         #     tf.train.write_graph(graph,".","/Users/shuubiasahi/Desktop/tensorflow/modelsavegraph/graph.db",as_text=False)
  119.         if i%500==0  and i>0:
  120.             graph = tf.graph_util.convert_variables_to_constants(session, session.graph_def,
  121.                                                                  ["keep_prob", "input_x", "score/pred_y"])
  122.             if rnn:
  123.                 tf.train.write_graph(graph, ".", modelpath+"graphrnn.model",
  124.                                      as_text=False)
  125.                 print("rnn模型在第{0}步已经保存".format(i))
  126.             else:
  127.                 tf.train.write_graph(graph, ".", modelpath+"graph.model",
  128.                                  as_text=False)
  129.                 print("cnn模型在第{0}步已经保存".format(i))
  130.         session.run(model.optim, feed_dict=feed_dict)  # 运行优化
  132.     # 最后在测试集上进行评估
  134.     session.close()
  136. if __name__ == '__main__':
  137.     #run_epoch(rnn=True)
  139.      run_epoch(rnn=True)

结果：

  1. Iter:    100, Train Loss:   0.33, Train Acc:  84.38%,  Time: 0:00:46
  2. Iter:    200, Train Loss:   0.24, Train Acc:  91.41%,  Time: 0:01:33
  3. Iter:    300, Train Loss:   0.22, Train Acc:  90.62%,  Time: 0:02:19
  4. Iter:    400, Train Loss:   0.28, Train Acc:  92.19%,  Time: 0:03:04
  5. Iter:    500, Train Loss:   0.11, Train Acc:  95.31%,  Time: 0:03:49
  6. Converted 11 variables to const ops.
  7. rnn模型在第500步已经保存
  8. Iter:    600, Train Loss:   0.12, Train Acc:  94.53%,  Time: 0:04:35
  9. Iter:    700, Train Loss:  0.086, Train Acc:  97.66%,  Time: 0:05:21
  10. Iter:    800, Train Loss:   0.21, Train Acc:  93.75%,  Time: 0:06:07
  11. Iter:    900, Train Loss:   0.12, Train Acc:  96.09%,  Time: 0:06:52
  12. Iter:   1000, Train Loss:  0.086, Train Acc:  97.66%,  Time: 0:07:38
训练过程中并没有发现在论文提到的快很多倍，不知道是什么原因，还是我写错了，求各位大神不吝赐教。
阅读全文
0 0
简明python教程豆瓣 简明python教程豆瓣
原创粉丝点击
热门IT博客
上海人工智能政策上海人工智能政策
淘宝其他来源是什么淘宝其他来源是什么
淘宝助理上传速度慢
传智播客mybatis源码
网络不行打什么电话
wpf 实时刷新数据
淘宝订单自己消失
tpo模考软件mac版下载
工资标准算法
淘宝网有线麦克风话筒
淘宝专业敲诈三无产品
centos6.5 yum网络配置
找工作的网站 知乎
百电通电话软件二维码
电脑双肩包推荐 知乎
商业源码网
php探针 安全
蓝光软件下载
sql server设置外键
mysql有可视化界面吗
热门问题 老师的惩罚 人脸识别 我在镇武司摸鱼那些年 重生之率土为王 我在大康的咸鱼生活 盘龙之生命进化 天生仙种 凡人之先天五行 春回大明朝 姑娘不必设防，我是瞎子 尿尿滴白 尿滴白的原因 尿滴白什么意思 尿前滴白怎么回事 尿滴白什么病 尿尿滴白怎么回事 小便后滴白怎么回事 男性尿滴白严重吗 滴白怎么回事 尿道口滴白怎么回事 尿滴白挂什么科 尿滴白会自己好吗 尿滴白的危害有哪些 小便有滴白 滴白是什么意思 尿道滴白怎么办 为什么会滴白 小便尿滴白 尿道滴白怎么回事 尿口滴白 女性尿滴白是怎么回事 大便滴白是怎么回事 尿中滴白 拉屎滴白 尿后有滴白 晨尿滴白 尿滴白照片 尿滴白怎么调理 尿滴白是怎么引起的 什么叫尿滴白 阴囊潮湿尿滴白 男人尿滴白 小便偶尔滴白 男性尿滴白是怎么回事 大便后滴白 男性滴白 小孩尿滴白 女性尿后滴白 大便尿道滴白 尿分叉尿滴白 尿滴白的危害

程序博客网，程序员的互联网技术博客家园。csdn论坛精品 msdn技术资料都在这里