学习笔记：SAE TensorFlow代码

栈式自编码：

一．基本原理

AE的原理是先通过一个encode层对输入进行编码，这个编码就是特征，然后利用encode乘第2层参数（也可以是encode层的参数的转置乘特征并加偏执），重构（解码）输入，然后用重构的输入和实际输入的损失训练参数。

对于我的应用来说，我要做的首先是抽取特征。AE的encode的过程很简单，是这样的：

SAE是这样的：

 

训练过程中，两个SAE分别训练，第一个SAE训练完之后，其encode的输出作为第二个SAE的输入，接着训练。训练完后，用第二层的特征，通过softmax分类器（由于分类器 还是得要带标签的数据来训练，所以，实际应用总，提取特征后不一定是拿来分类），将所有特征分为n类，得到n类标签。训练网络图

          step 1

              step 2

                      step 3    

 按照UFLDL的说明，在各自训练到快要收敛的时候，要对整个网络通过反向传播做微调，但是不能从一开始就整个网络调节。

两个SAE训练完后，每个encode的输出作为两个隐层的特征，然后重新构建网络，新网络不做训练，只做预测。网络如下：

 

from __future__ import division, print_function, absolute_importimport tensorflow as tfimport numpy as npimport matplotlib.pyplot as plt# 下载并导入MNIST数据集from tensorflow.examples.tutorials.mnist import input_datamnist = input_data.read_data_sets("MNIST_data", one_hot=True)# 参数learning_rate = 0.01#学习率training_epochs = 20#训练的周期batch_size = 256#每一批次训练的大小display_step = 1# 神经网络的参数n_hidden_1 = 256 # 隐层1的神经元个数n_hidden_2 = 100 # 隐层2神经元个数n_input = 784 # MNIST数据集的输出(img shape: 28*28)n_output=10# tf Graph input (only pictures)X = tf.placeholder("float", [None, n_input])weights = {    'encoder_h1': tf.Variable(tf.random_normal([n_input, n_hidden_1])),    'encoder_h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2])),    'decoder_h1': tf.Variable(tf.random_normal([n_hidden_1, n_input])),    'decoder_h2': tf.Variable(tf.random_normal([n_hidden_2, n_hidden_1])),    'softmax_w': tf.Variable(tf.random_normal([n_hidden_2, n_output])),}biases = {    'encoder_b1': tf.Variable(tf.random_normal([n_hidden_1])),    'encoder_b2': tf.Variable(tf.random_normal([n_hidden_2])),    'decoder_b1': tf.Variable(tf.random_normal([n_input])),    'decoder_b2': tf.Variable(tf.random_normal([n_hidden_1])),    'softmax_b': tf.Variable(tf.random_normal([n_output])),}#************************* 1st hidden layer **************X = tf.placeholder("float", [None, n_input])h1_out =tf.nn.sigmoid(tf.add(tf.matmul(X, weights['encoder_h1']),                                   biases['encoder_b1']))keep_prob = tf.placeholder("float")h1_out_drop = tf.nn.dropout(h1_out,keep_prob)X_1 = tf.nn.sigmoid(tf.matmul(h1_out_drop,weights['decoder_h1'])+biases['decoder_b1'])loss1 = tf.reduce_mean(tf.pow(X - X_1, 2))train_step_1 = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss1)sess=tf.Session()sess.run(tf.variables_initializer([weights['encoder_h1'],biases['encoder_b1'],                                  weights['decoder_h1'],biases['decoder_b1']]))# trainingfor i in range(training_epochs):    batch_x,batch_y =  mnist.train.next_batch(batch_size)    _,c=sess.run([train_step_1,loss1],feed_dict={X:batch_x, keep_prob:1.0})    if i%5==0:        print(c)#h1_out = tf.nn.sigmoid(tf.add(tf.matmul(X, weights['encoder_h1']),#                                  biases['encoder_b1']))#get result of 1st layer as well as the input of next layer#************************** 2nd hidden layer *************    h2_x = tf.placeholder("float", shape = [None, n_hidden_1])h2_out = tf.nn.sigmoid(tf.matmul(h2_x,weights['encoder_h2']) + biases['encoder_b2'])h2_out_drop = tf.nn.dropout(h2_out,keep_prob)h2_in_decode = tf.nn.sigmoid(tf.matmul(h2_out_drop, weights['decoder_h2']) +  biases['decoder_b2'])loss2 = tf.reduce_mean(tf.pow( h2_x- h2_in_decode, 2))train_step_2 = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss2)sess.run(tf.variables_initializer([weights['encoder_h2'],biases['encoder_b2'],                                   weights['decoder_h2'],biases['decoder_b2']]))for i in range(training_epochs):        #batch_x = numpy.reshape(batch_x,[batch_size,sample_length])    h1_out=[]    batch_x,batch_y =  mnist.train.next_batch(batch_size)    temp=tf.nn.sigmoid(tf.add(tf.matmul(batch_x, weights['encoder_h1']),                                   biases['encoder_b1']))    h1_out.extend(sess.run(temp))        _,c=sess.run([train_step_2,loss2],feed_dict={h2_x:h1_out,keep_prob:1.0})    if i%5==0:        print(c)#h2_out = tf.nn.sigmoid(tf.matmul(h1_out,weights['decoder_h2']) + biases['decoder_b2'])#get result of 2nd layer as well as the input of next layer#************************** softmax layer ****************y_ = tf.placeholder("float", shape = [None, n_output])soft_x = tf.placeholder("float", shape = [None, n_hidden_2])y_out = tf.nn.softmax(tf.matmul(soft_x, weights['softmax_w']) + biases['softmax_b'])cross_entropy = -tf.reduce_sum(y_ * tf.log(y_out))train_step_soft = tf.train.GradientDescentOptimizer(learning_rate).minimize(cross_entropy)correct_prediction = tf.equal(tf.argmax(y_out, 1), tf.argmax(y_, 1))accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))sess.run(tf.variables_initializer([weights['softmax_w'],biases['softmax_b']]))for i in range(training_epochs):    h2_out=[]    batch_x,batch_y =  mnist.train.next_batch(batch_size)    for i in range(batch_size):        temp=tf.nn.sigmoid(tf.add(tf.matmul(batch_x[i].reshape([1,784]), weights['encoder_h1']),                                       biases['encoder_b1']))        temp=tf.nn.sigmoid(tf.add(tf.matmul(temp, weights['encoder_h2']),                                       biases['encoder_b2']))        h2_out.extend(sess.run(temp))    sess.run(train_step_soft,feed_dict={soft_x:h2_out, y_:batch_y, keep_prob:1.0})    if i%5 == 0:        print(sess.run(accuracy, feed_dict={soft_x:h2_out, y_:batch_y, keep_prob:1.0}))#fine-tuningprint ("************************* fine-tuning ***************************")h1_out = tf.nn.sigmoid(tf.add(tf.matmul(X_1, weights['eecoder_h1']),                                   biases['encoder_b1']))h2_out = tf.nn.sigmoid(tf.matmul(h1_out,weights['encoder_h2']) + biases['encoder_b2'])        h2_out_drop = tf.nn.dropout(h2_out,keep_prob)y_out = tf.nn.softmax(tf.matmul(h2_out_drop, weights['softmax_w']) + biases['softmax_b'])cross_entropy = -tf.reduce_sum(y_ * tf.log(y_out))train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(cross_entropy)correct_prediction = tf.equal(tf.argmax(y_out, 1), tf.argmax(y_, 1))accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))for i in range(training_epochs):    batch_x,batch_y = mnist.train.next_batch(batch_size)    #batch_x = numpy.reshape(batch_x,[batch_size,sample_length])    sess.run(train_step,feed_dict={X:batch_x, Y:batch_y, keep_prob:0.5})    if i%5 == 0:        print (sess.run(accuracy,feed_dict={X:batch_x, Y:batch_y, keep_prob:0.5}))print (sess.run(accuracy,feed_dict={X:batch_x, Y:batch_y, keep_prob:0.5}))