TensorFlow：Chap5笔记总结（MNIST数字识别）

       MNIST是一个非常有名的手写体数字识别数据集，是NIST数据集的一个子集，它包含了60000张图片作为训练数据，10000张图片作为测试数据。在MNIST数据集中，每一张图片都代表了0~9中的一个数字。图片的大小都为28*28，且数字都会出现在图片正中间。

1.读取MNIST数据集

import tensorflow as tf#1.读取数据集，第一次TensorFlow会自动下载数据集到线面的路径中from tensorflow.examples.tutorials.mnist import input_datamnist = input_data.read_data_sets("/tmp/data/",one_hot=True)#2.数据集会自动被分成3个子集，train、validation和test。以下代码显示数据集的大小print("Training data size: ",mnist.train.num_examples)print("Validating data size: ",mnist.validation.num_examples)print("Testing data size: ",mnist.test.num_examples)#3.查看training数据集中某个成员的像素矩阵生成的一维数组和其属于的数字标签print("Example traning data: ",mnist.train.images[0])#print("Example training data label: ",mnist.train.lables[0])#4.使用mnist,train,next_batch来实现随机梯度下降batch_size = 100xs, ys = mnist.train.next_batch(batch_size)   #从train的集合中选取batch_size个训练数据print("X shape:", xs.shape)print("Y shape:", ys.shape)

2.使用三层全连接神经网络在TensorFlow下实现MNIST数据集

    该程序开始时设置了初始学习率、学习率衰减率、隐藏层节点数量、迭代轮数等不同的参数

import tensorflow as tffrom tensorflow.examples.tutorials.mnist import input_data# 1.设置输入和输出节点的个数,配置神经网络的参数INPUT_NODE = 784  # 输入节点,即图片像素OUTPUT_NODE = 10  # 输出节点，即类别数目。在MNIST数据集中需要区分的是0~9这10个数字，所以这里输出层节点数为10LAYER1_NODE = 500  # 隐藏层数.使用只有一个隐藏层的网络结构，这个隐藏层有500个节点BATCH_SIZE = 100  # 每次batch打包的样本个数。                  # 一个训练batch中的训练数据个数。数字越小，训练过程越接近随机梯度下降；数字越大，训练越接近梯度下降。# 模型相关的参数LEARNING_RATE_BASE = 0.8     # 基础的学习率LEARNING_RATE_DECAY = 0.99   # 学习率的衰减率REGULARAZTION_RATE = 0.0001  # 描述模型复杂度的正则化项在损失函数中的系数TRAINING_STEPS = 5000        # 训练轮数MOVING_AVERAGE_DECAY = 0.99  # 滑动平均衰减率# 2. 定义辅助函数来计算前向传播结果，使用ReLU做为激活函数。def inference(input_tensor, avg_class, weights1, biases1, weights2, biases2):    # 不使用滑动平均类    if avg_class == None:        layer1 = tf.nn.relu(tf.matmul(input_tensor, weights1) + biases1)        return tf.matmul(layer1, weights2) + biases2    else:        # 使用滑动平均类        layer1 = tf.nn.relu(tf.matmul(input_tensor, avg_class.average(weights1)) + avg_class.average(biases1))        return tf.matmul(layer1, avg_class.average(weights2)) + avg_class.average(biases2)# 3. 定义训练过程def train(mnist):    x = tf.placeholder(tf.float32, [None, INPUT_NODE], name='x-input')    y_ = tf.placeholder(tf.float32, [None, OUTPUT_NODE], name='y-input')    # 生成隐藏层的参数。    weights1 = tf.Variable(tf.truncated_normal([INPUT_NODE, LAYER1_NODE], stddev=0.1))    biases1 = tf.Variable(tf.constant(0.1, shape=[LAYER1_NODE]))    # 生成输出层的参数。    weights2 = tf.Variable(tf.truncated_normal([LAYER1_NODE, OUTPUT_NODE], stddev=0.1))    biases2 = tf.Variable(tf.constant(0.1, shape=[OUTPUT_NODE]))    # 计算不含滑动平均类的前向传播结果    y = inference(x, None, weights1, biases1, weights2, biases2)    # 定义训练轮数及相关的滑动平均类    global_step = tf.Variable(0, trainable=False)    # 不需要计算滑动平均值，所以这里指定这个变量为不可训练的变量(trainable=False)    #  给定训练轮数的变量可以加快训练早期变量的更新速度    variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)    variables_averages_op = variable_averages.apply(tf.trainable_variables())    # 在所有代表神经网络参数的变量上使用滑动平均    average_y = inference(x, variable_averages, weights1, biases1, weights2, biases2) # 计算使用了滑动平均之后的前向传播结果    # 计算交叉熵及其平均值    cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))    cross_entropy_mean = tf.reduce_mean(cross_entropy) # 计算在当前batch中所有样例的交叉熵平均值    # 损失函数的计算    regularizer = tf.contrib.layers.l2_regularizer(REGULARAZTION_RATE)    # L2正则化损失函数    regularaztion = regularizer(weights1) + regularizer(weights2)    # 计算模型的正则化损失    loss = cross_entropy_mean + regularaztion    # 总损失等于交叉熵损失和正则化损失的和    # 设置指数衰减的学习率。    learning_rate = tf.train.exponential_decay(        LEARNING_RATE_BASE,   # 基础的学习率        global_step,     # 当前迭代的轮数        mnist.train.num_examples / BATCH_SIZE,    # 过完所有的训练数据需要的迭代次数        LEARNING_RATE_DECAY,     # 学习率衰减速度        staircase=True)    # 优化损失函数    train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)    # 反向传播更新参数和更新每一个参数的滑动平均值    with tf.control_dependencies([train_step, variables_averages_op]):        train_op = tf.no_op(name='train')    # 计算正确率    correct_prediction = tf.equal(tf.argmax(average_y, 1), tf.argmax(y_, 1))    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))    # 初始化回话并开始训练过程。    with tf.Session() as sess:        tf.global_variables_initializer().run()        validate_feed = {x: mnist.validation.images, y_: mnist.validation.labels}    # 准备验证数据，来大致判断停止的条件和评判训练的效果        test_feed = {x: mnist.test.images, y_: mnist.test.labels}     # 准备测试数据。作为模型优劣的最后评价标准        # 循环的训练神经网络。        for i in range(TRAINING_STEPS):            if i % 1000 == 0:    # 每1000轮输出一次在验证数据集上的测试结果。                validate_acc = sess.run(accuracy, feed_dict=validate_feed)    # 计算滑动平均模型在验证数据上的结果                print("After %d training step(s), validation accuracy using average model is %g " % (i, validate_acc))            xs, ys = mnist.train.next_batch(BATCH_SIZE)    # 产生这一轮使用的一个batch的训练数据，并运行训练过程            sess.run(train_op, feed_dict={x: xs, y_: ys})        # 在训练结束之后，在测试数据上检测神经网络模型的最终正确率        test_acc = sess.run(accuracy, feed_dict=test_feed)        print(("After %d training step(s), test accuracy using average model is %g" % (TRAINING_STEPS, test_acc)))# 4. 主程序入口，这里设定模型训练次数为(5000)30000次。def main(argv=None):    mnist = input_data.read_data_sets("/tmp/MNIST_data", one_hot=True)    train(mnist)if __name__=='__main__':    main()

3.不使用正则化实现MNIST数据集

import tensorflow as tffrom tensorflow.examples.tutorials.mnist import input_data# 1.设置输入和输出节点的个数,配置神经网络的参数。INPUT_NODE = 784  # 输入节点OUTPUT_NODE = 10  # 输出节点LAYER1_NODE = 500  # 隐藏层数BATCH_SIZE = 100  # 每次batch打包的样本个数# 模型相关的参数LEARNING_RATE_BASE = 0.8LEARNING_RATE_DECAY = 0.99TRAINING_STEPS = 5000MOVING_AVERAGE_DECAY = 0.99# 2. 定义辅助函数来计算前向传播结果，使用ReLU做为激活函数def inference(input_tensor, avg_class, weights1, biases1, weights2, biases2):    # 不使用滑动平均类    if avg_class == None:        layer1 = tf.nn.relu(tf.matmul(input_tensor, weights1) + biases1)        return tf.matmul(layer1, weights2) + biases2    else:        # 使用滑动平均类        layer1 = tf.nn.relu(tf.matmul(input_tensor, avg_class.average(weights1)) + avg_class.average(biases1))        return tf.matmul(layer1, avg_class.average(weights2)) + avg_class.average(biases2)# 3. 定义训练过程。def train(mnist):    x = tf.placeholder(tf.float32, [None, INPUT_NODE], name='x-input')    y_ = tf.placeholder(tf.float32, [None, OUTPUT_NODE], name='y-input')    # 生成隐藏层的参数。    weights1 = tf.Variable(tf.truncated_normal([INPUT_NODE, LAYER1_NODE], stddev=0.1))    biases1 = tf.Variable(tf.constant(0.1, shape=[LAYER1_NODE]))    # 生成输出层的参数。    weights2 = tf.Variable(tf.truncated_normal([LAYER1_NODE, OUTPUT_NODE], stddev=0.1))    biases2 = tf.Variable(tf.constant(0.1, shape=[OUTPUT_NODE]))    # 计算不含滑动平均类的前向传播结果    y = inference(x, None, weights1, biases1, weights2, biases2)    # 定义训练轮数及相关的滑动平均类    global_step = tf.Variable(0, trainable=False)    variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)    variables_averages_op = variable_averages.apply(tf.trainable_variables())    average_y = inference(x, variable_averages, weights1, biases1, weights2, biases2)    # 计算交叉熵及其平均值    cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))    cross_entropy_mean = tf.reduce_mean(cross_entropy)    # 损失函数的计算    loss = cross_entropy_mean    # 设置指数衰减的学习率。    learning_rate = tf.train.exponential_decay(        LEARNING_RATE_BASE,        global_step,        mnist.train.num_examples / BATCH_SIZE,        LEARNING_RATE_DECAY,        staircase=True)    # 优化损失函数    train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)    # 反向传播更新参数和更新每一个参数的滑动平均值    with tf.control_dependencies([train_step, variables_averages_op]):        train_op = tf.no_op(name='train')    # 计算正确率    correct_prediction = tf.equal(tf.argmax(average_y, 1), tf.argmax(y_, 1))    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))    # 初始化回话并开始训练过程。    with tf.Session() as sess:        tf.global_variables_initializer().run()        validate_feed = {x: mnist.validation.images, y_: mnist.validation.labels}        test_feed = {x: mnist.test.images, y_: mnist.test.labels}        # 循环的训练神经网络。        for i in range(TRAINING_STEPS):            if i % 1000 == 0:                validate_acc = sess.run(accuracy, feed_dict=validate_feed)                print("After %d training step(s), validation accuracy using average model is %g " % (i, validate_acc))            xs, ys = mnist.train.next_batch(BATCH_SIZE)            sess.run(train_op, feed_dict={x: xs, y_: ys})        test_acc = sess.run(accuracy, feed_dict=test_feed)        print(("After %d training step(s), test accuracy using average model is %g" % (TRAINING_STEPS, test_acc)))# 4. 主程序入口，这里设定模型训练次数为5000次。def main(argv=None):    mnist = input_data.read_data_sets("/tmp/MNIST_data", one_hot=True)    train(mnist)if __name__=='__main__':    main()

4.不使用指数衰减学习率

import tensorflow as tffrom tensorflow.examples.tutorials.mnist import input_data# 1.设置输入和输出节点的个数,配置神经网络的参数INPUT_NODE = 784  # 输入节点OUTPUT_NODE = 10  # 输出节点LAYER1_NODE = 500  # 隐藏层数BATCH_SIZE = 100  # 每次batch打包的样本个数# 模型相关的参数LEARNING_RATE = 0.1REGULARAZTION_RATE = 0.0001TRAINING_STEPS = 5000MOVING_AVERAGE_DECAY = 0.99# 2. 定义辅助函数来计算前向传播结果，使用ReLU做为激活函数def inference(input_tensor, avg_class, weights1, biases1, weights2, biases2):    # 不使用滑动平均类    if avg_class == None:        layer1 = tf.nn.relu(tf.matmul(input_tensor, weights1) + biases1)        return tf.matmul(layer1, weights2) + biases2    else:        # 使用滑动平均类        layer1 = tf.nn.relu(tf.matmul(input_tensor, avg_class.average(weights1)) + avg_class.average(biases1))        return tf.matmul(layer1, avg_class.average(weights2)) + avg_class.average(biases2)# 3. 定义训练过程。def train(mnist):    x = tf.placeholder(tf.float32, [None, INPUT_NODE], name='x-input')    y_ = tf.placeholder(tf.float32, [None, OUTPUT_NODE], name='y-input')    # 生成隐藏层的参数。    weights1 = tf.Variable(tf.truncated_normal([INPUT_NODE, LAYER1_NODE], stddev=0.1))    biases1 = tf.Variable(tf.constant(0.1, shape=[LAYER1_NODE]))    # 生成输出层的参数。    weights2 = tf.Variable(tf.truncated_normal([LAYER1_NODE, OUTPUT_NODE], stddev=0.1))    biases2 = tf.Variable(tf.constant(0.1, shape=[OUTPUT_NODE]))    # 计算不含滑动平均类的前向传播结果    y = inference(x, None, weights1, biases1, weights2, biases2)    # 定义训练轮数及相关的滑动平均类    global_step = tf.Variable(0, trainable=False)    variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)    variables_averages_op = variable_averages.apply(tf.trainable_variables())    average_y = inference(x, variable_averages, weights1, biases1, weights2, biases2)    # 计算交叉熵及其平均值    cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))    cross_entropy_mean = tf.reduce_mean(cross_entropy)    # 损失函数的计算    regularizer = tf.contrib.layers.l2_regularizer(REGULARAZTION_RATE)    regularaztion = regularizer(weights1) + regularizer(weights2)    loss = cross_entropy_mean + regularaztion    # 优化损失函数    train_step = tf.train.GradientDescentOptimizer(LEARNING_RATE).minimize(loss, global_step=global_step)    # 反向传播更新参数和更新每一个参数的滑动平均值    with tf.control_dependencies([train_step, variables_averages_op]):        train_op = tf.no_op(name='train')    # 计算正确率    correct_prediction = tf.equal(tf.argmax(average_y, 1), tf.argmax(y_, 1))    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))    # 初始化回话并开始训练过程。    with tf.Session() as sess:        tf.global_variables_initializer().run()        validate_feed = {x: mnist.validation.images, y_: mnist.validation.labels}        test_feed = {x: mnist.test.images, y_: mnist.test.labels}        # 循环的训练神经网络。        for i in range(TRAINING_STEPS):            if i % 1000 == 0:                validate_acc = sess.run(accuracy, feed_dict=validate_feed)                print("After %d training step(s), validation accuracy using average model is %g " % (i, validate_acc))            xs, ys = mnist.train.next_batch(BATCH_SIZE)            sess.run(train_op, feed_dict={x: xs, y_: ys})        test_acc = sess.run(accuracy, feed_dict=test_feed)        print(("After %d training step(s), test accuracy using average model is %g" % (TRAINING_STEPS, test_acc)))# 4. 主程序入口，这里设定模型训练次数为5000次。def main(argv=None):    mnist = input_data.read_data_sets("/tmp/MNIST_data", one_hot=True)    train(mnist)if __name__=='__main__':    main()

5，不使用滑动平均模型

import tensorflow as tffrom tensorflow.examples.tutorials.mnist import input_data# 1.设置输入和输出节点的个数,配置神经网络的参数INPUT_NODE = 784  # 输入节点OUTPUT_NODE = 10  # 输出节点LAYER1_NODE = 500  # 隐藏层数BATCH_SIZE = 100  # 每次batch打包的样本个数# 模型相关的参数LEARNING_RATE_BASE = 0.8LEARNING_RATE_DECAY = 0.99REGULARAZTION_RATE = 0.0001TRAINING_STEPS = 5000# 2. 定义辅助函数来计算前向传播结果，使用ReLU做为激活函数def inference(input_tensor, avg_class, weights1, biases1, weights2, biases2):    # 不使用滑动平均类    if avg_class == None:        layer1 = tf.nn.relu(tf.matmul(input_tensor, weights1) + biases1)        return tf.matmul(layer1, weights2) + biases2    else:        # 使用滑动平均类        layer1 = tf.nn.relu(tf.matmul(input_tensor, avg_class.average(weights1)) + avg_class.average(biases1))        return tf.matmul(layer1, avg_class.average(weights2)) + avg_class.average(biases2)# 3. 定义训练过程def train(mnist):    x = tf.placeholder(tf.float32, [None, INPUT_NODE], name='x-input')    y_ = tf.placeholder(tf.float32, [None, OUTPUT_NODE], name='y-input')    # 生成隐藏层的参数。    weights1 = tf.Variable(tf.truncated_normal([INPUT_NODE, LAYER1_NODE], stddev=0.1))    biases1 = tf.Variable(tf.constant(0.1, shape=[LAYER1_NODE]))    # 生成输出层的参数。    weights2 = tf.Variable(tf.truncated_normal([LAYER1_NODE, OUTPUT_NODE], stddev=0.1))    biases2 = tf.Variable(tf.constant(0.1, shape=[OUTPUT_NODE]))    # 计算不含滑动平均类的前向传播结果    y = inference(x, None, weights1, biases1, weights2, biases2)    # 定义训练轮数及相关的滑动平均类    global_step = tf.Variable(0, trainable=False)    # 计算交叉熵及其平均值    cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))    cross_entropy_mean = tf.reduce_mean(cross_entropy)    # 损失函数的计算    regularizer = tf.contrib.layers.l2_regularizer(REGULARAZTION_RATE)    regularaztion = regularizer(weights1) + regularizer(weights2)    loss = cross_entropy_mean + regularaztion    # 设置指数衰减的学习率。    learning_rate = tf.train.exponential_decay(        LEARNING_RATE_BASE,        global_step,        mnist.train.num_examples / BATCH_SIZE,        LEARNING_RATE_DECAY,        staircase=True)    # 优化损失函数    train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)    # 反向传播更新参数    with tf.control_dependencies([train_step]):        train_op = tf.no_op(name='train')    # 计算正确率    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))    # 初始化回话并开始训练过程。    with tf.Session() as sess:        tf.global_variables_initializer().run()        validate_feed = {x: mnist.validation.images, y_: mnist.validation.labels}        test_feed = {x: mnist.test.images, y_: mnist.test.labels}        # 循环的训练神经网络。        for i in range(TRAINING_STEPS):            if i % 1000 == 0:                validate_acc = sess.run(accuracy, feed_dict=validate_feed)                print("After %d training step(s), validation accuracy using average model is %g " % (i, validate_acc))            xs, ys = mnist.train.next_batch(BATCH_SIZE)            sess.run(train_op, feed_dict={x: xs, y_: ys})        test_acc = sess.run(accuracy, feed_dict=test_feed)        print(("After %d training step(s), test accuracy using average model is %g" % (TRAINING_STEPS, test_acc)))# 4. 主程序入口，这里设定模型训练次数为5000次def main(argv=None):    mnist = input_data.read_data_sets("/tmp/MNIST_data", one_hot=True)    train(mnist)if __name__=='__main__':    main()

6.不使用隐藏层

import tensorflow as tffrom tensorflow.examples.tutorials.mnist import input_data# 1.设置输入和输出节点的个数,配置神经网络的参数INPUT_NODE = 784     # 输入节点OUTPUT_NODE = 10     # 输出节点BATCH_SIZE = 100     # 每次batch打包的样本个数# 模型相关的参数LEARNING_RATE_BASE = 0.8LEARNING_RATE_DECAY = 0.99REGULARAZTION_RATE = 0.0001TRAINING_STEPS = 5000MOVING_AVERAGE_DECAY = 0.99# 2. 定义辅助函数来计算前向传播结果，使用ReLU做为激活函数def inference(input_tensor, avg_class, weights1, biases1):    # 不使用滑动平均类    if avg_class == None:        layer1 = tf.nn.relu(tf.matmul(input_tensor, weights1) + biases1)        return layer1    else:        # 使用滑动平均类        layer1 = tf.nn.relu(tf.matmul(input_tensor, avg_class.average(weights1)) + avg_class.average(biases1))        return layer1# 3. 定义训练过程def train(mnist):    x = tf.placeholder(tf.float32, [None, INPUT_NODE], name='x-input')    y_ = tf.placeholder(tf.float32, [None, OUTPUT_NODE], name='y-input')    # 生成输出层的参数。    weights1 = tf.Variable(tf.truncated_normal([INPUT_NODE, OUTPUT_NODE], stddev=0.1))    biases1 = tf.Variable(tf.constant(0.1, shape=[OUTPUT_NODE]))    # 计算不含滑动平均类的前向传播结果    y = inference(x, None, weights1, biases1)    # 定义训练轮数及相关的滑动平均类    global_step = tf.Variable(0, trainable=False)    variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)    variables_averages_op = variable_averages.apply(tf.trainable_variables())    average_y = inference(x, variable_averages, weights1, biases1)    # 计算交叉熵及其平均值    cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))    cross_entropy_mean = tf.reduce_mean(cross_entropy)    # 损失函数的计算    regularizer = tf.contrib.layers.l2_regularizer(REGULARAZTION_RATE)    regularaztion = regularizer(weights1)    loss = cross_entropy_mean + regularaztion    # 设置指数衰减的学习率。    learning_rate = tf.train.exponential_decay(        LEARNING_RATE_BASE,        global_step,        mnist.train.num_examples / BATCH_SIZE,        LEARNING_RATE_DECAY,        staircase=True)    # 优化损失函数    train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)    # 反向传播更新参数和更新每一个参数的滑动平均值    with tf.control_dependencies([train_step, variables_averages_op]):        train_op = tf.no_op(name='train')    # 计算正确率    correct_prediction = tf.equal(tf.argmax(average_y, 1), tf.argmax(y_, 1))    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))    # 初始化回话并开始训练过程。    with tf.Session() as sess:        tf.global_variables_initializer().run()        validate_feed = {x: mnist.validation.images, y_: mnist.validation.labels}        test_feed = {x: mnist.test.images, y_: mnist.test.labels}        # 循环的训练神经网络。        for i in range(TRAINING_STEPS):            if i % 1000 == 0:                validate_acc = sess.run(accuracy, feed_dict=validate_feed)                print("After %d training step(s), validation accuracy using average model is %g " % (i, validate_acc))            xs, ys = mnist.train.next_batch(BATCH_SIZE)            sess.run(train_op, feed_dict={x: xs, y_: ys})        test_acc = sess.run(accuracy, feed_dict=test_feed)        print(("After %d training step(s), test accuracy using average model is %g" % (TRAINING_STEPS, test_acc)))# 4. 主程序入口，这里设定模型训练次数为5000次def main(argv=None):    mnist = input_data.read_data_sets("/tmp/MNIST_data", one_hot=True)    train(mnist)if __name__=='__main__':    main()

7.不使用激活函数

import tensorflow as tffrom tensorflow.examples.tutorials.mnist import input_data# 1.设置输入和输出节点的个数,配置神经网络的参数INPUT_NODE = 784  # 输入节点OUTPUT_NODE = 10  # 输出节点LAYER1_NODE = 500  # 隐藏层数BATCH_SIZE = 100  # 每次batch打包的样本个数# 模型相关的参数LEARNING_RATE_BASE = 0.8LEARNING_RATE_DECAY = 0.99REGULARAZTION_RATE = 0.0001TRAINING_STEPS = 5000MOVING_AVERAGE_DECAY = 0.99# 2. 定义辅助函数来计算前向传播结果，去掉ReLU激活函数def inference(input_tensor, avg_class, weights1, biases1, weights2, biases2):    # 不使用滑动平均类    if avg_class == None:        layer1 = tf.matmul(input_tensor, weights1) + biases1        return tf.matmul(layer1, weights2) + biases2    else:        # 使用滑动平均类        layer1 = tf.matmul(input_tensor, avg_class.average(weights1)) + avg_class.average(biases1)        return tf.matmul(layer1, avg_class.average(weights2)) + avg_class.average(biases2)# 3. 定义训练过程def train(mnist):    x = tf.placeholder(tf.float32, [None, INPUT_NODE], name='x-input')    y_ = tf.placeholder(tf.float32, [None, OUTPUT_NODE], name='y-input')    # 生成隐藏层的参数。    weights1 = tf.Variable(tf.truncated_normal([INPUT_NODE, LAYER1_NODE], stddev=0.1))    biases1 = tf.Variable(tf.constant(0.1, shape=[LAYER1_NODE]))    # 生成输出层的参数。    weights2 = tf.Variable(tf.truncated_normal([LAYER1_NODE, OUTPUT_NODE], stddev=0.1))    biases2 = tf.Variable(tf.constant(0.1, shape=[OUTPUT_NODE]))    # 计算不含滑动平均类的前向传播结果    y = inference(x, None, weights1, biases1, weights2, biases2)    # 定义训练轮数及相关的滑动平均类    global_step = tf.Variable(0, trainable=False)    variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)    variables_averages_op = variable_averages.apply(tf.trainable_variables())    average_y = inference(x, variable_averages, weights1, biases1, weights2, biases2)    # 计算交叉熵及其平均值    cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))    cross_entropy_mean = tf.reduce_mean(cross_entropy)    # 损失函数的计算    regularizer = tf.contrib.layers.l2_regularizer(REGULARAZTION_RATE)    regularaztion = regularizer(weights1) + regularizer(weights2)    loss = cross_entropy_mean + regularaztion    # 设置指数衰减的学习率。    learning_rate = tf.train.exponential_decay(        LEARNING_RATE_BASE,        global_step,        mnist.train.num_examples / BATCH_SIZE,        LEARNING_RATE_DECAY,        staircase=True)    # 优化损失函数    train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)    # 反向传播更新参数和更新每一个参数的滑动平均值    with tf.control_dependencies([train_step, variables_averages_op]):        train_op = tf.no_op(name='train')    # 计算正确率    correct_prediction = tf.equal(tf.argmax(average_y, 1), tf.argmax(y_, 1))    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))    # 初始化回话并开始训练过程。    with tf.Session() as sess:        tf.global_variables_initializer().run()        validate_feed = {x: mnist.validation.images, y_: mnist.validation.labels}        test_feed = {x: mnist.test.images, y_: mnist.test.labels}        # 循环的训练神经网络。        for i in range(TRAINING_STEPS):            if i % 1000 == 0:                validate_acc = sess.run(accuracy, feed_dict=validate_feed)                print("After %d training step(s), validation accuracy using average model is %g " % (i, validate_acc))            xs,ys=mnist.train.next_batch(BATCH_SIZE)            sess.run(train_op,feed_dict={x:xs,y_:ys})        test_acc=sess.run(accuracy,feed_dict=test_feed)        print(("After %d training step(s), test accuracy using average model is %g" %(TRAINING_STEPS, test_acc)))# 4. 主程序入口，这里设定模型训练次数为5000次def main(argv=None):    mnist = input_data.read_data_sets("/tmp/MNIST_data", one_hot=True)    train(mnist)if __name__=='__main__':    main()

8.小结

       通过MNIST数据集验证了常用的神经网络优化方法，对比了不同优化算法对模型在测试数据集上正确率的影响。不用模型效果对比如下图所示。

       由图可知，调整神经网络的结构对最终的正确率有很大的影响。尤其是没有使用激活函数时，模型的正确率只有大约3.8%。说明神经网络的结构对最终模型的效果有本质性的影响。使用了所有优化算法的模型和不使用滑动平均及指数衰减的学习率模型都可以达到大约98.3%的正确率，这是因为滑动平均模型和指数衰减的学习率在一定程度上都是限制神经网络中参数更新的速度，在MNIST数据上，因为模型收敛的速度很快，所以这两种优化对最终模型影响不大。

       下一篇将总结卷积神经网络，以及给出TensorFlow最佳实践程序实现类LeNet-5模型的卷积神经网络，来解决MNIST数字识别问题。