Tensorflow实战Google深度学习框架 笔记

1. 定义两个不同的图import tensorflow as tfg1 = tf.Graph()with g1.as_default():    v = tf.get_variable("v", [1], initializer = tf.zeros_initializer) # 设置初始值为0g2 = tf.Graph()with g2.as_default():    v = tf.get_variable("v", [1], initializer = tf.ones_initializer())  # 设置初始值为1    with tf.Session(graph = g1) as sess:    tf.global_variables_initializer().run()    with tf.variable_scope("", reuse=True):        print(sess.run(tf.get_variable("v")))with tf.Session(graph = g2) as sess:    tf.global_variables_initializer().run()    with tf.variable_scope("", reuse=True):        print(sess.run(tf.get_variable("v")))g=tf.Graph()with g.device("/gup:0"):result=a+b2. 张量的概念import tensorflow as tfa = tf.constant([1.0, 2.0], name="a")b = tf.constant([2.0, 3.0], name="b")result = a + bprint resultsess = tf.InteractiveSession ()print(result.eval())sess.close()3.1 创建和关闭会话# 创建一个会话。sess = tf.Session()# 使用会话得到之前计算的结果。print(sess.run(result))# 关闭会话使得本次运行中使用到的资源可以被释放。sess.close()3.2 试用with语句创建会话with tf.Session() as sess:    print(sess.run(result))3.3 制定默认会话sess = tf.Session()with sess.as_default():     print(result.eval()) sess = tf.Session()# 下面的两个命令有相同的功能。print(sess.run(result))print(result.eval(session=sess))4. 使用tf.InteractiveSession构建会话sess = tf.InteractiveSession ()print(result.eval())sess.close()5. 通过ConfigProto创建会话config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=True)sess1 = tf.InteractiveSession(config=config)sess2 = tf.Session(config=config) 三层简单神经网络 w1= tf.Variable(tf.random_normal([2, 3], stddev=1, seed=1))w2= tf.Variable(tf.random_normal([3, 1], stddev=1, seed=1))x = tf.constant([[0.7, 0.9]])  a = tf.matmul(x, w1)y = tf.matmul(a, w2)sess = tf.Session()sess.run(w1.initializer)  sess.run(w2.initializer)  print(sess.run(y))  sess.close()x = tf.placeholder(tf.float32, shape=(1, 2), name="input")a = tf.matmul(x, w1)y = tf.matmul(a, w2)sess = tf.Session()init_op = tf.global_variables_initializer()  sess.run(init_op)print(sess.run(y, feed_dict={x: [[0.7,0.9]]}))完整神经网络样例程序import tensorflow as tffrom numpy.random import RandomStatebatch_size = 8w1= tf.Variable(tf.random_normal([2, 3], stddev=1, seed=1))w2= tf.Variable(tf.random_normal([3, 1], stddev=1, seed=1))x = tf.placeholder(tf.float32, shape=(None, 2), name="x-input")y_= tf.placeholder(tf.float32, shape=(None, 1), name='y-input')a = tf.matmul(x, w1)y = tf.matmul(a, w2)cross_entropy = -tf.reduce_mean(y_ * tf.log(tf.clip_by_value(y, 1e-10, 1.0))) train_step = tf.train.AdamOptimizer(0.001).minimize(cross_entropy)rdm = RandomState(1)X = rdm.rand(128,2)Y = [[int(x1+x2 < 1)] for (x1, x2) in X]with tf.Session() as sess:    init_op = tf.global_variables_initializer()    sess.run(init_op)        # 输出目前（未经训练）的参数取值。    print "w1:", sess.run(w1)    print "w2:", sess.run(w2)    print "\n"        # 训练模型。    STEPS = 5000    for i in range(STEPS):        start = (i*batch_size) % 128        end = (i*batch_size) % 128 + batch_size        sess.run(train_step, feed_dict={x: X[start:end], y_: Y[start:end]})        if i % 1000 == 0:            total_cross_entropy = sess.run(cross_entropy, feed_dict={x: X, y_: Y})            print("After %d training step(s), cross entropy on all data is %g" % (i, total_cross_entropy))        # 输出训练后的参数取值。    print "\n"    print "w1:", sess.run(w1)    print "w2:", sess.run(w2)自定义损失函数import tensorflow as tffrom numpy.random import RandomStatebatch_size = 8x = tf.placeholder(tf.float32, shape=(None, 2), name="x-input")y_ = tf.placeholder(tf.float32, shape=(None, 1), name='y-input')w1= tf.Variable(tf.random_normal([2, 1], stddev=1, seed=1))y = tf.matmul(x, w1)# 定义损失函数使得预测少了的损失大，于是模型应该偏向多的方向预测。loss_less = 10loss_more = 1loss = tf.reduce_sum(tf.where(tf.greater(y, y_), (y - y_) * loss_more, (y_ - y) * loss_less))train_step = tf.train.AdamOptimizer(0.001).minimize(loss)rdm = RandomState(1)X = rdm.rand(128,2)Y = [[x1+x2+(rdm.rand()/10.0-0.05)] for (x1, x2) in X]with tf.Session() as sess:    init_op = tf.global_variables_initializer()    sess.run(init_op)    STEPS = 5000    for i in range(STEPS):        start = (i*batch_size) % 128        end = (i*batch_size) % 128 + batch_size        sess.run(train_step, feed_dict={x: X[start:end], y_: Y[start:end]})        if i % 1000 == 0:            print("After %d training step(s), w1 is: " % (i))            print sess.run(w1), "\n"    print "Final w1 is: \n", sess.run(w1)loss_less = 1loss_more = 10loss = tf.reduce_sum(tf.where(tf.greater(y, y_), (y - y_) * loss_more, (y_ - y) * loss_less))train_step = tf.train.AdamOptimizer(0.001).minimize(loss)with tf.Session() as sess:    init_op = tf.global_variables_initializer()    sess.run(init_op)    STEPS = 5000    for i in range(STEPS):        start = (i*batch_size) % 128        end = (i*batch_size) % 128 + batch_size        sess.run(train_step, feed_dict={x: X[start:end], y_: Y[start:end]})        if i % 1000 == 0:            print("After %d training step(s), w1 is: " % (i))            print sess.run(w1), "\n"    print "Final w1 is: \n", sess.run(w1)定义损失函数为MSEloss = tf.losses.mean_squared_error(y, y_)train_step = tf.train.AdamOptimizer(0.001).minimize(loss)with tf.Session() as sess:    init_op = tf.global_variables_initializer()    sess.run(init_op)    STEPS = 5000    for i in range(STEPS):        start = (i*batch_size) % 128        end = (i*batch_size) % 128 + batch_size        sess.run(train_step, feed_dict={x: X[start:end], y_: Y[start:end]})        if i % 1000 == 0:            print("After %d training step(s), w1 is: " % (i))            print sess.run(w1), "\n"    print "Final w1 is: \n", sess.run(w1)学习率的设置import tensorflow as tfTRAINING_STEPS = 10LEARNING_RATE = 1x = tf.Variable(tf.constant(5, dtype=tf.float32), name="x")y = tf.square(x)train_op = tf.train.GradientDescentOptimizer(LEARNING_RATE).minimize(y)with tf.Session() as sess:    sess.run(tf.global_variables_initializer())    for i in range(TRAINING_STEPS):        sess.run(train_op)        x_value = sess.run(x)        print "After %s iteration(s): x%s is %f."% (i+1, i+1, x_value) 指数衰减的学习率TRAINING_STEPS = 100global_step = tf.Variable(0)LEARNING_RATE = tf.train.exponential_decay(0.1, global_step, 1, 0.96, staircase=True)x = tf.Variable(tf.constant(5, dtype=tf.float32), name="x")y = tf.square(x)train_op = tf.train.GradientDescentOptimizer(LEARNING_RATE).minimize(y, global_step=global_step)with tf.Session() as sess:    sess.run(tf.global_variables_initializer())    for i in range(TRAINING_STEPS):        sess.run(train_op)        if i % 10 == 0:            LEARNING_RATE_value = sess.run(LEARNING_RATE)            x_value = sess.run(x)            print "After %s iteration(s): x%s is %f, learning rate is %f."% (i+1, i+1, x_value, LEARNING_RATE_value)正则化import tensorflow as tfimport matplotlib.pyplot as pltimport numpy as npdata = []label = []np.random.seed(0)# 以原点为圆心，半径为1的圆把散点划分成红蓝两部分，并加入随机噪音。for i in range(150):    x1 = np.random.uniform(-1,1)    x2 = np.random.uniform(0,2)    if x1**2 + x2**2 <= 1:        data.append([np.random.normal(x1, 0.1),np.random.normal(x2,0.1)])        label.append(0)    else:        data.append([np.random.normal(x1, 0.1), np.random.normal(x2, 0.1)])        label.append(1)        data = np.hstack(data).reshape(-1,2)label = np.hstack(label).reshape(-1, 1)plt.scatter(data[:,0], data[:,1], c=label,           cmap="RdBu", vmin=-.2, vmax=1.2, edgecolor="white")plt.show()def get_weight(shape, lambda1):    var = tf.Variable(tf.random_normal(shape), dtype=tf.float32)    tf.add_to_collection('losses', tf.contrib.layers.l2_regularizer(lambda1)(var))    return varx = tf.placeholder(tf.float32, shape=(None, 2))y_ = tf.placeholder(tf.float32, shape=(None, 1))sample_size = len(data)# 每层节点的个数layer_dimension = [2,10,5,3,1]n_layers = len(layer_dimension)cur_layer = xin_dimension = layer_dimension[0]# 循环生成网络结构for i in range(1, n_layers):    out_dimension = layer_dimension[i]    weight = get_weight([in_dimension, out_dimension], 0.003)    bias = tf.Variable(tf.constant(0.1, shape=[out_dimension]))    cur_layer = tf.nn.elu(tf.matmul(cur_layer, weight) + bias)    in_dimension = layer_dimension[i]y= cur_layer# 损失函数的定义。mse_loss = tf.reduce_sum(tf.pow(y_ - y, 2)) / sample_sizetf.add_to_collection('losses', mse_loss)loss = tf.add_n(tf.get_collection('losses'))# 定义训练的目标函数mse_loss，训练次数及训练模型train_op = tf.train.AdamOptimizer(0.001).minimize(mse_loss)TRAINING_STEPS = 40000with tf.Session() as sess:    tf.global_variables_initializer().run()    for i in range(TRAINING_STEPS):        sess.run(train_op, feed_dict={x: data, y_: label})        if i % 2000 == 0:            print("After %d steps, mse_loss: %f" % (i,sess.run(mse_loss, feed_dict={x: data, y_: label})))    # 画出训练后的分割曲线           xx, yy = np.mgrid[-1.2:1.2:.01, -0.2:2.2:.01]    grid = np.c_[xx.ravel(), yy.ravel()]    probs = sess.run(y, feed_dict={x:grid})    probs = probs.reshape(xx.shape)plt.scatter(data[:,0], data[:,1], c=label,           cmap="RdBu", vmin=-.2, vmax=1.2, edgecolor="white")plt.contour(xx, yy, probs, levels=[.5], cmap="Greys", vmin=0, vmax=.1)plt.show()# 定义训练的目标函数loss，训练次数及训练模型train_op = tf.train.AdamOptimizer(0.001).minimize(loss)TRAINING_STEPS = 40000with tf.Session() as sess:    tf.global_variables_initializer().run()    for i in range(TRAINING_STEPS):        sess.run(train_op, feed_dict={x: data, y_: label})        if i % 2000 == 0:            print("After %d steps, loss: %f" % (i, sess.run(loss, feed_dict={x: data, y_: label})))    # 画出训练后的分割曲线           xx, yy = np.mgrid[-1:1:.01, 0:2:.01]    grid = np.c_[xx.ravel(), yy.ravel()]    probs = sess.run(y, feed_dict={x:grid})    probs = probs.reshape(xx.shape)plt.scatter(data[:,0], data[:,1], c=label,           cmap="RdBu", vmin=-.2, vmax=1.2, edgecolor="white")plt.contour(xx, yy, probs, levels=[.5], cmap="Greys", vmin=0, vmax=.1)plt.show()滑动平均模型import tensorflow as tfv1 = tf.Variable(0, dtype=tf.float32)step = tf.Variable(0, trainable=False)ema = tf.train.ExponentialMovingAverage(0.99, step)maintain_averages_op = ema.apply([v1]) with tf.Session() as sess:        # 初始化    init_op = tf.global_variables_initializer()    sess.run(init_op)    print sess.run([v1, ema.average(v1)])        # 更新变量v1的取值    sess.run(tf.assign(v1, 5))    sess.run(maintain_averages_op)    print sess.run([v1, ema.average(v1)])         # 更新step和v1的取值    sess.run(tf.assign(step, 10000))      sess.run(tf.assign(v1, 10))    sess.run(maintain_averages_op)    print sess.run([v1, ema.average(v1)])               # 更新一次v1的滑动平均值    sess.run(maintain_averages_op)    print sess.run([v1, ema.average(v1)])       MNIST读取数据from tensorflow.examples.tutorials.mnist import input_datamnist = input_data.read_data_sets("../../datasets/MNIST_data/", one_hot=True)print "Training data size: ", mnist.train.num_examplesprint "Validating data size: ", mnist.validation.num_examplesprint "Testing data size: ", mnist.test.num_examplesbatch_size = 100xs, ys = mnist.train.next_batch(batch_size)    # 从train的集合中选取batch_size个训练数据。print "X shape:", xs.shape                     print "Y shape:", ys.shape   1. 全模型import tensorflow as tffrom tensorflow.examples.tutorials.mnist import input_dataINPUT_NODE = 784     # 输入节点OUTPUT_NODE = 10     # 输出节点LAYER1_NODE = 500    # 隐藏层数                                     BATCH_SIZE = 100     # 每次batch打包的样本个数        # 模型相关的参数LEARNING_RATE_BASE = 0.8      LEARNING_RATE_DECAY = 0.99    REGULARAZTION_RATE = 0.0001   TRAINING_STEPS = 5000        MOVING_AVERAGE_DECAY = 0.99  def inference(input_tensor, avg_class, weights1, biases1, weights2, biases2):    # 使用滑动平均类    if avg_class == None:        layer1 = tf.nn.relu(tf.matmul(input_tensor, weights1) + biases1)        return tf.matmul(layer1, weights2) + biases2    else:        # 不使用滑动平均类        layer1 = tf.nn.relu(tf.matmul(input_tensor, avg_class.average(weights1)) + avg_class.average(biases1))        return tf.matmul(layer1, avg_class.average(weights2)) + avg_class.average(biases2)  def train(mnist):    x = tf.placeholder(tf.float32, [None, INPUT_NODE], name='x-input')    y_ = tf.placeholder(tf.float32, [None, OUTPUT_NODE], name='y-input')    # 生成隐藏层的参数。    weights1 = tf.Variable(tf.truncated_normal([INPUT_NODE, LAYER1_NODE], stddev=0.1))    biases1 = tf.Variable(tf.constant(0.1, shape=[LAYER1_NODE]))    # 生成输出层的参数。    weights2 = tf.Variable(tf.truncated_normal([LAYER1_NODE, OUTPUT_NODE], stddev=0.1))    biases2 = tf.Variable(tf.constant(0.1, shape=[OUTPUT_NODE]))    # 计算不含滑动平均类的前向传播结果    y = inference(x, None, weights1, biases1, weights2, biases2)        # 定义训练轮数及相关的滑动平均类     global_step = tf.Variable(0, trainable=False)    variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)    variables_averages_op = variable_averages.apply(tf.trainable_variables())    average_y = inference(x, variable_averages, weights1, biases1, weights2, biases2)        # 计算交叉熵及其平均值    cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))    cross_entropy_mean = tf.reduce_mean(cross_entropy)        # 损失函数的计算    regularizer = tf.contrib.layers.l2_regularizer(REGULARAZTION_RATE)    regularaztion = regularizer(weights1) + regularizer(weights2)    loss = cross_entropy_mean + regularaztion        # 设置指数衰减的学习率。    learning_rate = tf.train.exponential_decay(        LEARNING_RATE_BASE,        global_step,        mnist.train.num_examples / BATCH_SIZE,        LEARNING_RATE_DECAY,        staircase=True)        # 优化损失函数    train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)        # 反向传播更新参数和更新每一个参数的滑动平均值    with tf.control_dependencies([train_step, variables_averages_op]):        train_op = tf.no_op(name='train')    # 计算正确率    correct_prediction = tf.equal(tf.argmax(average_y, 1), tf.argmax(y_, 1))    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))        # 初始化回话并开始训练过程。    with tf.Session() as sess:        tf.global_variables_initializer().run()        validate_feed = {x: mnist.validation.images, y_: mnist.validation.labels}        test_feed = {x: mnist.test.images, y_: mnist.test.labels}                 # 循环的训练神经网络。        for i in range(TRAINING_STEPS):            if i % 1000 == 0:                validate_acc = sess.run(accuracy, feed_dict=validate_feed)                print("After %d training step(s), validation accuracy using average model is %g " % (i, validate_acc))                        xs,ys=mnist.train.next_batch(BATCH_SIZE)            sess.run(train_op,feed_dict={x:xs,y_:ys})        test_acc=sess.run(accuracy,feed_dict=test_feed)        print(("After %d training step(s), test accuracy using average model is %g" %(TRAINING_STEPS, test_acc)))def main(argv=None):    mnist = input_data.read_data_sets("../../../datasets/MNIST_data", one_hot=True)    train(mnist)if __name__=='__main__':    main()卷积层，池化层样例import tensorflow as tfimport numpy as npM = np.array([        [[1],[-1],[0]],        [[-1],[2],[1]],        [[0],[2],[-2]]    ])print ("Matrix shape is: ",M.shape)filter_weight = tf.get_variable('weights', [2, 2, 1, 1], initializer = tf.constant_initializer([                                                                        [1, -1],                                                                        [0, 2]]))biases = tf.get_variable('biases', [1], initializer = tf.constant_initializer(1))M = np.asarray(M, dtype='float32')M = M.reshape(1, 3, 3, 1)x = tf.placeholder('float32', [1, None, None, 1])conv = tf.nn.conv2d(x, filter_weight, strides = [1, 2, 2, 1], padding = 'SAME')bias = tf.nn.bias_add(conv, biases)pool = tf.nn.avg_pool(x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')with tf.Session() as sess:    tf.global_variables_initializer().run()    convoluted_M = sess.run(bias,feed_dict={x:M})    pooled_M = sess.run(pool,feed_dict={x:M})        print ("convoluted_M: \n", convoluted_M)    print ("pooled_M: \n", pooled_M)LeNet5_infernece INPUT_NODE = 784OUTPUT_NODE = 10IMAGE_SIZE = 28NUM_CHANNELS = 1NUM_LABELS = 10CONV1_DEEP = 32CONV1_SIZE = 5CONV2_DEEP = 64CONV2_SIZE = 5FC_SIZE = 512def inference(input_tensor, train, regularizer):    with tf.variable_scope('layer1-conv1'):        conv1_weights = tf.get_variable(            "weight", [CONV1_SIZE, CONV1_SIZE, NUM_CHANNELS, CONV1_DEEP],            initializer=tf.truncated_normal_initializer(stddev=0.1))        conv1_biases = tf.get_variable("bias", [CONV1_DEEP], initializer=tf.constant_initializer(0.0))        conv1 = tf.nn.conv2d(input_tensor, conv1_weights, strides=[1, 1, 1, 1], padding='SAME')        relu1 = tf.nn.relu(tf.nn.bias_add(conv1, conv1_biases))    with tf.name_scope("layer2-pool1"):        pool1 = tf.nn.max_pool(relu1, ksize = [1,2,2,1],strides=[1,2,2,1],padding="SAME")    with tf.variable_scope("layer3-conv2"):        conv2_weights = tf.get_variable(            "weight", [CONV2_SIZE, CONV2_SIZE, CONV1_DEEP, CONV2_DEEP],            initializer=tf.truncated_normal_initializer(stddev=0.1))        conv2_biases = tf.get_variable("bias", [CONV2_DEEP], initializer=tf.constant_initializer(0.0))        conv2 = tf.nn.conv2d(pool1, conv2_weights, strides=[1, 1, 1, 1], padding='SAME')        relu2 = tf.nn.relu(tf.nn.bias_add(conv2, conv2_biases))    with tf.name_scope("layer4-pool2"):        pool2 = tf.nn.max_pool(relu2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')        pool_shape = pool2.get_shape().as_list()        nodes = pool_shape[1] * pool_shape[2] * pool_shape[3]        reshaped = tf.reshape(pool2, [pool_shape[0], nodes])    with tf.variable_scope('layer5-fc1'):        fc1_weights = tf.get_variable("weight", [nodes, FC_SIZE],                                      initializer=tf.truncated_normal_initializer(stddev=0.1))        if regularizer != None: tf.add_to_collection('losses', regularizer(fc1_weights))        fc1_biases = tf.get_variable("bias", [FC_SIZE], initializer=tf.constant_initializer(0.1))        fc1 = tf.nn.relu(tf.matmul(reshaped, fc1_weights) + fc1_biases)        if train: fc1 = tf.nn.dropout(fc1, 0.5)    with tf.variable_scope('layer6-fc2'):        fc2_weights = tf.get_variable("weight", [FC_SIZE, NUM_LABELS],                                      initializer=tf.truncated_normal_initializer(stddev=0.1))        if regularizer != None: tf.add_to_collection('losses', regularizer(fc2_weights))        fc2_biases = tf.get_variable("bias", [NUM_LABELS], initializer=tf.constant_initializer(0.1))        logit = tf.matmul(fc1, fc2_weights) + fc2_biases    return logitimport tensorflow as tffrom tensorflow.examples.tutorials.mnist import input_dataimport LeNet5_inferneceimport osimport numpy as npBATCH_SIZE = 100LEARNING_RATE_BASE = 0.01LEARNING_RATE_DECAY = 0.99REGULARIZATION_RATE = 0.0001TRAINING_STEPS = 6000MOVING_AVERAGE_DECAY = 0.99def train(mnist):    # 定义输出为4维矩阵的placeholder    x = tf.placeholder(tf.float32, [            BATCH_SIZE,            LeNet5_infernece.IMAGE_SIZE,            LeNet5_infernece.IMAGE_SIZE,            LeNet5_infernece.NUM_CHANNELS],        name='x-input')    y_ = tf.placeholder(tf.float32, [None, LeNet5_infernece.OUTPUT_NODE], name='y-input')        regularizer = tf.contrib.layers.l2_regularizer(REGULARIZATION_RATE)    y = LeNet5_infernece.inference(x,False,regularizer)    global_step = tf.Variable(0, trainable=False)    # 定义损失函数、学习率、滑动平均操作以及训练过程。    variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)    variables_averages_op = variable_averages.apply(tf.trainable_variables())    cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))    cross_entropy_mean = tf.reduce_mean(cross_entropy)    loss = cross_entropy_mean + tf.add_n(tf.get_collection('losses'))    learning_rate = tf.train.exponential_decay(        LEARNING_RATE_BASE,        global_step,        mnist.train.num_examples / BATCH_SIZE, LEARNING_RATE_DECAY,        staircase=True)    train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)    with tf.control_dependencies([train_step, variables_averages_op]):        train_op = tf.no_op(name='train')            # 初始化TensorFlow持久化类。    saver = tf.train.Saver()    with tf.Session() as sess:        tf.global_variables_initializer().run()        for i in range(TRAINING_STEPS):            xs, ys = mnist.train.next_batch(BATCH_SIZE)            reshaped_xs = np.reshape(xs, (                BATCH_SIZE,                LeNet5_infernece.IMAGE_SIZE,                LeNet5_infernece.IMAGE_SIZE,                LeNet5_infernece.NUM_CHANNELS))            _, loss_value, step = sess.run([train_op, loss, global_step], feed_dict={x: reshaped_xs, y_: ys})            if i % 1000 == 0:                print("After %d training step(s), loss on training batch is %g." % (step, loss_value))def main(argv=None):    mnist = input_data.read_data_sets("../../../datasets/MNIST_data", one_hot=True)    train(mnist)if __name__ == '__main__':    main()TFRecord样例程序import tensorflow as tffrom tensorflow.examples.tutorials.mnist import input_dataimport numpy as np# 定义函数转化变量类型。def _int64_feature(value):    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))def _bytes_feature(value):    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))# 读取mnist数据。mnist = input_data.read_data_sets("../../datasets/MNIST_data",dtype=tf.uint8, one_hot=True)images = mnist.train.imageslabels = mnist.train.labelspixels = images.shape[1]num_examples = mnist.train.num_examples# 输出TFRecord文件的地址。filename = "Records/output.tfrecords"writer = tf.python_io.TFRecordWriter(filename)for index in range(num_examples):    image_raw = images[index].tostring()    example = tf.train.Example(features=tf.train.Features(feature={        'pixels': _int64_feature(pixels),        'label': _int64_feature(np.argmax(labels[index])),        'image_raw': _bytes_feature(image_raw)    }))    writer.write(example.SerializeToString())writer.close()print ("TFRecord文件已保存。")# 读取文件。reader = tf.TFRecordReader()filename_queue = tf.train.string_input_producer(["Records/output.tfrecords"])_,serialized_example = reader.read(filename_queue)# 解析读取的样例。features = tf.parse_single_example(    serialized_example,    features={        'image_raw':tf.FixedLenFeature([],tf.string),        'pixels':tf.FixedLenFeature([],tf.int64),        'label':tf.FixedLenFeature([],tf.int64)    })images = tf.decode_raw(features['image_raw'],tf.uint8)labels = tf.cast(features['label'],tf.int32)pixels = tf.cast(features['pixels'],tf.int32)sess = tf.Session()# 启动多线程处理输入数据。coord = tf.train.Coordinator()threads = tf.train.start_queue_runners(sess=sess,coord=coord)for i in range(10):    image, label, pixel = sess.run([images, labels, pixels])TensorFlow图像处理函数import matplotlib.pyplot as pltimport tensorflow as tf   import numpy as npimage_raw_data = tf.gfile.FastGFile("../../datasets/cat.jpg",'r').read()with tf.Session() as sess:    img_data = tf.image.decode_jpeg(image_raw_data)        # 输出解码之后的三维矩阵。    print (img_data.eval())    img_data.set_shape([1797, 2673, 3])    print (img_data.get_shape())with tf.Session() as sess:    plt.imshow(img_data.eval())    plt.show()th tf.Session() as sess:        resized = tf.image.resize_images(img_data, [300, 300], method=0)        # TensorFlow的函数处理图片后存储的数据是float32格式的，需要转换成uint8才能正确打印图片。    print "Digital type: ", resized.dtype    cat = np.asarray(resized.eval(), dtype='uint8')    # tf.image.convert_image_dtype(rgb_image, tf.float32)    plt.imshow(cat)    plt.show()with tf.Session() as sess:        croped = tf.image.resize_image_with_crop_or_pad(img_data, 1000, 1000)    padded = tf.image.resize_image_with_crop_or_pad(img_data, 3000, 3000)    plt.imshow(croped.eval())    plt.show()    plt.imshow(padded.eval())    plt.show()with tf.Session() as sess:       central_cropped = tf.image.central_crop(img_data, 0.5)    plt.imshow(central_cropped.eval())    plt.show()with tf.Session() as sess:     # 上下翻转    #flipped1 = tf.image.flip_up_down(img_data)    # 左右翻转    #flipped2 = tf.image.flip_left_right(img_data)        #对角线翻转    transposed = tf.image.transpose_image(img_data)    plt.imshow(transposed.eval())    plt.show()        # 以一定概率上下翻转图片。    #flipped = tf.image.random_flip_up_down(img_data)    # 以一定概率左右翻转图片。    #flipped = tf.image.random_flip_left_right(img_data)with tf.Session() as sess:         # 将图片的亮度-0.5。    #adjusted = tf.image.adjust_brightness(img_data, -0.5)        # 将图片的亮度-0.5    #adjusted = tf.image.adjust_brightness(img_data, 0.5)        # 在[-max_delta, max_delta)的范围随机调整图片的亮度。    adjusted = tf.image.random_brightness(img_data, max_delta=0.5)        # 将图片的对比度-5    #adjusted = tf.image.adjust_contrast(img_data, -5)        # 将图片的对比度+5    #adjusted = tf.image.adjust_contrast(img_data, 5)        # 在[lower, upper]的范围随机调整图的对比度。    #adjusted = tf.image.random_contrast(img_data, lower, upper)    plt.imshow(adjusted.eval())    plt.show()with tf.Session() as sess:             adjusted = tf.image.adjust_hue(img_data, 0.1)    #adjusted = tf.image.adjust_hue(img_data, 0.3)    #adjusted = tf.image.adjust_hue(img_data, 0.6)    #adjusted = tf.image.adjust_hue(img_data, 0.9)        # 在[-max_delta, max_delta]的范围随机调整图片的色相。max_delta的取值在[0, 0.5]之间。    #adjusted = tf.image.random_hue(image, max_delta)        # 将图片的饱和度-5。    #adjusted = tf.image.adjust_saturation(img_data, -5)    # 将图片的饱和度+5。    #adjusted = tf.image.adjust_saturation(img_data, 5)    # 在[lower, upper]的范围随机调整图的饱和度。    #adjusted = tf.image.random_saturation(img_data, lower, upper)        # 将代表一张图片的三维矩阵中的数字均值变为0，方差变为1。    #adjusted = tf.image.per_image_whitening(img_data)        plt.imshow(adjusted.eval())    plt.show()with tf.Session() as sess:             boxes = tf.constant([[[0.05, 0.05, 0.9, 0.7], [0.35, 0.47, 0.5, 0.56]]])    begin, size, bbox_for_draw = tf.image.sample_distorted_bounding_box(        tf.shape(img_data), bounding_boxes=boxes)    batched = tf.expand_dims(tf.image.convert_image_dtype(img_data, tf.float32), 0)     image_with_box = tf.image.draw_bounding_boxes(batched, bbox_for_draw)        distorted_image = tf.slice(img_data, begin, size)    plt.imshow(distorted_image.eval())    plt.show()图像预处理完整样例import tensorflow as tfimport numpy as npimport matplotlib.pyplot as pltdef distort_color(image, color_ordering=0):    if color_ordering == 0:        image = tf.image.random_brightness(image, max_delta=32./255.)        image = tf.image.random_saturation(image, lower=0.5, upper=1.5)        image = tf.image.random_hue(image, max_delta=0.2)        image = tf.image.random_contrast(image, lower=0.5, upper=1.5)    else:        image = tf.image.random_saturation(image, lower=0.5, upper=1.5)        image = tf.image.random_brightness(image, max_delta=32./255.)        image = tf.image.random_contrast(image, lower=0.5, upper=1.5)        image = tf.image.random_hue(image, max_delta=0.2)    return tf.clip_by_value(image, 0.0, 1.0)def preprocess_for_train(image, height, width, bbox):    # 查看是否存在标注框。    if bbox is None:        bbox = tf.constant([0.0, 0.0, 1.0, 1.0], dtype=tf.float32, shape=[1, 1, 4])    if image.dtype != tf.float32:        image = tf.image.convert_image_dtype(image, dtype=tf.float32)            # 随机的截取图片中一个块。    bbox_begin, bbox_size, _ = tf.image.sample_distorted_bounding_box(        tf.shape(image), bounding_boxes=bbox)    bbox_begin, bbox_size, _ = tf.image.sample_distorted_bounding_box(        tf.shape(image), bounding_boxes=bbox)    distorted_image = tf.slice(image, bbox_begin, bbox_size)    # 将随机截取的图片调整为神经网络输入层的大小。    distorted_image = tf.image.resize_images(distorted_image, [height, width], method=np.random.randint(4))    distorted_image = tf.image.random_flip_left_right(distorted_image)    distorted_image = distort_color(distorted_image, np.random.randint(2))    return distorted_imageimage_raw_data = tf.gfile.FastGFile("../../datasets/cat.jpg", "r").read()with tf.Session() as sess:    img_data = tf.image.decode_jpeg(image_raw_data)    boxes = tf.constant([[[0.05, 0.05, 0.9, 0.7], [0.35, 0.47, 0.5, 0.56]]])    for i in range(9):        result = preprocess_for_train(img_data, 299, 299, boxes)        plt.imshow(result.eval())        plt.show()队列操作import tensorflow as tfq = tf.FIFOQueue(2, "int32")init = q.enqueue_many(([0, 10],))x = q.dequeue()y = x + 1q_inc = q.enqueue([y])with tf.Session() as sess:    init.run()    for _ in range(5):        v, _ = sess.run([x, q_inc])print(v)import numpy as npimport threadingimport timedef MyLoop(coord, worker_id):    while not coord.should_stop():        if np.random.rand()<0.1:            print "Stoping from id: %d\n" % worker_id,            coord.request_stop()        else:            print "Working on id: %d\n" % worker_id,         time.sleep(1)coord = tf.train.Coordinator()threads = [threading.Thread(target=MyLoop, args=(coord, i, )) for i in xrange(5)]for t in threads:t.start()coord.join(threads)       多线程队列操作 queue = tf.FIFOQueue(100,"float")enqueue_op = queue.enqueue([tf.random_normal([1])])qr = tf.train.QueueRunner(queue, [enqueue_op] * 5)tf.train.add_queue_runner(qr)out_tensor = queue.dequeue()with tf.Session() as sess:    coord = tf.train.Coordinator()    threads = tf.train.start_queue_runners(sess=sess, coord=coord)    for _ in range(3): print sess.run(out_tensor)[0]    coord.request_stop()    coord.join(threads)输入文件队列def _int64_feature(value):    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))num_shards = 2instances_per_shard = 2for i in range(num_shards):    filename = ('Records/data.tfrecords-%.5d-of-%.5d' % (i, num_shards))     # 将Example结构写入TFRecord文件。    writer = tf.python_io.TFRecordWriter(filename)    for j in range(instances_per_shard):    # Example结构仅包含当前样例属于第几个文件以及是当前文件的第几个样本。        example = tf.train.Example(features=tf.train.Features(feature={            'i': _int64_feature(i),            'j': _int64_feature(j)}))        writer.write(example.SerializeToString())    writer.close()  files = tf.train.match_filenames_once("Records/data.tfrecords-*")filename_queue = tf.train.string_input_producer(files, shuffle=False) reader = tf.TFRecordReader()_, serialized_example = reader.read(filename_queue)features = tf.parse_single_example(      serialized_example,      features={          'i': tf.FixedLenFeature([], tf.int64),          'j': tf.FixedLenFeature([], tf.int64),      })with tf.Session() as sess:    tf.global_variables_initializer().run()    print sess.run(files)    coord = tf.train.Coordinator()    threads = tf.train.start_queue_runners(sess=sess, coord=coord)    for i in range(6):        print sess.run([features['i'], features['j']])    coord.request_stop()    coord.join(threads)example, label = features['i'], features['j']batch_size = 2capacity = 1000 + 3 * batch_sizecapacity = 1000 + 3 * batch_sizeexample_batch, label_batch = tf.train.batch([example, label], batch_size=batch_size, capacity=capacity)with tf.Session() as sess:    tf.global_variables_initializer().run()    coord = tf.train.Coordinator()    threads = tf.train.start_queue_runners(sess=sess, coord=coord)    for i in range(3):        cur_example_batch, cur_label_batch = sess.run([example_batch, label_batch])        print (cur_example_batch, cur_label_batch)    coord.request_stop()    coord.join(threads)输入数据处理框架files = tf.train.match_filenames_once("Records/output.tfrecords")filename_queue = tf.train.string_input_producer(files, shuffle=False) # 读取文件。reader = tf.TFRecordReader()_,serialized_example = reader.read(filename_queue)# 解析读取的样例。features = tf.parse_single_example(    serialized_example,    features={        'image_raw':tf.FixedLenFeature([],tf.string),        'pixels':tf.FixedLenFeature([],tf.int64),        'label':tf.FixedLenFeature([],tf.int64)    })decoded_images = tf.decode_raw(features['image_raw'],tf.uint8)retyped_images = tf.cast(decoded_images, tf.float32)labels = tf.cast(features['label'],tf.int32)#pixels = tf.cast(features['pixels'],tf.int32)images = tf.reshape(retyped_images, [784])min_after_dequeue = 10000batch_size = 100capacity = min_after_dequeue + 3 * batch_sizeimage_batch, label_batch = tf.train.shuffle_batch([images, labels],                                                     batch_size=batch_size,                                                     capacity=capacity,                                                     min_after_dequeue=min_after_dequeue)def inference(input_tensor, weights1, biases1, weights2, biases2):        layer1 = tf.nn.relu(tf.matmul(input_tensor, weights1) + biases1)        return tf.matmul(layer1, weights2) + biases2# 模型相关的参数INPUT_NODE = 784OUTPUT_NODE = 10LAYER1_NODE = 500REGULARAZTION_RATE = 0.0001   TRAINING_STEPS = 5000        weights1 = tf.Variable(tf.truncated_normal([INPUT_NODE, LAYER1_NODE], stddev=0.1))biases1 = tf.Variable(tf.constant(0.1, shape=[LAYER1_NODE]))weights2 = tf.Variable(tf.truncated_normal([LAYER1_NODE, OUTPUT_NODE], stddev=0.1))biases2 = tf.Variable(tf.constant(0.1, shape=[OUTPUT_NODE]))y = inference(image_batch, weights1, biases1, weights2, biases2)    # 计算交叉熵及其平均值cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=label_batch)cross_entropy_mean = tf.reduce_mean(cross_entropy)    # 损失函数的计算regularizer = tf.contrib.layers.l2_regularizer(REGULARAZTION_RATE)regularaztion = regularizer(weights1) + regularizer(weights2)loss = cross_entropy_mean + regularaztion# 优化损失函数train_step = tf.train.GradientDescentOptimizer(0.01).minimize(loss)    # 初始化回话并开始训练过程。with tf.Session() as sess:    tf.global_variables_initializer().run()    coord = tf.train.Coordinator()    threads = tf.train.start_queue_runners(sess=sess, coord=coord)    # 循环的训练神经网络。    for i in range(TRAINING_STEPS):        if i % 1000 == 0:            print("After %d training step(s), loss is %g " % (i, sess.run(loss)))                          sess.run(train_step)     coord.request_stop()    coord.join(threads)       循环神经网络前向传播import numpy as npX = [1,2]state = [0.0, 0.0]w_cell_state = np.asarray([[0.1, 0.2], [0.3, 0.4]])w_cell_input = np.asarray([0.5, 0.6])b_cell = np.asarray([0.1, -0.1])w_output = np.asarray([[1.0], [2.0]])b_output = 0.1for i in range(len(X)):    before_activation = np.dot(state, w_cell_state) + X[i] * w_cell_input + b_cell    state = np.tanh(before_activation)    final_output = np.dot(state, w_output) + b_output    print ("before activation: ", before_activation)    print ("state: ", state)    print ("output: ", final_output)reader.py"""Utilities for parsing PTB text files."""from __future__ import absolute_importfrom __future__ import divisionfrom __future__ import print_functionimport collectionsimport osimport tensorflow as tfdef _read_words(filename):  with tf.gfile.GFile(filename, "r") as f:    return f.read().decode("utf-8").replace("\n", "<eos>").split()def _build_vocab(filename):  data = _read_words(filename)  counter = collections.Counter(data)  count_pairs = sorted(counter.items(), key=lambda x: (-x[1], x[0]))  words, _ = list(zip(*count_pairs))  word_to_id = dict(zip(words, range(len(words))))  return word_to_iddef _file_to_word_ids(filename, word_to_id):  data = _read_words(filename)  return [word_to_id[word] for word in data if word in word_to_id]def ptb_raw_data(data_path=None):  """Load PTB raw data from data directory "data_path".  Reads PTB text files, converts strings to integer ids,  and performs mini-batching of the inputs.  The PTB dataset comes from Tomas Mikolov's webpage:  http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz  Args:    data_path: string path to the directory where simple-examples.tgz has      been extracted.  Returns:    tuple (train_data, valid_data, test_data, vocabulary)    where each of the data objects can be passed to PTBIterator.  """  train_path = os.path.join(data_path, "ptb.train.txt")  valid_path = os.path.join(data_path, "ptb.valid.txt")  test_path = os.path.join(data_path, "ptb.test.txt")  word_to_id = _build_vocab(train_path)  train_data = _file_to_word_ids(train_path, word_to_id)  valid_data = _file_to_word_ids(valid_path, word_to_id)  test_data = _file_to_word_ids(test_path, word_to_id)  vocabulary = len(word_to_id)  return train_data, valid_data, test_data, vocabularydef ptb_producer(raw_data, batch_size, num_steps, name=None):  """Iterate on the raw PTB data.  This chunks up raw_data into batches of examples and returns Tensors that  are drawn from these batches.  Args:    raw_data: one of the raw data outputs from ptb_raw_data.    batch_size: int, the batch size.    num_steps: int, the number of unrolls.    name: the name of this operation (optional).  Returns:    A pair of Tensors, each shaped [batch_size, num_steps]. The second element    of the tuple is the same data time-shifted to the right by one.  Raises:    tf.errors.InvalidArgumentError: if batch_size or num_steps are too high.  """  with tf.name_scope(name, "PTBProducer", [raw_data, batch_size, num_steps]):    raw_data = tf.convert_to_tensor(raw_data, name="raw_data", dtype=tf.int32)    data_len = tf.size(raw_data)    batch_len = data_len // batch_size    data = tf.reshape(raw_data[0 : batch_size * batch_len],                      [batch_size, batch_len])    epoch_size = (batch_len - 1) // num_steps    assertion = tf.assert_positive(        epoch_size,        message="epoch_size == 0, decrease batch_size or num_steps")    with tf.control_dependencies([assertion]):      epoch_size = tf.identity(epoch_size, name="epoch_size")    i = tf.train.range_input_producer(epoch_size, shuffle=False).dequeue()    x = tf.strided_slice(data, [0, i * num_steps],                         [batch_size, (i + 1) * num_steps])    x.set_shape([batch_size, num_steps])    y = tf.strided_slice(data, [0, i * num_steps + 1],                         [batch_size, (i + 1) * num_steps + 1])    y.set_shape([batch_size, num_steps])    return x, yPTB数据集介绍DATA_PATH = "../../datasets/PTB_data"train_data, valid_data, test_data, _ = reader.ptb_raw_data(DATA_PATH)print len(train_data)print train_data[:100]# ptb_producer返回的为一个二维的tuple数据。result = reader.ptb_producer(train_data, 4, 5)# 通过队列依次读取batch。with tf.Session() as sess:    coord = tf.train.Coordinator()    threads = tf.train.start_queue_runners(sess=sess, coord=coord)    for i in range(3):        x, y = sess.run(result)        print "X%d: "%i, x        print "Y%d: "%i, y    coord.request_stop()    coord.join(threads) 使用循环神经网络实现语言模型 import numpy as npimport tensorflow as tfimport readerDATA_PATH = "../../datasets/PTB_data"HIDDEN_SIZE = 200NUM_LAYERS = 2VOCAB_SIZE = 10000LEARNING_RATE = 1.0TRAIN_BATCH_SIZE = 20TRAIN_NUM_STEP = 35EVAL_BATCH_SIZE = 1EVAL_NUM_STEP = 1NUM_EPOCH = 2KEEP_PROB = 0.5MAX_GRAD_NORM = 5class PTBModel(object):    def __init__(self, is_training, batch_size, num_steps):                self.batch_size = batch_size        self.num_steps = num_steps                # 定义输入层。        self.input_data = tf.placeholder(tf.int32, [batch_size, num_steps])        self.targets = tf.placeholder(tf.int32, [batch_size, num_steps])                # 定义使用LSTM结构及训练时使用dropout。        lstm_cell = tf.contrib.rnn.BasicLSTMCell(HIDDEN_SIZE)        if is_training:            lstm_cell = tf.contrib.rnn.DropoutWrapper(lstm_cell, output_keep_prob=KEEP_PROB)        cell = tf.contrib.rnn.MultiRNNCell([lstm_cell]*NUM_LAYERS)                # 初始化最初的状态。        self.initial_state = cell.zero_state(batch_size, tf.float32)        embedding = tf.get_variable("embedding", [VOCAB_SIZE, HIDDEN_SIZE])                # 将原本单词ID转为单词向量。        inputs = tf.nn.embedding_lookup(embedding, self.input_data)                if is_training:            inputs = tf.nn.dropout(inputs, KEEP_PROB)        # 定义输出列表。        outputs = []        state = self.initial_state        with tf.variable_scope("RNN"):            for time_step in range(num_steps):                if time_step > 0: tf.get_variable_scope().reuse_variables()                cell_output, state = cell(inputs[:, time_step, :], state)                outputs.append(cell_output)         output = tf.reshape(tf.concat(outputs, 1), [-1, HIDDEN_SIZE])        weight = tf.get_variable("weight", [HIDDEN_SIZE, VOCAB_SIZE])        bias = tf.get_variable("bias", [VOCAB_SIZE])        logits = tf.matmul(output, weight) + bias                # 定义交叉熵损失函数和平均损失。        loss = tf.contrib.legacy_seq2seq.sequence_loss_by_example(            [logits],            [tf.reshape(self.targets, [-1])],            [tf.ones([batch_size * num_steps], dtype=tf.float32)])        self.cost = tf.reduce_sum(loss) / batch_size        self.final_state = state                # 只在训练模型时定义反向传播操作。        if not is_training: return        trainable_variables = tf.trainable_variables()        # 控制梯度大小，定义优化方法和训练步骤。        grads, _ = tf.clip_by_global_norm(tf.gradients(self.cost, trainable_variables), MAX_GRAD_NORM)        optimizer = tf.train.GradientDescentOptimizer(LEARNING_RATE)        self.train_op = optimizer.apply_gradients(zip(grads, trainable_variables))def run_epoch(session, model, data, train_op, output_log, epoch_size):    total_costs = 0.0    iters = 0    state = session.run(model.initial_state)    # 训练一个epoch。    for step in range(epoch_size):        x, y = session.run(data)        cost, state, _ = session.run([model.cost, model.final_state, train_op],                                        {model.input_data: x, model.targets: y, model.initial_state: state})        total_costs += cost        iters += model.num_steps        if output_log and step % 100 == 0:            print("After %d steps, perplexity is %.3f" % (step, np.exp(total_costs / iters)))    return np.exp(total_costs / iters)def main():    train_data, valid_data, test_data, _ = reader.ptb_raw_data(DATA_PATH)    # 计算一个epoch需要训练的次数    train_data_len = len(train_data)    train_batch_len = train_data_len // TRAIN_BATCH_SIZE    train_epoch_size = (train_batch_len - 1) // TRAIN_NUM_STEP    valid_data_len = len(valid_data)    valid_batch_len = valid_data_len // EVAL_BATCH_SIZE    valid_epoch_size = (valid_batch_len - 1) // EVAL_NUM_STEP    test_data_len = len(test_data)    test_batch_len = test_data_len // EVAL_BATCH_SIZE    test_epoch_size = (test_batch_len - 1) // EVAL_NUM_STEP    initializer = tf.random_uniform_initializer(-0.05, 0.05)    with tf.variable_scope("language_model", reuse=None, initializer=initializer):        train_model = PTBModel(True, TRAIN_BATCH_SIZE, TRAIN_NUM_STEP)    with tf.variable_scope("language_model", reuse=True, initializer=initializer):        eval_model = PTBModel(False, EVAL_BATCH_SIZE, EVAL_NUM_STEP)    # 训练模型。    with tf.Session() as session:        tf.global_variables_initializer().run()        train_queue = reader.ptb_producer(train_data, train_model.batch_size, train_model.num_steps)        eval_queue = reader.ptb_producer(valid_data, eval_model.batch_size, eval_model.num_steps)        test_queue = reader.ptb_producer(test_data, eval_model.batch_size, eval_model.num_steps)        coord = tf.train.Coordinator()        threads = tf.train.start_queue_runners(sess=session, coord=coord)        for i in range(NUM_EPOCH):            print("In iteration: %d" % (i + 1))            run_epoch(session, train_model, train_queue, train_model.train_op, True, train_epoch_size)            valid_perplexity = run_epoch(session, eval_model, eval_queue, tf.no_op(), False, valid_epoch_size)            print("Epoch: %d Validation Perplexity: %.3f" % (i + 1, valid_perplexity))        test_perplexity = run_epoch(session, eval_model, test_queue, tf.no_op(), False, test_epoch_size)        print("Test Perplexity: %.3f" % test_perplexity)        coord.request_stop()        coord.join(threads)if __name__ == "__main__":    main()SKlearn封装例子from sklearn import model_selectionfrom sklearn import datasetsfrom sklearn import metricsimport tensorflow as tfimport numpy as npfrom tensorflow.contrib.learn.python.learn.estimators.estimator import SKCompatlearn = tf.contrib.learndef my_model(features, target):    target = tf.one_hot(target, 3, 1, 0)        # 计算预测值及损失函数。    logits = tf.contrib.layers.fully_connected(features, 3, tf.nn.softmax)    loss = tf.losses.softmax_cross_entropy(target, logits)        # 创建优化步骤。    train_op = tf.contrib.layers.optimize_loss(        loss,        tf.contrib.framework.get_global_step(),        optimizer='Adam',        learning_rate=0.01)    return tf.arg_max(logits, 1), loss, train_opiris = datasets.load_iris()x_train, x_test, y_train, y_test = model_selection.train_test_split(    iris.data, iris.target, test_size=0.2, random_state=0)x_train, x_test = map(np.float32, [x_train, x_test])classifier = SKCompat(learn.Estimator(model_fn=my_model, model_dir="Models/model_1"))classifier.fit(x_train, y_train, steps=800)y_predicted = [i for i in classifier.predict(x_test)]score = metrics.accuracy_score(y_test, y_predicted)print('Accuracy: %.2f%%' % (score * 100))预测正弦函数import numpy as npimport tensorflow as tffrom tensorflow.contrib.learn.python.learn.estimators.estimator import SKCompatfrom tensorflow.python.ops import array_ops as array_ops_import matplotlib.pyplot as pltlearn = tf.contrib.learnHIDDEN_SIZE = 30NUM_LAYERS = 2TIMESTEPS = 10TRAINING_STEPS = 3000BATCH_SIZE = 32TRAINING_EXAMPLES = 10000TESTING_EXAMPLES = 1000SAMPLE_GAP = 0.01def generate_data(seq):    X = []    y = []    for i in range(len(seq) - TIMESTEPS - 1):        X.append([seq[i: i + TIMESTEPS]])        y.append([seq[i + TIMESTEPS]])    return np.array(X, dtype=np.float32), np.array(y, dtype=np.float32)def lstm_model(X, y):    lstm_cell = tf.contrib.rnn.BasicLSTMCell(HIDDEN_SIZE, state_is_tuple=True)    cell = tf.contrib.rnn.MultiRNNCell([lstm_cell] * NUM_LAYERS)        output, _ = tf.nn.dynamic_rnn(cell, X, dtype=tf.float32)    output = tf.reshape(output, [-1, HIDDEN_SIZE])        # 通过无激活函数的全联接层计算线性回归，并将数据压缩成一维数组的结构。    predictions = tf.contrib.layers.fully_connected(output, 1, None)        # 将predictions和labels调整统一的shape    labels = tf.reshape(y, [-1])    predictions=tf.reshape(predictions, [-1])        loss = tf.losses.mean_squared_error(predictions, labels)        train_op = tf.contrib.layers.optimize_loss(        loss, tf.contrib.framework.get_global_step(),        optimizer="Adagrad", learning_rate=0.1)    return predictions, loss, train_op# 封装之前定义的lstm。regressor = SKCompat(learn.Estimator(model_fn=lstm_model,model_dir="Models/model_2"))# 生成数据。test_start = TRAINING_EXAMPLES * SAMPLE_GAPtest_end = (TRAINING_EXAMPLES + TESTING_EXAMPLES) * SAMPLE_GAPtrain_X, train_y = generate_data(np.sin(np.linspace(    0, test_start, TRAINING_EXAMPLES, dtype=np.float32)))test_X, test_y = generate_data(np.sin(np.linspace(    test_start, test_end, TESTING_EXAMPLES, dtype=np.float32)))# 拟合数据。regressor.fit(train_X, train_y, batch_size=BATCH_SIZE, steps=TRAINING_STEPS)# 计算预测值。predicted = [[pred] for pred in regressor.predict(test_X)]# 计算MSE。rmse = np.sqrt(((predicted - test_y) ** 2).mean(axis=0))print ("Mean Square Error is: %f" % rmse[0])plot_predicted, = plt.plot(predicted, label='predicted')plot_test, = plt.plot(test_y, label='real_sin')plt.legend([plot_predicted, plot_test],['predicted', 'real_sin'])plt.show()mnist_inference.pyimport tensorflow as tfINPUT_NODE = 784OUTPUT_NODE = 10LAYER1_NODE = 500def get_weight_variable(shape, regularizer):    weights = tf.get_variable("weights", shape, initializer=tf.truncated_normal_initializer(stddev=0.1))    if regularizer != None: tf.add_to_collection('losses', regularizer(weights))    return weightsdef inference(input_tensor, regularizer):    with tf.variable_scope('layer1'):        weights = get_weight_variable([INPUT_NODE, LAYER1_NODE], regularizer)        biases = tf.get_variable("biases", [LAYER1_NODE], initializer=tf.constant_initializer(0.0))        layer1 = tf.nn.relu(tf.matmul(input_tensor, weights) + biases)    with tf.variable_scope('layer2'):        weights = get_weight_variable([LAYER1_NODE, OUTPUT_NODE], regularizer)        biases = tf.get_variable("biases", [OUTPUT_NODE], initializer=tf.constant_initializer(0.0))        layer2 = tf.matmul(layer1, weights) + biases    return layer2命名空间import tensorflow as tfwith tf.variable_scope("foo"):    a = tf.get_variable("bar", [1])    print (a.name)with tf.variable_scope("bar"):    b = tf.get_variable("bar", [1])    print (b.name)with tf.name_scope("a"):    a = tf.Variable([1])    print (a.name)        a = tf.get_variable("b", [1])    print (a.name)with tf.name_scope("input1"):    input1 = tf.constant([1.0, 2.0, 3.0], name="input2")with tf.name_scope("input2"):    input2 = tf.Variable(tf.random_uniform([3]), name="input2")output = tf.add_n([input1, input2], name="add")writer = tf.summary.FileWriter("/log/simple_example.log", tf.get_default_graph())writer.close()改造后的mnist_trainimport tensorflow as tffrom tensorflow.examples.tutorials.mnist import input_dataimport mnist_inferenceBATCH_SIZE = 100LEARNING_RATE_BASE = 0.8LEARNING_RATE_DECAY = 0.99REGULARIZATION_RATE = 0.0001TRAINING_STEPS = 3000MOVING_AVERAGE_DECAY = 0.99def train(mnist):    #  输入数据的命名空间。    with tf.name_scope('input'):        x = tf.placeholder(tf.float32, [None, mnist_inference.INPUT_NODE], name='x-input')        y_ = tf.placeholder(tf.float32, [None, mnist_inference.OUTPUT_NODE], name='y-input')    regularizer = tf.contrib.layers.l2_regularizer(REGULARIZATION_RATE)    y = mnist_inference.inference(x, regularizer)    global_step = tf.Variable(0, trainable=False)        # 处理滑动平均的命名空间。    with tf.name_scope("moving_average"):        variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)        variables_averages_op = variable_averages.apply(tf.trainable_variables())       # 计算损失函数的命名空间。    with tf.name_scope("loss_function"):        cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))        cross_entropy_mean = tf.reduce_mean(cross_entropy)        loss = cross_entropy_mean + tf.add_n(tf.get_collection('losses'))        # 定义学习率、优化方法及每一轮执行训练的操作的命名空间。    with tf.name_scope("train_step"):        learning_rate = tf.train.exponential_decay(            LEARNING_RATE_BASE,            global_step,            mnist.train.num_examples / BATCH_SIZE, LEARNING_RATE_DECAY,            staircase=True)        train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)        with tf.control_dependencies([train_step, variables_averages_op]):            train_op = tf.no_op(name='train')                # 训练模型。    with tf.Session() as sess:        tf.global_variables_initializer().run()        for i in range(TRAINING_STEPS):            xs, ys = mnist.train.next_batch(BATCH_SIZE)            if i % 1000 == 0:                # 配置运行时需要记录的信息。                run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)                # 运行时记录运行信息的proto。                run_metadata = tf.RunMetadata()                _, loss_value, step = sess.run(                    [train_op, loss, global_step], feed_dict={x: xs, y_: ys},                    options=run_options, run_metadata=run_metadata)                print("After %d training step(s), loss on training batch is %g." % (step, loss_value))            else:                _, loss_value, step = sess.run([train_op, loss, global_step], feed_dict={x: xs, y_: ys})    writer = tf.summary.FileWriter("/log/modified_mnist_train.log", tf.get_default_graph())    writer.close()def main(argv=None):     mnist = input_data.read_data_sets("../../datasets/MNIST_data", one_hot=True)    train(mnist)if __name__ == '__main__':    main()监控指标可视化import tensorflow as tffrom tensorflow.examples.tutorials.mnist import input_dataSUMMARY_DIR = "/log/supervisor.log"BATCH_SIZE = 100TRAIN_STEPS = 3000def variable_summaries(var, name):    with tf.name_scope('summaries'):        tf.summary.histogram(name, var)        mean = tf.reduce_mean(var)        tf.summary.scalar('mean/' + name, mean)        stddev = tf.sqrt(tf.reduce_mean(tf.square(var - mean)))        tf.summary.scalar('stddev/' + name, stddev)  def nn_layer(input_tensor, input_dim, output_dim, layer_name, act=tf.nn.relu):    with tf.name_scope(layer_name):        with tf.name_scope('weights'):            weights = tf.Variable(tf.truncated_normal([input_dim, output_dim], stddev=0.1))            variable_summaries(weights, layer_name + '/weights')        with tf.name_scope('biases'):            biases = tf.Variable(tf.constant(0.0, shape=[output_dim]))            variable_summaries(biases, layer_name + '/biases')        with tf.name_scope('Wx_plus_b'):            preactivate = tf.matmul(input_tensor, weights) + biases            tf.summary.histogram(layer_name + '/pre_activations', preactivate)        activations = act(preactivate, name='activation')                        # 记录神经网络节点输出在经过激活函数之后的分布。        tf.summary.histogram(layer_name + '/activations', activations)        return activationsdef main():    mnist = input_data.read_data_sets("../../datasets/MNIST_data", one_hot=True)    with tf.name_scope('input'):        x = tf.placeholder(tf.float32, [None, 784], name='x-input')        y_ = tf.placeholder(tf.float32, [None, 10], name='y-input')    with tf.name_scope('input_reshape'):        image_shaped_input = tf.reshape(x, [-1, 28, 28, 1])        tf.summary.image('input', image_shaped_input, 10)    hidden1 = nn_layer(x, 784, 500, 'layer1')    y = nn_layer(hidden1, 500, 10, 'layer2', act=tf.identity)    with tf.name_scope('cross_entropy'):        cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=y, labels=y_))        tf.summary.scalar('cross_entropy', cross_entropy)    with tf.name_scope('train'):        train_step = tf.train.AdamOptimizer(0.001).minimize(cross_entropy)    with tf.name_scope('accuracy'):        with tf.name_scope('correct_prediction'):            correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))        with tf.name_scope('accuracy'):            accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))        tf.summary.scalar('accuracy', accuracy)    merged = tf.summary.merge_all()    with tf.Session() as sess:                summary_writer = tf.summary.FileWriter(SUMMARY_DIR, sess.graph)        tf.global_variables_initializer().run()        for i in range(TRAIN_STEPS):            xs, ys = mnist.train.next_batch(BATCH_SIZE)            # 运行训练步骤以及所有的日志生成操作，得到这次运行的日志。            summary, _ = sess.run([merged, train_step], feed_dict={x: xs, y_: ys})            # 将得到的所有日志写入日志文件，这样TensorBoard程序就可以拿到这次运行所对应的            # 运行信息。            summary_writer.add_summary(summary, i)    summary_writer.close()def nn_layer(input_tensor, input_dim, output_dim, layer_name, act=tf.nn.relu):    with tf.name_scope(layer_name):        with tf.name_scope('weights'):            weights = tf.Variable(tf.truncated_normal([input_dim, output_dim], stddev=0.1))            variable_summaries(weights, layer_name + '/weights')        with tf.name_scope('biases'):            biases = tf.Variable(tf.constant(0.0, shape=[output_dim]))            variable_summaries(biases, layer_name + '/biases')        with tf.name_scope('Wx_plus_b'):            preactivate = tf.matmul(input_tensor, weights) + biases            tf.summary.histogram(layer_name + '/pre_activations', preactivate)        activations = act(preactivate, name='activation')                        # 记录神经网络节点输出在经过激活函数之后的分布。        tf.summary.histogram(layer_name + '/activations', activations)        return activationsif __name__ == '__main__':    main()GPU基本操作.py# coding=utf-8import tensorflow as tfa = tf.constant([1.0, 2.0, 3.0], shape=[3], name='a')b = tf.constant([1.0, 2.0, 3.0], shape=[3], name='b')c = a + b# 通过log_device_placement参数来记录运行每一个运算的设备。sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))print sess.run(c)# 通过tf.device将运算指定到特定的设备上。with tf.device('/cpu:0'):a = tf.constant([1.0, 2.0, 3.0], shape=[3], name='a')b = tf.constant([1.0, 2.0, 3.0], shape=[3], name='b')with tf.device('/gpu:1'):    c = a + bsess = tf.Session(config=tf.ConfigProto(log_device_placement=True))print sess.run(c)a_cpu = tf.Variable(0, name="a_cpu")with tf.device('/gpu:0'):a_gpu = tf.Variable(0, name="a_gpu")# 通过allow_soft_placement参数自动将无法放在GPU上的操作放回CPU上。sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=True))sess.run(tf.global_variables_initializer()) 多GPU并行.py  # coding=utf-8from datetime import datetimeimport osimport timeimport tensorflow as tfimport mnist_inference# 定义训练神经网络时需要用到的参数。BATCH_SIZE = 100 LEARNING_RATE_BASE = 0.001LEARNING_RATE_DECAY = 0.99REGULARAZTION_RATE = 0.0001TRAINING_STEPS = 1000MOVING_AVERAGE_DECAY = 0.99 N_GPU = 4# 定义日志和模型输出的路径。MODEL_SAVE_PATH = "logs_and_models/"MODEL_NAME = "model.ckpt"DATA_PATH = "output.tfrecords" # 定义输入队列得到训练数据，具体细节可以参考第七章。def get_input():    filename_queue = tf.train.string_input_producer([DATA_PATH])     reader = tf.TFRecordReader()_, serialized_example = reader.read(filename_queue)# 定义数据解析格式。    features = tf.parse_single_example(        serialized_example,        features={            'image_raw': tf.FixedLenFeature([], tf.string),            'pixels': tf.FixedLenFeature([], tf.int64),            'label': tf.FixedLenFeature([], tf.int64),        })    # 解析图片和标签信息。    decoded_image = tf.decode_raw(features['image_raw'], tf.uint8)    reshaped_image = tf.reshape(decoded_image, [784])    retyped_image = tf.cast(reshaped_image, tf.float32)    label = tf.cast(features['label'], tf.int32)        # 定义输入队列并返回。    min_after_dequeue = 10000    capacity = min_after_dequeue + 3 * BATCH_SIZEreturn tf.train.shuffle_batch(    [retyped_image, label],     batch_size=BATCH_SIZE,     capacity=capacity,     min_after_dequeue=min_after_dequeue)# 定义损失函数。def get_loss(x, y_, regularizer, scope, reuse_variables=None):    with tf.variable_scope(tf.get_variable_scope(), reuse=reuse_variables):        y = mnist_inference.inference(x, regularizer)    cross_entropy = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(y, y_))    regularization_loss = tf.add_n(tf.get_collection('losses', scope))    loss = cross_entropy + regularization_loss    return loss# 计算每一个变量梯度的平均值。def average_gradients(tower_grads):    average_grads = []    # 枚举所有的变量和变量在不同GPU上计算得出的梯度。    for grad_and_vars in zip(*tower_grads):        # 计算所有GPU上的梯度平均值。        grads = []        for g, _ in grad_and_vars:            expanded_g = tf.expand_dims(g, 0)            grads.append(expanded_g)        grad = tf.concat(grads, 0)        grad = tf.reduce_mean(grad, 0)        v = grad_and_vars[0][1]        grad_and_var = (grad, v)        # 将变量和它的平均梯度对应起来。        average_grads.append(grad_and_var)    # 返回所有变量的平均梯度，这个将被用于变量的更新。    return average_grads# 主训练过程。def main(argv=None):     # 将简单的运算放在CPU上，只有神经网络的训练过程放在GPU上。with tf.Graph().as_default(), tf.device('/cpu:0'):  # 定义基本的训练过程        x, y_ = get_input()        regularizer = tf.contrib.layers.l2_regularizer(REGULARAZTION_RATE)                global_step = tf.get_variable('global_step', [], initializer=tf.constant_initializer(0), trainable=False)        learning_rate = tf.train.exponential_decay(            LEARNING_RATE_BASE, global_step, 60000 / BATCH_SIZE, LEARNING_RATE_DECAY)                       opt = tf.train.GradientDescentOptimizer(learning_rate)                tower_grads = []        reuse_variables = False        # 将神经网络的优化过程跑在不同的GPU上。        for i in range(N_GPU):            # 将优化过程指定在一个GPU上。            with tf.device('/gpu:%d' % i):                with tf.name_scope('GPU_%d' % i) as scope:                    cur_loss = get_loss(x, y_, regularizer, scope, reuse_variables)                    reuse_variables = True                    grads = opt.compute_gradients(cur_loss)                    tower_grads.append(grads)                # 计算变量的平均梯度。        grads = average_gradients(tower_grads)        for grad, var in grads:            if grad is not None:            tf.histogram_summary('gradients_on_average/%s' % var.op.name, grad)        # 使用平均梯度更新参数。        apply_gradient_op = opt.apply_gradients(grads, global_step=global_step)        for var in tf.trainable_variables():            tf.histogram_summary(var.op.name, var)        # 计算变量的滑动平均值。        variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)        variables_to_average = (tf.trainable_variables() +tf.moving_average_variables())        variables_averages_op = variable_averages.apply(variables_to_average)        # 每一轮迭代需要更新变量的取值并更新变量的滑动平均值。        train_op = tf.group(apply_gradient_op, variables_averages_op)        saver = tf.train.Saver(tf.all_variables())        summary_op = tf.merge_all_summaries()                init = tf.initialize_all_variables()        with tf.Session(config=tf.ConfigProto(                allow_soft_placement=True, log_device_placement=True)) as sess:            # 初始化所有变量并启动队列。            init.run()            coord = tf.train.Coordinator()            threads = tf.train.start_queue_runners(sess=sess, coord=coord)            summary_writer = tf.train.SummaryWriter(MODEL_SAVE_PATH, sess.graph)            for step in range(TRAINING_STEPS):                # 执行神经网络训练操作，并记录训练操作的运行时间。                start_time = time.time()                _, loss_value = sess.run([train_op, cur_loss])                duration = time.time() - start_time                                # 每隔一段时间数据当前的训练进度，并统计训练速度。                if step != 0 and step % 10 == 0:                    # 计算使用过的训练数据个数。                    num_examples_per_step = BATCH_SIZE * N_GPU                    examples_per_sec = num_examples_per_step / duration                    sec_per_batch = duration / N_GPU                        # 输出训练信息。                    format_str = ('%s: step %d, loss = %.2f (%.1f examples/sec; %.3f sec/batch)')                    print (format_str % (datetime.now(), step, loss_value, examples_per_sec, sec_per_batch))                                        # 通过TensorBoard可视化训练过程。                    summary = sess.run(summary_op)                    summary_writer.add_summary(summary, step)                    # 每隔一段时间保存当前的模型。                if step % 1000 == 0 or (step + 1) == TRAINING_STEPS:                    checkpoint_path = os.path.join(MODEL_SAVE_PATH, MODEL_NAME)                    saver.save(sess, checkpoint_path, global_step=step)                    coord.request_stop()            coord.join(threads)        if __name__ == '__main__':tf.app.run()分布式TensorFlow.pyimport tensorflow as tf# 创建一个本地集群。c = tf.constant("Hello, distributed TensorFlow!")# server = tf.train.Server.create_local_server()# sess = tf.Session(server.target)# print sess.run(c)# # 创建两个集群# c = tf.constant("Hello from server1!")# cluster = tf.train.ClusterSpec({"local": ["localhost:2222", "localhost:2223"]})# server = tf.train.Server(cluster, job_name="local", task_index=0)# sess = tf.Session(server.target, config=tf.ConfigProto(log_device_placement=True))# print （ sess.run(c)）# server.join()### import tensorflow as tf# c = tf.constant("Hello from server2!")# cluster = tf.train.ClusterSpec({"local": ["localhost:2222", "localhost:2223"]})# server = tf.train.Server(cluster, job_name="local", task_index=1)# sess = tf.Session(server.target, config=tf.ConfigProto(log_device_placement=True))# print （ sess.run(c)）# server.join()##异步更新模式样例程序.py# coding=utf-8import timeimport tensorflow as tffrom tensorflow.examples.tutorials.mnist import input_dataimport mnist_inference# 配置神经网络的参数。BATCH_SIZE = 100LEARNING_RATE_BASE = 0.01LEARNING_RATE_DECAY = 0.99REGULARAZTION_RATE = 0.0001TRAINING_STEPS = 1000MOVING_AVERAGE_DECAY = 0.99# 模型保存的路径和文件名。MODEL_SAVE_PATH = "logs/log_async"DATA_PATH = "../../datasets/MNIST_data"FLAGS = tf.app.flags.FLAGS# 指定当前程序是参数服务器还是计算服务器。tf.app.flags.DEFINE_string('job_name', 'worker', ' "ps" or "worker" ')# 指定集群中的参数服务器地址。tf.app.flags.DEFINE_string(    'ps_hosts', ' tf-ps0:2222,tf-ps1:1111',    'Comma-separated list of hostname:port for the parameter server jobs. e.g. "tf-ps0:2222,tf-ps1:1111" ')# 指定集群中的计算服务器地址。tf.app.flags.DEFINE_string(    'worker_hosts', ' tf-worker0:2222,tf-worker1:1111',    'Comma-separated list of hostname:port for the worker jobs. e.g. "tf-worker0:2222,tf-worker1:1111" ')# 指定当前程序的任务ID。tf.app.flags.DEFINE_integer('task_id', 0, 'Task ID of the worker/replica running the training.')# 定义TensorFlow的计算图，并返回每一轮迭代时需要运行的操作。def build_model(x, y_, is_chief):    regularizer = tf.contrib.layers.l2_regularizer(REGULARAZTION_RATE)    # 通过和5.5节给出的mnist_inference.py代码计算神经网络前向传播的结果。    y = mnist_inference.inference(x, regularizer)    global_step = tf.Variable(0, trainable=False)    # 计算损失函数并定义反向传播过程。    cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))    cross_entropy_mean = tf.reduce_mean(cross_entropy)    loss = cross_entropy_mean + tf.add_n(tf.get_collection('losses'))    learning_rate = tf.train.exponential_decay(        LEARNING_RATE_BASE, global_step, 60000 / BATCH_SIZE, LEARNING_RATE_DECAY)    train_op = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)    # 定义每一轮迭代需要运行的操作。    if is_chief:        # 计算变量的滑动平均值。           variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)        variables_averages_op = variable_averages.apply(tf.trainable_variables())        with tf.control_dependencies([variables_averages_op, train_op]):            train_op = tf.no_op()    return global_step, loss, train_opdef main(argv=None):    # 解析flags并通过tf.train.ClusterSpec配置TensorFlow集群。    ps_hosts = FLAGS.ps_hosts.split(',')    worker_hosts = FLAGS.worker_hosts.split(',')    cluster = tf.train.ClusterSpec({"ps": ps_hosts, "worker": worker_hosts})    # 通过tf.train.ClusterSpec以及当前任务创建tf.train.Server。    server = tf.train.Server(cluster, job_name = FLAGS.job_name, task_index=FLAGS.task_id)    # 参数服务器只需要管理TensorFlow中的变量，不需要执行训练的过程。server.join()会    # 一致停在这条语句上。    if FLAGS.job_name == 'ps':        with tf.device("/cpu:0"):            server.join()    # 定义计算服务器需要运行的操作。    is_chief = (FLAGS.task_id == 0)    mnist = input_data.read_data_sets(DATA_PATH, one_hot=True)    device_setter = tf.train.replica_device_setter(worker_device="/job:worker/task:%d" % FLAGS.task_id, cluster=cluster)    with tf.device(device_setter):        # 定义输入并得到每一轮迭代需要运行的操作。        x = tf.placeholder(tf.float32, [None, mnist_inference.INPUT_NODE], name='x-input')        y_ = tf.placeholder(tf.float32, [None, mnist_inference.OUTPUT_NODE], name='y-input')        global_step, loss, train_op = build_model(x, y_, is_chief)        # 定义用于保存模型的saver。        saver = tf.train.Saver()        # 定义日志输出操作。        summary_op = tf.summary.merge_all()        # 定义变量初始化操作。        init_op = tf.global_variables_initializer()        # 通过tf.train.Supervisor管理训练深度学习模型时的通用功能。        sv = tf.train.Supervisor(            is_chief=is_chief,            logdir=MODEL_SAVE_PATH,            init_op=init_op,            summary_op=summary_op,            saver=saver,            global_step=global_step,            save_model_secs=60,            save_summaries_secs=60)        sess_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)        # 通过tf.train.Supervisor生成会话。        sess = sv.prepare_or_wait_for_session(server.target, config=sess_config)        step = 0        start_time = time.time()        # 执行迭代过程。        while not sv.should_stop():            xs, ys = mnist.train.next_batch(BATCH_SIZE)            _, loss_value, global_step_value = sess.run([train_op, loss, global_step], feed_dict={x: xs, y_: ys})            if global_step_value >= TRAINING_STEPS: break           # 每隔一段时间输出训练信息。            if step > 0 and step % 100 == 0:                duration = time.time() - start_time                sec_per_batch = duration / global_step_value                format_str = "After %d training steps (%d global steps), loss on training batch is %g.  (%.3f sec/batch)"                print format_str % (step, global_step_value, loss_value, sec_per_batch)            step += 1    sv.stop()if __name__ == "__main__":    tf.app.run()同步更新模式样例程序.py# coding=utf-8import timeimport tensorflow as tffrom tensorflow.examples.tutorials.mnist import input_dataimport mnist_inference# 配置神经网络的参数。BATCH_SIZE = 100 LEARNING_RATE_BASE = 0.8LEARNING_RATE_DECAY = 0.99REGULARAZTION_RATE = 0.0001TRAINING_STEPS = 10000MOVING_AVERAGE_DECAY = 0.99 MODEL_SAVE_PATH = "logs/log_sync"DATA_PATH = "../../datasets/MNIST_data"# 和异步模式类似的设置flags。FLAGS = tf.app.flags.FLAGStf.app.flags.DEFINE_string('job_name', 'worker', ' "ps" or "worker" ')tf.app.flags.DEFINE_string(    'ps_hosts', ' tf-ps0:2222,tf-ps1:1111',    'Comma-separated list of hostname:port for the parameter server jobs. e.g. "tf-ps0:2222,tf-ps1:1111" ')tf.app.flags.DEFINE_string(    'worker_hosts', ' tf-worker0:2222,tf-worker1:1111','Comma-separated list of hostname:port for the worker jobs. e.g. "tf-worker0:2222,tf-worker1:1111" ')tf.app.flags.DEFINE_integer('task_id', 0, 'Task ID of the worker/replica running the training.')# 和异步模式类似的定义TensorFlow的计算图。唯一的区别在于使用# tf.train.SyncReplicasOptimizer函数处理同步更新。def build_model(x, y_, n_workers, is_chief):    regularizer = tf.contrib.layers.l2_regularizer(REGULARAZTION_RATE)    y = mnist_inference.inference(x, regularizer)    global_step = tf.Variable(0, trainable=False)    variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)    variables_averages_op = variable_averages.apply(tf.trainable_variables())    cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))    cross_entropy_mean = tf.reduce_mean(cross_entropy)    loss = cross_entropy_mean + tf.add_n(tf.get_collection('losses'))    learning_rate = tf.train.exponential_decay(        LEARNING_RATE_BASE, global_step, 60000 / BATCH_SIZE, LEARNING_RATE_DECAY)       # 通过tf.train.SyncReplicasOptimizer函数实现同步更新。    opt = tf.train.SyncReplicasOptimizer(        tf.train.GradientDescentOptimizer(learning_rate),        replicas_to_aggregate=n_workers,        total_num_replicas=n_workers)    train_op = opt.minimize(loss, global_step=global_step)         if is_chief:        variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)        variables_averages_op = variable_averages.apply(tf.trainable_variables())        with tf.control_dependencies([variables_averages_op, train_op]):            train_op = tf.no_op()    return global_step, loss, train_op, optdef main(argv=None):     # 和异步模式类似的创建TensorFlow集群。    ps_hosts = FLAGS.ps_hosts.split(',')    worker_hosts = FLAGS.worker_hosts.split(',')    print ('PS hosts are: %s' % ps_hosts)    print ('Worker hosts are: %s' % worker_hosts)    n_workers = len(worker_hosts)    cluster = tf.train.ClusterSpec({"ps": ps_hosts, "worker": worker_hosts})    server = tf.train.Server(        cluster, job_name = FLAGS.job_name, task_index=FLAGS.task_id)    if FLAGS.job_name == 'ps':        with tf.device("/cpu:0"):            server.join()    is_chief = (FLAGS.task_id == 0)    mnist = input_data.read_data_sets(DATA_PATH, one_hot=True)       with tf.device(tf.train.replica_device_setter(            worker_device="/job:worker/task:%d" % FLAGS.task_id, cluster=cluster)):        x = tf.placeholder(tf.float32, [None, mnist_inference.INPUT_NODE], name='x-input')        y_ = tf.placeholder(tf.float32, [None, mnist_inference.OUTPUT_NODE], name='y-input')        global_step, loss, train_op, opt = build_model(x, y_, n_workers, is_chief)        # 和异步模式类似的声明一些辅助函数。        saver = tf.train.Saver()        summary_op = tf.summary.merge_all()        init_op = tf.global_variables_initializer()        # 在同步模式下，主计算服务器需要协调不同计算服务器计算得到的参数梯度并最终更新参数。        # 这需要主计算服务器完成一些额外的初始化工作。        if is_chief:            # 获取协调不同计算服务器的队列。在更新参数之前，主计算服务器需要先启动这些队列。            chief_queue_runner = opt.get_chief_queue_runner()            # 初始化同步更新队列的操作。            init_tokens_op = opt.get_init_tokens_op(0)             # 和异步模式类似的声明tf.train.Supervisor。        sv = tf.train.Supervisor(is_chief=is_chief,                                logdir=MODEL_SAVE_PATH,                                init_op=init_op,                                summary_op=summary_op,                                saver = saver,                                global_step=global_step,                                save_model_secs=60,                                save_summaries_secs=60)        sess_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)        sess = sv.prepare_or_wait_for_session(server.target, config=sess_config)                # 在主计算服务器上启动协调同步更新的队列并执行初始化操作。        if is_chief:            sv.start_queue_runners(sess, [chief_queue_runner])            sess.run(init_tokens_op)             # 和异步模式类似的运行迭代的训练过程。        step = 0        start_time = time.time()        while not sv.should_stop():            xs, ys = mnist.train.next_batch(BATCH_SIZE)            _, loss_value, global_step_value = sess.run([train_op, loss, global_step], feed_dict={x: xs, y_: ys})            if global_step_value >= TRAINING_STEPS: break            if step > 0 and step % 100 == 0:                duration = time.time() - start_time                sec_per_batch = duration / (global_step_value * n_workers)                format_str = "After %d training steps (%d global steps), loss on training batch is %g.  (%.3f sec/batch)"                print format_str % (step, global_step_value, loss_value, sec_per_batch)            step += 1    sv.stop()       if __name__ == "__main__":    tf.app.run()