tensorflow15《TensorFlow实战Google深度学习框架》笔记-07图像数据处理 code

01 TFRecord样例程序

# 《TensorFlow实战Google深度学习框架》07 图像数据处理# win10 Tensorflow1.0.1 python3.5.3# CUDA v8.0 cudnn-8.0-windows10-x64-v5.1# filename:ts07.01.py # TFRecord样例程序import tensorflow as tffrom tensorflow.examples.tutorials.mnist import input_dataimport numpy as np# 1. 将输入转化成TFRecord格式并保存# 定义函数转化变量类型。def _int64_feature(value):    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))def _bytes_feature(value):    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))# 读取mnist数据。mnist = input_data.read_data_sets("../../datasets/MNIST_data",dtype=tf.uint8, one_hot=True)images = mnist.train.imageslabels = mnist.train.labelspixels = images.shape[1]num_examples = mnist.train.num_examples# 输出TFRecord文件的地址。filename = "Records/output.tfrecords" # 需要存在Records目录writer = tf.python_io.TFRecordWriter(filename)for index in range(num_examples):    image_raw = images[index].tostring()    example = tf.train.Example(features=tf.train.Features(feature={        'pixels': _int64_feature(pixels),        'label': _int64_feature(np.argmax(labels[index])),        'image_raw': _bytes_feature(image_raw)    }))    writer.write(example.SerializeToString())writer.close()print("TFRecord文件已保存。")# 2. 读取TFRecord文件# 读取文件。reader = tf.TFRecordReader()filename_queue = tf.train.string_input_producer(["Records/output.tfrecords"])_,serialized_example = reader.read(filename_queue)# 解析读取的样例。features = tf.parse_single_example(    serialized_example,    features={        'image_raw':tf.FixedLenFeature([],tf.string),        'pixels':tf.FixedLenFeature([],tf.int64),        'label':tf.FixedLenFeature([],tf.int64)    })images = tf.decode_raw(features['image_raw'],tf.uint8)labels = tf.cast(features['label'],tf.int32)pixels = tf.cast(features['pixels'],tf.int32)sess = tf.Session()# 启动多线程处理输入数据。coord = tf.train.Coordinator()threads = tf.train.start_queue_runners(sess=sess,coord=coord)for i in range(10):    image, label, pixel = sess.run([images, labels, pixels])

02 TensorFlow图像处理函数

# 《TensorFlow实战Google深度学习框架》07 图像数据处理# win10 Tensorflow1.0.1 python3.5.3# CUDA v8.0 cudnn-8.0-windows10-x64-v5.1# filename:ts07.02.py # TensorFlow图像处理函数import matplotlib.pyplot as pltimport tensorflow as tfimport numpy as np# 1. 读取图片, [tensorflow-tutorial\Deep_Learning_with_TensorFlow\datasets\cat.jpg]image_raw_data = tf.gfile.FastGFile("../../datasets/cat.jpg", 'r').read()with tf.Session() as sess:    img_data = tf.image.decode_jpeg(image_raw_data)    # 输出解码之后的三维矩阵。    print(img_data.eval())    img_data.set_shape([1797, 2673, 3])    print(img_data.get_shape())# 2. 打印图片with tf.Session() as sess:    plt.imshow(img_data.eval())    plt.show()# 3. 重新调整图片大小with tf.Session() as sess:    resized = tf.image.resize_images(img_data, [300, 300], method=0)    # TensorFlow的函数处理图片后存储的数据是float32格式的，需要转换成uint8才能正确打印图片。    print    "Digital type: ", resized.dtype    cat = np.asarray(resized.eval(), dtype='uint8')    # tf.image.convert_image_dtype(rgb_image, tf.float32)    plt.imshow(cat)    plt.show()# 4. 裁剪和填充图片with tf.Session() as sess:    croped = tf.image.resize_image_with_crop_or_pad(img_data, 1000, 1000)    padded = tf.image.resize_image_with_crop_or_pad(img_data, 3000, 3000)    plt.imshow(croped.eval())    plt.show()    plt.imshow(padded.eval())    plt.show()# 5. 截取中间50%的图片with tf.Session() as sess:    central_cropped = tf.image.central_crop(img_data, 0.5)    plt.imshow(central_cropped.eval())    plt.show()# 6. 翻转图片with tf.Session() as sess:    # 上下翻转    # flipped1 = tf.image.flip_up_down(img_data)    # 左右翻转    # flipped2 = tf.image.flip_left_right(img_data)    # 对角线翻转    transposed = tf.image.transpose_image(img_data)    plt.imshow(transposed.eval())    plt.show()    # 以一定概率上下翻转图片。    # flipped = tf.image.random_flip_up_down(img_data)    # 以一定概率左右翻转图片。    # flipped = tf.image.random_flip_left_right(img_data)# 7. 图片色彩调整with tf.Session() as sess:    # 将图片的亮度-0.5。    # adjusted = tf.image.adjust_brightness(img_data, -0.5)    # 将图片的亮度-0.5    # adjusted = tf.image.adjust_brightness(img_data, 0.5)    # 在[-max_delta, max_delta)的范围随机调整图片的亮度。    adjusted = tf.image.random_brightness(img_data, max_delta=0.5)    # 将图片的对比度-5    # adjusted = tf.image.adjust_contrast(img_data, -5)    # 将图片的对比度+5    # adjusted = tf.image.adjust_contrast(img_data, 5)    # 在[lower, upper]的范围随机调整图的对比度。    # adjusted = tf.image.random_contrast(img_data, lower, upper)    plt.imshow(adjusted.eval())    plt.show()# 8. 添加色相和饱和度with tf.Session() as sess:    adjusted = tf.image.adjust_hue(img_data, 0.1)    # adjusted = tf.image.adjust_hue(img_data, 0.3)    # adjusted = tf.image.adjust_hue(img_data, 0.6)    # adjusted = tf.image.adjust_hue(img_data, 0.9)    # 在[-max_delta, max_delta]的范围随机调整图片的色相。max_delta的取值在[0, 0.5]之间。    # adjusted = tf.image.random_hue(image, max_delta)    # 将图片的饱和度-5。    # adjusted = tf.image.adjust_saturation(img_data, -5)    # 将图片的饱和度+5。    # adjusted = tf.image.adjust_saturation(img_data, 5)    # 在[lower, upper]的范围随机调整图的饱和度。    # adjusted = tf.image.random_saturation(img_data, lower, upper)    # 将代表一张图片的三维矩阵中的数字均值变为0，方差变为1。    # adjusted = tf.image.per_image_whitening(img_data)    plt.imshow(adjusted.eval())    plt.show()# 9. 添加标注框并裁减with tf.Session() as sess:    boxes = tf.constant([[[0.05, 0.05, 0.9, 0.7], [0.35, 0.47, 0.5, 0.56]]])    begin, size, bbox_for_draw = tf.image.sample_distorted_bounding_box(        tf.shape(img_data), bounding_boxes=boxes)    batched = tf.expand_dims(tf.image.convert_image_dtype(img_data, tf.float32), 0)    image_with_box = tf.image.draw_bounding_boxes(batched, bbox_for_draw)    distorted_image = tf.slice(img_data, begin, size)    plt.imshow(distorted_image.eval())    plt.show()

03 图像预处理完整样例

# 《TensorFlow实战Google深度学习框架》07 图像数据处理# win10 Tensorflow1.0.1 python3.5.3# CUDA v8.0 cudnn-8.0-windows10-x64-v5.1# filename:ts07.03.py # 图像预处理完整样例import tensorflow as tfimport numpy as npimport matplotlib.pyplot as plt# 1. 随机调整图片的色彩，定义两种顺序def distort_color(image, color_ordering=0):    if color_ordering == 0:        image = tf.image.random_brightness(image, max_delta=32./255.)        image = tf.image.random_saturation(image, lower=0.5, upper=1.5)        image = tf.image.random_hue(image, max_delta=0.2)        image = tf.image.random_contrast(image, lower=0.5, upper=1.5)    else:        image = tf.image.random_saturation(image, lower=0.5, upper=1.5)        image = tf.image.random_brightness(image, max_delta=32./255.)        image = tf.image.random_contrast(image, lower=0.5, upper=1.5)        image = tf.image.random_hue(image, max_delta=0.2)    return tf.clip_by_value(image, 0.0, 1.0)# 2. 对图片进行预处理，将图片转化成神经网络的输入层数据def preprocess_for_train(image, height, width, bbox):    # 查看是否存在标注框。    if bbox is None:        bbox = tf.constant([0.0, 0.0, 1.0, 1.0], dtype=tf.float32, shape=[1, 1, 4])    if image.dtype != tf.float32:        image = tf.image.convert_image_dtype(image, dtype=tf.float32)    # 随机的截取图片中一个块。    bbox_begin, bbox_size, _ = tf.image.sample_distorted_bounding_box(        tf.shape(image), bounding_boxes=bbox)    bbox_begin, bbox_size, _ = tf.image.sample_distorted_bounding_box(        tf.shape(image), bounding_boxes=bbox)    distorted_image = tf.slice(image, bbox_begin, bbox_size)    # 将随机截取的图片调整为神经网络输入层的大小。    distorted_image = tf.image.resize_images(distorted_image, [height, width], method=np.random.randint(4))    distorted_image = tf.image.random_flip_left_right(distorted_image)    distorted_image = distort_color(distorted_image, np.random.randint(2))    return distorted_image# 3. 读取图片image_raw_data = tf.gfile.FastGFile("../../datasets/cat.jpg", "r").read()with tf.Session() as sess:    img_data = tf.image.decode_jpeg(image_raw_data)    boxes = tf.constant([[[0.05, 0.05, 0.9, 0.7], [0.35, 0.47, 0.5, 0.56]]])    for i in range(9):        result = preprocess_for_train(img_data, 299, 299, boxes)        plt.imshow(result.eval())        plt.show()

04 队列操作

# 《TensorFlow实战Google深度学习框架》07 图像数据处理# win10 Tensorflow1.0.1 python3.5.3# CUDA v8.0 cudnn-8.0-windows10-x64-v5.1# filename:ts07.04.py # 队列操作import tensorflow as tf# 1. 创建队列，并操作里面的元素q = tf.FIFOQueue(2, "int32")init = q.enqueue_many(([0, 10],))x = q.dequeue()y = x + 1q_inc = q.enqueue([y])with tf.Session() as sess:    init.run()    for _ in range(5):        v, _ = sess.run([x, q_inc])        print(v)'''0101112'''# 2. 这个程序每隔1秒判断是否需要停止并打印自己的IDimport numpy as npimport threadingimport timedef MyLoop(coord, worker_id):    while not coord.should_stop():        if np.random.rand()<0.1:            print("Stoping from id: %d\n" % worker_id, coord.request_stop())        else:            print("Working on id: %d\n" % worker_id, time.sleep(1))# 3. 创建、启动并退出线程coord = tf.train.Coordinator()threads = [threading.Thread(target=MyLoop, args=(coord, i, )) for i in range(5)]for t in threads:t.start()coord.join(threads)'''Stoping from id: 1 NoneWorking on id: 0 None'''

05 多线程队列操作

# 《TensorFlow实战Google深度学习框架》07 图像数据处理# win10 Tensorflow1.0.1 python3.5.3# CUDA v8.0 cudnn-8.0-windows10-x64-v5.1# filename:ts07.05.py # 多线程队列操作import tensorflow as tf# 1. 定义队列及其操作queue = tf.FIFOQueue(100,"float")enqueue_op = queue.enqueue([tf.random_normal([1])])qr = tf.train.QueueRunner(queue, [enqueue_op] * 5)tf.train.add_queue_runner(qr)out_tensor = queue.dequeue()# 2. 启动线程with tf.Session() as sess:    coord = tf.train.Coordinator()    threads = tf.train.start_queue_runners(sess=sess, coord=coord)    for _ in range(3):        print(sess.run(out_tensor)[0])    coord.request_stop()    coord.join(threads)'''-0.08383741.52686-0.267706'''

06 输入文件队列

# 《TensorFlow实战Google深度学习框架》07 图像数据处理# win10 Tensorflow1.0.1 python3.5.3# CUDA v8.0 cudnn-8.0-windows10-x64-v5.1# filename:ts07.06.py # 输入文件队列import tensorflow as tf# 1. 生成文件存储样例数据def _int64_feature(value):    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))num_shards = 2instances_per_shard = 2for i in range(num_shards):    filename = ('Records/data.tfrecords-%.5d-of-%.5d' % (i, num_shards))    # 将Example结构写入TFRecord文件。    writer = tf.python_io.TFRecordWriter(filename)    for j in range(instances_per_shard):    # Example结构仅包含当前样例属于第几个文件以及是当前文件的第几个样本。        example = tf.train.Example(features=tf.train.Features(feature={            'i': _int64_feature(i),            'j': _int64_feature(j)}))        writer.write(example.SerializeToString())    writer.close()# 2. 读取文件files = tf.train.match_filenames_once("Records/data.tfrecords-*")filename_queue = tf.train.string_input_producer(files, shuffle=False)reader = tf.TFRecordReader()_, serialized_example = reader.read(filename_queue)features = tf.parse_single_example(      serialized_example,      features={          'i': tf.FixedLenFeature([], tf.int64),          'j': tf.FixedLenFeature([], tf.int64),      })with tf.Session() as sess:    tf.global_variables_initializer().run()    print(sess.run(files))    coord = tf.train.Coordinator()    threads = tf.train.start_queue_runners(sess=sess, coord=coord)    for i in range(6):        print(sess.run([features['i'], features['j']]))    coord.request_stop()    coord.join(threads)'''[b'Records\\data.tfrecords-00000-of-00002' b'Records\\data.tfrecords-00001-of-00002'][0, 0][0, 1][1, 0][1, 1][0, 0][0, 1]'''# 3. 组合训练数据（Batching）example, label = features['i'], features['j']batch_size = 2capacity = 1000 + 3 * batch_sizecapacity = 1000 + 3 * batch_sizeexample_batch, label_batch = tf.train.batch([example, label], batch_size=batch_size, capacity=capacity)with tf.Session() as sess:    tf.global_variables_initializer().run()    coord = tf.train.Coordinator()    threads = tf.train.start_queue_runners(sess=sess, coord=coord)    for i in range(3):        cur_example_batch, cur_label_batch = sess.run([example_batch, label_batch])        print(cur_example_batch, cur_label_batch)    coord.request_stop()    coord.join(threads)'''[0 0] [0 1][1 1] [0 1][0 0] [0 1]'''

07 输入数据处理框架

# 《TensorFlow实战Google深度学习框架》07 图像数据处理# win10 Tensorflow1.0.1 python3.5.3# CUDA v8.0 cudnn-8.0-windows10-x64-v5.1# filename:ts07.07.py # 输入数据处理框架import tensorflow as tf# 1. 创建文件列表，通过文件列表创建输入文件队列，读取文件为本章第一节创建的文件files = tf.train.match_filenames_once("Records/output.tfrecords")filename_queue = tf.train.string_input_producer(files, shuffle=False)# 2. 解析TFRecord文件里的数据# 读取文件。reader = tf.TFRecordReader()_,serialized_example = reader.read(filename_queue)# 3. 解析读取的样例。features = tf.parse_single_example(    serialized_example,    features={        'image_raw':tf.FixedLenFeature([],tf.string),        'pixels':tf.FixedLenFeature([],tf.int64),        'label':tf.FixedLenFeature([],tf.int64)    })decoded_images = tf.decode_raw(features['image_raw'],tf.uint8)retyped_images = tf.cast(decoded_images, tf.float32)labels = tf.cast(features['label'],tf.int32)#pixels = tf.cast(features['pixels'],tf.int32)images = tf.reshape(retyped_images, [784])# 4. 将文件以100个为一组打包min_after_dequeue = 10000batch_size = 100capacity = min_after_dequeue + 3 * batch_sizeimage_batch, label_batch = tf.train.shuffle_batch([images, labels],                                                    batch_size=batch_size,                                                    capacity=capacity,                                                    min_after_dequeue=min_after_dequeue)# 5. 训练模型def inference(input_tensor, weights1, biases1, weights2, biases2):    layer1 = tf.nn.relu(tf.matmul(input_tensor, weights1) + biases1)    return tf.matmul(layer1, weights2) + biases2# 模型相关的参数INPUT_NODE = 784OUTPUT_NODE = 10LAYER1_NODE = 500REGULARAZTION_RATE = 0.0001TRAINING_STEPS = 5000weights1 = tf.Variable(tf.truncated_normal([INPUT_NODE, LAYER1_NODE], stddev=0.1))biases1 = tf.Variable(tf.constant(0.1, shape=[LAYER1_NODE]))weights2 = tf.Variable(tf.truncated_normal([LAYER1_NODE, OUTPUT_NODE], stddev=0.1))biases2 = tf.Variable(tf.constant(0.1, shape=[OUTPUT_NODE]))y = inference(image_batch, weights1, biases1, weights2, biases2)# 计算交叉熵及其平均值cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=label_batch)cross_entropy_mean = tf.reduce_mean(cross_entropy)# 损失函数的计算regularizer = tf.contrib.layers.l2_regularizer(REGULARAZTION_RATE)regularaztion = regularizer(weights1) + regularizer(weights2)loss = cross_entropy_mean + regularaztion# 优化损失函数train_step = tf.train.GradientDescentOptimizer(0.01).minimize(loss)# 初始化回话并开始训练过程。with tf.Session() as sess:    tf.global_variables_initializer().run()    coord = tf.train.Coordinator()    threads = tf.train.start_queue_runners(sess=sess, coord=coord)    # 循环的训练神经网络。    for i in range(TRAINING_STEPS):        if i % 1000 == 0:            print("After %d training step(s), loss is %g " % (i, sess.run(loss)))        sess.run(train_step)    coord.request_stop()    coord.join(threads)'''After 0 training step(s), loss is 425.467 After 1000 training step(s), loss is 2.40997 After 2000 training step(s), loss is 1.96758 After 3000 training step(s), loss is 1.87264 After 4000 training step(s), loss is 1.72888 '''