TensorLayer (TuneLayer) 实现 DropConnect

来源：互联网发布：挣钱的软件编辑：程序博客网时间：2024/05/01 11:26

DropConnect 是 Hinton DropOut 之后出现的一种 Regularization 方法，相比DropOut的好处是，它在模型较小的情况下依然能保证准确度。苦于网络上很难找到python例子，这里贴出一个用 TensorLayer 实现的代码，以供参考。

Paper of DropConnect

import tensorflow as tf
import tensorlayer as tl # 注意！！！ TensorLayer 现在更名为 TuneLayer 了 https://github.com/zsdonghao/tunelayer
from tensorlayer.layers import set_keep
import numpy as np
import time

X_train, y_train, X_val, y_val, X_test, y_test = \
tl.files.load_mnist_dataset(shape=(-1,784))

sess = tf.InteractiveSession()

# placeholder
x = tf.placeholder(tf.float32, shape=[None, 784], name='x')
y_ = tf.placeholder(tf.int64, shape=[None, ], name='y_')

network = tl.layers.InputLayer(x, name='input_layer')

network = tl.layers.DropconnectDenseLayer(network, keep = 0.8,
n_units=800, act = tf.nn.relu,
name='dropconnect_relu1')
network = tl.layers.DropconnectDenseLayer(network, keep = 0.5,
n_units=800, act = tf.nn.relu,
name='dropconnect_relu2')
network = tl.layers.DropconnectDenseLayer(network, keep = 0.5,
n_units=10,
act = tl.activation.identity,

name='output_layer')

y = network.outputs
y_op = tf.argmax(tf.nn.softmax(y), 1)
cost = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(y, y_))

params = network.all_params

# train
n_epoch = 500
batch_size = 128
learning_rate = 0.0001
print_freq = 10
train_op = tf.train.AdamOptimizer(learning_rate, beta1=0.9, beta2=0.999,
epsilon=1e-08, use_locking=False).minimize(cost)

sess.run(tf.initialize_all_variables()) # initialize all variables

network.print_params()
network.print_layers()

print(' learning_rate: %f' % learning_rate)
print(' batch_size: %d' % batch_size)

for epoch in range(n_epoch):
start_time = time.time()
for X_train_a, y_train_a in tl.iterate.minibatches(X_train, y_train,
batch_size, shuffle=True):
feed_dict = {x: X_train_a, y_: y_train_a}
feed_dict.update( network.all_drop ) # enable all dropout/dropconnect/denoising layers
sess.run(train_op, feed_dict=feed_dict)

if epoch + 1 == 1 or (epoch + 1) % print_freq == 0:
             print("Epoch %d of %d took %fs" % (epoch + 1, n_epoch, time.time() - start_time))
       dp_dict = tl.utils.dict_to_one( network.all_drop ) # disable all dropout/dropconnect/denoising layers
       feed_dict = {x: X_train, y_: y_train}
       feed_dict.update(dp_dict)
       print(" train loss: %f" % sess.run(cost, feed_dict=feed_dict))
       dp_dict = tl.utils.dict_to_one( network.all_drop )
       feed_dict = {x: X_val, y_: y_val}
       feed_dict.update(dp_dict)
       print(" val loss: %f" % sess.run(cost, feed_dict=feed_dict))
       print(" val acc: %f" % np.mean(y_val ==
       sess.run(y_op, feed_dict=feed_dict)))
       try:
       # You can visualize the weight of 1st hidden layer as follow.
       tl.visualize.W(network.all_params[0].eval(), second=10,
saveable=True, shape=[28, 28],
name='w1_'+str(epoch+1), fig_idx=2012)
       # You can also save the weight of 1st hidden layer to .npz file.
       # tl.files.save_npz([network.all_params[0]] , name='w1'+str(epoch+1)+'.npz')
       except:
       raise Exception("You should change visualize_W(), if you want \
       to save the feature images for different dataset")

print('Evaluation')
dp_dict = tl.utils.dict_to_one( network.all_drop )
feed_dict = {x: X_test, y_: y_test}
feed_dict.update(dp_dict)
print(" test loss: %f" % sess.run(cost, feed_dict=feed_dict))
print(" test acc: %f" % np.mean(y_test == sess.run(y_op,
feed_dict=feed_dict)))

tl.files.save_npz(network.all_params , name='model.npz')
tl.files.save_npz([network.all_params[0]] , name='model.npz')
# Then, restore the parameters as follow.
# load_params = tl.utils.load_npz(path='', name='model.npz')

# In the end, close TensorFlow session.
sess.close()

1 0