TensorFlow学习日记21

1. Recurrent Neural Network (LSTM)

解析：

# -*- coding: utf-8 -*-"""Simple example using LSTM recurrent neural network to classify IMDBsentiment dataset."""from __future__ import division, print_function, absolute_importimport tflearnfrom tflearn.data_utils import to_categorical, pad_sequencesfrom tflearn.datasets import imdb# IMDB Dataset loadingtrain, test, _ = imdb.load_data(path='imdb.pkl', n_words=10000,                                valid_portion=0.1)trainX, trainY = traintestX, testY = test# Data preprocessing# Sequence paddingtrainX = pad_sequences(trainX, maxlen=100, value=0.)testX = pad_sequences(testX, maxlen=100, value=0.)# Converting labels to binary vectorstrainY = to_categorical(trainY)testY = to_categorical(testY)# Network buildingnet = tflearn.input_data([None, 100])net = tflearn.embedding(net, input_dim=10000, output_dim=128)net = tflearn.lstm(net, 128, dropout=0.8)net = tflearn.fully_connected(net, 2, activation='softmax')net = tflearn.regression(net, optimizer='adam', learning_rate=0.001,                         loss='categorical_crossentropy')# Trainingmodel = tflearn.DNN(net, tensorboard_verbose=0)model.fit(trainX, trainY, validation_set=(testX, testY), show_metric=True,          batch_size=32)

2. Bi-Directional RNN (LSTM)

解析：

# -*- coding: utf-8 -*-"""Simple example using LSTM recurrent neural network to classify IMDBsentiment dataset."""from __future__ import division, print_function, absolute_importimport tflearnfrom tflearn.data_utils import to_categorical, pad_sequencesfrom tflearn.datasets import imdbfrom tflearn.layers.core import input_data, dropout, fully_connectedfrom tflearn.layers.embedding_ops import embeddingfrom tflearn.layers.recurrent import bidirectional_rnn, BasicLSTMCellfrom tflearn.layers.estimator import regression# IMDB Dataset loadingtrain, test, _ = imdb.load_data(path='imdb.pkl', n_words=10000,                                valid_portion=0.1)trainX, trainY = traintestX, testY = test# Data preprocessing# Sequence paddingtrainX = pad_sequences(trainX, maxlen=200, value=0.)testX = pad_sequences(testX, maxlen=200, value=0.)# Converting labels to binary vectorstrainY = to_categorical(trainY)testY = to_categorical(testY)# Network buildingnet = input_data(shape=[None, 200])net = embedding(net, input_dim=20000, output_dim=128)net = bidirectional_rnn(net, BasicLSTMCell(128), BasicLSTMCell(128))net = dropout(net, 0.5)net = fully_connected(net, 2, activation='softmax')net = regression(net, optimizer='adam', loss='categorical_crossentropy')# Trainingmodel = tflearn.DNN(net, clip_gradients=0., tensorboard_verbose=2)model.fit(trainX, trainY, validation_set=0.1, show_metric=True, batch_size=64)

3. Dynamic RNN (LSTM)

解析：

# -*- coding: utf-8 -*-"""Simple example using a Dynamic RNN (LSTM) to classify IMDB sentiment dataset.Dynamic computation are performed over sequences with variable length."""from __future__ import division, print_function, absolute_importimport tflearnfrom tflearn.data_utils import to_categorical, pad_sequencesfrom tflearn.datasets import imdb# IMDB Dataset loadingtrain, test, _ = imdb.load_data(path='imdb.pkl', n_words=10000,                                valid_portion=0.1)trainX, trainY = traintestX, testY = test# Data preprocessing# NOTE: Padding is required for dimension consistency. This will pad sequences# with 0 at the end, until it reaches the max sequence length. 0 is used as a# masking value by dynamic RNNs in TFLearn; a sequence length will be# retrieved by counting non zero elements in a sequence. Then dynamic RNN step# computation is performed according to that length.trainX = pad_sequences(trainX, maxlen=100, value=0.)testX = pad_sequences(testX, maxlen=100, value=0.)# Converting labels to binary vectorstrainY = to_categorical(trainY)testY = to_categorical(testY)# Network buildingnet = tflearn.input_data([None, 100])# Masking is not required for embedding, sequence length is computed prior to# the embedding op and assigned as 'seq_length' attribute to the returned Tensor.net = tflearn.embedding(net, input_dim=10000, output_dim=128)net = tflearn.lstm(net, 128, dropout=0.8, dynamic=True)net = tflearn.fully_connected(net, 2, activation='softmax')net = tflearn.regression(net, optimizer='adam', learning_rate=0.001,                         loss='categorical_crossentropy')# Trainingmodel = tflearn.DNN(net, tensorboard_verbose=0)model.fit(trainX, trainY, validation_set=(testX, testY), show_metric=True,          batch_size=32)

4. City Name Generation

解析：

from __future__ import absolute_import, division, print_functionimport osfrom six import movesimport sslimport tflearnfrom tflearn.data_utils import *path = "US_Cities.txt"if not os.path.isfile(path):    context = ssl._create_unverified_context()    moves.urllib.request.urlretrieve(        "https://raw.githubusercontent.com/tflearn/tflearn.github.io/master/resources/US_Cities.txt", path,        context=context)maxlen = 20string_utf8 = open(path, "r").read().decode('utf-8')X, Y, char_idx = \    string_to_semi_redundant_sequences(string_utf8, seq_maxlen=maxlen, redun_step=3)g = tflearn.input_data(shape=[None, maxlen, len(char_idx)])g = tflearn.lstm(g, 512, return_seq=True)g = tflearn.dropout(g, 0.5)g = tflearn.lstm(g, 512)g = tflearn.dropout(g, 0.5)g = tflearn.fully_connected(g, len(char_idx), activation='softmax')g = tflearn.regression(g, optimizer='adam', loss='categorical_crossentropy',                       learning_rate=0.001)m = tflearn.SequenceGenerator(g, dictionary=char_idx,                              seq_maxlen=maxlen,                              clip_gradients=5.0,                              checkpoint_path='model_us_cities')for i in range(40):    seed = random_sequence_from_string(string_utf8, maxlen)    m.fit(X, Y, validation_set=0.1, batch_size=128,          n_epoch=1, run_id='us_cities')    print("-- TESTING...")    print("-- Test with temperature of 1.2 --")    print(m.generate(30, temperature=1.2, seq_seed=seed).encode('utf-8'))    print("-- Test with temperature of 1.0 --")    print(m.generate(30, temperature=1.0, seq_seed=seed).encode('utf-8'))    print("-- Test with temperature of 0.5 --")    print(m.generate(30, temperature=0.5, seq_seed=seed).encode('utf-8'))

5. Shakespeare Scripts Generation

解析：

from __future__ import absolute_import, division, print_functionimport osimport picklefrom six.moves import urllibimport tflearnfrom tflearn.data_utils import *path = "shakespeare_input.txt"char_idx_file = 'char_idx.pickle'if not os.path.isfile(path):    urllib.request.urlretrieve(        "https://raw.githubusercontent.com/tflearn/tflearn.github.io/master/resources/shakespeare_input.txt", path)maxlen = 25char_idx = Noneif os.path.isfile(char_idx_file):    print('Loading previous char_idx')    char_idx = pickle.load(open(char_idx_file, 'rb'))X, Y, char_idx = \    textfile_to_semi_redundant_sequences(path, seq_maxlen=maxlen, redun_step=3,                                         pre_defined_char_idx=char_idx)pickle.dump(char_idx, open(char_idx_file, 'wb'))g = tflearn.input_data([None, maxlen, len(char_idx)])g = tflearn.lstm(g, 512, return_seq=True)g = tflearn.dropout(g, 0.5)g = tflearn.lstm(g, 512, return_seq=True)g = tflearn.dropout(g, 0.5)g = tflearn.lstm(g, 512)g = tflearn.dropout(g, 0.5)g = tflearn.fully_connected(g, len(char_idx), activation='softmax')g = tflearn.regression(g, optimizer='adam', loss='categorical_crossentropy',                       learning_rate=0.001)m = tflearn.SequenceGenerator(g, dictionary=char_idx,                              seq_maxlen=maxlen,                              clip_gradients=5.0,                              checkpoint_path='model_shakespeare')for i in range(50):    seed = random_sequence_from_textfile(path, maxlen)    m.fit(X, Y, validation_set=0.1, batch_size=128,          n_epoch=1, run_id='shakespeare')    print("-- TESTING...")    print("-- Test with temperature of 1.0 --")    print(m.generate(600, temperature=1.0, seq_seed=seed))    print("-- Test with temperature of 0.5 --")    print(m.generate(600, temperature=0.5, seq_seed=seed))

6. Seq2seq

解析：

'''Pedagogical example realization of seq2seq recurrent neural networks, using TensorFlow and TFLearn.More info at https://github.com/ichuang/tflearn_seq2seq'''from __future__ import division, print_functionimport osimport sysimport tflearnimport argparseimport jsonimport numpy as npimport tensorflow as tffrom tensorflow.contrib.legacy_seq2seq.python.ops import seq2seqfrom tensorflow.python.ops import rnn_cell# -----------------------------------------------------------------------------class SequencePattern(object):    INPUT_SEQUENCE_LENGTH = 10    OUTPUT_SEQUENCE_LENGTH = 10    INPUT_MAX_INT = 9    OUTPUT_MAX_INT = 9    PATTERN_NAME = "sorted"    def __init__(self, name=None, in_seq_len=None, out_seq_len=None):        if name is not None:            assert hasattr(self, "%s_sequence" % name)            self.PATTERN_NAME = name        if in_seq_len:            self.INPUT_SEQUENCE_LENGTH = in_seq_len        if out_seq_len:            self.OUTPUT_SEQUENCE_LENGTH = out_seq_len    def generate_output_sequence(self, x):        '''        For a given input sequence, generate the output sequence.  x is a 1D numpy array         of integers, with length INPUT_SEQUENCE_LENGTH.                Returns a 1D numpy array of length OUTPUT_SEQUENCE_LENGTH                This procedure defines the pattern which the seq2seq RNN will be trained to find.        '''        return getattr(self, "%s_sequence" % self.PATTERN_NAME)(x)    def maxmin_dup_sequence(self, x):        '''        Generate sequence with [max, min, rest of original entries]        '''        x = np.array(x)        y = [x.max(), x.min()] + list(x[2:])        return np.array(y)[:self.OUTPUT_SEQUENCE_LENGTH]  # truncate at out seq len    def sorted_sequence(self, x):        '''        Generate sorted version of original sequence        '''        return np.array(sorted(x))[:self.OUTPUT_SEQUENCE_LENGTH]    def reversed_sequence(self, x):        '''        Generate reversed version of original sequence        '''        return np.array(x[::-1])[:self.OUTPUT_SEQUENCE_LENGTH]# -----------------------------------------------------------------------------class TFLearnSeq2Seq(object):    '''    seq2seq recurrent neural network, implemented using TFLearn.    '''    AVAILABLE_MODELS = ["embedding_rnn", "embedding_attention"]    def __init__(self, sequence_pattern, seq2seq_model=None, verbose=None, name=None, data_dir=None):        '''        sequence_pattern_class = a SequencePattern class instance, which defines pattern parameters                                  (input, output lengths, name, generating function)        seq2seq_model = string specifying which seq2seq model to use, e.g. "embedding_rnn"        '''        self.sequence_pattern = sequence_pattern        self.seq2seq_model = seq2seq_model or "embedding_rnn"        assert self.seq2seq_model in self.AVAILABLE_MODELS        self.in_seq_len = self.sequence_pattern.INPUT_SEQUENCE_LENGTH        self.out_seq_len = self.sequence_pattern.OUTPUT_SEQUENCE_LENGTH        self.in_max_int = self.sequence_pattern.INPUT_MAX_INT        self.out_max_int = self.sequence_pattern.OUTPUT_MAX_INT        self.verbose = verbose or 0        self.n_input_symbols = self.in_max_int + 1        self.n_output_symbols = self.out_max_int + 2  # extra one for GO symbol        self.model_instance = None        self.name = name        self.data_dir = data_dir    def generate_trainig_data(self, num_points):        '''        Generate training dataset.  Produce random (integer) sequences X, and corresponding        expected output sequences Y = generate_output_sequence(X).        Return xy_data, y_data (both of type uint32)        xy_data = numpy array of shape [num_points, in_seq_len + out_seq_len], with each point being X + Y        y_data  = numpy array of shape [num_points, out_seq_len]        '''        x_data = np.random.randint(0, self.in_max_int,                                   size=(num_points, self.in_seq_len))  # shape [num_points, in_seq_len]        x_data = x_data.astype(np.uint32)  # ensure integer type        y_data = [self.sequence_pattern.generate_output_sequence(x) for x in x_data]        y_data = np.array(y_data)        xy_data = np.append(x_data, y_data, axis=1)  # shape [num_points, 2*seq_len]        return xy_data, y_data    def sequence_loss(self, y_pred, y_true):        '''        Loss function for the seq2seq RNN.  Reshape predicted and true (label) tensors, generate dummy weights,        then use seq2seq.sequence_loss to actually compute the loss function.        '''        if self.verbose > 2: print("my_sequence_loss y_pred=%s, y_true=%s" % (y_pred, y_true))        logits = tf.unstack(y_pred, axis=1)  # list of [-1, num_decoder_synbols] elements        targets = tf.unstack(y_true,                             axis=1)  # y_true has shape [-1, self.out_seq_len]; unpack to list of self.out_seq_len [-1] elements        if self.verbose > 2:            print("my_sequence_loss logits=%s" % (logits,))            print("my_sequence_loss targets=%s" % (targets,))        weights = [tf.ones_like(yp, dtype=tf.float32) for yp in targets]        if self.verbose > 4: print("my_sequence_loss weights=%s" % (weights,))        sl = seq2seq.sequence_loss(logits, targets, weights)        if self.verbose > 2: print("my_sequence_loss return = %s" % sl)        return sl    def accuracy(self, y_pred, y_true,                 x_in):  # y_pred is [-1, self.out_seq_len, num_decoder_symbols]; y_true is [-1, self.out_seq_len]        '''        Compute accuracy of the prediction, based on the true labels.  Use the average number of equal        values.        '''        pred_idx = tf.to_int32(tf.argmax(y_pred, 2))  # [-1, self.out_seq_len]        if self.verbose > 2: print("my_accuracy pred_idx = %s" % pred_idx)        accuracy = tf.reduce_mean(tf.cast(tf.equal(pred_idx, y_true), tf.float32), name='acc')        return accuracy    def model(self, mode="train", num_layers=1, cell_size=32, cell_type="BasicLSTMCell", embedding_size=20,              learning_rate=0.0001,              tensorboard_verbose=0, checkpoint_path=None):        '''        Build tensor specifying graph of operations for the seq2seq neural network model.        mode = string, either "train" or "predict"        cell_type = attribute of rnn_cell specifying which RNN cell type to use        cell_size = size for the hidden layer in the RNN cell        num_layers = number of RNN cell layers to use        Return TFLearn model instance.  Use DNN model for this.        '''        assert mode in ["train", "predict"]        checkpoint_path = checkpoint_path or (            "%s%ss2s_checkpoint.tfl" % (self.data_dir or "", "/" if self.data_dir else ""))        GO_VALUE = self.out_max_int + 1  # unique integer value used to trigger decoder outputs in the seq2seq RNN        network = tflearn.input_data(shape=[None, self.in_seq_len + self.out_seq_len], dtype=tf.int32, name="XY")        encoder_inputs = tf.slice(network, [0, 0], [-1, self.in_seq_len], name="enc_in")  # get encoder inputs        encoder_inputs = tf.unstack(encoder_inputs,                                    axis=1)  # transform into list of self.in_seq_len elements, each [-1]        decoder_inputs = tf.slice(network, [0, self.in_seq_len], [-1, self.out_seq_len],                                  name="dec_in")  # get decoder inputs        decoder_inputs = tf.unstack(decoder_inputs,                                    axis=1)  # transform into list of self.out_seq_len elements, each [-1]        go_input = tf.multiply(tf.ones_like(decoder_inputs[0], dtype=tf.int32),                               GO_VALUE)  # insert "GO" symbol as the first decoder input; drop the last decoder input        decoder_inputs = [go_input] + decoder_inputs[                                      : self.out_seq_len - 1]  # insert GO as first; drop last decoder input        feed_previous = not (mode == "train")        if self.verbose > 3:            print("feed_previous = %s" % str(feed_previous))            print("encoder inputs: %s" % str(encoder_inputs))            print("decoder inputs: %s" % str(decoder_inputs))            print("len decoder inputs: %s" % len(decoder_inputs))        self.n_input_symbols = self.in_max_int + 1  # default is integers from 0 to 9        self.n_output_symbols = self.out_max_int + 2  # extra "GO" symbol for decoder inputs        single_cell = getattr(rnn_cell, cell_type)(cell_size, state_is_tuple=True)        if num_layers == 1:            cell = single_cell        else:            cell = rnn_cell.MultiRNNCell([single_cell] * num_layers)        if self.seq2seq_model == "embedding_rnn":            model_outputs, states = seq2seq.embedding_rnn_seq2seq(encoder_inputs,                                                                  # encoder_inputs: A list of 2D Tensors [batch_size, input_size].                                                                  decoder_inputs,                                                                  cell,                                                                  num_encoder_symbols=self.n_input_symbols,                                                                  num_decoder_symbols=self.n_output_symbols,                                                                  embedding_size=embedding_size,                                                                  feed_previous=feed_previous)        elif self.seq2seq_model == "embedding_attention":            model_outputs, states = seq2seq.embedding_attention_seq2seq(encoder_inputs,                                                                        # encoder_inputs: A list of 2D Tensors [batch_size, input_size].                                                                        decoder_inputs,                                                                        cell,                                                                        num_encoder_symbols=self.n_input_symbols,                                                                        num_decoder_symbols=self.n_output_symbols,                                                                        embedding_size=embedding_size,                                                                        num_heads=1,                                                                        initial_state_attention=False,                                                                        feed_previous=feed_previous)        else:            raise Exception('[TFLearnSeq2Seq] Unknown seq2seq model %s' % self.seq2seq_model)        tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + "seq2seq_model",                             model_outputs)  # for TFLearn to know what to save and restore        # model_outputs: list of the same length as decoder_inputs of 2D Tensors with shape [batch_size x output_size] containing the generated outputs.        if self.verbose > 2: print("model outputs: %s" % model_outputs)        network = tf.stack(model_outputs,                           axis=1)  # shape [-1, n_decoder_inputs (= self.out_seq_len), num_decoder_symbols]        if self.verbose > 2: print("packed model outputs: %s" % network)        if self.verbose > 3:            all_vars = tf.get_collection(tf.GraphKeys.VARIABLES)            print("all_vars = %s" % all_vars)        with tf.name_scope("TargetsData"):  # placeholder for target variable (i.e. trainY input)            targetY = tf.placeholder(shape=[None, self.out_seq_len], dtype=tf.int32, name="Y")        network = tflearn.regression(network,                                     placeholder=targetY,                                     optimizer='adam',                                     learning_rate=learning_rate,                                     loss=self.sequence_loss,                                     metric=self.accuracy,                                     name="Y")        model = tflearn.DNN(network, tensorboard_verbose=tensorboard_verbose, checkpoint_path=checkpoint_path)        return model    def train(self, num_epochs=20, num_points=100000, model=None, model_params=None, weights_input_fn=None,              validation_set=0.1, snapshot_step=5000, batch_size=128, weights_output_fn=None):        '''        Train model, with specified number of epochs, and dataset size.        Use specified model, or create one if not provided.  Load initial weights from file weights_input_fn,         if provided. validation_set specifies what to use for the validation.        Returns logits for prediction, as an numpy array of shape [out_seq_len, n_output_symbols].        '''        trainXY, trainY = self.generate_trainig_data(num_points)        print("[TFLearnSeq2Seq] Training on %d point dataset (pattern '%s'), with %d epochs" % (num_points,                                                                                                self.sequence_pattern.PATTERN_NAME,                                                                                                num_epochs))        if self.verbose > 1:            print("  model parameters: %s" % json.dumps(model_params, indent=4))        model_params = model_params or {}        model = model or self.setup_model("train", model_params, weights_input_fn)        model.fit(trainXY, trainY,                  n_epoch=num_epochs,                  validation_set=validation_set,                  batch_size=batch_size,                  shuffle=True,                  show_metric=True,                  snapshot_step=snapshot_step,                  snapshot_epoch=False,                  run_id="TFLearnSeq2Seq"                  )        print("Done!")        if weights_output_fn is not None:            weights_output_fn = self.canonical_weights_fn(weights_output_fn)            model.save(weights_output_fn)            print("Saved %s" % weights_output_fn)            self.weights_output_fn = weights_output_fn        return model    def canonical_weights_fn(self, iteration_num=0):        '''        Construct canonical weights filename, based on model and pattern names.        '''        if not type(iteration_num) == int:            try:                iteration_num = int(iteration_num)            except Exception as err:                return iteration_num        model_name = self.name or "basic"        wfn = "ts2s__%s__%s_%s.tfl" % (model_name, self.sequence_pattern.PATTERN_NAME, iteration_num)        if self.data_dir:            wfn = "%s/%s" % (self.data_dir, wfn)        self.weights_filename = wfn        return wfn    def setup_model(self, mode, model_params=None, weights_input_fn=None):        '''        Setup a model instance, using the specified mode and model parameters.        Load the weights from the specified file, if it exists.        If weights_input_fn is an integer, use that the model name, and        the pattern name, to construct a canonical filename.        '''        model_params = model_params or {}        model = self.model_instance or self.model(mode=mode, **model_params)        self.model_instance = model        if weights_input_fn:            if type(weights_input_fn) == int:                weights_input_fn = self.canonical_weights_fn(weights_input_fn)            if os.path.exists(weights_input_fn):                model.load(weights_input_fn)                print("[TFLearnSeq2Seq] model weights loaded from %s" % weights_input_fn)            else:                print("[TFLearnSeq2Seq] MISSING model weights file %s" % weights_input_fn)        return model    def predict(self, Xin, model=None, model_params=None, weights_input_fn=None):        '''        Make a prediction, using the seq2seq model, for the given input sequence Xin.        If model is not provided, create one (or use last created instance).        Return prediction, y        prediction = array of integers, giving output prediction.  Length = out_seq_len        y = array of shape [out_seq_len, out_max_int], giving logits for output prediction        '''        if not model:            model = self.model_instance or self.setup_model("predict", model_params, weights_input_fn)        if self.verbose: print("Xin = %s" % str(Xin))        X = np.array(Xin).astype(np.uint32)        assert len(X) == self.in_seq_len        if self.verbose:            print("X Input shape=%s, data=%s" % (X.shape, X))            print("Expected output = %s" % str(self.sequence_pattern.generate_output_sequence(X)))        Yin = [0] * self.out_seq_len        XY = np.append(X, np.array(Yin).astype(np.float32))        XY = XY.reshape([-1, self.in_seq_len + self.out_seq_len])  # batch size 1        if self.verbose > 1: print("XY Input shape=%s, data=%s" % (XY.shape, XY))        res = model.predict(XY)        res = np.array(res)        if self.verbose > 1: print("prediction shape = %s" % str(res.shape))        y = res.reshape(self.out_seq_len, self.n_output_symbols)        prediction = np.argmax(y, axis=1)        if self.verbose:            print("Predicted output sequence: %s" % str(prediction))        return prediction, y# -----------------------------------------------------------------------------class VAction(argparse.Action):    def __call__(self, parser, args, values, option_string=None):        curval = getattr(args, self.dest, 0) or 0        values = values.count('v') + 1        setattr(args, self.dest, values + curval)# -----------------------------------------------------------------------------def CommandLine(args=None, arglist=None):    '''    Main command line.  Accepts args, to allow for simple unit testing.    '''    help_text = """Commands:train - give size of training set to use, as argumentpredict - give input sequence as argument (or specify inputs via --from-file <filename>)"""    parser = argparse.ArgumentParser(description=help_text, formatter_class=argparse.RawTextHelpFormatter)    parser.add_argument("cmd", help="command")    parser.add_argument("cmd_input", nargs='*', help="input to command")    parser.add_argument('-v', "--verbose", nargs=0,                        help="increase output verbosity (add more -v to increase versbosity)", action=VAction,                        dest='verbose')    parser.add_argument("-m", "--model",                        help="seq2seq model name: either embedding_rnn (default) or embedding_attention", default=None)    parser.add_argument("-r", "--learning-rate", type=float, help="learning rate (default 0.0001)", default=0.0001)    parser.add_argument("-e", "--epochs", type=int, help="number of trainig epochs", default=10)    parser.add_argument("-i", "--input-weights", type=str, help="tflearn file with network weights to load",                        default=None)    parser.add_argument("-o", "--output-weights", type=str,                        help="new tflearn file where network weights are to be saved", default=None)    parser.add_argument("-p", "--pattern-name", type=str, help="name of pattern to use for sequence", default=None)    parser.add_argument("-n", "--name", type=str, help="name of model, used when generating default weights filenames",                        default=None)    parser.add_argument("--in-len", type=int, help="input sequence length (default 10)", default=None)    parser.add_argument("--out-len", type=int, help="output sequence length (default 10)", default=None)    parser.add_argument("--from-file", type=str, help="name of file to take input data sequences from (json format)",                        default=None)    parser.add_argument("--iter-num", type=int,                        help="training iteration number; specify instead of input- or output-weights to use generated filenames",                        default=None)    parser.add_argument("--data-dir",                        help="directory to use for storing checkpoints (also used when generating default weights filenames)",                        default=None)    # model parameters    parser.add_argument("-L", "--num-layers", type=int, help="number of RNN layers to use in the model (default 1)",                        default=1)    parser.add_argument("--cell-size", type=int, help="size of RNN cell to use (default 32)", default=32)    parser.add_argument("--cell-type", type=str, help="type of RNN cell to use (default BasicLSTMCell)",                        default="BasicLSTMCell")    parser.add_argument("--embedding-size", type=int, help="size of embedding to use (default 20)", default=20)    parser.add_argument("--tensorboard-verbose", type=int, help="tensorboard verbosity level (default 0)", default=0)    if not args:        args = parser.parse_args(arglist)    if args.iter_num is not None:        args.input_weights = args.iter_num        args.output_weights = args.iter_num + 1    model_params = dict(num_layers=args.num_layers,                        cell_size=args.cell_size,                        cell_type=args.cell_type,                        embedding_size=args.embedding_size,                        learning_rate=args.learning_rate,                        tensorboard_verbose=args.tensorboard_verbose,                        )    if args.cmd == "train":        try:            num_points = int(args.cmd_input[0])        except:            raise Exception("Please specify the number of datapoints to use for training, as the first argument")        sp = SequencePattern(args.pattern_name, in_seq_len=args.in_len, out_seq_len=args.out_len)        ts2s = TFLearnSeq2Seq(sp, seq2seq_model=args.model, data_dir=args.data_dir, name=args.name,                              verbose=args.verbose)        ts2s.train(num_epochs=args.epochs, num_points=num_points, weights_output_fn=args.output_weights,                   weights_input_fn=args.input_weights, model_params=model_params)        return ts2s    elif args.cmd == "predict":        if args.from_file:            inputs = json.loads(args.from_file)        try:            input_x = map(int, args.cmd_input)            inputs = [input_x]        except:            raise Exception("Please provide a space-delimited input sequence as the argument")        sp = SequencePattern(args.pattern_name, in_seq_len=args.in_len, out_seq_len=args.out_len)        ts2s = TFLearnSeq2Seq(sp, seq2seq_model=args.model, data_dir=args.data_dir, name=args.name,                              verbose=args.verbose)        results = []        for x in inputs:            prediction, y = ts2s.predict(x, weights_input_fn=args.input_weights, model_params=model_params)            print("==> For input %s, prediction=%s (expected=%s)" % (x, prediction, sp.generate_output_sequence(x)))            results.append([prediction, y])        ts2s.prediction_results = results        return ts2s    else:        print("Unknown command %s" % args.cmd)# -----------------------------------------------------------------------------# unit testsdef test_sp1():    '''    Test two different SequencePattern instances    '''    sp = SequencePattern("maxmin_dup")    y = sp.generate_output_sequence(range(10))    assert all(y == np.array([9, 0, 2, 3, 4, 5, 6, 7, 8, 9]))    sp = SequencePattern("sorted")    y = sp.generate_output_sequence([5, 6, 1, 2, 9])    assert all(y == np.array([1, 2, 5, 6, 9]))    sp = SequencePattern("reversed")    y = sp.generate_output_sequence(range(10))    assert all(y == np.array([9, 8, 7, 6, 5, 4, 3, 2, 1, 0]))def test_sp2():    '''    Test two SequencePattern instance with lengths different from default    '''    sp = SequencePattern("sorted", in_seq_len=20, out_seq_len=5)    x = np.random.randint(0, 9, 20)    y = sp.generate_output_sequence(x)    assert len(y) == 5    y_exp = sorted(x)[:5]    assert all(y == y_exp)def test_train1():    '''    Test simple training of an embedding_rnn seq2seq model    '''    sp = SequencePattern()    ts2s = TFLearnSeq2Seq(sp)    ofn = "test_%s" % ts2s.canonical_weights_fn(0)    print("using weights filename %s" % ofn)    if os.path.exists(ofn):        os.unlink(ofn)    tf.reset_default_graph()    ts2s.train(num_epochs=1, num_points=10000, weights_output_fn=ofn)    assert os.path.exists(ofn)def test_predict1():    '''    Test simple preductions using weights just produced (in test_train1)    '''    sp = SequencePattern()    ts2s = TFLearnSeq2Seq(sp, verbose=1)    wfn = "test_%s" % ts2s.canonical_weights_fn(0)    print("using weights filename %s" % wfn)    tf.reset_default_graph()    prediction, y = ts2s.predict(Xin=range(10), weights_input_fn=wfn)    assert len(prediction == 10)def test_train_predict2():    '''    Test that the embedding_attention model works, with saving and loading of weights    '''    import tempfile    sp = SequencePattern()    tempdir = tempfile.mkdtemp()    ts2s = TFLearnSeq2Seq(sp, seq2seq_model="embedding_attention", data_dir=tempdir, name="attention")    tf.reset_default_graph()    ts2s.train(num_epochs=1, num_points=1000, weights_output_fn=1, weights_input_fn=0)    assert os.path.exists(ts2s.weights_output_fn)    tf.reset_default_graph()    ts2s = TFLearnSeq2Seq(sp, seq2seq_model="embedding_attention", data_dir="DATA", name="attention", verbose=1)    prediction, y = ts2s.predict(Xin=range(10), weights_input_fn=1)    assert len(prediction == 10)    os.system("rm -rf %s" % tempdir)def test_train_predict3():    '''    Test that a model trained on sequencees of one length can be used for predictions on other sequence lengths    '''    import tempfile    sp = SequencePattern("sorted", in_seq_len=10, out_seq_len=10)    tempdir = tempfile.mkdtemp()    ts2s = TFLearnSeq2Seq(sp, seq2seq_model="embedding_attention", data_dir=tempdir, name="attention")    tf.reset_default_graph()    ts2s.train(num_epochs=1, num_points=1000, weights_output_fn=1, weights_input_fn=0)    assert os.path.exists(ts2s.weights_output_fn)    tf.reset_default_graph()    sp = SequencePattern("sorted", in_seq_len=20, out_seq_len=8)    tf.reset_default_graph()    ts2s = TFLearnSeq2Seq(sp, seq2seq_model="embedding_attention", data_dir="DATA", name="attention", verbose=1)    x = np.random.randint(0, 9, 20)    prediction, y = ts2s.predict(x, weights_input_fn=1)    assert len(prediction == 8)    os.system("rm -rf %s" % tempdir)def test_main1():    '''    Integration test - training    '''    import tempfile    tempdir = tempfile.mkdtemp()    arglist = "--data-dir %s -e 2 --iter-num=1 -v -v --tensorboard-verbose=1 train 5000" % tempdir    arglist = arglist.split(' ')    tf.reset_default_graph()    ts2s = CommandLine(arglist=arglist)    assert os.path.exists(ts2s.weights_output_fn)    os.system("rm -rf %s" % tempdir)def test_main2():    '''    Integration test - training then prediction    '''    import tempfile    tempdir = tempfile.mkdtemp()    arglist = "--data-dir %s -e 2 --iter-num=1 -v -v --tensorboard-verbose=1 train 5000" % tempdir    arglist = arglist.split(' ')    tf.reset_default_graph()    ts2s = CommandLine(arglist=arglist)    wfn = ts2s.weights_output_fn    assert os.path.exists(wfn)    arglist = "-i %s predict 1 2 3 4 5 6 7 8 9 0" % wfn    arglist = arglist.split(' ')    tf.reset_default_graph()    ts2s = CommandLine(arglist=arglist)    assert len(ts2s.prediction_results[0][0]) == 10    os.system("rm -rf %s" % tempdir)def test_main3():    '''    Integration test - training then prediction: attention model    '''    import tempfile    wfn = "tmp_weights.tfl"    if os.path.exists(wfn):        os.unlink(wfn)    arglist = "-e 2 -o tmp_weights.tfl -v -v -v -v -m embedding_attention train 5000"    arglist = arglist.split(' ')    tf.reset_default_graph()    ts2s = CommandLine(arglist=arglist)    assert os.path.exists(wfn)    arglist = "-i tmp_weights.tfl -v -v -v -v -m embedding_attention predict 1 2 3 4 5 6 7 8 9 0"    arglist = arglist.split(' ')    tf.reset_default_graph()    ts2s = CommandLine(arglist=arglist)    assert len(ts2s.prediction_results[0][0]) == 10# -----------------------------------------------------------------------------if __name__ == "__main__":    CommandLine()

7. CNN Seq

解析：

# -*- coding: utf-8 -*-"""Simple example using convolutional neural network to classify IMDBsentiment dataset."""from __future__ import division, print_function, absolute_importimport tensorflow as tfimport tflearnfrom tflearn.layers.core import input_data, dropout, fully_connectedfrom tflearn.layers.conv import conv_1d, global_max_poolfrom tflearn.layers.merge_ops import mergefrom tflearn.layers.estimator import regressionfrom tflearn.data_utils import to_categorical, pad_sequencesfrom tflearn.datasets import imdb# IMDB Dataset loadingtrain, test, _ = imdb.load_data(path='imdb.pkl', n_words=10000,                                valid_portion=0.1)trainX, trainY = traintestX, testY = test# Data preprocessing# Sequence paddingtrainX = pad_sequences(trainX, maxlen=100, value=0.)testX = pad_sequences(testX, maxlen=100, value=0.)# Converting labels to binary vectorstrainY = to_categorical(trainY)testY = to_categorical(testY)# Building convolutional networknetwork = input_data(shape=[None, 100], name='input')network = tflearn.embedding(network, input_dim=10000, output_dim=128)branch1 = conv_1d(network, 128, 3, padding='valid', activation='relu', regularizer="L2")branch2 = conv_1d(network, 128, 4, padding='valid', activation='relu', regularizer="L2")branch3 = conv_1d(network, 128, 5, padding='valid', activation='relu', regularizer="L2")network = merge([branch1, branch2, branch3], mode='concat', axis=1)network = tf.expand_dims(network, 2)network = global_max_pool(network)network = dropout(network, 0.5)network = fully_connected(network, 2, activation='softmax')network = regression(network, optimizer='adam', learning_rate=0.001,                     loss='categorical_crossentropy', name='target')# Trainingmodel = tflearn.DNN(network, tensorboard_verbose=0)model.fit(trainX, trainY, n_epoch=5, shuffle=True, validation_set=(testX, testY), show_metric=True, batch_size=32)

参考文献：

[1] TFLearn Examples：https://github.com/tflearn/tflearn/tree/master/examples