Tensorflow之Basic word2vec代码详解（上）

Tensorflow上关于Vector Representations of Words里给出了word2vec两个源代码，本文解析基础的代码，代码地址为：https://github.com/tensorflow/tensorflow/blob/r1.3/tensorflow/examples/tutorials/word2vec/word2vec_basic.py

上篇为代码step1-3：讲解数据下载处理，与训练数据的生成。

from __future__ import absolute_import  from __future__ import division  from __future__ import print_function    import collections  import math  import os  import random  import zipfile    import numpy as np  from six.moves import urllib  from six.moves import xrange  # pylint: disable=redefined-builtin  import tensorflow as tf    # Step 1: 下载数据  url = 'http://mattmahoney.net/dc/'      def maybe_download(filename, expected_bytes):    """Download a file if not present, and make sure it's the right size."""    if not os.path.exists(filename):      filename, _ = urllib.request.urlretrieve(url + filename, filename)    statinfo = os.stat(filename)    if statinfo.st_size == expected_bytes:      print('Found and verified', filename)    else:      print(statinfo.st_size)      raise Exception(          'Failed to verify ' + filename + '. Can you get to it with a browser?')    return filename  #  filename = maybe_download('text8.zip', 31344016)      # 解压缩并读取数据转化到数组中.  def read_data(filename):    """Extract the first file enclosed in a zip file as a list of words."""    with zipfile.ZipFile(filename) as f:      data = tf.compat.as_str(f.read(f.namelist()[0])).split() #split(分割成序列)   return data    vocabulary = read_data(filename)  print('Data size', len(vocabulary))  #建立字典  # Step 2: Build the dictionary and replace rare words with UNK token.  vocabulary_size = 50000  def build_dataset(words, n_words):    """Process raw inputs into a dataset."""    count = [['UNK', -1]]    count.extend(collections.Counter(words).most_common(n_words - 1))#计数，取词频前50000个词，其余的为unk，    dictionary = dict()    for word, _ in count:      dictionary[word] = len(dictionary)#高频词排序给编号    data = list()    unk_count = 0    for word in words:      if word in dictionary:        index = dictionary[word]#给高频词一个索引      else:        index = 0  # 低频词索引为0        unk_count += 1  #统计低频词的个数    data.append(index)    count[0][1] = unk_count    reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys())) #逆词汇，键和值与dictionary相反   return data, count, dictionary, reversed_dictionary    data, count, dictionary, reverse_dictionary = build_dataset(vocabulary,                                                              vocabulary_size)  del vocabulary  # Hint to reduce memory.  print('Most common words (+UNK)', count[:5])  print('Sample data', data[:10], [reverse_dictionary[i] for i in data[:10]])  #如vocabulary（daefbmc……）其中：1.a词频：600；2.b词频：500；3.c词频：400；4.d词频：300；5.e词频：200；6.f词频：100；……unk:4148#count([UNK,4148],[a,600]，[b,500],[c,400],[d,300],[e,200,[f,100])#dictionary([a；1]，[b:2],[c:3],[d:4],[e；5],[f；6])#data:{4,1,5,6,2,0,3}#reversed_dictionary([1；a]，[2:b],[3:c],[4:d],[5；e],[6；f])#生成训练数据#从文本总体的第二次开始，每个单词一次作为输入，输出可以是上下文范围内的单词中的任何一个（一般不是取全部而是随机抽取其中几组，增加随机性）  data_index = 0   # Step 3: Function to generate a training batch for the skip-gram model.  def generate_batch(batch_size, num_skips, skip_window):  #batch_size：每次训练的词长度；num_skips：每个输入词重复的次数（一个输入产生多少个标签数据），skip_window：向左右取多少词  global data_index  #global：全局变量  assert batch_size % num_skips == 0  #assert：断言  assert num_skips <= 2 * skip_window      batch = np.ndarray(shape=(batch_size), dtype=np.int32)  #一维  labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)  #二维  span = 2 * skip_window + 1  # 左右两边各取skip_window，一共span个   buffer = collections.deque(maxlen=span)  #防止超长，挤出前面的数据，确保span个训练数据  if data_index + span > len(data):  #依次取span个词    data_index = 0    buffer.extend(data[data_index:data_index + span])    data_index += span    for i in range(batch_size // num_skips):      target = skip_window  #butter[skip_window]为输入数据    targets_to_avoid = [skip_window] #去除输入词自己本身     for j in range(num_skips):  #输入词重复num_skips次      while target in targets_to_avoid:          target = random.randint(0, span - 1) #随机生成 (0, span - 1)之间整数      targets_to_avoid.append(target)        batch[i * num_skips + j] = buffer[skip_window] #训练输入的序列       labels[i * num_skips + j, 0] = buffer[target]  #训练输出的序列（标签，对应词频的排序）    if data_index == len(data):  #超长时回到开始      buffer[:] = data[:span]        data_index = span      else:        buffer.append(data[data_index]) #挤掉开始几个，换一组词训练       data_index += 1    # Backtrack a little bit to avoid skipping words in the end of a batch    data_index = (data_index + len(data) - span) % len(data)    return batch, labels    batch, labels = generate_batch(batch_size=8, num_skips=2, skip_window=1)  for i in range(8):    print(batch[i], reverse_dictionary[batch[i]],          '->', labels[i, 0], reverse_dictionary[labels[i, 0]])   #如：vocabulary（m|daefbm|c……）取batch_size=6,num_skips=2,skip_window=1#batch = [4,4,1,1,5,5]#labels= [0,1,4,5,1,6]