Word2vec原理浅析及tensorflow实现

 1# These are all the modules we'll be using later. Make           sure you can import them
 2# before proceeding further.
 3%matplotlib inline
 4from __future__ import print_function
 5import collections
 6import math
 7import numpy as np
 8import os
 9import random
10import tensorflow as tf
11import zipfile
12from matplotlib import pylab
13from six.moves import range
14from six.moves.urllib.request import urlretrieve
15from sklearn.manifold import TSNE

 1url = 'http://mattmahoney.net/dc/'
 2def maybe_download(filename, expected_bytes):
 3"""Download a file if not present, and make sure it's the    right size."""
 4if not os.path.exists(filename):
 5filename, _ = urlretrieve(url + filename, filename)
 6statinfo = os.stat(filename)
 7if statinfo.st_size == expected_bytes:
 8print('Found and verified %s' % filename)
 9else:
10print(statinfo.st_size)
11raise Exception(
12'Failed to verify ' + filename + '. Can you get to it with a browser?')
13return filename
14filename = maybe_download('text8.zip', 31344016)
15Found and verified text8.zip

1def read_data(filename):
2"""Extract the first file enclosed in a zip file as a list of words"""
3with zipfile.ZipFile(filename) as f:
4data = tf.compat.as_str(f.read(f.namelist()[0])).split()
5return data
6words = read_data(filename)
7print('Data size %d' % len(words))
8Data size 17005207

 1vocabulary_size = 50000
 2def build_dataset(words):
 3count = [['UNK', -1]]
 4count.extend(collections.Counter(words).most_common(vocabulary_size - 1))
 5dictionary = dict()
 6for word, _ in count:
 7dictionary[word] = len(dictionary)
 8data = list()
 9unk_count = 0
10for word in words:
11if word in dictionary:
12index = dictionary[word]
13else:
14index = 0  # dictionary['UNK']
15unk_count = unk_count + 1
16data.append(index)
17count[0][1] = unk_count
18reverse_dictionary = dict(zip(dictionary.values(), dictionary.keys())) 
19return data, count, dictionary, reverse_dictionary
20data, count, dictionary, reverse_dictionary = build_dataset(words)
21print('Most common words (+UNK)', count[:5])
22print('Sample data', data[:10])
23del words  # Hint to reduce memory.

1Most common words (+UNK) [['UNK', 418391], ('the', 1061396), ('of', 593677), ('and', 416629), ('one', 411764)]
2Sample data [5239, 3084, 12, 6, 195, 2, 3137, 46, 59, 156]

 1data_index = 0
 2def generate_batch(batch_size, num_skips, skip_window):
 3global data_index
 4assert batch_size % num_skips == 0
 5assert num_skips <= 2 * skip_window
 6batch = np.ndarray(shape=(batch_size), dtype=np.int32)
 7labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
 8span = 2 * skip_window + 1 # [ skip_window target skip_window ]
 9buffer = collections.deque(maxlen=span)
10for _ in range(span):
11buffer.append(data[data_index])
12data_index = (data_index + 1) % len(data)
13for i in range(batch_size // num_skips):
14target = skip_window  # target label at the center of the buffer
15targets_to_avoid = [ skip_window ]
16for j in range(num_skips):
17    while target in targets_to_avoid:
18        target = random.randint(0, span - 1)
19    targets_to_avoid.append(target)
20    batch[i * num_skips + j] = buffer[skip_window]
21    labels[i * num_skips + j, 0] = buffer[target]
22buffer.append(data[data_index])
23data_index = (data_index + 1) % len(data)
24return batch, labels
25print('data:', [reverse_dictionary[di] for di in data[:8]])
26for num_skips, skip_window in [(2, 1), (4, 2)]:
27data_index = 0
28batch, labels = generate_batch(batch_size=8, num_skips=num_skips, skip_window=skip_window)
29print('\nwith num_skips = %d and skip_window = %d:' % (num_skips, skip_window))
30print('    batch:', [reverse_dictionary[bi] for bi in batch])
31print('    labels:', [reverse_dictionary[li] for li in labels.reshape(8)])

1data: ['anarchism', 'originated', 'as', 'a', 'term', 'of', 'abuse', 'first']
2with num_skips = 2 and skip_window = 1:
3batch: ['originated', 'originated', 'as', 'as', 'a', 'a', 'term', 'term']
4labels: ['anarchism', 'as', 'originated', 'a', 'as', 'term', 'a', 'of']
5with num_skips = 4 and skip_window = 2:
6batch: ['as', 'as', 'as', 'as', 'a', 'a', 'a', 'a']
7labels: ['originated', 'term', 'anarchism', 'a', 'of', 'as', 'originated', 'term']

 1batch_size = 128
 2embedding_size = 128 # Dimension of the embedding vector.
 3skip_window = 1 # How many words to consider left and right.
 4num_skips = 2 # How many times to reuse an input to generate a label.
 5# We pick a random validation set to sample nearest neighbors. here we limit the
 6# validation samples to the words that have a low numeric ID, which by
 7# construction are also the most frequent. 
 8valid_size = 16 # Random set of words to evaluate similarity on.
 9valid_window = 100 # Only pick dev samples in the head of the distribution.
10valid_examples = np.array(random.sample(range(valid_window), valid_size))
11#######important#########
12num_sampled = 64 # Number of negative examples to sample.
13graph = tf.Graph()
14with graph.as_default(), tf.device('/cpu:0'):
15# Input data.
16train_dataset = tf.placeholder(tf.int32, shape=[batch_size])
17train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
18valid_dataset =  tf.constant(valid_examples,dtype=tf.int32)
19# Variables.
20embeddings = tf.Variable(
21tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
22softmax_weights = tf.Variable(
23tf.truncated_normal([vocabulary_size, embedding_size],
24                 stddev=1.0 / math.sqrt(embedding_size)))
25softmax_biases = tf.Variable(tf.zeros([vocabulary_size]))
26# Model.
27# Look up embeddings for inputs.
28embed = tf.nn.embedding_lookup(embeddings, train_dataset)
29# Compute the softmax loss, using a sample of the negative labels each time.
30loss = tf.reduce_mean(
31tf.nn.sampled_softmax_loss(weights=softmax_weights,      biases=softmax_biases, inputs=embed,
32labels=train_labels,num_sampled=num_sampled, num_classes=vocabulary_size))
33# Optimizer.
34# Note: The optimizer will optimize the softmax_weights AND the embeddings.
35# This is because the embeddings are defined as a variable quantity and the
36# optimizer's `minimize` method will by default modify all variable quantities 
37# that contribute to the tensor it is passed.
38# See docs on `tf.train.Optimizer.minimize()` for more details.
39optimizer = tf.train.AdagradOptimizer(1.0).minimize(loss)
40# Compute the similarity between minibatch examples and all embeddings.
41# We use the cosine distance:
42norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
43normalized_embeddings = embeddings / norm
44valid_embeddings = tf.nn.embedding_lookup(normalized_embeddings, valid_dataset)
45similarity = tf.matmul(valid_embeddings, tf.transpose(normalized_embeddings))

 1num_steps = 100001
 2with tf.Session(graph=graph) as session:
 3tf.global_variables_initializer().run()
 4print('Initialized')
 5average_loss = 0
 6for step in range(num_steps):
 7batch_data, batch_labels = generate_batch(
 8batch_size, num_skips, skip_window)
 9feed_dict = {train_dataset : batch_data, train_labels : batch_labels}
10 _, l = session.run([optimizer, loss],feed_dict=feed_dict)average_loss += l
11if step % 2000 == 0:
12if step > 0:
13average_loss = average_loss / 2000
14# The average loss is an estimate of the loss over the last 2000 batches.
15 print('Average loss at step %d: %f' % (step,average_loss))
16 average_loss = 0
17# note that this is expensive (~20% slowdown if computed every 500 steps)
18if step % 10000 == 0:
19sim = similarity.eval()
20for i in range(valid_size):
21valid_word = reverse_dictionary[valid_examples[i]]
22top_k = 8 # number of nearest neighbors
23nearest = (-sim[i, :]).argsort()[1:top_k+1]
24log = 'Nearest to %s:' % valid_word
25for k in range(top_k):
26  close_word = reverse_dictionary[nearest[k]]
27  log = '%s %s,' % (log, close_word)
28print(log)
29final_embeddings = normalized_embeddings.eval()

1num_points = 400
2tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000)
3two_d_embeddings = tsne.fit_transform(final_embeddings[1:num_points+1, :])

 1def plot(embeddings, labels):
 2assert embeddings.shape[0] >= len(labels), 'More labels than embeddings'
 3pylab.figure(figsize=(15,15))  # in inches
 4for i, label in enumerate(labels):
 5x, y = embeddings[i,:]
 6pylab.scatter(x, y)
 7pylab.annotate(label, xy=(x, y), xytext=(5, 2),textcoords='offset points',
 8           ha='right', va='bottom')
 9pylab.show()
10words = [reverse_dictionary[i] for i in range(1, num_points+1)]
11plot(two_d_embeddings, words)

 1data_index_cbow = 0
 2def get_cbow_batch(batch_size, num_skips, skip_window):
 3global data_index_cbow
 4assert batch_size % num_skips == 0
 5assert num_skips <= 2 * skip_window
 6batch = np.ndarray(shape=(batch_size), dtype=np.int32)
 7labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
 8span = 2 * skip_window + 1 # [ skip_window target skip_window ]
 9buffer = collections.deque(maxlen=span)
10for _ in range(span):
11buffer.append(data[data_index_cbow])
12data_index_cbow = (data_index_cbow + 1) % len(data)
13for i in range(batch_size // num_skips):
14target = skip_window  # target label at the center of the buffer
15targets_to_avoid = [ skip_window ]
16for j in range(num_skips):
17    while target in targets_to_avoid:
18        target = random.randint(0, span - 1)
19    targets_to_avoid.append(target)
20    batch[i * num_skips + j] = buffer[skip_window]
21    labels[i * num_skips + j, 0] = buffer[target]
22buffer.append(data[data_index_cbow])
23data_index_cbow = (data_index_cbow + 1) % len(data)
24cbow_batch = np.ndarray(shape=(batch_size), dtype=np.int32)
25cbow_labels = np.ndarray(shape=(batch_size // (skip_window * 2), 1), dtype=np.int32)
26for i in range(batch_size):
27cbow_batch[i] = labels[i]
28cbow_batch = np.reshape(cbow_batch, [batch_size // (skip_window * 2), skip_window * 2])
29for i in range(batch_size // (skip_window * 2)):
30# center word
31cbow_labels[i] = batch[2 * skip_window * i]
32return cbow_batch, cbow_labels

 1# actual batch_size = batch_size // (2 * skip_window)
 2batch_size = 128
 3embedding_size = 128 # Dimension of the embedding vector.
 4skip_window = 1 # How many words to consider left and right.
 5num_skips = 2 # How many times to reuse an input to generate a label.
 6# We pick a random validation set to sample nearest neighbors. here we limit the
 7# validation samples to the words that have a low numeric ID, which by
 8# construction are also the most frequent. 
 9valid_size = 16 # Random set of words to evaluate similarity on.
10valid_window = 100 # Only pick dev samples in the head of the distribution.
11valid_examples = np.array(random.sample(range(valid_window), valid_size))
12#######important#########
13num_sampled = 64 # Number of negative examples to sample.
14graph = tf.Graph()
15with graph.as_default(), tf.device('/cpu:0'):
16# Input data.
17train_dataset = tf.placeholder(tf.int32, shape=[batch_size // (skip_window * 2), skip_window * 2])
18train_labels = tf.placeholder(tf.int32, shape=[batch_size // (skip_window * 2), 1])
19valid_dataset =   tf.constant(valid_examples,dtype=tf.int32)
20# Variables.
21embeddings = tf.Variable(
22tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
23softmax_weights = tf.Variable(
24tf.truncated_normal([vocabulary_size, embedding_size],
25                 stddev=1.0 / math.sqrt(embedding_size)))
26softmax_biases = tf.Variable(tf.zeros([vocabulary_size]))
27# Model.
28# Look up embeddings for inputs.
29embed = tf.nn.embedding_lookup(embeddings, train_dataset)
30# reshape embed
31embed = tf.reshape(embed, (skip_window * 2, batch_size // (skip_window * 2), embedding_size))
32# average embedembed = tf.reduce_mean(embed, 0)
33# Compute the softmax loss, using a sample of the negative labels each time.
34loss = tf.reduce_mean(
35tf.nn.sampled_softmax_loss(weights=softmax_weights, biases=softmax_biases, inputs=embed,labels=train_labels,    num_sampled=num_sampled, num_classes=vocabulary_size))
36# Optimizer.
37# Note: The optimizer will optimize the softmax_weights AND the embeddings.
38# This is because the embeddings are defined as a variable quantity and the
39# optimizer's `minimize` method will by default modify all variable quantities 
40# that contribute to the tensor it is passed.
41# See docs on `tf.train.Optimizer.minimize()` for more details.
42optimizer = tf.train.AdagradOptimizer(1.0).minimize(loss)
43# Compute the similarity between minibatch examples and all embeddings.
44# We use the cosine distance:
45norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
46normalized_embeddings = embeddings / norm
47valid_embeddings = tf.nn.embedding_lookup(
48normalized_embeddings, valid_dataset)
49similarity = tf.matmul(valid_embeddings,tf.transpose(normalized_embeddings))

 1num_steps = 100001
 2with tf.Session(graph=graph) as session:
 3tf.global_variables_initializer().run()
 4print('Initialized')
 5average_loss = 0
 6for step in range(num_steps):
 7batch_data, batch_labels = get_cbow_batch(
 8batch_size, num_skips, skip_window)
 9feed_dict = {train_dataset : batch_data, train_labels : batch_labels}
10_, l = session.run([optimizer, loss],feed_dict=feed_dict)
11average_loss += l
12if step % 2000 == 0:
13if step > 0:
14average_loss = average_loss / 2000
15# The average loss is an estimate of the loss over the last 2000 batches.
16print('Average loss at step %d: %f' % (step, average_loss))
17average_loss = 0
18# note that this is expensive (~20% slowdown if computed every 500 steps)
19if step % 10000 == 0:
20sim = similarity.eval()
21for i in range(valid_size):
22valid_word = reverse_dictionary[valid_examples[i]]
23top_k = 8 # number of nearest neighbors
24nearest = (-sim[i, :]).argsort()[1:top_k+1]
25log = 'Nearest to %s:' % valid_word
26for k in range(top_k):
27  close_word = reverse_dictionary[nearest[k]]
28  log = '%s %s,' % (log, close_word)
29print(log)
30final_embeddings = normalized_embeddings.eval()

1num_points = 400
2tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000)
3two_d_embeddings = tsne.fit_transform(final_embeddings[1:num_points+1, :])
4words = [reverse_dictionary[i] for i in range(200, num_points+1)]
5plot(two_d_embeddings, words)