【专知-PyTorch手把手深度学习教程06】NLP-Word Embedding快速理解与PyTorch实现: 图文+代码

# 上下文宽度
CONTEXT_SIZE = 2
# embedding 维度
EMBEDDING_DIM = 10
# 训练语料
test_sentence = """When forty winters shall besiege thy brow,
And dig deep trenches in thy beauty's field,
Thy youth's proud livery so gazed on now,
Will be a totter'd weed of small worth held:
Then being asked, where all thy beauty lies,
Where all the treasure of thy lusty days;
To say, within thine own deep sunken eyes,
Were an all-eating shame, and thriftless praise.
How much more praise deserv'd thy beauty's use,
If thou couldst answer 'This fair child of mine
Shall sum my count, and make my old excuse,'
Proving his beauty by succession thine!
This were to be new made when thou art old,
And see thy blood warm when thou feel'st it cold.""".split()


# 我们把训练语料拆成三元组, 如:
# [(['When', 'forty'], 'winters'), (['forty', 'winters'], 'shall'), (['winters', 'shall'], 'besiege')]

trigrams = [ ([test_sentence[i], test_sentence[i+1]], test_sentence[i+2]) for i in xrange(len(test_sentence) - 2) ]

# 构建词语索引字典,
# 比如:word_to_ix = { "hello": 0, "world": 1 }

vocab = set(test_sentence)
word_to_ix = { word: i for i, word in enumerate(vocab) }

# 集成nn.Module 实现Embedding过程
class NGramLanguageModeler(nn.Module):
    
    def __init__(self, vocab_size, embedding_dim, context_size):
        super(NGramLanguageModeler, self).__init__()
        # 定义Embedding层
        self.embeddings = nn.Embedding(vocab_size, embedding_dim)
        # 定义线性隐层
        self.linear1 = nn.Linear(context_size * embedding_dim, 128)
        # 输出层
        self.linear2 = nn.Linear(128, vocab_size)
        
    # 编写前向过程
    def forward(self, inputs):
        embeds = self.embeddings(inputs).view((1, -1))
        out = F.relu(self.linear1(embeds))
        out = self.linear2(out)
        log_probs = F.log_softmax(out)
        return log_probs

# 选择损失函数, 优化方法, 实例化网络
losses = []
loss_function = nn.NLLLoss()
model = NGramLanguageModeler(len(vocab), EMBEDDING_DIM, CONTEXT_SIZE)
optimizer = optim.SGD(model.parameters(), lr=0.001)

# 对于每一个epoch
for epoch in xrange(10):
    total_loss = torch.Tensor([0])
    for context, target in trigrams:
    
        # 准备数据, 打包成Variable
        context_idxs = map(lambda w: word_to_ix[w], context)
        context_var = autograd.Variable( torch.LongTensor(context_idxs) )
    
        # 清空梯度缓存
        model.zero_grad()
    
        # 计算前向过程
        log_probs = model(context_var)
    
        #  计算损失
        loss = loss_function(log_probs, autograd.Variable(torch.LongTensor([word_to_ix[target]])))
    
        #  反向传播, 优化一步
        loss.backward()
        optimizer.step()
    
        total_loss += loss.data
    losses.append(total_loss)

# 打印一下损失
print losses 

############################
[
 518.8207
[torch.FloatTensor of size 1]
, 
 516.3852
[torch.FloatTensor of size 1]
, 
 513.9670
[torch.FloatTensor of size 1]
, 
 511.5646
[torch.FloatTensor of size 1]
, 
 509.1782
[torch.FloatTensor of size 1]
, 
 506.8095
[torch.FloatTensor of size 1]
, 
 504.4555
[torch.FloatTensor of size 1]
, 
 502.1131
[torch.FloatTensor of size 1]
, 
 499.7835
[torch.FloatTensor of size 1]
, 
 497.4669
[torch.FloatTensor of size 1]
]