# 三分钟带你对 Softmax 划重点

## 1. 什么是Softmax

Softmax 在机器学习和深度学习中有着非常广泛的应用。尤其在处理多分类（C > 2）问题，分类器最后的输出单元需要Softmax 函数进行数值处理。关于Softmax 函数的定义如下所示：

Si=eVi∑CieViSi=eVi∑iCeVi

S_i=\frac{e^{V_i}}{\sum_i^Ce^{V_i}}

V=⎡⎣⎢⎢⎢−32−10⎤⎦⎥⎥⎥V=[−32−10]

V=\left[ \begin{matrix} -3 \\ 2 \\ -1 \\ 0 \end{matrix} \right]

S=⎡⎣⎢⎢⎢0.00570.83900.04180.1135⎤⎦⎥⎥⎥S=[0.00570.83900.04180.1135]

S=\left[ \begin{matrix} 0.0057 \\ 0.8390 \\ 0.0418 \\ 0.1135 \end{matrix} \right]

D=max(V)D=max(V)

D=max(V)

Si=eVi−D∑CieVi−DSi=eVi−D∑iCeVi−D

S_i=\frac{e^{V_i-D}}{\sum_i^Ce^{V_i-D}}

scores = np.array([123, 456, 789])    # example with 3 classes and each having large scores
scores -= np.max(scores)    # scores becomes [-666, -333, 0]
p = np.exp(scores) / np.sum(np.exp(scores))

## 2. Softmax 损失函数

Si=eSyi∑Cj=1eSjSi=eSyi∑j=1CeSj

S_i=\frac{e^{S_{y_i}}}{\sum_{j=1}^Ce^{S_j}}

Si=logeSyi∑Cj=1eSjSi=logeSyi∑j=1CeSj

S_i=log\frac{e^{S_{y_i}}}{\sum_{j=1}^Ce^{S_j}}

Li=−Si=−logeSyi∑Cj=1eSjLi=−Si=−logeSyi∑j=1CeSj

L_i=-S_i=-log\frac{e^{S_{y_i}}}{\sum_{j=1}^Ce^{S_j}}

Li=−logeSyi∑Cj=1eSj=−(syi−log∑j=1Cesj)=−syi+log∑j=1CesjLi=−logeSyi∑j=1CeSj=−(syi−log∑j=1Cesj)=−syi+log∑j=1Cesj

L_i=-log\frac{e^{S_{y_i}}}{\sum_{j=1}^Ce^{S_j}}=-(s_{y_i}-log\sum_{j=1}^Ce^{s_j})=-s_{y_i}+log\sum_{j=1}^Ce^{s_j}

V=⎡⎣⎢⎢⎢−32−10⎤⎦⎥⎥⎥V=[−32−10]

V=\left[ \begin{matrix} -3 \\ 2 \\ -1 \\ 0 \end{matrix} \right]

Li=−2+log(e−3+e2+e−1+e0)=0.1755Li=−2+log(e−3+e2+e−1+e0)=0.1755

L_i=-2+log(e^{-3}+e^2+e^{-1}+e^0)=0.1755

Li=3+log(e−3+e2+e−1+e0)=5.1755Li=3+log(e−3+e2+e−1+e0)=5.1755

L_i=3+log(e^{-3}+e^2+e^{-1}+e^0)=5.1755

## 3. Softmax 反向梯度

Softmax 线性分类器中，线性输出为：

Si=WxiSi=Wxi

S_i=Wx_i

def softmax_loss_naive(W, X, y, reg):
"""
Softmax loss function, naive implementation (with loops)

Inputs have dimension D, there are C classes, and we operate on minibatches
of N examples.

Inputs:
- W: A numpy array of shape (D, C) containing weights.
- X: A numpy array of shape (N, D) containing a minibatch of data.
- y: A numpy array of shape (N,) containing training labels; y[i] = c means
that X[i] has label c, where 0 <= c < C.
- reg: (float) regularization strength

Returns a tuple of:
- loss as single float
- gradient with respect to weights W; an array of same shape as W
"""
# Initialize the loss and gradient to zero.
loss = 0.0
dW = np.zeros_like(W)

num_train = X.shape[0]
num_classes = W.shape[1]
for i in xrange(num_train):
scores = X[i,:].dot(W)
scores_shift = scores - np.max(scores)
right_class = y[i]
loss += -scores_shift[right_class] + np.log(np.sum(np.exp(scores_shift)))
for j in xrange(num_classes):
softmax_output = np.exp(scores_shift[j]) / np.sum(np.exp(scores_shift))
if j == y[i]:
dW[:,j] += (-1 + softmax_output) * X[i,:]
else:
dW[:,j] += softmax_output * X[i,:]

loss /= num_train
loss += 0.5 * reg * np.sum(W * W)
dW /= num_train
dW += reg * W

return loss, dW

def softmax_loss_vectorized(W, X, y, reg):
"""
Softmax loss function, vectorized version.

Inputs and outputs are the same as softmax_loss_naive.
"""
# Initialize the loss and gradient to zero.
loss = 0.0
dW = np.zeros_like(W)

num_train = X.shape[0]
num_classes = W.shape[1]
scores = X.dot(W)
scores_shift = scores - np.max(scores, axis = 1).reshape(-1,1)
softmax_output = np.exp(scores_shift) / np.sum(np.exp(scores_shift), axis=1).reshape(-1,1)
loss = -np.sum(np.log(softmax_output[range(num_train), list(y)]))
loss /= num_train
loss += 0.5 * reg * np.sum(W * W)

dS = softmax_output.copy()
dS[range(num_train), list(y)] += -1
dW = (X.T).dot(dS)
dW = dW / num_train + reg * W

return loss, dW

tic = time.time()
loss_naive, grad_naive = softmax_loss_naive(W, X_train, y_train, 0.000005)
toc = time.time()
print('naive loss: %e computed in %fs' % (loss_naive, toc - tic))

tic = time.time()
loss_vectorized, grad_vectorized = softmax_loss_vectorized(W, X_train, y_train, 0.000005)
toc = time.time()
print('vectorized loss: %e computed in %fs' % (loss_vectorized, toc - tic))

print('Loss difference: %f' % np.abs(loss_naive - loss_vectorized))
print('Gradient difference: %f' % grad_difference)

naive loss: 2.362135e+00 computed in 14.680000s vectorized loss: 2.362135e+00 computed in 0.242000s Loss difference: 0.000000 Gradient difference: 0.000000

## 4. Softmax 与 SVM

Softmax线性分类器的损失函数计算相对概率，又称交叉熵损失「Cross Entropy Loss」。线性 SVM 分类器和 Softmax 线性分类器的主要区别在于损失函数不同。SVM 使用 hinge loss，更关注分类正确样本和错误样本之间的距离「Δ = 1」，只要距离大于 Δ，就不在乎到底距离相差多少，忽略细节。而 Softmax 中每个类别的得分函数都会影响其损失函数的大小。举个例子来说明，类别个数 C = 3，两个样本的得分函数分别为[10, -10, -10]，[10, 9, 9]，真实标签为第0类。对于 SVM 来说，这两个 Li 都为0；但对于Softmax来说，这两个 Li 分别为0.00和0.55，差别很大。

## 5. Softmax 实际应用

# Use the validation set to tune hyperparameters (regularization strength and
# learning rate). You should experiment with different ranges for the learning
# rates and regularization strengths; if you are careful you should be able to
# get a classification accuracy of over 0.35 on the validation set.
results = {}
best_val = -1
best_softmax = None
learning_rates = [1.4e-7, 1.5e-7, 1.6e-7]
regularization_strengths = [8000.0, 9000.0, 10000.0, 11000.0, 18000.0, 19000.0, 20000.0, 21000.0]

for lr in learning_rates:
for reg in regularization_strengths:
softmax = Softmax()
loss = softmax.train(X_train, y_train, learning_rate=lr, reg=reg, num_iters=3000)
y_train_pred = softmax.predict(X_train)
training_accuracy = np.mean(y_train == y_train_pred)
y_val_pred = softmax.predict(X_val)
val_accuracy = np.mean(y_val == y_val_pred)
if val_accuracy > best_val:
best_val = val_accuracy
best_softmax = softmax
results[(lr, reg)] = training_accuracy, val_accuracy

# Print out results.
for lr, reg in sorted(results):
train_accuracy, val_accuracy = results[(lr, reg)]
print('lr %e reg %e train accuracy: %f val accuracy: %f' % (
lr, reg, train_accuracy, val_accuracy))

print('best validation accuracy achieved during cross-validation: %f' % best_val)

# evaluate on test set
# Evaluate the best softmax on test set
y_test_pred = best_softmax.predict(X_test)
test_accuracy = np.mean(y_test == y_test_pred)
print('softmax on raw pixels final test set accuracy: %f' % (test_accuracy, ))

softmax on raw pixels final test set accuracy: 0.386000

# Visualize the learned weights for each class
w = best_softmax.W[:-1,:] # strip out the bias
w = w.reshape(32, 32, 3, 10)

w_min, w_max = np.min(w), np.max(w)

classes = ['plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']
for i in range(10):
plt.subplot(2, 5, i + 1)

# Rescale the weights to be between 0 and 255
wimg = 255.0 * (w[:, :, :, i].squeeze() - w_min) / (w_max - w_min)
plt.imshow(wimg.astype('uint8'))
plt.axis('off')
plt.title(classes[i])

http://cs231n.github.io/linear-classify/

95 篇文章38 人订阅

0 条评论