【动手学深度学习笔记】之通过权重衰减法解决过拟合问题

根据模型，随机生成特征值，计算得到标签。
n_train,n_test,num_inputs = 20,100,200
true_w,true_b = torch.ones(num_inputs,1)*0.01,0.05
#设置为与输入数据同形，方便计算
features = torch.randn((n_train+n_test,num_inputs))
#随机生成训练集和测试集中的特征值
labels = torch.matmul(features,true_w)+true_b
labels += torch.tensor(np.random.normal(0,0.01,size = labels.size()),dtype = torch.float)
#生成训练集和测试集中的标签
train_features = features[:n_train,:]
test_features = features[n_train:,:]
train_labels = labels[:n_train,:]
test_labels = labels[n_train:,:]
#分割测试集和训练集

def init():
	w = torch.randn((num_inputs,1),requires_grad = True)
    b = torch.zeros(1,requires_grad = True)
    return [w,b]

def linear(x,w,b):    return torch.mm(x,w)+b

#平方误差损失函数
def square_loss(y_hat,y):
    return (y_hat-y.view(y_hat.size()))**2/2

#小批量随机梯度下降
def sgd(params,lr,batch_size):
    for param in params:
        param.data -= lr*param.grad/batch_size

def l2(w):    return (w**2).sum()/2

batch_size,num_epochs,lr = 1,100,0.003
net,loss = linear,square_loss

dataset = torch.utils.data.TensorDataset(train_features,train_labels)
train_iter = torch.utils.data.DataLoader(dataset,batch_size,shuffle = True)

def train(lambda1):
#通过设置lambda=0，可以实现过拟合效果。设置lambda=3，实现权重衰减，减轻过拟合。
    w,b = init()
    train_ls,test_ls = [],[]
    for epoch in range(num_epochs+1):
        for x,y in train_iter:
            l = loss(net(x,w,b),y) +lambda1*l2(w)
            l = l.sum()
            
            if w.grad is not None:
                #如果权重参数的梯度信息不是None,代表已经开始计算，需要进行梯度清零
                w.grad.data.zero_()
                b.grad.data.zero_()
            l.backward()
            sgd([w,b],lr,batch_size)
        train_ls.append(loss(net(train_features,w,b),train_labels).mean().item())
        test_ls.append(loss(net(test_features,w,b),test_labels).mean().item())
        print('权重参数的L2范数',w.norm().item())

#令lambda=0，L2范数为0。即不开启权重衰减
train(0)

#令lambda=3，开启权重衰减
train(3)

不难看出，使用权重衰减法后， 过拟合现象得到一定程度的缓解。