这次再尝试解决过拟合,把残差模块的数量减少到2个,自适应参数化ReLU激活函数里面第一个全连接层的权重数量,减少为之前的1/8,批量大小设置为1000(主要是为了省时间)。
自适应参数化ReLU激活函数的基本原理如下:
Keras程序如下:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Apr 14 04:17:45 2020
Implemented using TensorFlow 1.0.1 and Keras 2.2.1
Minghang Zhao, Shisheng Zhong, Xuyun Fu, Baoping Tang, Shaojiang Dong, Michael Pecht,
Deep Residual Networks with Adaptively Parametric Rectifier Linear Units for Fault Diagnosis,
IEEE Transactions on Industrial Electronics, 2020, DOI: 10.1109/TIE.2020.2972458
@author: Minghang Zhao
"""
from __future__ import print_function
import keras
import numpy as np
from keras.datasets import cifar10
from keras.layers import Dense, Conv2D, BatchNormalization, Activation, Minimum
from keras.layers import AveragePooling2D, Input, GlobalAveragePooling2D, Concatenate, Reshape
from keras.regularizers import l2
from keras import backend as K
from keras.models import Model
from keras import optimizers
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import LearningRateScheduler
K.set_learning_phase(1)
# The data, split between train and test sets
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
# Noised data
x_train = x_train.astype('float32') / 255.
x_test = x_test.astype('float32') / 255.
x_test = x_test-np.mean(x_train)
x_train = x_train-np.mean(x_train)
print('x_train shape:', x_train.shape)
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')
# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, 10)
y_test = keras.utils.to_categorical(y_test, 10)
# Schedule the learning rate, multiply 0.1 every 1500 epoches
def scheduler(epoch):
if epoch % 1500 == 0 and epoch != 0:
lr = K.get_value(model.optimizer.lr)
K.set_value(model.optimizer.lr, lr * 0.1)
print("lr changed to {}".format(lr * 0.1))
return K.get_value(model.optimizer.lr)
# An adaptively parametric rectifier linear unit (APReLU)
def aprelu(inputs):
# get the number of channels
channels = inputs.get_shape().as_list()[-1]
# get a zero feature map
zeros_input = keras.layers.subtract([inputs, inputs])
# get a feature map with only positive features
pos_input = Activation('relu')(inputs)
# get a feature map with only negative features
neg_input = Minimum()([inputs,zeros_input])
# define a network to obtain the scaling coefficients
scales_p = GlobalAveragePooling2D()(pos_input)
scales_n = GlobalAveragePooling2D()(neg_input)
scales = Concatenate()([scales_n, scales_p])
scales = Dense(channels//8, activation='linear', kernel_initializer='he_normal', kernel_regularizer=l2(1e-4))(scales)
scales = BatchNormalization(momentum=0.9, gamma_regularizer=l2(1e-4))(scales)
scales = Activation('relu')(scales)
scales = Dense(channels, activation='linear', kernel_initializer='he_normal', kernel_regularizer=l2(1e-4))(scales)
scales = BatchNormalization(momentum=0.9, gamma_regularizer=l2(1e-4))(scales)
scales = Activation('sigmoid')(scales)
scales = Reshape((1,1,channels))(scales)
# apply a paramtetric relu
neg_part = keras.layers.multiply([scales, neg_input])
return keras.layers.add([pos_input, neg_part])
# Residual Block
def residual_block(incoming, nb_blocks, out_channels, downsample=False,
downsample_strides=2):
residual = incoming
in_channels = incoming.get_shape().as_list()[-1]
for i in range(nb_blocks):
identity = residual
if not downsample:
downsample_strides = 1
residual = BatchNormalization(momentum=0.9, gamma_regularizer=l2(1e-4))(residual)
residual = aprelu(residual)
residual = Conv2D(out_channels, 3, strides=(downsample_strides, downsample_strides),
padding='same', kernel_initializer='he_normal',
kernel_regularizer=l2(1e-4))(residual)
residual = BatchNormalization(momentum=0.9, gamma_regularizer=l2(1e-4))(residual)
residual = aprelu(residual)
residual = Conv2D(out_channels, 3, padding='same', kernel_initializer='he_normal',
kernel_regularizer=l2(1e-4))(residual)
# Downsampling
if downsample_strides > 1:
identity = AveragePooling2D(pool_size=(1,1), strides=(2,2))(identity)
# Zero_padding to match channels
if in_channels != out_channels:
zeros_identity = keras.layers.subtract([identity, identity])
identity = keras.layers.concatenate([identity, zeros_identity])
in_channels = out_channels
residual = keras.layers.add([residual, identity])
return residual
# define and train a model
inputs = Input(shape=(32, 32, 3))
net = Conv2D(16, 3, padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(1e-4))(inputs)
# net = residual_block(net, 3, 16, downsample=False)
net = residual_block(net, 1, 32, downsample=True)
# net = residual_block(net, 2, 32, downsample=False)
net = residual_block(net, 1, 64, downsample=True)
# net = residual_block(net, 2, 64, downsample=False)
net = BatchNormalization(momentum=0.9, gamma_regularizer=l2(1e-4))(net)
net = Activation('relu')(net)
net = GlobalAveragePooling2D()(net)
outputs = Dense(10, activation='softmax', kernel_initializer='he_normal', kernel_regularizer=l2(1e-4))(net)
model = Model(inputs=inputs, outputs=outputs)
sgd = optimizers.SGD(lr=0.1, decay=0., momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
# data augmentation
datagen = ImageDataGenerator(
# randomly rotate images in the range (deg 0 to 180)
rotation_range=30,
# Range for random zoom
zoom_range = 0.2,
# shear angle in counter-clockwise direction in degrees
shear_range = 30,
# randomly flip images
horizontal_flip=True,
# randomly shift images horizontally
width_shift_range=0.125,
# randomly shift images vertically
height_shift_range=0.125)
reduce_lr = LearningRateScheduler(scheduler)
# fit the model on the batches generated by datagen.flow().
model.fit_generator(datagen.flow(x_train, y_train, batch_size=1000),
validation_data=(x_test, y_test), epochs=5000,
verbose=1, callbacks=[reduce_lr], workers=4)
# get results
K.set_learning_phase(0)
DRSN_train_score = model.evaluate(x_train, y_train, batch_size=1000, verbose=0)
print('Train loss:', DRSN_train_score[0])
print('Train accuracy:', DRSN_train_score[1])
DRSN_test_score = model.evaluate(x_test, y_test, batch_size=1000, verbose=0)
print('Test loss:', DRSN_test_score[0])
print('Test accuracy:', DRSN_test_score[1])
实验结果如下(为了方便看,删掉了一部分等号):
Epoch 2336/5000
50/50 [=======] - 10s 191ms/step - loss: 0.5059 - acc: 0.8646 - val_loss: 0.5191 - val_acc: 0.8616
Epoch 2337/5000
50/50 [=======] - 10s 191ms/step - loss: 0.5013 - acc: 0.8675 - val_loss: 0.5189 - val_acc: 0.8629
Epoch 2338/5000
50/50 [=======] - 9s 189ms/step - loss: 0.5069 - acc: 0.8645 - val_loss: 0.5244 - val_acc: 0.8630
Epoch 2339/5000
50/50 [=======] - 10s 190ms/step - loss: 0.5085 - acc: 0.8638 - val_loss: 0.5189 - val_acc: 0.8609
Epoch 2340/5000
50/50 [=======] - 9s 184ms/step - loss: 0.5009 - acc: 0.8653 - val_loss: 0.5210 - val_acc: 0.8609
Epoch 2341/5000
50/50 [=======] - 9s 187ms/step - loss: 0.5008 - acc: 0.8657 - val_loss: 0.5144 - val_acc: 0.8628
Epoch 2342/5000
50/50 [=======] - 9s 184ms/step - loss: 0.5017 - acc: 0.8661 - val_loss: 0.5251 - val_acc: 0.8606
Epoch 2343/5000
50/50 [=======] - 9s 186ms/step - loss: 0.5069 - acc: 0.8646 - val_loss: 0.5204 - val_acc: 0.8608
Epoch 2344/5000
50/50 [=======] - 9s 189ms/step - loss: 0.4997 - acc: 0.8675 - val_loss: 0.5203 - val_acc: 0.8651
Epoch 2345/5000
50/50 [=======] - 9s 189ms/step - loss: 0.5015 - acc: 0.8672 - val_loss: 0.5220 - val_acc: 0.8610
Epoch 2346/5000
50/50 [=======] - 10s 192ms/step - loss: 0.5005 - acc: 0.8656 - val_loss: 0.5200 - val_acc: 0.8620
Epoch 2347/5000
50/50 [=======] - 9s 186ms/step - loss: 0.5070 - acc: 0.8636 - val_loss: 0.5230 - val_acc: 0.8607
Epoch 2348/5000
50/50 [=======] - 9s 189ms/step - loss: 0.5018 - acc: 0.8657 - val_loss: 0.5175 - val_acc: 0.8611
Epoch 2349/5000
50/50 [=======] - 9s 182ms/step - loss: 0.5042 - acc: 0.8656 - val_loss: 0.5244 - val_acc: 0.8603
Epoch 2350/5000
50/50 [=======] - 9s 187ms/step - loss: 0.5062 - acc: 0.8656 - val_loss: 0.5182 - val_acc: 0.8614
Epoch 2351/5000
50/50 [=======] - 9s 184ms/step - loss: 0.5053 - acc: 0.8642 - val_loss: 0.5183 - val_acc: 0.8611
Epoch 2352/5000
50/50 [=======] - 10s 192ms/step - loss: 0.5094 - acc: 0.8640 - val_loss: 0.5202 - val_acc: 0.8589
Epoch 2353/5000
50/50 [=======] - 10s 191ms/step - loss: 0.5077 - acc: 0.8648 - val_loss: 0.5176 - val_acc: 0.8630
Epoch 2354/5000
50/50 [=======] - 9s 189ms/step - loss: 0.5082 - acc: 0.8635 - val_loss: 0.5217 - val_acc: 0.8620
Epoch 2355/5000
50/50 [=======] - 9s 189ms/step - loss: 0.5024 - acc: 0.8656 - val_loss: 0.5174 - val_acc: 0.8627
Epoch 2356/5000
50/50 [=======] - 9s 185ms/step - loss: 0.5014 - acc: 0.8653 - val_loss: 0.5185 - val_acc: 0.8615
Epoch 2357/5000
50/50 [=======] - 9s 183ms/step - loss: 0.5070 - acc: 0.8650 - val_loss: 0.5207 - val_acc: 0.8630
Epoch 2358/5000
50/50 [=======] - 9s 182ms/step - loss: 0.5031 - acc: 0.8648 - val_loss: 0.5190 - val_acc: 0.8649
Epoch 2359/5000
50/50 [=======] - 9s 186ms/step - loss: 0.5080 - acc: 0.8630 - val_loss: 0.5172 - val_acc: 0.8636
Epoch 2360/5000
50/50 [=======] - 9s 189ms/step - loss: 0.5041 - acc: 0.8661 - val_loss: 0.5220 - val_acc: 0.8597
Epoch 2361/5000
50/50 [=======] - 10s 191ms/step - loss: 0.5029 - acc: 0.8654 - val_loss: 0.5226 - val_acc: 0.8602
Epoch 2362/5000
50/50 [=======] - 9s 188ms/step - loss: 0.4963 - acc: 0.8706 - val_loss: 0.5257 - val_acc: 0.8605
Epoch 2363/5000
50/50 [=======] - 9s 189ms/step - loss: 0.4999 - acc: 0.8673 - val_loss: 0.5233 - val_acc: 0.8563
Epoch 2364/5000
50/50 [=======] - 9s 184ms/step - loss: 0.5031 - acc: 0.8658 - val_loss: 0.5160 - val_acc: 0.8612
Epoch 2365/5000
50/50 [=======] - 9s 184ms/step - loss: 0.5002 - acc: 0.8673 - val_loss: 0.5206 - val_acc: 0.8605
Epoch 2366/5000
50/50 [=======] - 9s 185ms/step - loss: 0.5039 - acc: 0.8649 - val_loss: 0.5190 - val_acc: 0.8608
Epoch 2367/5000
50/50 [=======] - 9s 185ms/step - loss: 0.5082 - acc: 0.8639 - val_loss: 0.5184 - val_acc: 0.8623
Epoch 2368/5000
50/50 [=======] - 9s 189ms/step - loss: 0.5051 - acc: 0.8658 - val_loss: 0.5175 - val_acc: 0.8611
Epoch 2369/5000
50/50 [=======] - 10s 191ms/step - loss: 0.5079 - acc: 0.8635 - val_loss: 0.5172 - val_acc: 0.8612
Epoch 2370/5000
50/50 [=======] - 9s 189ms/step - loss: 0.5056 - acc: 0.8647 - val_loss: 0.5243 - val_acc: 0.8609
Epoch 2371/5000
50/50 [=======] - 9s 186ms/step - loss: 0.5034 - acc: 0.8654 - val_loss: 0.5149 - val_acc: 0.8632
Epoch 2372/5000
50/50 [=======] - 9s 185ms/step - loss: 0.5055 - acc: 0.8625 - val_loss: 0.5204 - val_acc: 0.8613
Epoch 2373/5000
50/50 [=======] - 9s 183ms/step - loss: 0.5015 - acc: 0.8663 - val_loss: 0.5233 - val_acc: 0.8620
Epoch 2374/5000
50/50 [=======] - 9s 185ms/step - loss: 0.5034 - acc: 0.8649 - val_loss: 0.5199 - val_acc: 0.8624
Epoch 2375/5000
50/50 [=======] - 9s 186ms/step - loss: 0.5043 - acc: 0.8667 - val_loss: 0.5198 - val_acc: 0.8643
Epoch 2376/5000
50/50 [=======] - 10s 190ms/step - loss: 0.5060 - acc: 0.8655 - val_loss: 0.5159 - val_acc: 0.8641
Epoch 2377/5000
50/50 [=======] - 10s 190ms/step - loss: 0.5025 - acc: 0.8661 - val_loss: 0.5171 - val_acc: 0.8600
Epoch 2378/5000
50/50 [=======] - 9s 190ms/step - loss: 0.5058 - acc: 0.8643 - val_loss: 0.5229 - val_acc: 0.8596
Epoch 2379/5000
50/50 [=======] - 9s 186ms/step - loss: 0.5067 - acc: 0.8649 - val_loss: 0.5204 - val_acc: 0.8585
Epoch 2380/5000
50/50 [=======] - 9s 185ms/step - loss: 0.5039 - acc: 0.8648 - val_loss: 0.5211 - val_acc: 0.8616
Epoch 2381/5000
50/50 [=======] - 9s 186ms/step - loss: 0.5031 - acc: 0.8650 - val_loss: 0.5184 - val_acc: 0.8609
Epoch 2382/5000
50/50 [=======] - 9s 185ms/step - loss: 0.5069 - acc: 0.8644 - val_loss: 0.5213 - val_acc: 0.8594
Epoch 2383/5000
50/50 [=======] - 9s 189ms/step - loss: 0.5071 - acc: 0.8639 - val_loss: 0.5227 - val_acc: 0.8589
Epoch 2384/5000
50/50 [=======] - 9s 189ms/step - loss: 0.5073 - acc: 0.8663 - val_loss: 0.5190 - val_acc: 0.8623
Epoch 2385/5000
50/50 [=======] - 10s 193ms/step - loss: 0.5056 - acc: 0.8662 - val_loss: 0.5171 - val_acc: 0.8633
Epoch 2386/5000
50/50 [=======] - 9s 188ms/step - loss: 0.5086 - acc: 0.8638 - val_loss: 0.5144 - val_acc: 0.8637
Epoch 2387/5000
50/50 [=======] - 9s 188ms/step - loss: 0.4997 - acc: 0.8667 - val_loss: 0.5166 - val_acc: 0.8624
Epoch 2388/5000
50/50 [=======] - 9s 184ms/step - loss: 0.4999 - acc: 0.8663 - val_loss: 0.5198 - val_acc: 0.8641
Epoch 2389/5000
50/50 [=======] - 9s 185ms/step - loss: 0.5057 - acc: 0.8638 - val_loss: 0.5206 - val_acc: 0.8614
Epoch 2390/5000
50/50 [=======] - 9s 185ms/step - loss: 0.5030 - acc: 0.8666 - val_loss: 0.5173 - val_acc: 0.8608
Epoch 2391/5000
50/50 [=======] - 9s 188ms/step - loss: 0.5021 - acc: 0.8650 - val_loss: 0.5237 - val_acc: 0.8592
Epoch 2392/5000
50/50 [=======] - 9s 190ms/step - loss: 0.5059 - acc: 0.8660 - val_loss: 0.5201 - val_acc: 0.8632
Epoch 2393/5000
50/50 [=======] - 10s 192ms/step - loss: 0.5029 - acc: 0.8650 - val_loss: 0.5208 - val_acc: 0.8596
Epoch 2394/5000
50/50 [=======] - 10s 191ms/step - loss: 0.5014 - acc: 0.8664 - val_loss: 0.5260 - val_acc: 0.8575
Epoch 2395/5000
50/50 [=======] - 9s 185ms/step - loss: 0.5004 - acc: 0.8677 - val_loss: 0.5158 - val_acc: 0.8635
Epoch 2396/5000
50/50 [=======] - 9s 187ms/step - loss: 0.5001 - acc: 0.8665 - val_loss: 0.5234 - val_acc: 0.8596
Epoch 2397/5000
50/50 [=======] - 9s 182ms/step - loss: 0.5038 - acc: 0.8644 - val_loss: 0.5157 - val_acc: 0.8631
Epoch 2398/5000
50/50 [=======] - 9s 187ms/step - loss: 0.5047 - acc: 0.8655 - val_loss: 0.5172 - val_acc: 0.8619
Epoch 2399/5000
50/50 [=======] - 9s 190ms/step - loss: 0.5061 - acc: 0.8647 - val_loss: 0.5225 - val_acc: 0.8602
Epoch 2400/5000
50/50 [=======] - 10s 191ms/step - loss: 0.5054 - acc: 0.8632 - val_loss: 0.5205 - val_acc: 0.8616
Epoch 2401/5000
50/50 [=======] - 10s 193ms/step - loss: 0.5061 - acc: 0.8649 - val_loss: 0.5271 - val_acc: 0.8580
程序还没运行完,看这样子,估计是欠拟合了。
Minghang Zhao, Shisheng Zhong, Xuyun Fu, Baoping Tang, Shaojiang Dong, Michael Pecht, Deep Residual Networks with Adaptively Parametric Rectifier Linear Units for Fault Diagnosis, IEEE Transactions on Industrial Electronics, 2020, DOI: 10.1109/TIE.2020.2972458
https://ieeexplore.ieee.org/document/8998530
————————————————
版权声明:本文为CSDN博主「dangqing1988」的原创文章,遵循CC 4.0 BY-SA版权协议,转载请附上原文出处链接及本声明。
原文链接:https://blog.csdn.net/dangqing1988/article/details/105831139
本文系转载,前往查看
如有侵权,请联系 cloudcommunity@tencent.com 删除。
本文系转载,前往查看
如有侵权,请联系 cloudcommunity@tencent.com 删除。