【哈工大版】动态ReLU：自适应参数化ReLU及Keras代码（调参记录18）Cifar10~94.28%

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Apr 14 04:17:45 2020
Implemented using TensorFlow 1.0.1 and Keras 2.2.1

Minghang Zhao, Shisheng Zhong, Xuyun Fu, Baoping Tang, Shaojiang Dong, Michael Pecht,
Deep Residual Networks with Adaptively Parametric Rectifier Linear Units for Fault Diagnosis, 
IEEE Transactions on Industrial Electronics, DOI: 10.1109/TIE.2020.2972458,
Date of Publication: 13 February 2020

@author: Minghang Zhao
"""

from __future__ import print_function
import keras
import numpy as np
from keras.datasets import cifar10
from keras.layers import Dense, Conv2D, BatchNormalization, Activation, Minimum
from keras.layers import AveragePooling2D, Input, GlobalAveragePooling2D, Concatenate, Reshape
from keras.regularizers import l2
from keras import backend as K
from keras.models import Model
from keras import optimizers
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import LearningRateScheduler
K.set_learning_phase(1)

# The data, split between train and test sets
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
x_train = x_train.astype('float32') / 255.
x_test = x_test.astype('float32') / 255.
x_test = x_test-np.mean(x_train)
x_train = x_train-np.mean(x_train)
print('x_train shape:', x_train.shape)
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')

# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, 10)
y_test = keras.utils.to_categorical(y_test, 10)

# Schedule the learning rate, multiply 0.1 every 1500 epoches
def scheduler(epoch):
    if epoch % 1500 == 0 and epoch != 0:
        lr = K.get_value(model.optimizer.lr)
        K.set_value(model.optimizer.lr, lr * 0.1)
        print("lr changed to {}".format(lr * 0.1))
    return K.get_value(model.optimizer.lr)

# An adaptively parametric rectifier linear unit (APReLU)
def aprelu(inputs):
    # get the number of channels
    channels = inputs.get_shape().as_list()[-1]
    # get a zero feature map
    zeros_input = keras.layers.subtract([inputs, inputs])
    # get a feature map with only positive features
    pos_input = Activation('relu')(inputs)
    # get a feature map with only negative features
    neg_input = Minimum()([inputs,zeros_input])
    # define a network to obtain the scaling coefficients
    scales_p = GlobalAveragePooling2D()(pos_input)
    scales_n = GlobalAveragePooling2D()(neg_input)
    scales = Concatenate()([scales_n, scales_p])
    scales = Dense(channels//16, activation='linear', kernel_initializer='he_normal', kernel_regularizer=l2(1e-4))(scales)
    scales = BatchNormalization(momentum=0.9, gamma_regularizer=l2(1e-4))(scales)
    scales = Activation('relu')(scales)
    scales = Dense(channels, activation='linear', kernel_initializer='he_normal', kernel_regularizer=l2(1e-4))(scales)
    scales = BatchNormalization(momentum=0.9, gamma_regularizer=l2(1e-4))(scales)
    scales = Activation('sigmoid')(scales)
    scales = Reshape((1,1,channels))(scales)
    # apply a paramtetric relu
    neg_part = keras.layers.multiply([scales, neg_input])
    return keras.layers.add([pos_input, neg_part])

# Residual Block
def residual_block(incoming, nb_blocks, out_channels, downsample=False,
                   downsample_strides=2):
    
    residual = incoming
    in_channels = incoming.get_shape().as_list()[-1]
    
    for i in range(nb_blocks):
        
        identity = residual
        
        if not downsample:
            downsample_strides = 1
        
        residual = BatchNormalization(momentum=0.9, gamma_regularizer=l2(1e-4))(residual)
        residual = aprelu(residual)
        residual = Conv2D(out_channels, 3, strides=(downsample_strides, downsample_strides), 
                          padding='same', kernel_initializer='he_normal', 
                          kernel_regularizer=l2(1e-4))(residual)
        
        residual = BatchNormalization(momentum=0.9, gamma_regularizer=l2(1e-4))(residual)
        residual = aprelu(residual)
        residual = Conv2D(out_channels, 3, padding='same', kernel_initializer='he_normal', 
                          kernel_regularizer=l2(1e-4))(residual)
        
        # Downsampling
        if downsample_strides > 1:
            identity = AveragePooling2D(pool_size=(1,1), strides=(2,2))(identity)
            
        # Zero_padding to match channels
        if in_channels != out_channels:
            zeros_identity = keras.layers.subtract([identity, identity])
            identity = keras.layers.concatenate([identity, zeros_identity])
            in_channels = out_channels
        
        residual = keras.layers.add([residual, identity])
    
    return residual


# define and train a model
inputs = Input(shape=(32, 32, 3))
net = Conv2D(16, 3, padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(1e-4))(inputs)
net = residual_block(net, 9, 32, downsample=False)
net = residual_block(net, 1, 32, downsample=True)
net = residual_block(net, 8, 32, downsample=False)
net = residual_block(net, 1, 64, downsample=True)
net = residual_block(net, 8, 64, downsample=False)
net = BatchNormalization(momentum=0.9, gamma_regularizer=l2(1e-4))(net)
net = aprelu(net)
net = GlobalAveragePooling2D()(net)
outputs = Dense(10, activation='softmax', kernel_initializer='he_normal', kernel_regularizer=l2(1e-4))(net)
model = Model(inputs=inputs, outputs=outputs)
sgd = optimizers.SGD(lr=0.1, decay=0., momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])

# data augmentation
datagen = ImageDataGenerator(
    # randomly rotate images in the range (deg 0 to 180)
    rotation_range=30,
    # Range for random zoom
    zoom_range = 0.2,
    # shear angle in counter-clockwise direction in degrees
    shear_range = 30,
    # randomly flip images
    horizontal_flip=True,
    # randomly shift images horizontally
    width_shift_range=0.125,
    # randomly shift images vertically
    height_shift_range=0.125)

reduce_lr = LearningRateScheduler(scheduler)
# fit the model on the batches generated by datagen.flow().
model.fit_generator(datagen.flow(x_train, y_train, batch_size=100),
                    validation_data=(x_test, y_test), epochs=5000, 
                    verbose=1, callbacks=[reduce_lr], workers=4)

# get results
K.set_learning_phase(0)
DRSN_train_score = model.evaluate(x_train, y_train, batch_size=100, verbose=0)
print('Train loss:', DRSN_train_score[0])
print('Train accuracy:', DRSN_train_score[1])
DRSN_test_score = model.evaluate(x_test, y_test, batch_size=100, verbose=0)
print('Test loss:', DRSN_test_score[0])
print('Test accuracy:', DRSN_test_score[1])

Using TensorFlow backend.
x_train shape: (50000, 32, 32, 3)
50000 train samples
10000 test samples
Epoch 1/5000
107s 215ms/step - loss: 2.9739 - acc: 0.3942 - val_loss: 2.5752 - val_acc: 0.5134
Epoch 2/5000
77s 155ms/step - loss: 2.4558 - acc: 0.5143 - val_loss: 2.1374 - val_acc: 0.6048
Epoch 3/5000
78s 155ms/step - loss: 2.1359 - acc: 0.5744 - val_loss: 1.8827 - val_acc: 0.6355
Epoch 4/5000
78s 155ms/step - loss: 1.9044 - acc: 0.6088 - val_loss: 1.6888 - val_acc: 0.6620
Epoch 5/5000
78s 155ms/step - loss: 1.7154 - acc: 0.6381 - val_loss: 1.5106 - val_acc: 0.6946
Epoch 6/5000
77s 155ms/step - loss: 1.5849 - acc: 0.6528 - val_loss: 1.3992 - val_acc: 0.7031
Epoch 7/5000
77s 155ms/step - loss: 1.4700 - acc: 0.6686 - val_loss: 1.2869 - val_acc: 0.7208
Epoch 8/5000
77s 155ms/step - loss: 1.3735 - acc: 0.6851 - val_loss: 1.2068 - val_acc: 0.7400
Epoch 9/5000
77s 155ms/step - loss: 1.2940 - acc: 0.6980 - val_loss: 1.1169 - val_acc: 0.7588
Epoch 10/5000
77s 155ms/step - loss: 1.2318 - acc: 0.7058 - val_loss: 1.0696 - val_acc: 0.7647
Epoch 11/5000
77s 155ms/step - loss: 1.1735 - acc: 0.7202 - val_loss: 1.0117 - val_acc: 0.7757
Epoch 12/5000
77s 155ms/step - loss: 1.1366 - acc: 0.7240 - val_loss: 0.9968 - val_acc: 0.7770
Epoch 13/5000
77s 155ms/step - loss: 1.0974 - acc: 0.7308 - val_loss: 0.9548 - val_acc: 0.7824
Epoch 14/5000
77s 155ms/step - loss: 1.0653 - acc: 0.7390 - val_loss: 0.9196 - val_acc: 0.7885
Epoch 15/5000
77s 155ms/step - loss: 1.0409 - acc: 0.7419 - val_loss: 0.9444 - val_acc: 0.7775
Epoch 16/5000
78s 155ms/step - loss: 1.0122 - acc: 0.7526 - val_loss: 0.9039 - val_acc: 0.7893
Epoch 17/5000
78s 155ms/step - loss: 0.9970 - acc: 0.7561 - val_loss: 0.8913 - val_acc: 0.7952
Epoch 18/5000
77s 155ms/step - loss: 0.9819 - acc: 0.7562 - val_loss: 0.8748 - val_acc: 0.7995
Epoch 19/5000
77s 155ms/step - loss: 0.9638 - acc: 0.7615 - val_loss: 0.8445 - val_acc: 0.8063
Epoch 20/5000
78s 155ms/step - loss: 0.9516 - acc: 0.7662 - val_loss: 0.8522 - val_acc: 0.8011
Epoch 21/5000
78s 155ms/step - loss: 0.9401 - acc: 0.7698 - val_loss: 0.8325 - val_acc: 0.8145
Epoch 22/5000
78s 155ms/step - loss: 0.9302 - acc: 0.7731 - val_loss: 0.8266 - val_acc: 0.8141
Epoch 23/5000
78s 155ms/step - loss: 0.9231 - acc: 0.7747 - val_loss: 0.8390 - val_acc: 0.8043
Epoch 24/5000
77s 155ms/step - loss: 0.9105 - acc: 0.7786 - val_loss: 0.8009 - val_acc: 0.8191
Epoch 25/5000
77s 155ms/step - loss: 0.9027 - acc: 0.7795 - val_loss: 0.7903 - val_acc: 0.8218
Epoch 26/5000
77s 155ms/step - loss: 0.8946 - acc: 0.7863 - val_loss: 0.7812 - val_acc: 0.8271
Epoch 27/5000
78s 155ms/step - loss: 0.8923 - acc: 0.7852 - val_loss: 0.7933 - val_acc: 0.8226
Epoch 28/5000
78s 155ms/step - loss: 0.8808 - acc: 0.7890 - val_loss: 0.7597 - val_acc: 0.8346
Epoch 29/5000
78s 155ms/step - loss: 0.8826 - acc: 0.7881 - val_loss: 0.7639 - val_acc: 0.8332
Epoch 30/5000
77s 155ms/step - loss: 0.8685 - acc: 0.7941 - val_loss: 0.7807 - val_acc: 0.8278
Epoch 31/5000
78s 155ms/step - loss: 0.8647 - acc: 0.7936 - val_loss: 0.7686 - val_acc: 0.8304
Epoch 32/5000
77s 155ms/step - loss: 0.8611 - acc: 0.7971 - val_loss: 0.7619 - val_acc: 0.8324
Epoch 33/5000
77s 155ms/step - loss: 0.8584 - acc: 0.7965 - val_loss: 0.7632 - val_acc: 0.8323
Epoch 34/5000
77s 155ms/step - loss: 0.8513 - acc: 0.8000 - val_loss: 0.7575 - val_acc: 0.8363
Epoch 35/5000
77s 155ms/step - loss: 0.8494 - acc: 0.7990 - val_loss: 0.7669 - val_acc: 0.8293
Epoch 36/5000
77s 155ms/step - loss: 0.8486 - acc: 0.8024 - val_loss: 0.7269 - val_acc: 0.8488
Epoch 37/5000
78s 155ms/step - loss: 0.8432 - acc: 0.8032 - val_loss: 0.7633 - val_acc: 0.8355
Epoch 38/5000
78s 155ms/step - loss: 0.8380 - acc: 0.8061 - val_loss: 0.7554 - val_acc: 0.8356
Epoch 39/5000
77s 155ms/step - loss: 0.8340 - acc: 0.8071 - val_loss: 0.7587 - val_acc: 0.8378
Epoch 40/5000
77s 155ms/step - loss: 0.8310 - acc: 0.8098 - val_loss: 0.7495 - val_acc: 0.8345
Epoch 41/5000
78s 155ms/step - loss: 0.8313 - acc: 0.8092 - val_loss: 0.7370 - val_acc: 0.8446
Epoch 42/5000
77s 155ms/step - loss: 0.8239 - acc: 0.8118 - val_loss: 0.7481 - val_acc: 0.8398
Epoch 43/5000
78s 155ms/step - loss: 0.8244 - acc: 0.8125 - val_loss: 0.7351 - val_acc: 0.8473
Epoch 44/5000
77s 155ms/step - loss: 0.8197 - acc: 0.8122 - val_loss: 0.7396 - val_acc: 0.8444
Epoch 45/5000
78s 155ms/step - loss: 0.8152 - acc: 0.8143 - val_loss: 0.7500 - val_acc: 0.8444
Epoch 46/5000
78s 155ms/step - loss: 0.8158 - acc: 0.8162 - val_loss: 0.7657 - val_acc: 0.8358
Epoch 47/5000
77s 155ms/step - loss: 0.8120 - acc: 0.8173 - val_loss: 0.7409 - val_acc: 0.8431
Epoch 48/5000
77s 155ms/step - loss: 0.8141 - acc: 0.8155 - val_loss: 0.7370 - val_acc: 0.8458
Epoch 49/5000
77s 155ms/step - loss: 0.8054 - acc: 0.8202 - val_loss: 0.7485 - val_acc: 0.8412
Epoch 50/5000
77s 154ms/step - loss: 0.8030 - acc: 0.8189 - val_loss: 0.7240 - val_acc: 0.8494
Epoch 51/5000
77s 155ms/step - loss: 0.8020 - acc: 0.8212 - val_loss: 0.7283 - val_acc: 0.8472
Epoch 52/5000
77s 155ms/step - loss: 0.7991 - acc: 0.8221 - val_loss: 0.7443 - val_acc: 0.8416
Epoch 53/5000
77s 155ms/step - loss: 0.7915 - acc: 0.8252 - val_loss: 0.7322 - val_acc: 0.8468
Epoch 54/5000
77s 155ms/step - loss: 0.7956 - acc: 0.8231 - val_loss: 0.7171 - val_acc: 0.8502
Epoch 55/5000
77s 155ms/step - loss: 0.7890 - acc: 0.8257 - val_loss: 0.7102 - val_acc: 0.8563
Epoch 56/5000
77s 155ms/step - loss: 0.7930 - acc: 0.8233 - val_loss: 0.7081 - val_acc: 0.8569
Epoch 57/5000
77s 155ms/step - loss: 0.7912 - acc: 0.8261 - val_loss: 0.7117 - val_acc: 0.8597
Epoch 58/5000
77s 154ms/step - loss: 0.7878 - acc: 0.8259 - val_loss: 0.7217 - val_acc: 0.8504
Epoch 59/5000
78s 155ms/step - loss: 0.7851 - acc: 0.8294 - val_loss: 0.6983 - val_acc: 0.8614
Epoch 60/5000
77s 155ms/step - loss: 0.7844 - acc: 0.8276 - val_loss: 0.7018 - val_acc: 0.8598
Epoch 61/5000
77s 155ms/step - loss: 0.7840 - acc: 0.8290 - val_loss: 0.7333 - val_acc: 0.8507
Epoch 62/5000
77s 154ms/step - loss: 0.7848 - acc: 0.8288 - val_loss: 0.7279 - val_acc: 0.8503
Epoch 63/5000
77s 155ms/step - loss: 0.7819 - acc: 0.8305 - val_loss: 0.7187 - val_acc: 0.8520
Epoch 64/5000
77s 155ms/step - loss: 0.7822 - acc: 0.8290 - val_loss: 0.7123 - val_acc: 0.8568
Epoch 65/5000
77s 154ms/step - loss: 0.7768 - acc: 0.8323 - val_loss: 0.6983 - val_acc: 0.8621
Epoch 66/5000
77s 155ms/step - loss: 0.7759 - acc: 0.8318 - val_loss: 0.7027 - val_acc: 0.8614
Epoch 67/5000
77s 155ms/step - loss: 0.7783 - acc: 0.8329 - val_loss: 0.7249 - val_acc: 0.8558
Epoch 68/5000
77s 155ms/step - loss: 0.7772 - acc: 0.8324 - val_loss: 0.7089 - val_acc: 0.8595
Epoch 69/5000
77s 155ms/step - loss: 0.7760 - acc: 0.8341 - val_loss: 0.7116 - val_acc: 0.8562
Epoch 70/5000
77s 155ms/step - loss: 0.7720 - acc: 0.8344 - val_loss: 0.7378 - val_acc: 0.8486
Epoch 71/5000
77s 155ms/step - loss: 0.7721 - acc: 0.8357 - val_loss: 0.6936 - val_acc: 0.8653
Epoch 72/5000
77s 155ms/step - loss: 0.7706 - acc: 0.8353 - val_loss: 0.7158 - val_acc: 0.8580
Epoch 73/5000
77s 155ms/step - loss: 0.7727 - acc: 0.8345 - val_loss: 0.7100 - val_acc: 0.8605
Epoch 74/5000
77s 155ms/step - loss: 0.7682 - acc: 0.8385 - val_loss: 0.6988 - val_acc: 0.8621
Epoch 75/5000
77s 155ms/step - loss: 0.7705 - acc: 0.8353 - val_loss: 0.7062 - val_acc: 0.8553
Epoch 76/5000
77s 155ms/step - loss: 0.7682 - acc: 0.8365 - val_loss: 0.7227 - val_acc: 0.8571
Epoch 77/5000
77s 155ms/step - loss: 0.7621 - acc: 0.8388 - val_loss: 0.6973 - val_acc: 0.8635
Epoch 78/5000
77s 155ms/step - loss: 0.7617 - acc: 0.8393 - val_loss: 0.7023 - val_acc: 0.8591
Epoch 79/5000
77s 155ms/step - loss: 0.7618 - acc: 0.8374 - val_loss: 0.6919 - val_acc: 0.8656
Epoch 80/5000
77s 155ms/step - loss: 0.7658 - acc: 0.8372 - val_loss: 0.7192 - val_acc: 0.8559
Epoch 81/5000
77s 155ms/step - loss: 0.7600 - acc: 0.8415 - val_loss: 0.7004 - val_acc: 0.8628
Epoch 82/5000
77s 155ms/step - loss: 0.7623 - acc: 0.8378 - val_loss: 0.6683 - val_acc: 0.8748
Epoch 83/5000
77s 154ms/step - loss: 0.7579 - acc: 0.8407 - val_loss: 0.7009 - val_acc: 0.8604
Epoch 84/5000
77s 155ms/step - loss: 0.7569 - acc: 0.8407 - val_loss: 0.6992 - val_acc: 0.8611
Epoch 85/5000
77s 155ms/step - loss: 0.7515 - acc: 0.8431 - val_loss: 0.7052 - val_acc: 0.8606
Epoch 86/5000
77s 155ms/step - loss: 0.7581 - acc: 0.8420 - val_loss: 0.7094 - val_acc: 0.8549
Epoch 87/5000
77s 155ms/step - loss: 0.7555 - acc: 0.8413 - val_loss: 0.7164 - val_acc: 0.8573
Epoch 88/5000
77s 154ms/step - loss: 0.7555 - acc: 0.8413 - val_loss: 0.7003 - val_acc: 0.8671
Epoch 89/5000
77s 155ms/step - loss: 0.7523 - acc: 0.8468 - val_loss: 0.6850 - val_acc: 0.8698
Epoch 90/5000
77s 155ms/step - loss: 0.7511 - acc: 0.8451 - val_loss: 0.6796 - val_acc: 0.8720
Epoch 91/5000
77s 155ms/step - loss: 0.7539 - acc: 0.8430 - val_loss: 0.6969 - val_acc: 0.8632
Epoch 92/5000
77s 155ms/step - loss: 0.7519 - acc: 0.8438 - val_loss: 0.7286 - val_acc: 0.8538
Epoch 93/5000
77s 155ms/step - loss: 0.7509 - acc: 0.8443 - val_loss: 0.6878 - val_acc: 0.8669
Epoch 94/5000
77s 155ms/step - loss: 0.7487 - acc: 0.8455 - val_loss: 0.6985 - val_acc: 0.8661
Epoch 95/5000
77s 155ms/step - loss: 0.7497 - acc: 0.8444 - val_loss: 0.6983 - val_acc: 0.8646
Epoch 96/5000
77s 155ms/step - loss: 0.7501 - acc: 0.8437 - val_loss: 0.6890 - val_acc: 0.8677
Epoch 97/5000
77s 155ms/step - loss: 0.7471 - acc: 0.8461 - val_loss: 0.6912 - val_acc: 0.8662
Epoch 98/5000
77s 154ms/step - loss: 0.7472 - acc: 0.8466 - val_loss: 0.6830 - val_acc: 0.8695
Epoch 99/5000
77s 155ms/step - loss: 0.7476 - acc: 0.8451 - val_loss: 0.7109 - val_acc: 0.8630
Epoch 100/5000
77s 155ms/step - loss: 0.7452 - acc: 0.8473 - val_loss: 0.6877 - val_acc: 0.8684
Epoch 101/5000
78s 155ms/step - loss: 0.7436 - acc: 0.8483 - val_loss: 0.7104 - val_acc: 0.8597
Epoch 102/5000
77s 155ms/step - loss: 0.7479 - acc: 0.8471 - val_loss: 0.6644 - val_acc: 0.8757
Epoch 103/5000
77s 154ms/step - loss: 0.7395 - acc: 0.8489 - val_loss: 0.6755 - val_acc: 0.8717
Epoch 104/5000
77s 155ms/step - loss: 0.7394 - acc: 0.8501 - val_loss: 0.6735 - val_acc: 0.8696
Epoch 105/5000
77s 155ms/step - loss: 0.7442 - acc: 0.8465 - val_loss: 0.6777 - val_acc: 0.8741
Epoch 106/5000
77s 154ms/step - loss: 0.7429 - acc: 0.8485 - val_loss: 0.6860 - val_acc: 0.8713
Epoch 107/5000
77s 155ms/step - loss: 0.7420 - acc: 0.8494 - val_loss: 0.6858 - val_acc: 0.8711
Epoch 108/5000
77s 155ms/step - loss: 0.7438 - acc: 0.8483 - val_loss: 0.6807 - val_acc: 0.8702
Epoch 109/5000
77s 154ms/step - loss: 0.7385 - acc: 0.8506 - val_loss: 0.6778 - val_acc: 0.8713
Epoch 110/5000
77s 155ms/step - loss: 0.7409 - acc: 0.8482 - val_loss: 0.7039 - val_acc: 0.8640
Epoch 111/5000
77s 155ms/step - loss: 0.7376 - acc: 0.8501 - val_loss: 0.6737 - val_acc: 0.8745
Epoch 112/5000
77s 155ms/step - loss: 0.7388 - acc: 0.8497 - val_loss: 0.6785 - val_acc: 0.8711
Epoch 113/5000
77s 155ms/step - loss: 0.7307 - acc: 0.8534 - val_loss: 0.6741 - val_acc: 0.8719
Epoch 114/5000
77s 155ms/step - loss: 0.7443 - acc: 0.8496 - val_loss: 0.6898 - val_acc: 0.8699
Epoch 115/5000
77s 155ms/step - loss: 0.7340 - acc: 0.8514 - val_loss: 0.6953 - val_acc: 0.8666
Epoch 116/5000
77s 155ms/step - loss: 0.7367 - acc: 0.8516 - val_loss: 0.6951 - val_acc: 0.8656
Epoch 117/5000
77s 155ms/step - loss: 0.7367 - acc: 0.8508 - val_loss: 0.6809 - val_acc: 0.8709
Epoch 118/5000
77s 155ms/step - loss: 0.7324 - acc: 0.8521 - val_loss: 0.6809 - val_acc: 0.8746
Epoch 119/5000
77s 155ms/step - loss: 0.7335 - acc: 0.8528 - val_loss: 0.6919 - val_acc: 0.8663
Epoch 120/5000
77s 155ms/step - loss: 0.7363 - acc: 0.8507 - val_loss: 0.6922 - val_acc: 0.8693
Epoch 121/5000
77s 155ms/step - loss: 0.7341 - acc: 0.8530 - val_loss: 0.6859 - val_acc: 0.8717
Epoch 122/5000
77s 155ms/step - loss: 0.7334 - acc: 0.8533 - val_loss: 0.7199 - val_acc: 0.8596
Epoch 123/5000
77s 155ms/step - loss: 0.7328 - acc: 0.8527 - val_loss: 0.6934 - val_acc: 0.8658
Epoch 124/5000
77s 155ms/step - loss: 0.7278 - acc: 0.8532 - val_loss: 0.6822 - val_acc: 0.8684
Epoch 125/5000
77s 155ms/step - loss: 0.7338 - acc: 0.8532 - val_loss: 0.6865 - val_acc: 0.8700
Epoch 126/5000
77s 155ms/step - loss: 0.7319 - acc: 0.8528 - val_loss: 0.6759 - val_acc: 0.8732
Epoch 127/5000
78s 155ms/step - loss: 0.7335 - acc: 0.8519 - val_loss: 0.6776 - val_acc: 0.8744
Epoch 128/5000
77s 155ms/step - loss: 0.7302 - acc: 0.8521 - val_loss: 0.6883 - val_acc: 0.8678
Epoch 129/5000
77s 155ms/step - loss: 0.7299 - acc: 0.8543 - val_loss: 0.6868 - val_acc: 0.8697
Epoch 130/5000
77s 155ms/step - loss: 0.7318 - acc: 0.8527 - val_loss: 0.6784 - val_acc: 0.8735
Epoch 131/5000
77s 155ms/step - loss: 0.7280 - acc: 0.8559 - val_loss: 0.6750 - val_acc: 0.8745
Epoch 132/5000
77s 155ms/step - loss: 0.7301 - acc: 0.8543 - val_loss: 0.6815 - val_acc: 0.8721
Epoch 133/5000
77s 155ms/step - loss: 0.7319 - acc: 0.8536 - val_loss: 0.6647 - val_acc: 0.8762
Epoch 134/5000
77s 155ms/step - loss: 0.7338 - acc: 0.8532 - val_loss: 0.6733 - val_acc: 0.8767
Epoch 135/5000
77s 155ms/step - loss: 0.7330 - acc: 0.8533 - val_loss: 0.6891 - val_acc: 0.8705
Epoch 136/5000
77s 155ms/step - loss: 0.7251 - acc: 0.8563 - val_loss: 0.6765 - val_acc: 0.8781
Epoch 137/5000
77s 154ms/step - loss: 0.7315 - acc: 0.8543 - val_loss: 0.6875 - val_acc: 0.8701
Epoch 138/5000
77s 155ms/step - loss: 0.7290 - acc: 0.8546 - val_loss: 0.6824 - val_acc: 0.8701
Epoch 139/5000
77s 155ms/step - loss: 0.7256 - acc: 0.8565 - val_loss: 0.6717 - val_acc: 0.8731
Epoch 140/5000
77s 155ms/step - loss: 0.7240 - acc: 0.8548 - val_loss: 0.6846 - val_acc: 0.8727
Epoch 141/5000
77s 155ms/step - loss: 0.7244 - acc: 0.8580 - val_loss: 0.6681 - val_acc: 0.8793
Epoch 142/5000
77s 155ms/step - loss: 0.7270 - acc: 0.8555 - val_loss: 0.6734 - val_acc: 0.8781
Epoch 143/5000
77s 154ms/step - loss: 0.7295 - acc: 0.8567 - val_loss: 0.6731 - val_acc: 0.8783
Epoch 144/5000
77s 154ms/step - loss: 0.7271 - acc: 0.8549 - val_loss: 0.6803 - val_acc: 0.8758
Epoch 145/5000
77s 154ms/step - loss: 0.7257 - acc: 0.8544 - val_loss: 0.6675 - val_acc: 0.8763
Epoch 146/5000
78s 157ms/step - loss: 0.7257 - acc: 0.8547 - val_loss: 0.6824 - val_acc: 0.8709
Epoch 147/5000
78s 156ms/step - loss: 0.7259 - acc: 0.8550 - val_loss: 0.6763 - val_acc: 0.8733
Epoch 148/5000
77s 155ms/step - loss: 0.7291 - acc: 0.8542 - val_loss: 0.6908 - val_acc: 0.8683
Epoch 149/5000
78s 156ms/step - loss: 0.7262 - acc: 0.8571 - val_loss: 0.6593 - val_acc: 0.8818
Epoch 150/5000
78s 157ms/step - loss: 0.7260 - acc: 0.8565 - val_loss: 0.6801 - val_acc: 0.8763
Epoch 151/5000
78s 156ms/step - loss: 0.7256 - acc: 0.8574 - val_loss: 0.6660 - val_acc: 0.8800
Epoch 152/5000
78s 155ms/step - loss: 0.7201 - acc: 0.8602 - val_loss: 0.6659 - val_acc: 0.8799
Epoch 153/5000
78s 156ms/step - loss: 0.7257 - acc: 0.8561 - val_loss: 0.6740 - val_acc: 0.8764
Epoch 154/5000
77s 154ms/step - loss: 0.7273 - acc: 0.8552 - val_loss: 0.6778 - val_acc: 0.8751
Epoch 155/5000
77s 154ms/step - loss: 0.7214 - acc: 0.8579 - val_loss: 0.6649 - val_acc: 0.8803
Epoch 156/5000
77s 154ms/step - loss: 0.7220 - acc: 0.8558 - val_loss: 0.6748 - val_acc: 0.8782
Epoch 157/5000
77s 154ms/step - loss: 0.7227 - acc: 0.8584 - val_loss: 0.6671 - val_acc: 0.8779
Epoch 158/5000
77s 154ms/step - loss: 0.7264 - acc: 0.8558 - val_loss: 0.6607 - val_acc: 0.8780
Epoch 159/5000
77s 154ms/step - loss: 0.7202 - acc: 0.8597 - val_loss: 0.6702 - val_acc: 0.8785
Epoch 160/5000
77s 154ms/step - loss: 0.7211 - acc: 0.8563 - val_loss: 0.6630 - val_acc: 0.8812
Epoch 161/5000
77s 154ms/step - loss: 0.7229 - acc: 0.8579 - val_loss: 0.6743 - val_acc: 0.8769
Epoch 162/5000
77s 154ms/step - loss: 0.7216 - acc: 0.8571 - val_loss: 0.6772 - val_acc: 0.8761
Epoch 163/5000
77s 154ms/step - loss: 0.7168 - acc: 0.8599 - val_loss: 0.6661 - val_acc: 0.8786
Epoch 164/5000
77s 154ms/step - loss: 0.7208 - acc: 0.8573 - val_loss: 0.6801 - val_acc: 0.8763
Epoch 165/5000
77s 154ms/step - loss: 0.7240 - acc: 0.8590 - val_loss: 0.6678 - val_acc: 0.8781
Epoch 166/5000
79s 158ms/step - loss: 0.7154 - acc: 0.8621 - val_loss: 0.6768 - val_acc: 0.8744
Epoch 167/5000
78s 155ms/step - loss: 0.7177 - acc: 0.8600 - val_loss: 0.6536 - val_acc: 0.8816
Epoch 168/5000
78s 155ms/step - loss: 0.7160 - acc: 0.8585 - val_loss: 0.7057 - val_acc: 0.8640
Epoch 169/5000
77s 155ms/step - loss: 0.7196 - acc: 0.8598 - val_loss: 0.6517 - val_acc: 0.8854
Epoch 170/5000
77s 154ms/step - loss: 0.7153 - acc: 0.8581 - val_loss: 0.6592 - val_acc: 0.8844
Epoch 171/5000
77s 154ms/step - loss: 0.7189 - acc: 0.8587 - val_loss: 0.6797 - val_acc: 0.8774
Epoch 172/5000
77s 154ms/step - loss: 0.7182 - acc: 0.8583 - val_loss: 0.6625 - val_acc: 0.8767
Epoch 173/5000
77s 154ms/step - loss: 0.7161 - acc: 0.8597 - val_loss: 0.6863 - val_acc: 0.8726
Epoch 174/5000
77s 154ms/step - loss: 0.7171 - acc: 0.8602 - val_loss: 0.6963 - val_acc: 0.8669
Epoch 175/5000
77s 154ms/step - loss: 0.7211 - acc: 0.8589 - val_loss: 0.6638 - val_acc: 0.8830
Epoch 176/5000
77s 154ms/step - loss: 0.7149 - acc: 0.8607 - val_loss: 0.6717 - val_acc: 0.8723
Epoch 177/5000
77s 153ms/step - loss: 0.7135 - acc: 0.8619 - val_loss: 0.7025 - val_acc: 0.8681
Epoch 178/5000
77s 154ms/step - loss: 0.7175 - acc: 0.8587 - val_loss: 0.6832 - val_acc: 0.8726
Epoch 179/5000
77s 154ms/step - loss: 0.7155 - acc: 0.8598 - val_loss: 0.6633 - val_acc: 0.8790
Epoch 180/5000
77s 154ms/step - loss: 0.7189 - acc: 0.8598 - val_loss: 0.6783 - val_acc: 0.8756
Epoch 181/5000
77s 154ms/step - loss: 0.7187 - acc: 0.8605 - val_loss: 0.6682 - val_acc: 0.8805
Epoch 182/5000
77s 154ms/step - loss: 0.7174 - acc: 0.8607 - val_loss: 0.6748 - val_acc: 0.8792
Epoch 183/5000
77s 154ms/step - loss: 0.7133 - acc: 0.8619 - val_loss: 0.6691 - val_acc: 0.8773
Epoch 184/5000
77s 154ms/step - loss: 0.7180 - acc: 0.8589 - val_loss: 0.6698 - val_acc: 0.8804
Epoch 185/5000
77s 154ms/step - loss: 0.7133 - acc: 0.8629 - val_loss: 0.6720 - val_acc: 0.8753
Epoch 186/5000
77s 154ms/step - loss: 0.7128 - acc: 0.8615 - val_loss: 0.6919 - val_acc: 0.8699
Epoch 187/5000
77s 154ms/step - loss: 0.7196 - acc: 0.8598 - val_loss: 0.6849 - val_acc: 0.8721
Epoch 188/5000
77s 154ms/step - loss: 0.7207 - acc: 0.8581 - val_loss: 0.6754 - val_acc: 0.8780
Epoch 189/5000
77s 154ms/step - loss: 0.7150 - acc: 0.8619 - val_loss: 0.6571 - val_acc: 0.8837
Epoch 190/5000
77s 154ms/step - loss: 0.7159 - acc: 0.8617 - val_loss: 0.6698 - val_acc: 0.8783
Epoch 191/5000
77s 154ms/step - loss: 0.7158 - acc: 0.8606 - val_loss: 0.6782 - val_acc: 0.8772
Epoch 192/5000
77s 153ms/step - loss: 0.7121 - acc: 0.8609 - val_loss: 0.6745 - val_acc: 0.8786
Epoch 193/5000
77s 154ms/step - loss: 0.7137 - acc: 0.8624 - val_loss: 0.6800 - val_acc: 0.8765
Epoch 194/5000
77s 154ms/step - loss: 0.7158 - acc: 0.8607 - val_loss: 0.6836 - val_acc: 0.8734
Epoch 195/5000
77s 153ms/step - loss: 0.7067 - acc: 0.8640 - val_loss: 0.6691 - val_acc: 0.8784
Epoch 196/5000
77s 154ms/step - loss: 0.7111 - acc: 0.8629 - val_loss: 0.6716 - val_acc: 0.8759
Epoch 197/5000
77s 154ms/step - loss: 0.7139 - acc: 0.8611 - val_loss: 0.6686 - val_acc: 0.8777
Epoch 198/5000
77s 154ms/step - loss: 0.7134 - acc: 0.8621 - val_loss: 0.6709 - val_acc: 0.8767
Epoch 199/5000
77s 154ms/step - loss: 0.7101 - acc: 0.8625 - val_loss: 0.6513 - val_acc: 0.8863
Epoch 200/5000
77s 154ms/step - loss: 0.7128 - acc: 0.8617 - val_loss: 0.6713 - val_acc: 0.8768
Epoch 201/5000
77s 154ms/step - loss: 0.7171 - acc: 0.8620 - val_loss: 0.6620 - val_acc: 0.8830
Epoch 202/5000
77s 153ms/step - loss: 0.7122 - acc: 0.8617 - val_loss: 0.6694 - val_acc: 0.8821
Epoch 203/5000
77s 154ms/step - loss: 0.7127 - acc: 0.8604 - val_loss: 0.6717 - val_acc: 0.8797
Epoch 204/5000
77s 154ms/step - loss: 0.7103 - acc: 0.8634 - val_loss: 0.6683 - val_acc: 0.8826
Epoch 205/5000
77s 154ms/step - loss: 0.7123 - acc: 0.8612 - val_loss: 0.6576 - val_acc: 0.8831
Epoch 206/5000
77s 154ms/step - loss: 0.7078 - acc: 0.8632 - val_loss: 0.6453 - val_acc: 0.8842
Epoch 207/5000
77s 154ms/step - loss: 0.7135 - acc: 0.8623 - val_loss: 0.6683 - val_acc: 0.8804
Epoch 208/5000
77s 154ms/step - loss: 0.7038 - acc: 0.8655 - val_loss: 0.6795 - val_acc: 0.8755
Epoch 209/5000
77s 154ms/step - loss: 0.7160 - acc: 0.8607 - val_loss: 0.6709 - val_acc: 0.8773
Epoch 210/5000
77s 154ms/step - loss: 0.7122 - acc: 0.8629 - val_loss: 0.6917 - val_acc: 0.8718
Epoch 211/5000
77s 154ms/step - loss: 0.7064 - acc: 0.8643 - val_loss: 0.6831 - val_acc: 0.8701
Epoch 212/5000
77s 154ms/step - loss: 0.7090 - acc: 0.8629 - val_loss: 0.6475 - val_acc: 0.8875
Epoch 213/5000
77s 154ms/step - loss: 0.7086 - acc: 0.8621 - val_loss: 0.6740 - val_acc: 0.8741
Epoch 214/5000
77s 154ms/step - loss: 0.7105 - acc: 0.8611 - val_loss: 0.6682 - val_acc: 0.8792
Epoch 215/5000
77s 154ms/step - loss: 0.7129 - acc: 0.8616 - val_loss: 0.6930 - val_acc: 0.8682
Epoch 216/5000
77s 154ms/step - loss: 0.7105 - acc: 0.8632 - val_loss: 0.6647 - val_acc: 0.8776
Epoch 217/5000
77s 154ms/step - loss: 0.7086 - acc: 0.8634 - val_loss: 0.6555 - val_acc: 0.8832
Epoch 218/5000
77s 154ms/step - loss: 0.7128 - acc: 0.8634 - val_loss: 0.6665 - val_acc: 0.8780
Epoch 219/5000
77s 153ms/step - loss: 0.7050 - acc: 0.8641 - val_loss: 0.6698 - val_acc: 0.8800
Epoch 220/5000
77s 154ms/step - loss: 0.7042 - acc: 0.8628 - val_loss: 0.6882 - val_acc: 0.8706
Epoch 221/5000
77s 154ms/step - loss: 0.7098 - acc: 0.8615 - val_loss: 0.6649 - val_acc: 0.8802
Epoch 222/5000
77s 154ms/step - loss: 0.7053 - acc: 0.8648 - val_loss: 0.6672 - val_acc: 0.8808
Epoch 223/5000
77s 154ms/step - loss: 0.7064 - acc: 0.8638 - val_loss: 0.6609 - val_acc: 0.8823
Epoch 224/5000
77s 154ms/step - loss: 0.7071 - acc: 0.8627 - val_loss: 0.6677 - val_acc: 0.8812
Epoch 225/5000
77s 154ms/step - loss: 0.7049 - acc: 0.8655 - val_loss: 0.6656 - val_acc: 0.8784
Epoch 226/5000
77s 154ms/step - loss: 0.7065 - acc: 0.8649 - val_loss: 0.6582 - val_acc: 0.8839
Epoch 227/5000
77s 154ms/step - loss: 0.7049 - acc: 0.8644 - val_loss: 0.6812 - val_acc: 0.8763
Epoch 228/5000
77s 154ms/step - loss: 0.7052 - acc: 0.8657 - val_loss: 0.6639 - val_acc: 0.8794
Epoch 229/5000
77s 154ms/step - loss: 0.7054 - acc: 0.8654 - val_loss: 0.6662 - val_acc: 0.8816
Epoch 230/5000
77s 154ms/step - loss: 0.7096 - acc: 0.8639 - val_loss: 0.6789 - val_acc: 0.8752
Epoch 231/5000
77s 154ms/step - loss: 0.7049 - acc: 0.8667 - val_loss: 0.6748 - val_acc: 0.8768
Epoch 232/5000
77s 154ms/step - loss: 0.7047 - acc: 0.8654 - val_loss: 0.6710 - val_acc: 0.8773
Epoch 233/5000
78s 155ms/step - loss: 0.7051 - acc: 0.8648 - val_loss: 0.6516 - val_acc: 0.8839
Epoch 234/5000
77s 155ms/step - loss: 0.7056 - acc: 0.8619 - val_loss: 0.6996 - val_acc: 0.8685
Epoch 235/5000
77s 155ms/step - loss: 0.7043 - acc: 0.8652 - val_loss: 0.6481 - val_acc: 0.8843
Epoch 236/5000
77s 155ms/step - loss: 0.7038 - acc: 0.8638 - val_loss: 0.6760 - val_acc: 0.8808
Epoch 237/5000
77s 155ms/step - loss: 0.7035 - acc: 0.8660 - val_loss: 0.6597 - val_acc: 0.8833
Epoch 238/5000
77s 155ms/step - loss: 0.7042 - acc: 0.8642 - val_loss: 0.6615 - val_acc: 0.8825
Epoch 239/5000
77s 155ms/step - loss: 0.7061 - acc: 0.8645 - val_loss: 0.6715 - val_acc: 0.8753
Epoch 240/5000
77s 155ms/step - loss: 0.7057 - acc: 0.8648 - val_loss: 0.6609 - val_acc: 0.8834
Epoch 241/5000
77s 155ms/step - loss: 0.6990 - acc: 0.8671 - val_loss: 0.6591 - val_acc: 0.8849
Epoch 242/5000
77s 155ms/step - loss: 0.7092 - acc: 0.8638 - val_loss: 0.6439 - val_acc: 0.8888
Epoch 243/5000
77s 155ms/step - loss: 0.7043 - acc: 0.8650 - val_loss: 0.6699 - val_acc: 0.8803
Epoch 244/5000
77s 155ms/step - loss: 0.7055 - acc: 0.8641 - val_loss: 0.6740 - val_acc: 0.8780
Epoch 245/5000
78s 155ms/step - loss: 0.7010 - acc: 0.8668 - val_loss: 0.6673 - val_acc: 0.8807
Epoch 246/5000
78s 155ms/step - loss: 0.7043 - acc: 0.8649 - val_loss: 0.6574 - val_acc: 0.8857
Epoch 247/5000
77s 155ms/step - loss: 0.7071 - acc: 0.8643 - val_loss: 0.6640 - val_acc: 0.8831
Epoch 248/5000
78s 155ms/step - loss: 0.7048 - acc: 0.8664 - val_loss: 0.6647 - val_acc: 0.8825
Epoch 249/5000
77s 155ms/step - loss: 0.7033 - acc: 0.8649 - val_loss: 0.6821 - val_acc: 0.8750
Epoch 250/5000
77s 155ms/step - loss: 0.7035 - acc: 0.8665 - val_loss: 0.6776 - val_acc: 0.8757
Epoch 251/5000
77s 155ms/step - loss: 0.7050 - acc: 0.8643 - val_loss: 0.6857 - val_acc: 0.8721
Epoch 252/5000
78s 155ms/step - loss: 0.7044 - acc: 0.8664 - val_loss: 0.6780 - val_acc: 0.8746
Epoch 253/5000
78s 155ms/step - loss: 0.7033 - acc: 0.8657 - val_loss: 0.6739 - val_acc: 0.8799
Epoch 254/5000
78s 155ms/step - loss: 0.7057 - acc: 0.8670 - val_loss: 0.6654 - val_acc: 0.8812
Epoch 255/5000
77s 155ms/step - loss: 0.7065 - acc: 0.8656 - val_loss: 0.6798 - val_acc: 0.8752
Epoch 256/5000
78s 155ms/step - loss: 0.7033 - acc: 0.8656 - val_loss: 0.6670 - val_acc: 0.8814
Epoch 257/5000
78s 155ms/step - loss: 0.7041 - acc: 0.8648 - val_loss: 0.6699 - val_acc: 0.8823
Epoch 258/5000
77s 155ms/step - loss: 0.6975 - acc: 0.8682 - val_loss: 0.6650 - val_acc: 0.8792
Epoch 259/5000
78s 155ms/step - loss: 0.7007 - acc: 0.8681 - val_loss: 0.6572 - val_acc: 0.8843
Epoch 260/5000
77s 155ms/step - loss: 0.6987 - acc: 0.8668 - val_loss: 0.6633 - val_acc: 0.8820
Epoch 261/5000
77s 155ms/step - loss: 0.7003 - acc: 0.8667 - val_loss: 0.6728 - val_acc: 0.8835
Epoch 262/5000
77s 155ms/step - loss: 0.7007 - acc: 0.8669 - val_loss: 0.6813 - val_acc: 0.8791
Epoch 263/5000
78s 155ms/step - loss: 0.7036 - acc: 0.8659 - val_loss: 0.6670 - val_acc: 0.8805
Epoch 264/5000
78s 155ms/step - loss: 0.7001 - acc: 0.8667 - val_loss: 0.6954 - val_acc: 0.8714
Epoch 265/5000
77s 155ms/step - loss: 0.7016 - acc: 0.8678 - val_loss: 0.6727 - val_acc: 0.8774
Epoch 266/5000
78s 155ms/step - loss: 0.7005 - acc: 0.8664 - val_loss: 0.6804 - val_acc: 0.8751
Epoch 267/5000
77s 155ms/step - loss: 0.7004 - acc: 0.8665 - val_loss: 0.6690 - val_acc: 0.8828
...
Epoch 1491/5000
77s 154ms/step - loss: 0.6771 - acc: 0.8807 - val_loss: 0.6669 - val_acc: 0.8886
Epoch 1492/5000
77s 154ms/step - loss: 0.6735 - acc: 0.8827 - val_loss: 0.6475 - val_acc: 0.8920
Epoch 1493/5000
77s 154ms/step - loss: 0.6773 - acc: 0.8814 - val_loss: 0.6660 - val_acc: 0.8891
Epoch 1494/5000
77s 154ms/step - loss: 0.6807 - acc: 0.8792 - val_loss: 0.6478 - val_acc: 0.8925
Epoch 1495/5000
77s 154ms/step - loss: 0.6784 - acc: 0.8819 - val_loss: 0.6947 - val_acc: 0.8754
Epoch 1496/5000
77s 154ms/step - loss: 0.6815 - acc: 0.8802 - val_loss: 0.6643 - val_acc: 0.8882
Epoch 1497/5000
77s 154ms/step - loss: 0.6844 - acc: 0.8787 - val_loss: 0.6254 - val_acc: 0.9019
Epoch 1498/5000
77s 154ms/step - loss: 0.6783 - acc: 0.8815 - val_loss: 0.6634 - val_acc: 0.8876
Epoch 1499/5000
77s 154ms/step - loss: 0.6787 - acc: 0.8800 - val_loss: 0.6708 - val_acc: 0.8862
Epoch 1500/5000
77s 154ms/step - loss: 0.6815 - acc: 0.8810 - val_loss: 0.6554 - val_acc: 0.8919
Epoch 1501/5000
lr changed to 0.010000000149011612
77s 154ms/step - loss: 0.5717 - acc: 0.9188 - val_loss: 0.5618 - val_acc: 0.9242
Epoch 1502/5000
77s 154ms/step - loss: 0.5116 - acc: 0.9373 - val_loss: 0.5485 - val_acc: 0.9261
Epoch 1503/5000
77s 154ms/step - loss: 0.4922 - acc: 0.9420 - val_loss: 0.5352 - val_acc: 0.9278
Epoch 1504/5000
77s 154ms/step - loss: 0.4747 - acc: 0.9466 - val_loss: 0.5291 - val_acc: 0.9268
Epoch 1505/5000
77s 154ms/step - loss: 0.4657 - acc: 0.9481 - val_loss: 0.5215 - val_acc: 0.9288
Epoch 1506/5000
77s 154ms/step - loss: 0.4528 - acc: 0.9497 - val_loss: 0.5122 - val_acc: 0.9292
Epoch 1507/5000
77s 154ms/step - loss: 0.4453 - acc: 0.9513 - val_loss: 0.5116 - val_acc: 0.9304
Epoch 1508/5000
77s 154ms/step - loss: 0.4360 - acc: 0.9530 - val_loss: 0.5029 - val_acc: 0.9325
Epoch 1509/5000
77s 154ms/step - loss: 0.4239 - acc: 0.9547 - val_loss: 0.5028 - val_acc: 0.9297
Epoch 1510/5000
77s 154ms/step - loss: 0.4195 - acc: 0.9549 - val_loss: 0.4906 - val_acc: 0.9322
Epoch 1511/5000
77s 154ms/step - loss: 0.4109 - acc: 0.9565 - val_loss: 0.4917 - val_acc: 0.9308
Epoch 1512/5000
77s 154ms/step - loss: 0.4071 - acc: 0.9561 - val_loss: 0.4811 - val_acc: 0.9324
Epoch 1513/5000
77s 154ms/step - loss: 0.3979 - acc: 0.9580 - val_loss: 0.4752 - val_acc: 0.9348
Epoch 1514/5000
77s 154ms/step - loss: 0.3896 - acc: 0.9590 - val_loss: 0.4839 - val_acc: 0.9303
Epoch 1515/5000
77s 154ms/step - loss: 0.3846 - acc: 0.9599 - val_loss: 0.4811 - val_acc: 0.9307
Epoch 1516/5000
77s 153ms/step - loss: 0.3815 - acc: 0.9592 - val_loss: 0.4753 - val_acc: 0.9317
Epoch 1517/5000
77s 154ms/step - loss: 0.3730 - acc: 0.9610 - val_loss: 0.4688 - val_acc: 0.9302
Epoch 1518/5000
77s 154ms/step - loss: 0.3661 - acc: 0.9624 - val_loss: 0.4653 - val_acc: 0.9324
Epoch 1519/5000
77s 154ms/step - loss: 0.3621 - acc: 0.9618 - val_loss: 0.4602 - val_acc: 0.9334
Epoch 1520/5000
77s 154ms/step - loss: 0.3581 - acc: 0.9636 - val_loss: 0.4604 - val_acc: 0.9310
Epoch 1521/5000
77s 154ms/step - loss: 0.3542 - acc: 0.9622 - val_loss: 0.4598 - val_acc: 0.9299
Epoch 1522/5000
77s 154ms/step - loss: 0.3498 - acc: 0.9632 - val_loss: 0.4564 - val_acc: 0.9307
Epoch 1523/5000
77s 153ms/step - loss: 0.3445 - acc: 0.9631 - val_loss: 0.4461 - val_acc: 0.9310
Epoch 1524/5000
77s 154ms/step - loss: 0.3386 - acc: 0.9641 - val_loss: 0.4460 - val_acc: 0.9319
Epoch 1525/5000
77s 154ms/step - loss: 0.3348 - acc: 0.9636 - val_loss: 0.4432 - val_acc: 0.9323
Epoch 1526/5000
77s 154ms/step - loss: 0.3320 - acc: 0.9652 - val_loss: 0.4571 - val_acc: 0.9249
Epoch 1527/5000
77s 154ms/step - loss: 0.3278 - acc: 0.9650 - val_loss: 0.4385 - val_acc: 0.9324
Epoch 1528/5000
77s 154ms/step - loss: 0.3279 - acc: 0.9639 - val_loss: 0.4325 - val_acc: 0.9330
Epoch 1529/5000
77s 154ms/step - loss: 0.3228 - acc: 0.9647 - val_loss: 0.4303 - val_acc: 0.9326
Epoch 1530/5000
77s 154ms/step - loss: 0.3179 - acc: 0.9658 - val_loss: 0.4323 - val_acc: 0.9314
...
Epoch 2955/5000
77s 154ms/step - loss: 0.2282 - acc: 0.9719 - val_loss: 0.3926 - val_acc: 0.9239
Epoch 2956/5000
77s 154ms/step - loss: 0.2209 - acc: 0.9742 - val_loss: 0.3995 - val_acc: 0.9219
Epoch 2957/5000
77s 154ms/step - loss: 0.2250 - acc: 0.9729 - val_loss: 0.4095 - val_acc: 0.9195
Epoch 2958/5000
77s 154ms/step - loss: 0.2200 - acc: 0.9744 - val_loss: 0.4053 - val_acc: 0.9213
Epoch 2959/5000
77s 154ms/step - loss: 0.2259 - acc: 0.9725 - val_loss: 0.3938 - val_acc: 0.9243
Epoch 2960/5000
77s 154ms/step - loss: 0.2225 - acc: 0.9733 - val_loss: 0.4226 - val_acc: 0.9149
Epoch 2961/5000
77s 154ms/step - loss: 0.2228 - acc: 0.9733 - val_loss: 0.4012 - val_acc: 0.9231
Epoch 2962/5000
77s 154ms/step - loss: 0.2266 - acc: 0.9722 - val_loss: 0.3923 - val_acc: 0.9220
Epoch 2963/5000
77s 154ms/step - loss: 0.2257 - acc: 0.9720 - val_loss: 0.4094 - val_acc: 0.9207
Epoch 2964/5000
77s 154ms/step - loss: 0.2232 - acc: 0.9719 - val_loss: 0.4030 - val_acc: 0.9195
Epoch 2965/5000
77s 154ms/step - loss: 0.2257 - acc: 0.9726 - val_loss: 0.4015 - val_acc: 0.9208
Epoch 2966/5000
77s 154ms/step - loss: 0.2260 - acc: 0.9720 - val_loss: 0.3923 - val_acc: 0.9252
Epoch 2967/5000
77s 154ms/step - loss: 0.2298 - acc: 0.9702 - val_loss: 0.4055 - val_acc: 0.9212
Epoch 2968/5000
77s 154ms/step - loss: 0.2341 - acc: 0.9689 - val_loss: 0.3956 - val_acc: 0.9204
Epoch 2969/5000
78s 155ms/step - loss: 0.2216 - acc: 0.9743 - val_loss: 0.4065 - val_acc: 0.9227
Epoch 2970/5000
78s 155ms/step - loss: 0.2271 - acc: 0.9713 - val_loss: 0.4127 - val_acc: 0.9191
Epoch 2971/5000
77s 155ms/step - loss: 0.2275 - acc: 0.9716 - val_loss: 0.3983 - val_acc: 0.9206
Epoch 2972/5000
78s 155ms/step - loss: 0.2237 - acc: 0.9723 - val_loss: 0.3819 - val_acc: 0.9278
Epoch 2973/5000
78s 155ms/step - loss: 0.2237 - acc: 0.9740 - val_loss: 0.3824 - val_acc: 0.9241
Epoch 2974/5000
78s 155ms/step - loss: 0.2282 - acc: 0.9711 - val_loss: 0.4099 - val_acc: 0.9220
Epoch 2975/5000
77s 155ms/step - loss: 0.2226 - acc: 0.9730 - val_loss: 0.3993 - val_acc: 0.9249
Epoch 2976/5000
77s 155ms/step - loss: 0.2274 - acc: 0.9716 - val_loss: 0.3942 - val_acc: 0.9241
Epoch 2977/5000
78s 155ms/step - loss: 0.2303 - acc: 0.9703 - val_loss: 0.3946 - val_acc: 0.9228
Epoch 2978/5000
78s 155ms/step - loss: 0.2236 - acc: 0.9729 - val_loss: 0.4192 - val_acc: 0.9195
Epoch 2979/5000
77s 155ms/step - loss: 0.2303 - acc: 0.9711 - val_loss: 0.4105 - val_acc: 0.9204
Epoch 2980/5000
77s 155ms/step - loss: 0.2281 - acc: 0.9708 - val_loss: 0.4206 - val_acc: 0.9166
Epoch 2981/5000
78s 155ms/step - loss: 0.2293 - acc: 0.9714 - val_loss: 0.3983 - val_acc: 0.9233
Epoch 2982/5000
77s 155ms/step - loss: 0.2212 - acc: 0.9744 - val_loss: 0.4093 - val_acc: 0.9220
Epoch 2983/5000
78s 155ms/step - loss: 0.2282 - acc: 0.9713 - val_loss: 0.3909 - val_acc: 0.9266
Epoch 2984/5000
78s 155ms/step - loss: 0.2220 - acc: 0.9738 - val_loss: 0.4007 - val_acc: 0.9222
Epoch 2985/5000
77s 155ms/step - loss: 0.2246 - acc: 0.9728 - val_loss: 0.4016 - val_acc: 0.9231
Epoch 2986/5000
78s 155ms/step - loss: 0.2263 - acc: 0.9714 - val_loss: 0.3954 - val_acc: 0.9229
Epoch 2987/5000
77s 155ms/step - loss: 0.2253 - acc: 0.9724 - val_loss: 0.3986 - val_acc: 0.9254
Epoch 2988/5000
77s 155ms/step - loss: 0.2278 - acc: 0.9718 - val_loss: 0.3938 - val_acc: 0.9237
Epoch 2989/5000
77s 155ms/step - loss: 0.2291 - acc: 0.9715 - val_loss: 0.4007 - val_acc: 0.9208
Epoch 2990/5000
78s 155ms/step - loss: 0.2288 - acc: 0.9708 - val_loss: 0.3921 - val_acc: 0.9235
Epoch 2991/5000
77s 155ms/step - loss: 0.2186 - acc: 0.9752 - val_loss: 0.4077 - val_acc: 0.9234
Epoch 2992/5000
78s 155ms/step - loss: 0.2271 - acc: 0.9711 - val_loss: 0.3884 - val_acc: 0.9262
Epoch 2993/5000
78s 155ms/step - loss: 0.2236 - acc: 0.9722 - val_loss: 0.3975 - val_acc: 0.9241
Epoch 2994/5000
77s 155ms/step - loss: 0.2231 - acc: 0.9732 - val_loss: 0.3987 - val_acc: 0.9255
Epoch 2995/5000
77s 155ms/step - loss: 0.2234 - acc: 0.9729 - val_loss: 0.4180 - val_acc: 0.9175
Epoch 2996/5000
77s 155ms/step - loss: 0.2252 - acc: 0.9726 - val_loss: 0.4069 - val_acc: 0.9222
Epoch 2997/5000
78s 155ms/step - loss: 0.2273 - acc: 0.9727 - val_loss: 0.3979 - val_acc: 0.9229
Epoch 2998/5000
77s 155ms/step - loss: 0.2260 - acc: 0.9722 - val_loss: 0.4036 - val_acc: 0.9208
Epoch 2999/5000
78s 155ms/step - loss: 0.2254 - acc: 0.9721 - val_loss: 0.3880 - val_acc: 0.9252
Epoch 3000/5000
78s 155ms/step - loss: 0.2262 - acc: 0.9725 - val_loss: 0.4040 - val_acc: 0.9217
Epoch 3001/5000
lr changed to 0.0009999999776482583
78s 155ms/step - loss: 0.2012 - acc: 0.9814 - val_loss: 0.3627 - val_acc: 0.9343
Epoch 3002/5000
78s 155ms/step - loss: 0.1846 - acc: 0.9875 - val_loss: 0.3587 - val_acc: 0.9344
Epoch 3003/5000
78s 155ms/step - loss: 0.1793 - acc: 0.9893 - val_loss: 0.3559 - val_acc: 0.9357
Epoch 3004/5000
78s 155ms/step - loss: 0.1770 - acc: 0.9901 - val_loss: 0.3559 - val_acc: 0.9361
Epoch 3005/5000
78s 155ms/step - loss: 0.1752 - acc: 0.9909 - val_loss: 0.3563 - val_acc: 0.9363
Epoch 3006/5000
78s 155ms/step - loss: 0.1730 - acc: 0.9916 - val_loss: 0.3536 - val_acc: 0.9366
Epoch 3007/5000
78s 155ms/step - loss: 0.1744 - acc: 0.9905 - val_loss: 0.3543 - val_acc: 0.9388
Epoch 3008/5000
78s 155ms/step - loss: 0.1692 - acc: 0.9927 - val_loss: 0.3553 - val_acc: 0.9374
Epoch 3009/5000
78s 155ms/step - loss: 0.1693 - acc: 0.9925 - val_loss: 0.3557 - val_acc: 0.9370
Epoch 3010/5000
78s 155ms/step - loss: 0.1688 - acc: 0.9926 - val_loss: 0.3541 - val_acc: 0.9389
Epoch 3011/5000
78s 155ms/step - loss: 0.1664 - acc: 0.9934 - val_loss: 0.3545 - val_acc: 0.9384
Epoch 3012/5000
78s 155ms/step - loss: 0.1671 - acc: 0.9931 - val_loss: 0.3572 - val_acc: 0.9387
Epoch 3013/5000
78s 155ms/step - loss: 0.1644 - acc: 0.9940 - val_loss: 0.3561 - val_acc: 0.9387
Epoch 3014/5000
77s 155ms/step - loss: 0.1646 - acc: 0.9937 - val_loss: 0.3563 - val_acc: 0.9388
Epoch 3015/5000
78s 155ms/step - loss: 0.1644 - acc: 0.9936 - val_loss: 0.3564 - val_acc: 0.9379
Epoch 3016/5000
78s 155ms/step - loss: 0.1632 - acc: 0.9939 - val_loss: 0.3542 - val_acc: 0.9376
Epoch 3017/5000
78s 155ms/step - loss: 0.1622 - acc: 0.9941 - val_loss: 0.3562 - val_acc: 0.9380
Epoch 3018/5000
78s 155ms/step - loss: 0.1619 - acc: 0.9943 - val_loss: 0.3545 - val_acc: 0.9385
Epoch 3019/5000
77s 155ms/step - loss: 0.1609 - acc: 0.9947 - val_loss: 0.3553 - val_acc: 0.9393
Epoch 3020/5000
78s 155ms/step - loss: 0.1607 - acc: 0.9945 - val_loss: 0.3575 - val_acc: 0.9373
...
Epoch 4492/5000
77s 154ms/step - loss: 0.0646 - acc: 0.9953 - val_loss: 0.3020 - val_acc: 0.9352
Epoch 4493/5000
77s 153ms/step - loss: 0.0649 - acc: 0.9947 - val_loss: 0.3092 - val_acc: 0.9344
Epoch 4494/5000
77s 154ms/step - loss: 0.0642 - acc: 0.9952 - val_loss: 0.2993 - val_acc: 0.9351
Epoch 4495/5000
77s 154ms/step - loss: 0.0645 - acc: 0.9949 - val_loss: 0.2898 - val_acc: 0.9371
Epoch 4496/5000
77s 154ms/step - loss: 0.0652 - acc: 0.9952 - val_loss: 0.3040 - val_acc: 0.9364
Epoch 4497/5000
77s 154ms/step - loss: 0.0646 - acc: 0.9949 - val_loss: 0.3002 - val_acc: 0.9357
Epoch 4498/5000
77s 153ms/step - loss: 0.0655 - acc: 0.9947 - val_loss: 0.2957 - val_acc: 0.9385
Epoch 4499/5000
77s 153ms/step - loss: 0.0648 - acc: 0.9947 - val_loss: 0.2993 - val_acc: 0.9371
Epoch 4500/5000
77s 153ms/step - loss: 0.0657 - acc: 0.9944 - val_loss: 0.3034 - val_acc: 0.9349
Epoch 4501/5000
lr changed to 9.999999310821295e-05
77s 154ms/step - loss: 0.0631 - acc: 0.9958 - val_loss: 0.2941 - val_acc: 0.9384
Epoch 4502/5000
77s 154ms/step - loss: 0.0606 - acc: 0.9963 - val_loss: 0.2887 - val_acc: 0.9393
Epoch 4503/5000
77s 154ms/step - loss: 0.0585 - acc: 0.9971 - val_loss: 0.2866 - val_acc: 0.9398
Epoch 4504/5000
77s 154ms/step - loss: 0.0579 - acc: 0.9974 - val_loss: 0.2861 - val_acc: 0.9408
Epoch 4505/5000
77s 153ms/step - loss: 0.0568 - acc: 0.9976 - val_loss: 0.2838 - val_acc: 0.9415
Epoch 4506/5000
77s 154ms/step - loss: 0.0579 - acc: 0.9975 - val_loss: 0.2825 - val_acc: 0.9419
Epoch 4507/5000
77s 154ms/step - loss: 0.0567 - acc: 0.9977 - val_loss: 0.2818 - val_acc: 0.9417
Epoch 4508/5000
77s 154ms/step - loss: 0.0570 - acc: 0.9977 - val_loss: 0.2814 - val_acc: 0.9416
Epoch 4509/5000
77s 154ms/step - loss: 0.0557 - acc: 0.9982 - val_loss: 0.2794 - val_acc: 0.9421
Epoch 4510/5000
77s 154ms/step - loss: 0.0555 - acc: 0.9982 - val_loss: 0.2793 - val_acc: 0.9428
Epoch 4511/5000
77s 153ms/step - loss: 0.0565 - acc: 0.9979 - val_loss: 0.2789 - val_acc: 0.9418
Epoch 4512/5000
77s 154ms/step - loss: 0.0555 - acc: 0.9981 - val_loss: 0.2806 - val_acc: 0.9408
Epoch 4513/5000
77s 153ms/step - loss: 0.0561 - acc: 0.9977 - val_loss: 0.2801 - val_acc: 0.9417
Epoch 4514/5000
77s 153ms/step - loss: 0.0552 - acc: 0.9983 - val_loss: 0.2791 - val_acc: 0.9415
Epoch 4515/5000
77s 154ms/step - loss: 0.0550 - acc: 0.9985 - val_loss: 0.2796 - val_acc: 0.9423
Epoch 4516/5000
77s 153ms/step - loss: 0.0547 - acc: 0.9982 - val_loss: 0.2817 - val_acc: 0.9416
Epoch 4517/5000
77s 153ms/step - loss: 0.0559 - acc: 0.9980 - val_loss: 0.2821 - val_acc: 0.9423
Epoch 4518/5000
77s 154ms/step - loss: 0.0548 - acc: 0.9985 - val_loss: 0.2807 - val_acc: 0.9430
Epoch 4519/5000
77s 154ms/step - loss: 0.0545 - acc: 0.9986 - val_loss: 0.2801 - val_acc: 0.9433
Epoch 4520/5000
77s 154ms/step - loss: 0.0550 - acc: 0.9980 - val_loss: 0.2814 - val_acc: 0.9422
Epoch 4521/5000
77s 154ms/step - loss: 0.0554 - acc: 0.9981 - val_loss: 0.2811 - val_acc: 0.9422
Epoch 4522/5000
77s 154ms/step - loss: 0.0545 - acc: 0.9984 - val_loss: 0.2792 - val_acc: 0.9432
Epoch 4523/5000
77s 154ms/step - loss: 0.0541 - acc: 0.9986 - val_loss: 0.2798 - val_acc: 0.9421
Epoch 4524/5000
77s 154ms/step - loss: 0.0543 - acc: 0.9985 - val_loss: 0.2789 - val_acc: 0.9435
Epoch 4525/5000
77s 154ms/step - loss: 0.0543 - acc: 0.9986 - val_loss: 0.2788 - val_acc: 0.9428
Epoch 4526/5000
77s 154ms/step - loss: 0.0547 - acc: 0.9984 - val_loss: 0.2786 - val_acc: 0.9429
Epoch 4527/5000
77s 154ms/step - loss: 0.0541 - acc: 0.9985 - val_loss: 0.2791 - val_acc: 0.9425
Epoch 4528/5000
77s 155ms/step - loss: 0.0539 - acc: 0.9986 - val_loss: 0.2797 - val_acc: 0.9427
Epoch 4529/5000
78s 156ms/step - loss: 0.0553 - acc: 0.9980 - val_loss: 0.2777 - val_acc: 0.9432
Epoch 4530/5000
78s 155ms/step - loss: 0.0542 - acc: 0.9985 - val_loss: 0.2787 - val_acc: 0.9422
...
Epoch 4975/5000
77s 154ms/step - loss: 0.0484 - acc: 0.9993 - val_loss: 0.2867 - val_acc: 0.9438
Epoch 4976/5000
77s 154ms/step - loss: 0.0480 - acc: 0.9993 - val_loss: 0.2857 - val_acc: 0.9439
Epoch 4977/5000
77s 154ms/step - loss: 0.0483 - acc: 0.9992 - val_loss: 0.2848 - val_acc: 0.9436
Epoch 4978/5000
77s 154ms/step - loss: 0.0483 - acc: 0.9992 - val_loss: 0.2860 - val_acc: 0.9440
Epoch 4979/5000
77s 154ms/step - loss: 0.0481 - acc: 0.9993 - val_loss: 0.2872 - val_acc: 0.9433
Epoch 4980/5000
77s 154ms/step - loss: 0.0482 - acc: 0.9993 - val_loss: 0.2867 - val_acc: 0.9436
Epoch 4981/5000
77s 154ms/step - loss: 0.0480 - acc: 0.9994 - val_loss: 0.2871 - val_acc: 0.9431
Epoch 4982/5000
77s 154ms/step - loss: 0.0477 - acc: 0.9995 - val_loss: 0.2867 - val_acc: 0.9431
Epoch 4983/5000
77s 154ms/step - loss: 0.0485 - acc: 0.9992 - val_loss: 0.2864 - val_acc: 0.9423
Epoch 4984/5000
77s 153ms/step - loss: 0.0480 - acc: 0.9994 - val_loss: 0.2852 - val_acc: 0.9437
Epoch 4985/5000
77s 154ms/step - loss: 0.0479 - acc: 0.9994 - val_loss: 0.2846 - val_acc: 0.9437
Epoch 4986/5000
77s 154ms/step - loss: 0.0488 - acc: 0.9990 - val_loss: 0.2852 - val_acc: 0.9433
Epoch 4987/5000
77s 154ms/step - loss: 0.0484 - acc: 0.9990 - val_loss: 0.2831 - val_acc: 0.9443
Epoch 4988/5000
77s 154ms/step - loss: 0.0484 - acc: 0.9993 - val_loss: 0.2850 - val_acc: 0.9440
Epoch 4989/5000
77s 154ms/step - loss: 0.0484 - acc: 0.9991 - val_loss: 0.2871 - val_acc: 0.9431
Epoch 4990/5000
77s 154ms/step - loss: 0.0479 - acc: 0.9993 - val_loss: 0.2863 - val_acc: 0.9433
Epoch 4991/5000
77s 153ms/step - loss: 0.0483 - acc: 0.9993 - val_loss: 0.2868 - val_acc: 0.9434
Epoch 4992/5000
77s 154ms/step - loss: 0.0482 - acc: 0.9992 - val_loss: 0.2847 - val_acc: 0.9431
Epoch 4993/5000
77s 154ms/step - loss: 0.0485 - acc: 0.9992 - val_loss: 0.2853 - val_acc: 0.9417
Epoch 4994/5000
77s 154ms/step - loss: 0.0481 - acc: 0.9994 - val_loss: 0.2835 - val_acc: 0.9422
Epoch 4995/5000
77s 154ms/step - loss: 0.0481 - acc: 0.9994 - val_loss: 0.2836 - val_acc: 0.9432
Epoch 4996/5000
77s 153ms/step - loss: 0.0483 - acc: 0.9991 - val_loss: 0.2852 - val_acc: 0.9427
Epoch 4997/5000
77s 154ms/step - loss: 0.0481 - acc: 0.9994 - val_loss: 0.2855 - val_acc: 0.9427
Epoch 4998/5000
77s 154ms/step - loss: 0.0481 - acc: 0.9993 - val_loss: 0.2839 - val_acc: 0.9430
Epoch 4999/5000
77s 154ms/step - loss: 0.0478 - acc: 0.9994 - val_loss: 0.2855 - val_acc: 0.9424
Epoch 5000/5000
77s 154ms/step - loss: 0.0480 - acc: 0.9993 - val_loss: 0.2855 - val_acc: 0.9428
Train loss: 0.04765264599025249
Train accuracy: 0.9993600006103516
Test loss: 0.2855186524987221
Test accuracy: 0.9428000026941299