【哈工大版】Dynamic ReLU：Adaptively Parametric ReLU及Keras代码（调参记录17）

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Apr 14 04:17:45 2020
Implemented using TensorFlow 1.0.1 and Keras 2.2.1

Minghang Zhao, Shisheng Zhong, Xuyun Fu, Baoping Tang, Shaojiang Dong, Michael Pecht,
Deep Residual Networks with Adaptively Parametric Rectifier Linear Units for Fault Diagnosis, 
IEEE Transactions on Industrial Electronics, DOI: 10.1109/TIE.2020.2972458,
Date of Publication: 13 February 2020

@author: Minghang Zhao
"""

from __future__ import print_function
import keras
import numpy as np
from keras.datasets import cifar10
from keras.layers import Dense, Conv2D, BatchNormalization, Activation, Minimum
from keras.layers import AveragePooling2D, Input, GlobalAveragePooling2D, Concatenate, Reshape
from keras.regularizers import l2
from keras import backend as K
from keras.models import Model
from keras import optimizers
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import LearningRateScheduler
K.set_learning_phase(1)

# The data, split between train and test sets
(x_train, y_train), (x_test, y_test) = cifar10.load_data()
x_train = x_train.astype('float32') / 255.
x_test = x_test.astype('float32') / 255.
x_test = x_test-np.mean(x_train)
x_train = x_train-np.mean(x_train)
print('x_train shape:', x_train.shape)
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')

# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, 10)
y_test = keras.utils.to_categorical(y_test, 10)

# Schedule the learning rate, multiply 0.1 every 1500 epoches
def scheduler(epoch):
    if epoch % 1500 == 0 and epoch != 0:
        lr = K.get_value(model.optimizer.lr)
        K.set_value(model.optimizer.lr, lr * 0.1)
        print("lr changed to {}".format(lr * 0.1))
    return K.get_value(model.optimizer.lr)

# An adaptively parametric rectifier linear unit (APReLU)
def aprelu(inputs):
    # get the number of channels
    channels = inputs.get_shape().as_list()[-1]
    # get a zero feature map
    zeros_input = keras.layers.subtract([inputs, inputs])
    # get a feature map with only positive features
    pos_input = Activation('relu')(inputs)
    # get a feature map with only negative features
    neg_input = Minimum()([inputs,zeros_input])
    # define a network to obtain the scaling coefficients
    scales_p = GlobalAveragePooling2D()(pos_input)
    scales_n = GlobalAveragePooling2D()(neg_input)
    scales = Concatenate()([scales_n, scales_p])
    scales = Dense(channels//16, activation='linear', kernel_initializer='he_normal', kernel_regularizer=l2(1e-4))(scales)
    scales = BatchNormalization(momentum=0.9, gamma_regularizer=l2(1e-4))(scales)
    scales = Activation('relu')(scales)
    scales = Dense(channels, activation='linear', kernel_initializer='he_normal', kernel_regularizer=l2(1e-4))(scales)
    scales = BatchNormalization(momentum=0.9, gamma_regularizer=l2(1e-4))(scales)
    scales = Activation('sigmoid')(scales)
    scales = Reshape((1,1,channels))(scales)
    # apply a paramtetric relu
    neg_part = keras.layers.multiply([scales, neg_input])
    return keras.layers.add([pos_input, neg_part])

# Residual Block
def residual_block(incoming, nb_blocks, out_channels, downsample=False,
                   downsample_strides=2):
    
    residual = incoming
    in_channels = incoming.get_shape().as_list()[-1]
    
    for i in range(nb_blocks):
        
        identity = residual
        
        if not downsample:
            downsample_strides = 1
        
        residual = BatchNormalization(momentum=0.9, gamma_regularizer=l2(1e-4))(residual)
        residual = aprelu(residual)
        residual = Conv2D(out_channels, 3, strides=(downsample_strides, downsample_strides), 
                          padding='same', kernel_initializer='he_normal', 
                          kernel_regularizer=l2(1e-4))(residual)
        
        residual = BatchNormalization(momentum=0.9, gamma_regularizer=l2(1e-4))(residual)
        residual = aprelu(residual)
        residual = Conv2D(out_channels, 3, padding='same', kernel_initializer='he_normal', 
                          kernel_regularizer=l2(1e-4))(residual)
        
        # Downsampling
        if downsample_strides > 1:
            identity = AveragePooling2D(pool_size=(1,1), strides=(2,2))(identity)
            
        # Zero_padding to match channels
        if in_channels != out_channels:
            zeros_identity = keras.layers.subtract([identity, identity])
            identity = keras.layers.concatenate([identity, zeros_identity])
            in_channels = out_channels
        
        residual = keras.layers.add([residual, identity])
    
    return residual


# define and train a model
inputs = Input(shape=(32, 32, 3))
net = Conv2D(16, 3, padding='same', kernel_initializer='he_normal', kernel_regularizer=l2(1e-4))(inputs)
net = residual_block(net, 1, 32, downsample=False)
net = residual_block(net, 1, 32, downsample=True)
net = residual_block(net, 1, 32, downsample=False)
net = residual_block(net, 1, 64, downsample=True)
net = residual_block(net, 1, 64, downsample=False)
net = BatchNormalization(momentum=0.9, gamma_regularizer=l2(1e-4))(net)
net = aprelu(net)
net = GlobalAveragePooling2D()(net)
outputs = Dense(10, activation='softmax', kernel_initializer='he_normal', kernel_regularizer=l2(1e-4))(net)
model = Model(inputs=inputs, outputs=outputs)
sgd = optimizers.SGD(lr=0.1, decay=0., momentum=0.9, nesterov=True)
model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])

# data augmentation
datagen = ImageDataGenerator(
    # randomly rotate images in the range (deg 0 to 180)
    rotation_range=30,
    # Range for random zoom
    zoom_range = 0.2,
    # shear angle in counter-clockwise direction in degrees
    shear_range = 30,
    # randomly flip images
    horizontal_flip=True,
    # randomly shift images horizontally
    width_shift_range=0.125,
    # randomly shift images vertically
    height_shift_range=0.125)

reduce_lr = LearningRateScheduler(scheduler)
# fit the model on the batches generated by datagen.flow().
model.fit_generator(datagen.flow(x_train, y_train, batch_size=100),
                    validation_data=(x_test, y_test), epochs=5000, 
                    verbose=1, callbacks=[reduce_lr], workers=4)

# get results
K.set_learning_phase(0)
DRSN_train_score = model.evaluate(x_train, y_train, batch_size=100, verbose=0)
print('Train loss:', DRSN_train_score[0])
print('Train accuracy:', DRSN_train_score[1])
DRSN_test_score = model.evaluate(x_test, y_test, batch_size=100, verbose=0)
print('Test loss:', DRSN_test_score[0])
print('Test accuracy:', DRSN_test_score[1])

Using TensorFlow backend.
x_train shape: (50000, 32, 32, 3)
50000 train samples
10000 test samples
Epoch 1/5000
20s 41ms/step - loss: 1.9490 - acc: 0.3869 - val_loss: 1.6784 - val_acc: 0.4900
Epoch 2/5000
14s 29ms/step - loss: 1.6703 - acc: 0.4833 - val_loss: 1.4725 - val_acc: 0.5484
Epoch 3/5000
14s 29ms/step - loss: 1.5095 - acc: 0.5392 - val_loss: 1.3049 - val_acc: 0.6056
Epoch 4/5000
14s 29ms/step - loss: 1.3979 - acc: 0.5784 - val_loss: 1.2282 - val_acc: 0.6386
Epoch 5/5000
14s 29ms/step - loss: 1.3169 - acc: 0.6028 - val_loss: 1.1671 - val_acc: 0.6576
Epoch 6/5000
14s 29ms/step - loss: 1.2644 - acc: 0.6202 - val_loss: 1.1295 - val_acc: 0.6625
Epoch 7/5000
14s 29ms/step - loss: 1.2118 - acc: 0.6399 - val_loss: 1.0974 - val_acc: 0.6794
Epoch 8/5000
14s 29ms/step - loss: 1.1754 - acc: 0.6510 - val_loss: 1.0352 - val_acc: 0.7042
Epoch 9/5000
14s 29ms/step - loss: 1.1392 - acc: 0.6640 - val_loss: 1.0079 - val_acc: 0.7117
Epoch 10/5000
14s 29ms/step - loss: 1.1119 - acc: 0.6746 - val_loss: 0.9513 - val_acc: 0.7395
Epoch 11/5000
14s 29ms/step - loss: 1.0894 - acc: 0.6835 - val_loss: 0.9600 - val_acc: 0.7335
Epoch 12/5000
14s 29ms/step - loss: 1.0705 - acc: 0.6928 - val_loss: 0.9051 - val_acc: 0.7574
Epoch 13/5000
14s 29ms/step - loss: 1.0536 - acc: 0.6997 - val_loss: 0.9088 - val_acc: 0.7560
Epoch 14/5000
14s 29ms/step - loss: 1.0392 - acc: 0.7067 - val_loss: 0.8895 - val_acc: 0.7610
Epoch 15/5000
14s 29ms/step - loss: 1.0240 - acc: 0.7122 - val_loss: 0.9020 - val_acc: 0.7570
Epoch 16/5000
14s 29ms/step - loss: 1.0169 - acc: 0.7169 - val_loss: 0.9082 - val_acc: 0.7576
Epoch 17/5000
14s 29ms/step - loss: 1.0000 - acc: 0.7205 - val_loss: 0.8647 - val_acc: 0.7762
Epoch 18/5000
14s 29ms/step - loss: 0.9946 - acc: 0.7258 - val_loss: 0.8520 - val_acc: 0.7816
Epoch 19/5000
14s 29ms/step - loss: 0.9874 - acc: 0.7300 - val_loss: 0.8447 - val_acc: 0.7781
Epoch 20/5000
14s 29ms/step - loss: 0.9796 - acc: 0.7331 - val_loss: 0.8503 - val_acc: 0.7834
Epoch 21/5000
14s 29ms/step - loss: 0.9739 - acc: 0.7345 - val_loss: 0.8347 - val_acc: 0.7905
Epoch 22/5000
14s 29ms/step - loss: 0.9654 - acc: 0.7408 - val_loss: 0.8485 - val_acc: 0.7847
Epoch 23/5000
14s 29ms/step - loss: 0.9562 - acc: 0.7437 - val_loss: 0.8187 - val_acc: 0.8000
Epoch 24/5000
14s 29ms/step - loss: 0.9529 - acc: 0.7453 - val_loss: 0.8432 - val_acc: 0.7849
Epoch 25/5000
14s 29ms/step - loss: 0.9468 - acc: 0.7479 - val_loss: 0.8176 - val_acc: 0.7991
Epoch 26/5000
14s 29ms/step - loss: 0.9374 - acc: 0.7529 - val_loss: 0.8107 - val_acc: 0.8008
Epoch 27/5000
14s 29ms/step - loss: 0.9401 - acc: 0.7521 - val_loss: 0.8046 - val_acc: 0.8056
Epoch 28/5000
14s 29ms/step - loss: 0.9357 - acc: 0.7532 - val_loss: 0.7972 - val_acc: 0.8110
Epoch 29/5000
14s 29ms/step - loss: 0.9268 - acc: 0.7579 - val_loss: 0.8300 - val_acc: 0.7964
Epoch 30/5000
14s 29ms/step - loss: 0.9260 - acc: 0.7603 - val_loss: 0.8116 - val_acc: 0.7988
Epoch 31/5000
14s 29ms/step - loss: 0.9204 - acc: 0.7607 - val_loss: 0.7832 - val_acc: 0.8146
Epoch 32/5000
14s 29ms/step - loss: 0.9200 - acc: 0.7617 - val_loss: 0.7900 - val_acc: 0.8138
Epoch 33/5000
14s 29ms/step - loss: 0.9179 - acc: 0.7647 - val_loss: 0.8021 - val_acc: 0.8061
Epoch 34/5000
14s 28ms/step - loss: 0.9111 - acc: 0.7656 - val_loss: 0.8005 - val_acc: 0.8052
Epoch 35/5000
14s 29ms/step - loss: 0.9107 - acc: 0.7671 - val_loss: 0.7924 - val_acc: 0.8160
Epoch 36/5000
14s 29ms/step - loss: 0.9084 - acc: 0.7690 - val_loss: 0.7984 - val_acc: 0.8104
Epoch 37/5000
14s 29ms/step - loss: 0.9018 - acc: 0.7695 - val_loss: 0.8222 - val_acc: 0.8028
Epoch 38/5000
14s 29ms/step - loss: 0.9068 - acc: 0.7711 - val_loss: 0.7934 - val_acc: 0.8156
Epoch 39/5000
14s 29ms/step - loss: 0.8978 - acc: 0.7737 - val_loss: 0.7866 - val_acc: 0.8163
Epoch 40/5000
14s 29ms/step - loss: 0.9013 - acc: 0.7723 - val_loss: 0.8102 - val_acc: 0.8108
Epoch 41/5000
14s 29ms/step - loss: 0.8921 - acc: 0.7756 - val_loss: 0.7937 - val_acc: 0.8163
Epoch 42/5000
14s 29ms/step - loss: 0.8922 - acc: 0.7764 - val_loss: 0.7852 - val_acc: 0.8196
Epoch 43/5000
14s 29ms/step - loss: 0.8946 - acc: 0.7751 - val_loss: 0.8259 - val_acc: 0.8046
Epoch 44/5000
14s 29ms/step - loss: 0.8908 - acc: 0.7771 - val_loss: 0.7840 - val_acc: 0.8193
Epoch 45/5000
14s 29ms/step - loss: 0.8865 - acc: 0.7795 - val_loss: 0.8009 - val_acc: 0.8101
Epoch 46/5000
14s 29ms/step - loss: 0.8877 - acc: 0.7812 - val_loss: 0.7900 - val_acc: 0.8169
Epoch 47/5000
14s 29ms/step - loss: 0.8811 - acc: 0.7818 - val_loss: 0.7807 - val_acc: 0.8206
Epoch 48/5000
14s 29ms/step - loss: 0.8779 - acc: 0.7845 - val_loss: 0.7634 - val_acc: 0.8273
Epoch 49/5000
14s 29ms/step - loss: 0.8800 - acc: 0.7827 - val_loss: 0.7734 - val_acc: 0.8242
Epoch 50/5000
14s 29ms/step - loss: 0.8772 - acc: 0.7835 - val_loss: 0.7841 - val_acc: 0.8193
Epoch 51/5000
14s 29ms/step - loss: 0.8736 - acc: 0.7843 - val_loss: 0.7970 - val_acc: 0.8190
Epoch 52/5000
14s 29ms/step - loss: 0.8768 - acc: 0.7852 - val_loss: 0.7855 - val_acc: 0.8178
Epoch 53/5000
14s 29ms/step - loss: 0.8741 - acc: 0.7835 - val_loss: 0.7851 - val_acc: 0.8209
Epoch 54/5000
14s 29ms/step - loss: 0.8722 - acc: 0.7858 - val_loss: 0.7825 - val_acc: 0.8184
Epoch 55/5000
14s 29ms/step - loss: 0.8697 - acc: 0.7891 - val_loss: 0.7771 - val_acc: 0.8227
Epoch 56/5000
14s 29ms/step - loss: 0.8711 - acc: 0.7873 - val_loss: 0.7677 - val_acc: 0.8236
Epoch 57/5000
14s 29ms/step - loss: 0.8711 - acc: 0.7878 - val_loss: 0.7872 - val_acc: 0.8155
Epoch 58/5000
14s 29ms/step - loss: 0.8678 - acc: 0.7886 - val_loss: 0.7936 - val_acc: 0.8193
Epoch 59/5000
14s 29ms/step - loss: 0.8650 - acc: 0.7904 - val_loss: 0.7778 - val_acc: 0.8287
Epoch 60/5000
14s 29ms/step - loss: 0.8655 - acc: 0.7905 - val_loss: 0.7670 - val_acc: 0.8283
Epoch 61/5000
14s 29ms/step - loss: 0.8640 - acc: 0.7906 - val_loss: 0.7757 - val_acc: 0.8247
Epoch 62/5000
14s 29ms/step - loss: 0.8645 - acc: 0.7894 - val_loss: 0.7600 - val_acc: 0.8312
Epoch 63/5000
14s 29ms/step - loss: 0.8620 - acc: 0.7916 - val_loss: 0.7823 - val_acc: 0.8211
Epoch 64/5000
14s 29ms/step - loss: 0.8584 - acc: 0.7914 - val_loss: 0.7873 - val_acc: 0.8187
Epoch 65/5000
14s 29ms/step - loss: 0.8567 - acc: 0.7930 - val_loss: 0.7810 - val_acc: 0.8231
Epoch 66/5000
14s 29ms/step - loss: 0.8639 - acc: 0.7912 - val_loss: 0.7653 - val_acc: 0.8298
Epoch 67/5000
14s 29ms/step - loss: 0.8594 - acc: 0.7939 - val_loss: 0.7763 - val_acc: 0.8265
Epoch 68/5000
14s 29ms/step - loss: 0.8585 - acc: 0.7921 - val_loss: 0.7711 - val_acc: 0.8277
Epoch 69/5000
14s 29ms/step - loss: 0.8571 - acc: 0.7937 - val_loss: 0.7650 - val_acc: 0.8278
Epoch 70/5000
14s 29ms/step - loss: 0.8590 - acc: 0.7942 - val_loss: 0.7646 - val_acc: 0.8290
Epoch 71/5000
14s 29ms/step - loss: 0.8548 - acc: 0.7960 - val_loss: 0.7720 - val_acc: 0.8247
Epoch 72/5000
14s 29ms/step - loss: 0.8521 - acc: 0.7967 - val_loss: 0.7601 - val_acc: 0.8321
Epoch 73/5000
14s 29ms/step - loss: 0.8558 - acc: 0.7944 - val_loss: 0.7635 - val_acc: 0.8326
Epoch 74/5000
14s 29ms/step - loss: 0.8483 - acc: 0.7973 - val_loss: 0.7519 - val_acc: 0.8357
Epoch 75/5000
14s 29ms/step - loss: 0.8506 - acc: 0.7966 - val_loss: 0.7509 - val_acc: 0.8339
Epoch 76/5000
14s 29ms/step - loss: 0.8447 - acc: 0.7989 - val_loss: 0.7960 - val_acc: 0.8171
Epoch 77/5000
14s 29ms/step - loss: 0.8497 - acc: 0.7991 - val_loss: 0.7699 - val_acc: 0.8282
Epoch 78/5000
14s 29ms/step - loss: 0.8464 - acc: 0.7996 - val_loss: 0.7747 - val_acc: 0.8269
Epoch 79/5000
14s 29ms/step - loss: 0.8509 - acc: 0.7971 - val_loss: 0.7450 - val_acc: 0.8379
Epoch 80/5000
14s 29ms/step - loss: 0.8449 - acc: 0.7981 - val_loss: 0.7784 - val_acc: 0.8251
Epoch 81/5000
14s 29ms/step - loss: 0.8446 - acc: 0.7985 - val_loss: 0.7689 - val_acc: 0.8312
Epoch 82/5000
14s 29ms/step - loss: 0.8481 - acc: 0.7992 - val_loss: 0.7573 - val_acc: 0.8306
Epoch 83/5000
14s 29ms/step - loss: 0.8417 - acc: 0.7996 - val_loss: 0.7677 - val_acc: 0.8297
Epoch 84/5000
14s 29ms/step - loss: 0.8475 - acc: 0.7991 - val_loss: 0.7583 - val_acc: 0.8287
Epoch 85/5000
14s 29ms/step - loss: 0.8464 - acc: 0.7983 - val_loss: 0.7552 - val_acc: 0.8322
Epoch 86/5000
14s 29ms/step - loss: 0.8413 - acc: 0.8003 - val_loss: 0.7631 - val_acc: 0.8272
Epoch 87/5000
14s 29ms/step - loss: 0.8400 - acc: 0.8001 - val_loss: 0.7308 - val_acc: 0.8473
Epoch 88/5000
14s 29ms/step - loss: 0.8438 - acc: 0.8011 - val_loss: 0.7709 - val_acc: 0.8336
Epoch 89/5000
14s 28ms/step - loss: 0.8425 - acc: 0.8016 - val_loss: 0.7587 - val_acc: 0.8323
Epoch 90/5000
14s 29ms/step - loss: 0.8343 - acc: 0.8021 - val_loss: 0.7741 - val_acc: 0.8234
Epoch 91/5000
14s 29ms/step - loss: 0.8438 - acc: 0.7996 - val_loss: 0.7556 - val_acc: 0.8351
Epoch 92/5000
14s 29ms/step - loss: 0.8312 - acc: 0.8047 - val_loss: 0.7597 - val_acc: 0.8329
Epoch 93/5000
14s 29ms/step - loss: 0.8412 - acc: 0.8006 - val_loss: 0.7615 - val_acc: 0.8289
Epoch 94/5000
14s 29ms/step - loss: 0.8381 - acc: 0.8022 - val_loss: 0.7550 - val_acc: 0.8356
Epoch 95/5000
14s 29ms/step - loss: 0.8362 - acc: 0.8025 - val_loss: 0.7662 - val_acc: 0.8306
Epoch 96/5000
14s 29ms/step - loss: 0.8378 - acc: 0.8015 - val_loss: 0.7676 - val_acc: 0.8299
Epoch 97/5000
14s 29ms/step - loss: 0.8378 - acc: 0.8032 - val_loss: 0.7482 - val_acc: 0.8382
Epoch 98/5000
14s 28ms/step - loss: 0.8357 - acc: 0.8020 - val_loss: 0.7348 - val_acc: 0.8398
Epoch 99/5000
14s 29ms/step - loss: 0.8369 - acc: 0.8018 - val_loss: 0.7620 - val_acc: 0.8332
Epoch 100/5000
14s 29ms/step - loss: 0.8390 - acc: 0.8032 - val_loss: 0.7553 - val_acc: 0.8336
Epoch 101/5000
14s 29ms/step - loss: 0.8316 - acc: 0.8059 - val_loss: 0.7575 - val_acc: 0.8314
Epoch 102/5000
14s 29ms/step - loss: 0.8347 - acc: 0.8044 - val_loss: 0.7530 - val_acc: 0.8338
Epoch 103/5000
14s 29ms/step - loss: 0.8327 - acc: 0.8043 - val_loss: 0.7527 - val_acc: 0.8376
Epoch 104/5000
14s 29ms/step - loss: 0.8349 - acc: 0.8051 - val_loss: 0.7427 - val_acc: 0.8376
Epoch 105/5000
14s 29ms/step - loss: 0.8320 - acc: 0.8050 - val_loss: 0.7632 - val_acc: 0.8333
Epoch 106/5000
14s 29ms/step - loss: 0.8307 - acc: 0.8051 - val_loss: 0.7351 - val_acc: 0.8399
Epoch 107/5000
14s 29ms/step - loss: 0.8311 - acc: 0.8061 - val_loss: 0.7481 - val_acc: 0.8351
Epoch 108/5000
14s 29ms/step - loss: 0.8304 - acc: 0.8057 - val_loss: 0.7464 - val_acc: 0.8383
Epoch 109/5000
14s 29ms/step - loss: 0.8292 - acc: 0.8068 - val_loss: 0.7460 - val_acc: 0.8399
Epoch 110/5000
14s 29ms/step - loss: 0.8288 - acc: 0.8068 - val_loss: 0.7730 - val_acc: 0.8277
Epoch 111/5000
14s 29ms/step - loss: 0.8307 - acc: 0.8046 - val_loss: 0.7451 - val_acc: 0.8381
Epoch 112/5000
14s 29ms/step - loss: 0.8258 - acc: 0.8079 - val_loss: 0.7317 - val_acc: 0.8452
Epoch 113/5000
14s 29ms/step - loss: 0.8304 - acc: 0.8067 - val_loss: 0.7715 - val_acc: 0.8274
Epoch 114/5000
14s 29ms/step - loss: 0.8317 - acc: 0.8041 - val_loss: 0.7588 - val_acc: 0.8337
Epoch 115/5000
14s 29ms/step - loss: 0.8293 - acc: 0.8044 - val_loss: 0.7498 - val_acc: 0.8366
Epoch 116/5000
14s 29ms/step - loss: 0.8330 - acc: 0.8057 - val_loss: 0.7491 - val_acc: 0.8317
Epoch 117/5000
14s 29ms/step - loss: 0.8215 - acc: 0.8074 - val_loss: 0.7496 - val_acc: 0.8365
Epoch 118/5000
14s 29ms/step - loss: 0.8276 - acc: 0.8073 - val_loss: 0.7538 - val_acc: 0.8362
Epoch 119/5000
14s 29ms/step - loss: 0.8291 - acc: 0.8069 - val_loss: 0.7536 - val_acc: 0.8353
Epoch 120/5000
14s 29ms/step - loss: 0.8283 - acc: 0.8057 - val_loss: 0.7564 - val_acc: 0.8338
Epoch 121/5000
14s 29ms/step - loss: 0.8248 - acc: 0.8079 - val_loss: 0.7387 - val_acc: 0.8420
Epoch 122/5000
14s 29ms/step - loss: 0.8241 - acc: 0.8077 - val_loss: 0.7784 - val_acc: 0.8272
Epoch 123/5000
14s 29ms/step - loss: 0.8257 - acc: 0.8077 - val_loss: 0.7649 - val_acc: 0.8289
Epoch 124/5000
14s 29ms/step - loss: 0.8194 - acc: 0.8097 - val_loss: 0.7383 - val_acc: 0.8386
Epoch 125/5000
14s 29ms/step - loss: 0.8246 - acc: 0.8063 - val_loss: 0.7556 - val_acc: 0.8373
Epoch 126/5000
14s 29ms/step - loss: 0.8253 - acc: 0.8072 - val_loss: 0.7366 - val_acc: 0.8380
Epoch 127/5000
14s 29ms/step - loss: 0.8204 - acc: 0.8103 - val_loss: 0.7359 - val_acc: 0.8400
Epoch 128/5000
14s 29ms/step - loss: 0.8223 - acc: 0.8089 - val_loss: 0.7587 - val_acc: 0.8358
Epoch 129/5000
14s 29ms/step - loss: 0.8210 - acc: 0.8077 - val_loss: 0.7663 - val_acc: 0.8323
Epoch 130/5000
14s 28ms/step - loss: 0.8232 - acc: 0.8083 - val_loss: 0.7408 - val_acc: 0.8400
Epoch 131/5000
14s 29ms/step - loss: 0.8270 - acc: 0.8074 - val_loss: 0.7589 - val_acc: 0.8339
Epoch 132/5000
14s 29ms/step - loss: 0.8245 - acc: 0.8106 - val_loss: 0.7649 - val_acc: 0.8324
Epoch 133/5000
14s 29ms/step - loss: 0.8216 - acc: 0.8092 - val_loss: 0.7717 - val_acc: 0.8322
Epoch 134/5000
14s 29ms/step - loss: 0.8218 - acc: 0.8099 - val_loss: 0.7480 - val_acc: 0.8341
Epoch 135/5000
14s 29ms/step - loss: 0.8220 - acc: 0.8093 - val_loss: 0.7613 - val_acc: 0.8310
Epoch 136/5000
14s 29ms/step - loss: 0.8208 - acc: 0.8096 - val_loss: 0.7516 - val_acc: 0.8367
Epoch 137/5000
14s 29ms/step - loss: 0.8132 - acc: 0.8130 - val_loss: 0.7599 - val_acc: 0.8323
Epoch 138/5000
14s 29ms/step - loss: 0.8183 - acc: 0.8109 - val_loss: 0.7543 - val_acc: 0.8345
Epoch 139/5000
14s 29ms/step - loss: 0.8213 - acc: 0.8090 - val_loss: 0.7256 - val_acc: 0.8475
Epoch 140/5000
14s 29ms/step - loss: 0.8207 - acc: 0.8094 - val_loss: 0.7671 - val_acc: 0.8319
Epoch 141/5000
14s 29ms/step - loss: 0.8181 - acc: 0.8129 - val_loss: 0.7379 - val_acc: 0.8425
Epoch 142/5000
14s 29ms/step - loss: 0.8168 - acc: 0.8102 - val_loss: 0.7349 - val_acc: 0.8420
Epoch 143/5000
14s 29ms/step - loss: 0.8204 - acc: 0.8106 - val_loss: 0.7240 - val_acc: 0.8473
Epoch 144/5000
14s 29ms/step - loss: 0.8152 - acc: 0.8119 - val_loss: 0.7530 - val_acc: 0.8353
Epoch 145/5000
14s 29ms/step - loss: 0.8173 - acc: 0.8134 - val_loss: 0.7306 - val_acc: 0.8451
Epoch 146/5000
14s 29ms/step - loss: 0.8162 - acc: 0.8130 - val_loss: 0.7702 - val_acc: 0.8250
Epoch 147/5000
14s 29ms/step - loss: 0.8133 - acc: 0.8126 - val_loss: 0.7580 - val_acc: 0.8365
Epoch 148/5000
14s 29ms/step - loss: 0.8170 - acc: 0.8108 - val_loss: 0.7546 - val_acc: 0.8344
Epoch 149/5000
14s 29ms/step - loss: 0.8143 - acc: 0.8109 - val_loss: 0.7413 - val_acc: 0.8371
Epoch 150/5000
14s 29ms/step - loss: 0.8168 - acc: 0.8106 - val_loss: 0.7591 - val_acc: 0.8348
Epoch 151/5000
14s 29ms/step - loss: 0.8184 - acc: 0.8087 - val_loss: 0.7205 - val_acc: 0.8462
Epoch 152/5000
14s 29ms/step - loss: 0.8173 - acc: 0.8111 - val_loss: 0.7353 - val_acc: 0.8407
Epoch 153/5000
14s 29ms/step - loss: 0.8132 - acc: 0.8144 - val_loss: 0.7446 - val_acc: 0.8339
Epoch 154/5000
14s 29ms/step - loss: 0.8166 - acc: 0.8106 - val_loss: 0.7216 - val_acc: 0.8496
Epoch 155/5000
14s 29ms/step - loss: 0.8203 - acc: 0.8103 - val_loss: 0.7505 - val_acc: 0.8340
Epoch 156/5000
14s 29ms/step - loss: 0.8138 - acc: 0.8115 - val_loss: 0.7481 - val_acc: 0.8363
Epoch 157/5000
14s 29ms/step - loss: 0.8158 - acc: 0.8109 - val_loss: 0.7553 - val_acc: 0.8353
Epoch 158/5000
14s 29ms/step - loss: 0.8133 - acc: 0.8131 - val_loss: 0.7236 - val_acc: 0.8460
Epoch 159/5000
14s 29ms/step - loss: 0.8155 - acc: 0.8104 - val_loss: 0.7323 - val_acc: 0.8432
Epoch 160/5000
14s 28ms/step - loss: 0.8107 - acc: 0.8135 - val_loss: 0.7567 - val_acc: 0.8351
Epoch 161/5000
14s 29ms/step - loss: 0.8114 - acc: 0.8138 - val_loss: 0.7379 - val_acc: 0.8399
Epoch 162/5000
14s 29ms/step - loss: 0.8095 - acc: 0.8148 - val_loss: 0.7653 - val_acc: 0.8350
Epoch 163/5000
14s 29ms/step - loss: 0.8136 - acc: 0.8132 - val_loss: 0.7310 - val_acc: 0.8416
Epoch 164/5000
14s 29ms/step - loss: 0.8074 - acc: 0.8149 - val_loss: 0.7142 - val_acc: 0.8510
Epoch 165/5000
14s 29ms/step - loss: 0.8148 - acc: 0.8118 - val_loss: 0.7520 - val_acc: 0.8398
Epoch 166/5000
14s 29ms/step - loss: 0.8153 - acc: 0.8122 - val_loss: 0.7357 - val_acc: 0.8398
Epoch 167/5000
14s 29ms/step - loss: 0.8086 - acc: 0.8139 - val_loss: 0.7406 - val_acc: 0.8405
Epoch 168/5000
14s 29ms/step - loss: 0.8110 - acc: 0.8135 - val_loss: 0.7361 - val_acc: 0.8427
Epoch 169/5000
14s 29ms/step - loss: 0.8107 - acc: 0.8130 - val_loss: 0.7491 - val_acc: 0.8362
Epoch 170/5000
14s 29ms/step - loss: 0.8105 - acc: 0.8130 - val_loss: 0.7632 - val_acc: 0.8316
Epoch 171/5000
14s 29ms/step - loss: 0.8125 - acc: 0.8145 - val_loss: 0.7676 - val_acc: 0.8293
Epoch 172/5000
14s 29ms/step - loss: 0.8102 - acc: 0.8146 - val_loss: 0.7597 - val_acc: 0.8353
Epoch 173/5000
14s 29ms/step - loss: 0.8098 - acc: 0.8125 - val_loss: 0.7485 - val_acc: 0.8389
Epoch 174/5000
14s 29ms/step - loss: 0.8108 - acc: 0.8131 - val_loss: 0.7655 - val_acc: 0.8329
Epoch 175/5000
14s 29ms/step - loss: 0.8066 - acc: 0.8166 - val_loss: 0.7373 - val_acc: 0.8422
Epoch 176/5000
14s 29ms/step - loss: 0.8074 - acc: 0.8133 - val_loss: 0.7478 - val_acc: 0.8356
Epoch 177/5000
14s 29ms/step - loss: 0.8065 - acc: 0.8154 - val_loss: 0.7594 - val_acc: 0.8379
Epoch 178/5000
14s 29ms/step - loss: 0.8104 - acc: 0.8150 - val_loss: 0.7273 - val_acc: 0.8477
Epoch 179/5000
14s 29ms/step - loss: 0.8053 - acc: 0.8162 - val_loss: 0.7233 - val_acc: 0.8457
Epoch 180/5000
14s 29ms/step - loss: 0.8067 - acc: 0.8153 - val_loss: 0.7389 - val_acc: 0.8433
Epoch 181/5000
14s 29ms/step - loss: 0.8089 - acc: 0.8153 - val_loss: 0.7563 - val_acc: 0.8371
Epoch 182/5000
14s 29ms/step - loss: 0.8097 - acc: 0.8143 - val_loss: 0.7127 - val_acc: 0.8539
Epoch 183/5000
14s 29ms/step - loss: 0.8092 - acc: 0.8155 - val_loss: 0.7457 - val_acc: 0.8378
Epoch 184/5000
14s 29ms/step - loss: 0.8085 - acc: 0.8138 - val_loss: 0.7481 - val_acc: 0.8390
Epoch 185/5000
14s 29ms/step - loss: 0.8130 - acc: 0.8125 - val_loss: 0.7404 - val_acc: 0.8399
Epoch 186/5000
14s 29ms/step - loss: 0.8106 - acc: 0.8137 - val_loss: 0.7346 - val_acc: 0.8386
Epoch 187/5000
14s 29ms/step - loss: 0.8029 - acc: 0.8188 - val_loss: 0.7304 - val_acc: 0.8427
Epoch 188/5000
14s 29ms/step - loss: 0.8038 - acc: 0.8168 - val_loss: 0.7381 - val_acc: 0.8420
Epoch 189/5000
14s 29ms/step - loss: 0.8064 - acc: 0.8163 - val_loss: 0.7607 - val_acc: 0.8323
Epoch 190/5000
14s 29ms/step - loss: 0.8120 - acc: 0.8136 - val_loss: 0.7491 - val_acc: 0.8408
Epoch 191/5000
14s 29ms/step - loss: 0.8080 - acc: 0.8159 - val_loss: 0.7317 - val_acc: 0.8477
Epoch 192/5000
14s 29ms/step - loss: 0.8058 - acc: 0.8151 - val_loss: 0.7431 - val_acc: 0.8367
Epoch 193/5000
14s 29ms/step - loss: 0.8114 - acc: 0.8142 - val_loss: 0.7348 - val_acc: 0.8399
Epoch 194/5000
14s 29ms/step - loss: 0.7997 - acc: 0.8172 - val_loss: 0.7295 - val_acc: 0.8434
Epoch 195/5000
14s 29ms/step - loss: 0.8110 - acc: 0.8143 - val_loss: 0.7138 - val_acc: 0.8516
Epoch 196/5000
14s 29ms/step - loss: 0.8069 - acc: 0.8127 - val_loss: 0.7544 - val_acc: 0.8377
Epoch 197/5000
14s 29ms/step - loss: 0.8031 - acc: 0.8177 - val_loss: 0.7436 - val_acc: 0.8442
Epoch 198/5000
14s 29ms/step - loss: 0.8027 - acc: 0.8170 - val_loss: 0.7417 - val_acc: 0.8372
Epoch 199/5000
14s 29ms/step - loss: 0.8024 - acc: 0.8155 - val_loss: 0.7525 - val_acc: 0.8345
Epoch 200/5000
14s 29ms/step - loss: 0.8033 - acc: 0.8170 - val_loss: 0.7385 - val_acc: 0.8423
Epoch 201/5000
14s 29ms/step - loss: 0.8066 - acc: 0.8155 - val_loss: 0.7318 - val_acc: 0.8452
Epoch 202/5000
14s 29ms/step - loss: 0.8076 - acc: 0.8165 - val_loss: 0.7371 - val_acc: 0.8439
Epoch 203/5000
14s 29ms/step - loss: 0.8030 - acc: 0.8166 - val_loss: 0.7295 - val_acc: 0.8447
Epoch 204/5000
14s 29ms/step - loss: 0.8019 - acc: 0.8164 - val_loss: 0.7153 - val_acc: 0.8485
Epoch 205/5000
14s 29ms/step - loss: 0.8073 - acc: 0.8142 - val_loss: 0.7385 - val_acc: 0.8428
Epoch 206/5000
14s 29ms/step - loss: 0.8034 - acc: 0.8164 - val_loss: 0.7435 - val_acc: 0.8428
Epoch 207/5000
14s 29ms/step - loss: 0.8040 - acc: 0.8167 - val_loss: 0.7462 - val_acc: 0.8398
Epoch 208/5000
14s 28ms/step - loss: 0.8017 - acc: 0.8141 - val_loss: 0.7423 - val_acc: 0.8387
Epoch 209/5000
14s 29ms/step - loss: 0.8069 - acc: 0.8141 - val_loss: 0.7276 - val_acc: 0.8446
Epoch 210/5000
14s 28ms/step - loss: 0.7999 - acc: 0.8182 - val_loss: 0.7280 - val_acc: 0.8451
Epoch 211/5000
14s 29ms/step - loss: 0.8038 - acc: 0.8173 - val_loss: 0.7510 - val_acc: 0.8375
Epoch 212/5000
14s 29ms/step - loss: 0.8054 - acc: 0.8154 - val_loss: 0.7397 - val_acc: 0.8402
Epoch 213/5000
14s 29ms/step - loss: 0.8063 - acc: 0.8174 - val_loss: 0.7352 - val_acc: 0.8399
Epoch 214/5000
14s 29ms/step - loss: 0.8059 - acc: 0.8167 - val_loss: 0.7599 - val_acc: 0.8330
Epoch 215/5000
14s 29ms/step - loss: 0.8095 - acc: 0.8139 - val_loss: 0.7212 - val_acc: 0.8494
Epoch 216/5000
14s 29ms/step - loss: 0.8039 - acc: 0.8165 - val_loss: 0.7612 - val_acc: 0.8321
Epoch 217/5000
14s 29ms/step - loss: 0.8084 - acc: 0.8142 - val_loss: 0.7265 - val_acc: 0.8488
Epoch 218/5000
14s 29ms/step - loss: 0.8024 - acc: 0.8164 - val_loss: 0.7552 - val_acc: 0.8385
Epoch 219/5000
14s 29ms/step - loss: 0.8067 - acc: 0.8140 - val_loss: 0.7338 - val_acc: 0.8453
Epoch 220/5000
14s 29ms/step - loss: 0.8021 - acc: 0.8185 - val_loss: 0.7373 - val_acc: 0.8473
Epoch 221/5000
14s 29ms/step - loss: 0.8006 - acc: 0.8187 - val_loss: 0.7413 - val_acc: 0.8401
Epoch 222/5000
14s 29ms/step - loss: 0.7987 - acc: 0.8192 - val_loss: 0.7323 - val_acc: 0.8426
Epoch 223/5000
14s 29ms/step - loss: 0.8001 - acc: 0.8177 - val_loss: 0.7311 - val_acc: 0.8450
Epoch 224/5000
14s 29ms/step - loss: 0.8025 - acc: 0.8179 - val_loss: 0.7417 - val_acc: 0.8415
Epoch 225/5000
14s 29ms/step - loss: 0.8026 - acc: 0.8182 - val_loss: 0.7238 - val_acc: 0.8520
Epoch 226/5000
14s 29ms/step - loss: 0.8008 - acc: 0.8174 - val_loss: 0.7348 - val_acc: 0.8471
Epoch 227/5000
14s 29ms/step - loss: 0.8020 - acc: 0.8173 - val_loss: 0.7414 - val_acc: 0.8392
Epoch 228/5000
14s 29ms/step - loss: 0.8049 - acc: 0.8159 - val_loss: 0.7642 - val_acc: 0.8351
Epoch 229/5000
14s 29ms/step - loss: 0.8000 - acc: 0.8202 - val_loss: 0.7448 - val_acc: 0.8428
Epoch 230/5000
14s 29ms/step - loss: 0.8026 - acc: 0.8165 - val_loss: 0.7576 - val_acc: 0.8365
Epoch 231/5000
14s 29ms/step - loss: 0.8023 - acc: 0.8173 - val_loss: 0.7490 - val_acc: 0.8394
Epoch 232/5000
14s 29ms/step - loss: 0.8002 - acc: 0.8186 - val_loss: 0.7445 - val_acc: 0.8411
Epoch 233/5000
14s 29ms/step - loss: 0.8010 - acc: 0.8179 - val_loss: 0.7350 - val_acc: 0.8431
Epoch 234/5000
14s 29ms/step - loss: 0.8057 - acc: 0.8156 - val_loss: 0.7372 - val_acc: 0.8453
Epoch 235/5000
14s 29ms/step - loss: 0.7973 - acc: 0.8185 - val_loss: 0.7230 - val_acc: 0.8490
Epoch 236/5000
14s 29ms/step - loss: 0.8048 - acc: 0.8169 - val_loss: 0.7283 - val_acc: 0.8421
Epoch 237/5000
14s 29ms/step - loss: 0.8001 - acc: 0.8179 - val_loss: 0.7345 - val_acc: 0.8444
Epoch 238/5000
14s 29ms/step - loss: 0.7995 - acc: 0.8173 - val_loss: 0.7595 - val_acc: 0.8340
Epoch 239/5000
14s 29ms/step - loss: 0.8022 - acc: 0.8173 - val_loss: 0.7389 - val_acc: 0.8398
Epoch 240/5000
14s 28ms/step - loss: 0.8003 - acc: 0.8181 - val_loss: 0.7388 - val_acc: 0.8429
Epoch 241/5000
14s 29ms/step - loss: 0.8006 - acc: 0.8187 - val_loss: 0.7415 - val_acc: 0.8413
Epoch 242/5000
14s 29ms/step - loss: 0.8020 - acc: 0.8167 - val_loss: 0.7296 - val_acc: 0.8506
Epoch 243/5000
14s 29ms/step - loss: 0.7966 - acc: 0.8192 - val_loss: 0.7433 - val_acc: 0.8383
Epoch 244/5000
14s 29ms/step - loss: 0.8014 - acc: 0.8186 - val_loss: 0.7347 - val_acc: 0.8423
Epoch 245/5000
14s 29ms/step - loss: 0.8037 - acc: 0.8164 - val_loss: 0.7479 - val_acc: 0.8413
Epoch 246/5000
14s 29ms/step - loss: 0.7996 - acc: 0.8185 - val_loss: 0.7393 - val_acc: 0.8413
Epoch 247/5000
14s 29ms/step - loss: 0.7986 - acc: 0.8198 - val_loss: 0.7266 - val_acc: 0.8473
Epoch 248/5000
14s 29ms/step - loss: 0.7990 - acc: 0.8187 - val_loss: 0.7422 - val_acc: 0.8415
Epoch 249/5000
14s 28ms/step - loss: 0.7970 - acc: 0.8193 - val_loss: 0.7325 - val_acc: 0.8450
Epoch 250/5000
14s 29ms/step - loss: 0.8014 - acc: 0.8166 - val_loss: 0.7342 - val_acc: 0.8443
Epoch 251/5000
14s 29ms/step - loss: 0.8032 - acc: 0.8176 - val_loss: 0.7330 - val_acc: 0.8453
Epoch 252/5000
14s 29ms/step - loss: 0.8051 - acc: 0.8161 - val_loss: 0.7535 - val_acc: 0.8367
Epoch 253/5000
14s 29ms/step - loss: 0.7980 - acc: 0.8211 - val_loss: 0.7217 - val_acc: 0.8507
Epoch 254/5000
14s 29ms/step - loss: 0.8033 - acc: 0.8163 - val_loss: 0.7390 - val_acc: 0.8423
Epoch 255/5000
14s 29ms/step - loss: 0.7991 - acc: 0.8179 - val_loss: 0.7291 - val_acc: 0.8468
Epoch 256/5000
14s 29ms/step - loss: 0.7978 - acc: 0.8189 - val_loss: 0.7384 - val_acc: 0.8454
Epoch 257/5000
14s 29ms/step - loss: 0.7993 - acc: 0.8190 - val_loss: 0.7049 - val_acc: 0.8536
Epoch 258/5000
14s 29ms/step - loss: 0.8019 - acc: 0.8183 - val_loss: 0.7616 - val_acc: 0.8394
Epoch 259/5000
14s 29ms/step - loss: 0.7873 - acc: 0.8211 - val_loss: 0.7415 - val_acc: 0.8396
Epoch 260/5000
14s 29ms/step - loss: 0.8046 - acc: 0.8170 - val_loss: 0.7395 - val_acc: 0.8447
Epoch 261/5000
14s 29ms/step - loss: 0.8046 - acc: 0.8175 - val_loss: 0.7319 - val_acc: 0.8470
Epoch 262/5000
14s 29ms/step - loss: 0.7945 - acc: 0.8204 - val_loss: 0.7372 - val_acc: 0.8398
Epoch 263/5000
14s 29ms/step - loss: 0.7995 - acc: 0.8181 - val_loss: 0.7467 - val_acc: 0.8421
Epoch 264/5000
14s 29ms/step - loss: 0.8025 - acc: 0.8168 - val_loss: 0.7216 - val_acc: 0.8483
Epoch 265/5000
14s 29ms/step - loss: 0.8051 - acc: 0.8167 - val_loss: 0.7334 - val_acc: 0.8440
Epoch 266/5000
14s 29ms/step - loss: 0.7917 - acc: 0.8221 - val_loss: 0.7452 - val_acc: 0.8417
Epoch 267/5000
14s 29ms/step - loss: 0.8009 - acc: 0.8181 - val_loss: 0.7270 - val_acc: 0.8465
Epoch 268/5000
14s 29ms/step - loss: 0.7983 - acc: 0.8188 - val_loss: 0.7242 - val_acc: 0.8477
Epoch 269/5000
14s 29ms/step - loss: 0.8044 - acc: 0.8162 - val_loss: 0.7402 - val_acc: 0.8418
Epoch 270/5000
14s 29ms/step - loss: 0.7993 - acc: 0.8196 - val_loss: 0.7411 - val_acc: 0.8402
Epoch 271/5000
14s 29ms/step - loss: 0.7981 - acc: 0.8188 - val_loss: 0.7554 - val_acc: 0.8427
Epoch 272/5000
14s 29ms/step - loss: 0.8031 - acc: 0.8176 - val_loss: 0.7506 - val_acc: 0.8383
Epoch 273/5000
14s 29ms/step - loss: 0.7973 - acc: 0.8203 - val_loss: 0.7229 - val_acc: 0.8523
Epoch 274/5000
14s 29ms/step - loss: 0.7960 - acc: 0.8208 - val_loss: 0.7330 - val_acc: 0.8437
Epoch 275/5000
14s 29ms/step - loss: 0.7953 - acc: 0.8189 - val_loss: 0.7349 - val_acc: 0.8428
Epoch 276/5000
14s 29ms/step - loss: 0.7989 - acc: 0.8173 - val_loss: 0.7498 - val_acc: 0.8386
Epoch 277/5000
14s 29ms/step - loss: 0.7967 - acc: 0.8187 - val_loss: 0.7153 - val_acc: 0.8524
Epoch 278/5000
14s 29ms/step - loss: 0.8038 - acc: 0.8175 - val_loss: 0.7033 - val_acc: 0.8495
Epoch 279/5000
14s 29ms/step - loss: 0.7964 - acc: 0.8188 - val_loss: 0.7159 - val_acc: 0.8529
Epoch 280/5000
14s 29ms/step - loss: 0.7979 - acc: 0.8176 - val_loss: 0.7406 - val_acc: 0.8419
Epoch 281/5000
14s 29ms/step - loss: 0.7985 - acc: 0.8196 - val_loss: 0.7250 - val_acc: 0.8493
Epoch 282/5000
14s 29ms/step - loss: 0.7990 - acc: 0.8180 - val_loss: 0.7351 - val_acc: 0.8448
Epoch 283/5000
14s 29ms/step - loss: 0.7967 - acc: 0.8202 - val_loss: 0.7554 - val_acc: 0.8332
Epoch 284/5000
14s 29ms/step - loss: 0.7939 - acc: 0.8195 - val_loss: 0.7553 - val_acc: 0.8359
Epoch 285/5000
14s 29ms/step - loss: 0.7956 - acc: 0.8200 - val_loss: 0.7495 - val_acc: 0.8414
Epoch 286/5000
14s 29ms/step - loss: 0.7963 - acc: 0.8196 - val_loss: 0.7406 - val_acc: 0.8407
Epoch 287/5000
14s 29ms/step - loss: 0.7997 - acc: 0.8182 - val_loss: 0.7359 - val_acc: 0.8465
...
Epoch 581/5000
14s 29ms/step - loss: 0.7889 - acc: 0.8222 - val_loss: 0.7353 - val_acc: 0.8437
Epoch 582/5000
14s 29ms/step - loss: 0.7835 - acc: 0.8240 - val_loss: 0.7153 - val_acc: 0.8461
Epoch 583/5000
14s 29ms/step - loss: 0.7886 - acc: 0.8208 - val_loss: 0.7357 - val_acc: 0.8421
Epoch 584/5000
14s 29ms/step - loss: 0.7904 - acc: 0.8230 - val_loss: 0.7176 - val_acc: 0.8446
Epoch 585/5000
14s 29ms/step - loss: 0.7861 - acc: 0.8230 - val_loss: 0.7678 - val_acc: 0.8285
Epoch 586/5000
14s 29ms/step - loss: 0.7901 - acc: 0.8217 - val_loss: 0.7487 - val_acc: 0.8363
Epoch 587/5000
14s 28ms/step - loss: 0.7887 - acc: 0.8211 - val_loss: 0.7400 - val_acc: 0.8426
Epoch 588/5000
14s 29ms/step - loss: 0.7829 - acc: 0.8234 - val_loss: 0.7117 - val_acc: 0.8529
Epoch 589/5000
14s 29ms/step - loss: 0.7845 - acc: 0.8238 - val_loss: 0.7595 - val_acc: 0.8350
Epoch 590/5000
14s 29ms/step - loss: 0.7875 - acc: 0.8226 - val_loss: 0.7112 - val_acc: 0.8499
Epoch 591/5000
14s 29ms/step - loss: 0.7838 - acc: 0.8233 - val_loss: 0.7156 - val_acc: 0.8485
Epoch 592/5000
14s 29ms/step - loss: 0.7843 - acc: 0.8225 - val_loss: 0.7237 - val_acc: 0.8474
Epoch 593/5000
14s 29ms/step - loss: 0.7888 - acc: 0.8225 - val_loss: 0.7290 - val_acc: 0.8455
Epoch 594/5000
14s 29ms/step - loss: 0.7876 - acc: 0.8233 - val_loss: 0.7346 - val_acc: 0.8438
Epoch 595/5000
14s 29ms/step - loss: 0.7906 - acc: 0.8214 - val_loss: 0.7223 - val_acc: 0.8450
Epoch 596/5000
14s 29ms/step - loss: 0.7828 - acc: 0.8233 - val_loss: 0.7248 - val_acc: 0.8466
Epoch 597/5000
14s 29ms/step - loss: 0.7805 - acc: 0.8251 - val_loss: 0.7219 - val_acc: 0.8479
Epoch 598/5000
14s 29ms/step - loss: 0.7873 - acc: 0.8230 - val_loss: 0.7362 - val_acc: 0.8438
Epoch 599/5000
14s 29ms/step - loss: 0.7867 - acc: 0.8247 - val_loss: 0.7001 - val_acc: 0.8563
Epoch 600/5000
14s 29ms/step - loss: 0.7870 - acc: 0.8247 - val_loss: 0.7105 - val_acc: 0.8518
Epoch 601/5000
14s 29ms/step - loss: 0.7811 - acc: 0.8256 - val_loss: 0.7327 - val_acc: 0.8407
Epoch 602/5000
14s 29ms/step - loss: 0.7863 - acc: 0.8213 - val_loss: 0.7418 - val_acc: 0.8377
Epoch 603/5000
14s 29ms/step - loss: 0.7815 - acc: 0.8256 - val_loss: 0.7202 - val_acc: 0.8453
Epoch 604/5000
14s 29ms/step - loss: 0.7885 - acc: 0.8217 - val_loss: 0.7289 - val_acc: 0.8461
Epoch 605/5000
14s 29ms/step - loss: 0.7861 - acc: 0.8238 - val_loss: 0.7215 - val_acc: 0.8449
Epoch 606/5000
14s 29ms/step - loss: 0.7815 - acc: 0.8248 - val_loss: 0.7062 - val_acc: 0.8526
Epoch 607/5000
14s 29ms/step - loss: 0.7814 - acc: 0.8233 - val_loss: 0.7171 - val_acc: 0.8495
Epoch 608/5000
14s 29ms/step - loss: 0.7878 - acc: 0.8215 - val_loss: 0.7230 - val_acc: 0.8442
Epoch 609/5000
14s 29ms/step - loss: 0.7821 - acc: 0.8244 - val_loss: 0.7295 - val_acc: 0.8476
Epoch 610/5000
14s 28ms/step - loss: 0.7858 - acc: 0.8218 - val_loss: 0.7366 - val_acc: 0.8411
Epoch 611/5000
14s 29ms/step - loss: 0.7887 - acc: 0.8218 - val_loss: 0.7110 - val_acc: 0.8499
Epoch 612/5000
14s 29ms/step - loss: 0.7917 - acc: 0.8206 - val_loss: 0.7094 - val_acc: 0.8540
Epoch 613/5000
14s 28ms/step - loss: 0.7826 - acc: 0.8241 - val_loss: 0.6942 - val_acc: 0.8548
Epoch 614/5000
14s 28ms/step - loss: 0.7845 - acc: 0.8233 - val_loss: 0.7119 - val_acc: 0.8496
Epoch 615/5000
14s 28ms/step - loss: 0.7824 - acc: 0.8249 - val_loss: 0.7234 - val_acc: 0.8471
Epoch 616/5000
14s 28ms/step - loss: 0.7837 - acc: 0.8220 - val_loss: 0.7226 - val_acc: 0.8481
Epoch 617/5000
14s 28ms/step - loss: 0.7834 - acc: 0.8233 - val_loss: 0.7059 - val_acc: 0.8516
Epoch 618/5000
14s 28ms/step - loss: 0.7806 - acc: 0.8244 - val_loss: 0.7116 - val_acc: 0.8511
Epoch 619/5000
14s 28ms/step - loss: 0.7861 - acc: 0.8225 - val_loss: 0.7207 - val_acc: 0.8437
Epoch 620/5000
14s 28ms/step - loss: 0.7830 - acc: 0.8234 - val_loss: 0.7345 - val_acc: 0.8428
Epoch 621/5000
14s 28ms/step - loss: 0.7830 - acc: 0.8232 - val_loss: 0.7335 - val_acc: 0.8431
Epoch 622/5000
14s 28ms/step - loss: 0.7852 - acc: 0.8231 - val_loss: 0.7109 - val_acc: 0.8528
Epoch 623/5000
14s 28ms/step - loss: 0.7849 - acc: 0.8220 - val_loss: 0.7252 - val_acc: 0.8443
Epoch 624/5000
14s 28ms/step - loss: 0.7858 - acc: 0.8227 - val_loss: 0.7056 - val_acc: 0.8544
Epoch 625/5000
14s 28ms/step - loss: 0.7814 - acc: 0.8242 - val_loss: 0.7218 - val_acc: 0.8459
Epoch 626/5000
14s 28ms/step - loss: 0.7824 - acc: 0.8256 - val_loss: 0.7179 - val_acc: 0.8509
Epoch 627/5000
14s 28ms/step - loss: 0.7840 - acc: 0.8230 - val_loss: 0.7445 - val_acc: 0.8378
Epoch 628/5000
14s 28ms/step - loss: 0.7890 - acc: 0.8221 - val_loss: 0.7291 - val_acc: 0.8432
Epoch 629/5000
14s 28ms/step - loss: 0.7780 - acc: 0.8242 - val_loss: 0.7305 - val_acc: 0.8440
Epoch 630/5000
14s 28ms/step - loss: 0.7870 - acc: 0.8220 - val_loss: 0.7329 - val_acc: 0.8427
Epoch 631/5000
14s 28ms/step - loss: 0.7901 - acc: 0.8223 - val_loss: 0.7152 - val_acc: 0.8513
Epoch 632/5000
14s 28ms/step - loss: 0.7849 - acc: 0.8222 - val_loss: 0.7248 - val_acc: 0.8465
Epoch 633/5000
14s 28ms/step - loss: 0.7769 - acc: 0.8270 - val_loss: 0.7084 - val_acc: 0.8510
Epoch 634/5000
14s 28ms/step - loss: 0.7813 - acc: 0.8237 - val_loss: 0.7297 - val_acc: 0.8467
Epoch 635/5000
14s 28ms/step - loss: 0.7853 - acc: 0.8225 - val_loss: 0.7219 - val_acc: 0.8479
Epoch 636/5000
14s 28ms/step - loss: 0.7814 - acc: 0.8251 - val_loss: 0.7101 - val_acc: 0.8508
Epoch 637/5000
14s 28ms/step - loss: 0.7818 - acc: 0.8251 - val_loss: 0.7232 - val_acc: 0.8498
Epoch 638/5000
14s 28ms/step - loss: 0.7867 - acc: 0.8222 - val_loss: 0.7100 - val_acc: 0.8483
Epoch 639/5000
14s 28ms/step - loss: 0.7847 - acc: 0.8233 - val_loss: 0.7252 - val_acc: 0.8462
Epoch 640/5000
14s 28ms/step - loss: 0.7837 - acc: 0.8233 - val_loss: 0.7194 - val_acc: 0.8511
...
Epoch 907/5000
14s 28ms/step - loss: 0.7814 - acc: 0.8254 - val_loss: 0.7039 - val_acc: 0.8514
Epoch 908/5000
14s 28ms/step - loss: 0.7799 - acc: 0.8246 - val_loss: 0.7347 - val_acc: 0.8395
Epoch 909/5000
14s 28ms/step - loss: 0.7739 - acc: 0.8261 - val_loss: 0.7010 - val_acc: 0.8542
Epoch 910/5000
14s 28ms/step - loss: 0.7797 - acc: 0.8248 - val_loss: 0.7322 - val_acc: 0.8428
Epoch 911/5000
14s 28ms/step - loss: 0.7788 - acc: 0.8253 - val_loss: 0.7366 - val_acc: 0.8418
Epoch 912/5000
14s 28ms/step - loss: 0.7861 - acc: 0.8226 - val_loss: 0.7011 - val_acc: 0.8568
Epoch 913/5000
14s 28ms/step - loss: 0.7801 - acc: 0.8244 - val_loss: 0.6968 - val_acc: 0.8560
Epoch 914/5000
14s 28ms/step - loss: 0.7812 - acc: 0.8238 - val_loss: 0.7325 - val_acc: 0.8383
Epoch 915/5000
14s 28ms/step - loss: 0.7781 - acc: 0.8239 - val_loss: 0.7323 - val_acc: 0.8416
Epoch 916/5000
14s 28ms/step - loss: 0.7773 - acc: 0.8245 - val_loss: 0.7187 - val_acc: 0.8464
Epoch 917/5000
14s 28ms/step - loss: 0.7746 - acc: 0.8262 - val_loss: 0.7141 - val_acc: 0.8484
Epoch 918/5000
14s 28ms/step - loss: 0.7820 - acc: 0.8229 - val_loss: 0.7188 - val_acc: 0.8476
Epoch 919/5000
14s 28ms/step - loss: 0.7798 - acc: 0.8227 - val_loss: 0.7166 - val_acc: 0.8459
Epoch 920/5000
14s 28ms/step - loss: 0.7801 - acc: 0.8248 - val_loss: 0.7265 - val_acc: 0.8438
Epoch 921/5000
14s 28ms/step - loss: 0.7773 - acc: 0.8257 - val_loss: 0.7139 - val_acc: 0.8456
Epoch 922/5000
14s 28ms/step - loss: 0.7853 - acc: 0.8236 - val_loss: 0.7258 - val_acc: 0.8431
Epoch 923/5000
14s 28ms/step - loss: 0.7806 - acc: 0.8232 - val_loss: 0.7302 - val_acc: 0.8440
Epoch 924/5000
14s 28ms/step - loss: 0.7784 - acc: 0.8245 - val_loss: 0.7190 - val_acc: 0.8476
Epoch 925/5000
14s 28ms/step - loss: 0.7818 - acc: 0.8227 - val_loss: 0.7139 - val_acc: 0.8480
Epoch 926/5000
14s 28ms/step - loss: 0.7779 - acc: 0.8267 - val_loss: 0.7152 - val_acc: 0.8533
Epoch 927/5000
15s 29ms/step - loss: 0.7790 - acc: 0.8242 - val_loss: 0.7014 - val_acc: 0.8520
Epoch 928/5000
14s 28ms/step - loss: 0.7787 - acc: 0.8264 - val_loss: 0.7274 - val_acc: 0.8445
Epoch 929/5000
14s 28ms/step - loss: 0.7769 - acc: 0.8246 - val_loss: 0.7101 - val_acc: 0.8457
Epoch 930/5000
14s 29ms/step - loss: 0.7826 - acc: 0.8242 - val_loss: 0.7131 - val_acc: 0.8502
Epoch 931/5000
14s 29ms/step - loss: 0.7770 - acc: 0.8260 - val_loss: 0.7165 - val_acc: 0.8492
Epoch 932/5000
14s 29ms/step - loss: 0.7812 - acc: 0.8240 - val_loss: 0.7143 - val_acc: 0.8517
Epoch 933/5000
14s 29ms/step - loss: 0.7774 - acc: 0.8250 - val_loss: 0.6973 - val_acc: 0.8551
Epoch 934/5000
14s 29ms/step - loss: 0.7747 - acc: 0.8256 - val_loss: 0.7229 - val_acc: 0.8469
Epoch 935/5000
14s 29ms/step - loss: 0.7803 - acc: 0.8248 - val_loss: 0.7103 - val_acc: 0.8532
Epoch 936/5000
14s 29ms/step - loss: 0.7770 - acc: 0.8273 - val_loss: 0.7221 - val_acc: 0.8475
Epoch 937/5000
14s 28ms/step - loss: 0.7828 - acc: 0.8241 - val_loss: 0.7053 - val_acc: 0.8541
Epoch 938/5000
14s 29ms/step - loss: 0.7760 - acc: 0.8251 - val_loss: 0.7360 - val_acc: 0.8417
Epoch 939/5000
14s 29ms/step - loss: 0.7789 - acc: 0.8248 - val_loss: 0.7297 - val_acc: 0.8446
Epoch 940/5000
14s 29ms/step - loss: 0.7808 - acc: 0.8250 - val_loss: 0.7342 - val_acc: 0.8445
Epoch 941/5000
14s 29ms/step - loss: 0.7798 - acc: 0.8247 - val_loss: 0.7191 - val_acc: 0.8469
Epoch 942/5000
14s 29ms/step - loss: 0.7810 - acc: 0.8243 - val_loss: 0.7038 - val_acc: 0.8537
Epoch 943/5000
14s 29ms/step - loss: 0.7766 - acc: 0.8234 - val_loss: 0.7158 - val_acc: 0.8497
Epoch 944/5000
14s 28ms/step - loss: 0.7749 - acc: 0.8285 - val_loss: 0.7142 - val_acc: 0.8502
Epoch 945/5000
14s 28ms/step - loss: 0.7757 - acc: 0.8276 - val_loss: 0.7203 - val_acc: 0.8459
Epoch 946/5000
14s 28ms/step - loss: 0.7853 - acc: 0.8229 - val_loss: 0.7134 - val_acc: 0.8510
Epoch 947/5000
14s 28ms/step - loss: 0.7785 - acc: 0.8253 - val_loss: 0.7214 - val_acc: 0.8443
Epoch 948/5000
14s 28ms/step - loss: 0.7776 - acc: 0.8260 - val_loss: 0.7191 - val_acc: 0.8467
Epoch 949/5000
14s 28ms/step - loss: 0.7815 - acc: 0.8235 - val_loss: 0.7117 - val_acc: 0.8541
Epoch 950/5000
14s 28ms/step - loss: 0.7805 - acc: 0.8250 - val_loss: 0.7047 - val_acc: 0.8485
Epoch 951/5000
14s 28ms/step - loss: 0.7816 - acc: 0.8244 - val_loss: 0.7113 - val_acc: 0.8516
Epoch 952/5000
14s 28ms/step - loss: 0.7752 - acc: 0.8263 - val_loss: 0.7217 - val_acc: 0.8486
Epoch 953/5000
14s 28ms/step - loss: 0.7805 - acc: 0.8238 - val_loss: 0.7264 - val_acc: 0.8471
Epoch 954/5000
14s 28ms/step - loss: 0.7809 - acc: 0.8240 - val_loss: 0.7176 - val_acc: 0.8461
Epoch 955/5000
14s 28ms/step - loss: 0.7737 - acc: 0.8266 - val_loss: 0.7265 - val_acc: 0.8421
Epoch 956/5000
14s 28ms/step - loss: 0.7777 - acc: 0.8255 - val_loss: 0.7178 - val_acc: 0.8502
Epoch 957/5000
14s 28ms/step - loss: 0.7841 - acc: 0.8241 - val_loss: 0.7262 - val_acc: 0.8425
Epoch 958/5000
14s 28ms/step - loss: 0.7793 - acc: 0.8251 - val_loss: 0.7250 - val_acc: 0.8396
Epoch 959/5000
14s 28ms/step - loss: 0.7811 - acc: 0.8235 - val_loss: 0.7398 - val_acc: 0.8405
Epoch 960/5000
14s 28ms/step - loss: 0.7792 - acc: 0.8248 - val_loss: 0.7114 - val_acc: 0.8484
...
Epoch 1132/5000
14s 28ms/step - loss: 0.7840 - acc: 0.8225 - val_loss: 0.7195 - val_acc: 0.8467
Epoch 1133/5000
14s 28ms/step - loss: 0.7818 - acc: 0.8247 - val_loss: 0.7167 - val_acc: 0.8462
Epoch 1134/5000
14s 28ms/step - loss: 0.7802 - acc: 0.8243 - val_loss: 0.7180 - val_acc: 0.8485
Epoch 1135/5000
14s 28ms/step - loss: 0.7771 - acc: 0.8255 - val_loss: 0.7354 - val_acc: 0.8404
Epoch 1136/5000
14s 28ms/step - loss: 0.7804 - acc: 0.8241 - val_loss: 0.7151 - val_acc: 0.8499
Epoch 1137/5000
14s 28ms/step - loss: 0.7722 - acc: 0.8276 - val_loss: 0.7180 - val_acc: 0.8478
Epoch 1138/5000
14s 28ms/step - loss: 0.7803 - acc: 0.8242 - val_loss: 0.7294 - val_acc: 0.8413
Epoch 1139/5000
14s 28ms/step - loss: 0.7787 - acc: 0.8240 - val_loss: 0.7112 - val_acc: 0.8497
Epoch 1140/5000
14s 28ms/step - loss: 0.7827 - acc: 0.8220 - val_loss: 0.7258 - val_acc: 0.8462
Epoch 1141/5000
14s 28ms/step - loss: 0.7783 - acc: 0.8255 - val_loss: 0.6970 - val_acc: 0.8542
Epoch 1142/5000
14s 28ms/step - loss: 0.7835 - acc: 0.8218 - val_loss: 0.7143 - val_acc: 0.8519
Epoch 1143/5000
14s 28ms/step - loss: 0.7795 - acc: 0.8235 - val_loss: 0.7167 - val_acc: 0.8486
Epoch 1144/5000
14s 28ms/step - loss: 0.7782 - acc: 0.8266 - val_loss: 0.7335 - val_acc: 0.8416
Epoch 1145/5000
14s 28ms/step - loss: 0.7783 - acc: 0.8251 - val_loss: 0.7014 - val_acc: 0.8528
Epoch 1146/5000
14s 28ms/step - loss: 0.7766 - acc: 0.8234 - val_loss: 0.7010 - val_acc: 0.8550
Epoch 1147/5000
14s 28ms/step - loss: 0.7820 - acc: 0.8245 - val_loss: 0.7037 - val_acc: 0.8545
Epoch 1148/5000
14s 28ms/step - loss: 0.7740 - acc: 0.8264 - val_loss: 0.7141 - val_acc: 0.8519
Epoch 1149/5000
14s 28ms/step - loss: 0.7791 - acc: 0.8246 - val_loss: 0.7495 - val_acc: 0.8421
Epoch 1150/5000
14s 28ms/step - loss: 0.7785 - acc: 0.8252 - val_loss: 0.7398 - val_acc: 0.8373
Epoch 1151/5000
14s 28ms/step - loss: 0.7832 - acc: 0.8237 - val_loss: 0.7199 - val_acc: 0.8472
Epoch 1152/5000
14s 28ms/step - loss: 0.7823 - acc: 0.8240 - val_loss: 0.7073 - val_acc: 0.8535
Epoch 1153/5000
14s 28ms/step - loss: 0.7805 - acc: 0.8251 - val_loss: 0.7187 - val_acc: 0.8499
Epoch 1154/5000
14s 28ms/step - loss: 0.7820 - acc: 0.8228 - val_loss: 0.7399 - val_acc: 0.8407
Epoch 1155/5000
15s 29ms/step - loss: 0.7829 - acc: 0.8224 - val_loss: 0.7405 - val_acc: 0.8377
Epoch 1156/5000
14s 29ms/step - loss: 0.7805 - acc: 0.8248 - val_loss: 0.7229 - val_acc: 0.8470
Epoch 1157/5000
14s 28ms/step - loss: 0.7794 - acc: 0.8263 - val_loss: 0.7350 - val_acc: 0.8440
Epoch 1158/5000
14s 28ms/step - loss: 0.7799 - acc: 0.8241 - val_loss: 0.7069 - val_acc: 0.8525
Epoch 1159/5000
14s 28ms/step - loss: 0.7801 - acc: 0.8253 - val_loss: 0.7056 - val_acc: 0.8543
Epoch 1160/5000
14s 28ms/step - loss: 0.7777 - acc: 0.8258 - val_loss: 0.7226 - val_acc: 0.8501
Epoch 1161/5000
14s 28ms/step - loss: 0.7806 - acc: 0.8246 - val_loss: 0.7263 - val_acc: 0.8458
Epoch 1162/5000
14s 28ms/step - loss: 0.7793 - acc: 0.8243 - val_loss: 0.7063 - val_acc: 0.8522
Epoch 1163/5000
14s 28ms/step - loss: 0.7783 - acc: 0.8263 - val_loss: 0.7391 - val_acc: 0.8430
Epoch 1164/5000
14s 28ms/step - loss: 0.7753 - acc: 0.8263 - val_loss: 0.7254 - val_acc: 0.8465
Epoch 1165/5000
14s 28ms/step - loss: 0.7838 - acc: 0.8232 - val_loss: 0.7287 - val_acc: 0.8479
Epoch 1166/5000
14s 28ms/step - loss: 0.7827 - acc: 0.8229 - val_loss: 0.7269 - val_acc: 0.8464
Epoch 1167/5000
14s 28ms/step - loss: 0.7782 - acc: 0.8262 - val_loss: 0.7305 - val_acc: 0.8438
Epoch 1168/5000
14s 28ms/step - loss: 0.7778 - acc: 0.8251 - val_loss: 0.7065 - val_acc: 0.8501
Epoch 1169/5000
14s 28ms/step - loss: 0.7768 - acc: 0.8249 - val_loss: 0.7039 - val_acc: 0.8540
Epoch 1170/5000
14s 28ms/step - loss: 0.7797 - acc: 0.8261 - val_loss: 0.7052 - val_acc: 0.8547
Epoch 1171/5000
14s 29ms/step - loss: 0.7799 - acc: 0.8245 - val_loss: 0.6993 - val_acc: 0.8564
Epoch 1172/5000
14s 29ms/step - loss: 0.7768 - acc: 0.8253 - val_loss: 0.7237 - val_acc: 0.8473