print("Execution time with earlier stopping: %s minutes" % ((end_time[5] - start_time[5])/60))

Execution time with earlier stopping: 31.909461422761282 minutes

fig, (ax1, ax2) = plt.subplots(1, 2, sharey=True, figsize=(16, 5))
nn_plot(hist[4], 17, subplot=ax1, loss=True, title="Reference: structure 1")
nn_plot(hist[5], 17, subplot=ax2, loss=True, title="structure 2")

fig, (ax1, ax2) = plt.subplots(1, 2, sharey=True, figsize=(16, 5))
nn_plot(hist[4], 17, subplot=ax1, acc=True, title="Reference: structure 1")
nn_plot(hist[5], 17, subplot=ax2, acc=True, title="structure 2")

# Evaluate the model on the test data using 'evaluate'
print("Evaluate on test data")
results.append(model[-1].evaluate(x_test, y_test_label, batch_size=64))
print("test loss, test acc:", results[3])

Evaluate on test data
157/157 [==============================] - 8s 53ms/step - loss: 0.0333 - categorical_accuracy: 0.9919
test loss, test acc: [0.03331693261861801, 0.9919000267982483]

The dual convolutional layers before the pooling layers improved the model, to an accuracy of 99.19% over the test set.

Structure 3: LeNet-5

Our next model will be based on LeNet-5 architecture. This structure was proposed by Yann LeCun, in 1989, and was one of the earliest convolutional neural networks. Since the model was designed for 32x32 input data and ours is 28x28, we will apply padding to the first layer. This model applies the hyperbolic tangent activation function and uses strides for average-pooling.

model.append(tf.keras.models.Sequential([
    tf.keras.layers.Conv2D(6, (5, 5), padding='same', activation=tf.keras.activations.tanh,
                           input_shape=(28, 28, 1)),
    tf.keras.layers.AveragePooling2D(pool_size=(2, 2), strides=2),
    tf.keras.layers.Conv2D(16, (5, 5), activation=tf.keras.activations.tanh),
    tf.keras.layers.AveragePooling2D(pool_size=(2, 2), strides=2),
    tf.keras.layers.Conv2D(120, (5, 5), activation=tf.keras.activations.tanh),
    tf.keras.layers.Flatten(),
    tf.keras.layers.Dense(84, activation=tf.keras.activations.tanh),
    tf.keras.layers.Dense(10, activation='softmax')
]))

nn_compile(model[-1], params[-1])

model[-1].summary()

Model: "sequential_7"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 conv2d_19 (Conv2D)          (None, 28, 28, 6)         156       
                                                                 
 average_pooling2d (AverageP  (None, 14, 14, 6)        0         
 ooling2D)                                                       
                                                                 
 conv2d_20 (Conv2D)          (None, 10, 10, 16)        2416      
                                                                 
 average_pooling2d_1 (Averag  (None, 5, 5, 16)         0         
 ePooling2D)                                                     
                                                                 
 conv2d_21 (Conv2D)          (None, 1, 1, 120)         48120     
                                                                 
 flatten_6 (Flatten)         (None, 120)               0         
                                                                 
 dense_12 (Dense)            (None, 84)                10164     
                                                                 
 dense_13 (Dense)            (None, 10)                850       
                                                                 
=================================================================
Total params: 61,706
Trainable params: 61,706
Non-trainable params: 0
_________________________________________________________________

tf.keras.utils.plot_model(model[-1], show_shapes=True, show_layer_names=True)