print("Execution time with earlier stopping: %s minutes" % ((end_time[6] - start_time[6])/60))

Execution time with earlier stopping: 8.783784981568655 minutes

fig, (ax1, ax2) = plt.subplots(1, 2, sharey=True, figsize=(16, 5))
nn_plot(hist[4], 14, subplot=ax1, loss=True, title="Reference: structure 1")
nn_plot(hist[6], 14, subplot=ax2, loss=True, title="structure 3")

fig, (ax1, ax2) = plt.subplots(1, 2, sharey=True, figsize=(16, 5))
nn_plot(hist[4], 14, subplot=ax1, acc=True, title="Reference: structure 1")
nn_plot(hist[6], 14, subplot=ax2, acc=True, title="structure 3")

# Evaluate the model on the test data using 'evaluate'
print("Evaluate on test data")
results.append(model[-1].evaluate(x_test, y_test_label, batch_size=64))
print("test loss, test acc:", results[4])

Evaluate on test data
157/157 [==============================] - 4s 26ms/step - loss: 0.0552 - categorical_accuracy: 0.9855
test loss, test acc: [0.05515224114060402, 0.9854999780654907]

This model achieved an accuracy of 98.55% over the test set. It was a pioneer at the time and also helped promoting deep learning. However nowadays, with more computing power, it is possible to create and train larger and more complex models, more accurate as well.

Structure 4: Yassine's model (99.7% acc)

We will apply now a structure based on Yassine Ghouzam's model over the Kaggle competition. It is similar to our previous model, but with different hyperparameters and additional dropout layers. Also the filters use padding, mantaining the kernel size. The model applies 2 convolutional layers followed by pooling and dropout and another 2 convolutional layers followed by pooling and dropout.

model.append(tf.keras.models.Sequential([
    tf.keras.layers.Conv2D(32, (5, 5), padding='same', activation='relu', input_shape=(28, 28, 1)),
    tf.keras.layers.Conv2D(32, (5, 5), padding='same', activation='relu'),
    tf.keras.layers.MaxPooling2D(pool_size=(2, 2)),
    tf.keras.layers.Dropout(0.25),
    tf.keras.layers.Conv2D(64, (3, 3), padding='same', activation='relu'),
    tf.keras.layers.Conv2D(64, (3, 3), padding='same', activation='relu'),
    tf.keras.layers.MaxPooling2D(pool_size=(2, 2), strides=2),
    tf.keras.layers.Dropout(0.5),
    tf.keras.layers.Flatten(),
    tf.keras.layers.Dense(256, activation='relu'),
    tf.keras.layers.Dropout(0.5),
    tf.keras.layers.Dense(10, activation='softmax')
]))

nn_compile(model[-1], params[-1])

model[-1].summary()

Model: "sequential_8"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 conv2d_22 (Conv2D)          (None, 28, 28, 32)        832       
                                                                 
 conv2d_23 (Conv2D)          (None, 28, 28, 32)        25632     
                                                                 
 max_pooling2d_12 (MaxPoolin  (None, 14, 14, 32)       0         
 g2D)                                                            
                                                                 
 dropout (Dropout)           (None, 14, 14, 32)        0         
                                                                 
 conv2d_24 (Conv2D)          (None, 14, 14, 64)        18496     
                                                                 
 conv2d_25 (Conv2D)          (None, 14, 14, 64)        36928     
                                                                 
 max_pooling2d_13 (MaxPoolin  (None, 7, 7, 64)         0         
 g2D)                                                            
                                                                 
 dropout_1 (Dropout)         (None, 7, 7, 64)          0         
                                                                 
 flatten_7 (Flatten)         (None, 3136)              0         
                                                                 
 dense_14 (Dense)            (None, 256)               803072    
                                                                 
 dropout_2 (Dropout)         (None, 256)               0         
                                                                 
 dense_15 (Dense)            (None, 10)                2570      
                                                                 
=================================================================
Total params: 887,530
Trainable params: 887,530
Non-trainable params: 0
_________________________________________________________________

tf.keras.utils.plot_model(model[-1], show_shapes=True, show_layer_names=True)