print("Execution time with earlier stopping: %s minutes" % ((end_time[4] - start_time[4])/60))

Execution time with earlier stopping: 8.33876238266627 minutes

fig, (ax1, ax2) = plt.subplots(1, 2, sharey=True, figsize=(16, 5))
nn_plot(hist[3], 15, subplot=ax1, loss=True, title="Reference: simple convolution filters")
nn_plot(hist[4], 15, subplot=ax2, loss=True, title="filters with activation function")

fig, (ax1, ax2) = plt.subplots(1, 2, sharey=True, figsize=(16, 5))
nn_plot(hist[3], 15, subplot=ax1, acc=True, title="Reference: simple convolution filters")
nn_plot(hist[4], 15, subplot=ax2, acc=True, title="filters with activation function")

# Evaluate the model on the test data using 'evaluate'
print("Evaluate on test data")
results.append(model[-1].evaluate(x_test, y_test_label, batch_size=64))
print("test loss, test acc:", results[2])

Evaluate on test data
157/157 [==============================] - 1s 8ms/step - loss: 0.0425 - categorical_accuracy: 0.9899
test loss, test acc: [0.04253215715289116, 0.9898999929428101]

The accuracy over the test set is 98.99%. It improved a little compared to previous model.

Different network structures

We proceed now to investigate some different structures of convolutional network.

Structure 2: 2x(2xConv + Pool)

Oposed to our first model, we will apply two convolutional layers before a pooling layer and another two convolutional layers before a second pooling layer. This configuration is intended for more feature extraction.

model.append(tf.keras.models.Sequential([
    tf.keras.layers.Conv2D(16, (3, 3), activation='relu', input_shape=(28, 28, 1)),
    tf.keras.layers.Conv2D(32, (3, 3), activation='relu'),
    tf.keras.layers.MaxPooling2D(pool_size=(2, 2)),
    tf.keras.layers.Conv2D(32, (3, 3), activation='relu'),
    tf.keras.layers.Conv2D(32, (3, 3), activation='relu'),
    tf.keras.layers.MaxPooling2D(pool_size=(2, 2)),
    tf.keras.layers.Flatten(),
    tf.keras.layers.Dense(32, activation='relu'),
    tf.keras.layers.Dense(10, activation='softmax')
]))

nn_compile(model[-1], params[-1])

model[-1].summary()

Model: "sequential_6"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 conv2d_15 (Conv2D)          (None, 26, 26, 16)        160       
                                                                 
 conv2d_16 (Conv2D)          (None, 24, 24, 32)        4640      
                                                                 
 max_pooling2d_10 (MaxPoolin  (None, 12, 12, 32)       0         
 g2D)                                                            
                                                                 
 conv2d_17 (Conv2D)          (None, 10, 10, 32)        9248      
                                                                 
 conv2d_18 (Conv2D)          (None, 8, 8, 32)          9248      
                                                                 
 max_pooling2d_11 (MaxPoolin  (None, 4, 4, 32)         0         
 g2D)                                                            
                                                                 
 flatten_5 (Flatten)         (None, 512)               0         
                                                                 
 dense_10 (Dense)            (None, 32)                16416     
                                                                 
 dense_11 (Dense)            (None, 10)                330       
                                                                 
=================================================================
Total params: 40,042
Trainable params: 40,042
Non-trainable params: 0
_________________________________________________________________

tf.keras.utils.plot_model(model[-1], show_shapes=True, show_layer_names=True)