fig, (ax1, ax2) = plt.subplots(1, 2, sharey=True, sharex=True, figsize=(16, 5))
nn_plot(hist[17], 200, subplot=ax1, loss=True, title="Reference: without dropout")
nn_plot(hist[25], 200, subplot=ax2, loss=True, title="with dropout")

fig, (ax1, ax2) = plt.subplots(1, 2, sharey=True, sharex=True, figsize=(16, 5))
nn_plot(hist[17], 200, subplot=ax1, acc=True, title="Reference: without dropout")
nn_plot(hist[25], 200, subplot=ax2, acc=True, title="with dropout")

It can be seen that the dropout reduced overfitting effect over the training set. Since the dropout is applied only during training step, it doesn't directly affect the validation results, although the model fitting is influenced by dropout.

Full training set: Dropout

For the full training set, we will apply a dropout layer after each hidden nodes layer, with the L2 factor $\lambda = 0.005$ found previously.

model.append(tf.keras.models.Sequential([
    tf.keras.layers.Dense(100, kernel_regularizer=tf.keras.regularizers.l2(0.005),
                          activation='relu', input_shape=(784,), name='hidden_1_layer'),
    tf.keras.layers.Dropout(0.5, name='dropout_1'),
    tf.keras.layers.Dense(100, kernel_regularizer=tf.keras.regularizers.l2(0.005),
                          activation='relu', name='hidden_2_layer'),
    tf.keras.layers.Dropout(0.5, name='dropout_2'),
    tf.keras.layers.Dense(10, activation='softmax', name='output_layer')
]))

nn_compile(model[-1], params[3])

model[-1].summary()

Model: "sequential_27"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 hidden_1_layer (Dense)      (None, 100)               78500     
                                                                 
 dropout_1 (Dropout)         (None, 100)               0         
                                                                 
 hidden_2_layer (Dense)      (None, 100)               10100     
                                                                 
 dropout_2 (Dropout)         (None, 100)               0         
                                                                 
 output_layer (Dense)        (None, 10)                1010      
                                                                 
=================================================================
Total params: 89,610
Trainable params: 89,610
Non-trainable params: 0
_________________________________________________________________

tf.keras.utils.plot_model(model[-1], show_shapes=True, show_layer_names=True)