options(repr.plot.width=12, repr.plot.height=7)
train_set %>% mutate(y = factor(y)) %>%
  ggplot(aes(x_1, x_2, fill = y, color=y)) +
  geom_point(show.legend = FALSE) + stat_ellipse(type="norm")

Classification tree

The following techniques are applied to the two outcomes: 2 and 7 again.
The classification tree allow us to use a higher number of predictors and also allow these regions to take more complex shapes, by partitioning the predictor space.
It uses cross-validation to find the best complexity parameter (cp) for the model, calculated by the Gini Index or the Entropy.

train_rpart <- train(y ~ ., method = "rpart",
                     tuneGrid = data.frame(cp = seq(0.0, 0.1, len = 25)),
                     data = mnist_27$train)
train_rpart$bestTune
train_rpart$finalModel

A data.frame: 1 × 1

	cp
	<dbl>
3	0.008333333

n= 800 

node), split, n, loss, yval, (yprob)
      * denotes terminal node

 1) root 800 379 7 (0.47375000 0.52625000)  
   2) x_1< 0.1901286 423 108 2 (0.74468085 0.25531915)  
     4) x_2>=0.1954991 344  55 2 (0.84011628 0.15988372)  
       8) x_2< 0.392281 328  42 2 (0.87195122 0.12804878) *
       9) x_2>=0.392281 16   3 7 (0.18750000 0.81250000) *
     5) x_2< 0.1954991 79  26 7 (0.32911392 0.67088608)  
      10) x_1< 0.09045455 22   5 2 (0.77272727 0.22727273) *
      11) x_1>=0.09045455 57   9 7 (0.15789474 0.84210526) *
   3) x_1>=0.1901286 377  64 7 (0.16976127 0.83023873)  
     6) x_1< 0.2423059 171  46 7 (0.26900585 0.73099415)  
      12) x_2>=0.2283192 111  45 7 (0.40540541 0.59459459)  
        24) x_2< 0.392507 91  44 7 (0.48351648 0.51648352)  
          48) x_2>=0.3045177 32   9 2 (0.71875000 0.28125000) *
          49) x_2< 0.3045177 59  21 7 (0.35593220 0.64406780) *
        25) x_2>=0.392507 20   1 7 (0.05000000 0.95000000) *
      13) x_2< 0.2283192 60   1 7 (0.01666667 0.98333333) *
     7) x_1>=0.2423059 206  18 7 (0.08737864 0.91262136) *

plot(train_rpart$finalModel, margin = 0.1)
text(train_rpart$finalModel, cex = 1)

y_hat_rpart <- predict(train_rpart, mnist_27$test, type = "raw")
confusionMatrix(data = y_hat_rpart, reference = mnist_27$test$y)

Confusion Matrix and Statistics

          Reference
Prediction  2  7
         2 92 22
         7 14 72
                                          
               Accuracy : 0.82            
                 95% CI : (0.7596, 0.8706)
    No Information Rate : 0.53            
    P-Value [Acc > NIR] : <2e-16          
                                          
                  Kappa : 0.637           
                                          
 Mcnemar's Test P-Value : 0.2433          
                                          
            Sensitivity : 0.8679          
            Specificity : 0.7660          
         Pos Pred Value : 0.8070          
         Neg Pred Value : 0.8372          
             Prevalence : 0.5300          
         Detection Rate : 0.4600          
   Detection Prevalence : 0.5700          
      Balanced Accuracy : 0.8169          
                                          
       'Positive' Class : 2               
                                          

confusionMatrix(data = y_hat_rpart, reference = mnist_27$test$y)$overall["Accuracy"]
cat("F1 Score:", F_meas(data = y_hat_rpart, reference = mnist_27$test$y))

Accuracy: 0.82

F1 Score: 0.8363636

ggplot(train_rpart, highlight = TRUE)

y_hat_rpart <- predict(train_rpart, mnist_27$true_p, type = "prob")[,2]

grid.arrange(plot_cond_prob(),
             plot_cond_prob(y_hat_rpart, title = "Classification Tree"),
             ncol = 2)

Random Forest

Random Forest improves prediction performance and reduces instability of decision tree by averaging multiple decision trees, obtained through bootstrap samples.

train_rf <- randomForest(y ~ ., data=mnist_27$train)
train_rf

Call:
 randomForest(formula = y ~ ., data = mnist_27$train) 
               Type of random forest: classification
                     Number of trees: 500
No. of variables tried at each split: 1

        OOB estimate of  error rate: 16.88%
Confusion matrix:
    2   7 class.error
2 311  68   0.1794195
7  67 354   0.1591449

y_hat_rf <- predict(train_rf, mnist_27$test)
confusionMatrix(data = y_hat_rf, reference = mnist_27$test$y)

Confusion Matrix and Statistics

          Reference
Prediction  2  7
         2 88 23
         7 18 71
                                          
               Accuracy : 0.795           
                 95% CI : (0.7323, 0.8487)
    No Information Rate : 0.53            
    P-Value [Acc > NIR] : 5.821e-15       
                                          
                  Kappa : 0.5873          
                                          
 Mcnemar's Test P-Value : 0.5322          
                                          
            Sensitivity : 0.8302          
            Specificity : 0.7553          
         Pos Pred Value : 0.7928          
         Neg Pred Value : 0.7978          
             Prevalence : 0.5300          
         Detection Rate : 0.4400          
   Detection Prevalence : 0.5550          
      Balanced Accuracy : 0.7928          
                                          
       'Positive' Class : 2               
                                          

confusionMatrix(data = y_hat_rf, reference = mnist_27$test$y)$overall["Accuracy"]
cat("F1 Score:", F_meas(data = y_hat_rf, reference = mnist_27$test$y))

Accuracy: 0.795

F1 Score: 0.8110599

One significant metric for random forest is the variable importance. However since this example uses only 2 predictors, both will be important. Here the upper left quadrant (x_1) is more important than the lower right quadrant (x_2).

varImp(train_rf)

A data.frame: 2 × 1

	Overall
	<dbl>
x_1	232.2584
x_2	161.5105