Recommender Systems: Evaluation
Michael Hahsler
Wed May 25 14:54:35 2016
This work is licensed under the Creative Commons Attribution 4.0 International License. For questions please contact Michael Hahsler.
library(recommenderlab)## Loading required package: Matrix
## Loading required package: arules
##
## Attaching package: 'arules'
##
## The following objects are masked from 'package:base':
##
## abbreviate, write
##
## Loading required package: proxy
##
## Attaching package: 'proxy'
##
## The following object is masked from 'package:Matrix':
##
## as.matrix
##
## The following objects are masked from 'package:stats':
##
## as.dist, dist
##
## The following object is masked from 'package:base':
##
## as.matrix
##
## Loading required package: registrydata("Jester5k")
Jester5k## 5000 x 100 rating matrix of class 'realRatingMatrix' with 362106 ratings.Create an Evaluation Scheme
Here we use 4-fold cross validation and a given-10 scheme. Note: goodRating is used to determine positive ratings.
es <- evaluationScheme(Jester5k, method="cross-validation",
k=4, given=10, goodRating=5)
es## Evaluation scheme with 10 items given
## Method: 'cross-validation' with 4 run(s).
## Good ratings: >=5.000000
## Data set: 5000 x 100 rating matrix of class 'realRatingMatrix' with 362106 ratings.Compare Several Algorithms
algorithms <- list(
RANDOM = list(name = "RANDOM", param = NULL),
POPULAR = list(name = "POPULAR", param = NULL),
IBCF = list(name = "IBCF", param = NULL),
UBCF = list(name = "UBCF", param = NULL),
SVD = list(name = "SVD", param = NULL)
)
# Predict top-N recommendation lists
ev <- evaluate(es, algorithms, type="topNList", n=c(1, 3, 5, 10))## RANDOM run fold/sample [model time/prediction time]
## 1 [0.008sec/0.228sec]
## 2 [0.004sec/0.228sec]
## 3 [0.004sec/0.22sec]
## 4 [0.004sec/0.212sec]
## POPULAR run fold/sample [model time/prediction time]
## 1 [0.036sec/2.092sec]
## 2 [0.036sec/1.88sec]
## 3 [0.032sec/1.904sec]
## 4 [0.032sec/1.844sec]
## IBCF run fold/sample [model time/prediction time]
## 1 [0.252sec/0.296sec]
## 2 [0.252sec/0.392sec]
## 3 [0.232sec/0.28sec]
## 4 [0.236sec/0.296sec]
## UBCF run fold/sample [model time/prediction time]
## 1 [0.028sec/5.708sec]
## 2 [0.032sec/5.384sec]
## 3 [0.028sec/5.38sec]
## 4 [0.028sec/5.368sec]
## SVD run fold/sample [model time/prediction time]
## 1 [0.168sec/0.216sec]
## 2 [0.18sec/0.216sec]
## 3 [0.176sec/0.216sec]
## 4 [0.188sec/0.22sec]avg(ev)## $RANDOM
## TP FP FN TN precision recall TPR
## 1 0.1788 0.8212 15.8110 73.1890 0.1788000 0.01121724 0.01121724
## 3 0.5518 2.4482 15.4380 71.5620 0.1839333 0.03489294 0.03489294
## 5 0.9108 4.0892 15.0790 69.9210 0.1821600 0.05735469 0.05735469
## 10 1.7966 8.2034 14.1932 65.8068 0.1796600 0.11332066 0.11332066
## FPR
## 1 0.01107279
## 3 0.03302675
## 5 0.05516435
## 10 0.11081516
##
## $POPULAR
## TP FP FN TN precision recall TPR
## 1 0.4376 0.5624 15.5522 73.4478 0.43760 0.03662579 0.03662579
## 3 1.2642 1.7358 14.7256 72.2744 0.42140 0.10221764 0.10221764
## 5 2.0668 2.9332 13.9230 71.0770 0.41336 0.16565647 0.16565647
## 10 3.9652 6.0348 12.0246 67.9754 0.39652 0.31162899 0.31162899
## FPR
## 1 0.007261672
## 3 0.022458989
## 5 0.037983735
## 10 0.078251005
##
## $IBCF
## TP FP FN TN precision recall TPR
## 1 0.1786 0.8214 15.8112 73.1888 0.17860 0.01062117 0.01062117
## 3 0.5580 2.4420 15.4318 71.5682 0.18600 0.03250471 0.03250471
## 5 0.9930 4.0070 14.9968 70.0032 0.19860 0.05952597 0.05952597
## 10 2.1942 7.8058 13.7956 66.2044 0.21942 0.14146203 0.14146203
## FPR
## 1 0.01096116
## 3 0.03263682
## 5 0.05344010
## 10 0.10381372
##
## $UBCF
## TP FP FN TN precision recall TPR
## 1 0.4354 0.5646 15.5544 73.4456 0.43540 0.03545452 0.03545452
## 3 1.2612 1.7388 14.7286 72.2714 0.42040 0.10076502 0.10076502
## 5 2.0540 2.9460 13.9358 71.0642 0.41080 0.16453975 0.16453975
## 10 3.8848 6.1152 12.1050 67.8950 0.38848 0.30187845 0.30187845
## FPR
## 1 0.00729654
## 3 0.02245960
## 5 0.03812575
## 10 0.07928566
##
## $SVD
## TP FP FN TN precision recall TPR
## 1 0.3356 0.6644 15.6542 73.3458 0.3356000 0.02750627 0.02750627
## 3 0.9916 2.0084 14.9982 72.0018 0.3305333 0.07729627 0.07729627
## 5 1.6248 3.3752 14.3650 70.6350 0.3249600 0.12268035 0.12268035
## 10 3.0868 6.9132 12.9030 67.0970 0.3086800 0.22968521 0.22968521
## FPR
## 1 0.008774236
## 3 0.026602447
## 5 0.044569620
## 10 0.091281550plot(ev, annotate=TRUE)
plot(ev, annotate=TRUE, avg = FALSE)
plot(ev, "prec/rec", annotate=TRUE)
# Predict missing ratings
ev <- evaluate(es, algorithms, type="ratings")## RANDOM run fold/sample [model time/prediction time]
## 1 [0.004sec/0.024sec]
## 2 [0sec/0.024sec]
## 3 [0.004sec/0.024sec]
## 4 [0sec/0.024sec]
## POPULAR run fold/sample [model time/prediction time]
## 1 [0.036sec/0.02sec]
## 2 [0.032sec/0.02sec]
## 3 [0.032sec/0.016sec]
## 4 [0.032sec/0.016sec]
## IBCF run fold/sample [model time/prediction time]
## 1 [0.24sec/0.088sec]
## 2 [0.248sec/0.092sec]
## 3 [0.236sec/0.084sec]
## 4 [0.232sec/0.088sec]
## UBCF run fold/sample [model time/prediction time]
## 1 [0.028sec/5.248sec]
## 2 [0.028sec/5.212sec]
## 3 [0.028sec/5.236sec]
## 4 [0.028sec/5.188sec]
## SVD run fold/sample [model time/prediction time]
## 1 [0.164sec/0.028sec]
## 2 [0.164sec/0.032sec]
## 3 [0.276sec/0.028sec]
## 4 [0.168sec/0.028sec]avg(ev)## $RANDOM
## RMSE MSE MAE
## res 7.795522 60.77039 6.368337
##
## $POPULAR
## RMSE MSE MAE
## res 4.572787 20.91263 3.593896
##
## $IBCF
## RMSE MSE MAE
## res 4.930812 24.3156 3.688436
##
## $UBCF
## RMSE MSE MAE
## res 4.580047 20.97882 3.588258
##
## $SVD
## RMSE MSE MAE
## res 4.747848 22.54437 3.761621plot(ev)