+++ /dev/null
-library(agghoo)
-
-data(iris) #already there
-library(mlbench)
-data(PimaIndiansDiabetes)
-
-# Run only agghoo on iris dataset (split into train/test, etc).
-# Default parameters: see ?agghoo and ?AgghooCV
-compareTo(iris[,-5], iris[,5], agghoo_run)
-
-# Run both agghoo and standard CV, specifiying some parameters.
-compareTo(iris[,-5], iris[,5], list(agghoo_run, standardCV_run), gmodel="tree")
-compareTo(iris[,-5], iris[,5], list(agghoo_run, standardCV_run),
- gmodel="knn", params=c(3, 7, 13, 17, 23, 31),
- CV = list(type="vfold", V=5, shuffle=T))
-
-# Run both agghoo and standard CV, averaging errors over N=10 runs
-# (possible for a single method but wouldn't make much sense...).
-compareMulti(PimaIndiansDiabetes[,-9], PimaIndiansDiabetes[,9],
- list(agghoo_run, standardCV_run), N=10, gmodel="rf")
-
-# Compare several values of V
-compareRange(PimaIndiansDiabetes[,-9], PimaIndiansDiabetes[,9],
- list(agghoo_run, standardCV_run), N=10, V_range=c(10, 20, 30))
-
-# For example to use average of squared differences.
-# Default is "mean(abs(y1 - y2))".
-loss2 <- function(y1, y2) mean((y1 - y2)^2)
-
-# In regression on artificial datasets (TODO: real data?)
-data <- mlbench.twonorm(300, 3)$x
-target <- rowSums(data)
-compareMulti(data, target, list(agghoo_run, standardCV_run),
- N=10, gmodel="tree", params=c(1, 3, 5, 7, 9), loss=loss2,
- CV = list(type="MC", V=12, test_size=0.3))
-
-compareMulti(data, target, list(agghoo_run, standardCV_run),
- N=10, floss=loss2, CV = list(type="vfold", V=10, shuffle=F))
-
-# Random tests to check that method doesn't fail in 1D case
-M <- matrix(rnorm(200), ncol=2)
-compareTo(as.matrix(M[,-2]), M[,2], list(agghoo_run, standardCV_run), gmodel="knn")
-compareTo(as.matrix(M[,-2]), M[,2], list(agghoo_run, standardCV_run), gmodel="tree")
--- /dev/null
+Encoding: UTF-8
+Package: agghoo
+Title: Aggregated Hold-Out Cross Validation
+Version: 0.1-1
+Description: The 'agghoo' procedure is an alternative to usual cross-validation.
+ Instead of choosing the best model trained on V subsamples, it determines
+ a winner model for each subsample, and then aggregate the V outputs.
+ For the details, see "Aggregated hold-out" by Guillaume Maillard,
+ Sylvain Arlot, Matthieu Lerasle (2021) <doi:10.48550/arXiv.1909.04890>
+ published in Journal of Machine Learning Research 22(20):1--55.
+Authors@R: c(person(given = "Sylvain",
+ family = "Arlot",
+ role = c("cph", "ctb"),
+ email = "sylvain.arlot@universite-paris-saclay.fr"),
+ person(given = "Benjamin",
+ family = "Auder",
+ role = c("aut", "cre", "cph"),
+ email = "benjamin.auder@universite-paris-saclay.fr"),
+ person(given = "Melina",
+ family = "Gallopin",
+ role = c("cph", "ctb"),
+ email = "melina.gallopin@universite-paris-saclay.fr"),
+ person(given = "Matthieu",
+ family = "Lerasle",
+ role = c("cph", "ctb"),
+ email = "matthieu.lerasle@universite-paris-saclay.fr"),
+ person(given = "Guillaume",
+ family = "Maillard",
+ role = c("cph", "ctb"),
+ email = "guillaume.maillard@uni.lu"))
+Depends:
+ R (>= 3.5.0)
+Imports:
+ class,
+ parallel,
+ R6,
+ rpart,
+ FNN
+Suggests:
+ mlbench,
+ roxygen2
+URL: https://git.auder.net/?p=agghoo.git
+License: MIT + file LICENSE
+Collate:
+ 'compareTo.R'
+ 'agghoo.R'
+ 'R6_AgghooCV.R'
+ 'R6_Model.R'
+ 'checks.R'
+ 'utils.R'
+ 'A_NAMESPACE.R'
+Config/roxygen2/version: 8.1.1
#' standardCV_core
#'
#' Cross-validation method, added here as an example.
-#' Parameters are described in ?agghoo and ?AgghooCV
+#'
+#' @param data Matrix or data.frame
+#' @param target Vector of targets (generally numeric or factor)
+#' @param task "regression" or "classification".
+#' Default: classification if target not numeric.
+#' @param gmodel Generic model returning a predictive function
+#' Default: tree if mixed data, knn/ppr otherwise.
+#' @param params List of parameters for cross-validation (each defining a model)
+#' @param loss Function assessing the error of a prediction
+#' Default: error rate or mean(abs(error)).
+#' @param CV List describing cross-validation to run. Slots: \cr
+#' - type: 'vfold' or 'MC' for Monte-Carlo (default: MC) \cr
+#' - V: number of runs (default: 10) \cr
+#' - test_size: percentage of data in the test dataset, for MC
+#' (irrelevant for V-fold). Default: 0.2. \cr
+#' - shuffle: wether or not to shuffle data before V-fold.
+#' Irrelevant for Monte-Carlo; default: TRUE \cr
+#' Default (if NULL): type="MC", V=10, test_size=0.2
+#'
standardCV_core <- function(data, target, task, gmodel, params, loss, CV) {
n <- nrow(data)
shuffle_inds <- NULL
#' CVvoting_core
#'
#' "voting" cross-validation method, added here as an example.
-#' Parameters are described in ?agghoo and ?AgghooCV
+#'
+#' @inheritParams standardCV_core
+#'
CVvoting_core <- function(data, target, task, gmodel, params, loss, CV) {
CV <- checkCV(CV)
n <- nrow(data)
#' standardCV_run
#'
#' Run and eval the standard cross-validation procedure.
-#' Parameters are rather explicit except "floss", which corresponds to the
-#' "final" loss function, applied to compute the error on testing dataset.
+#'
+#' @param dataTrain Training data
+#' @param dataTest Testing data
+#' @param targetTrain Training target
+#' @param targetTest Testing target
+#' @param floss loss function applied on the testing dataset
+#' @param verbose TRUE for a verbose output
+#' @param ... Arguments of [standardCV_core()]
+#'
standardCV_run <- function(
dataTrain, dataTest, targetTrain, targetTest, floss, verbose, ...
) {
#' CVvoting_run
#'
#' Run and eval the voting cross-validation procedure.
-#' Parameters are rather explicit except "floss", which corresponds to the
-#' "final" loss function, applied to compute the error on testing dataset.
+#'
+#' @inheritParams standardCV_run
+#' @param ... Arguments of [CVvoting_core()]
+#'
CVvoting_run <- function(
dataTrain, dataTest, targetTrain, targetTest, floss, verbose, ...
) {
#' agghoo_run
#'
#' Run and eval the agghoo procedure.
-#' Parameters are rather explicit except "floss", which corresponds to the
-#' "final" loss function, applied to compute the error on testing dataset.
+#'
+#' @inheritParams standardCV_run
+#' @param ... Optional arguments of [agghoo()] : task, gmodel, params, loss, and CV (see [standardCV_core()])
+#'
agghoo_run <- function(
dataTrain, dataTest, targetTrain, targetTest, floss, verbose, ...
) {
#' @param verbose TRUE to request methods to be verbose.
#' @param ... arguments passed to method_s function(s)
#'
+#' @examples
+#' data(iris)
+#'
+#' # Run only agghoo on iris dataset (split into train/test, etc).
+#' # Default parameters: see ?agghoo and ?AgghooCV
+#' compareTo(iris[,-5], iris[,5], agghoo_run)
+#'
+#' # Run both agghoo and standard CV, specifiying some parameters.
+#' compareTo(iris[,-5], iris[,5], list(agghoo_run, standardCV_run), gmodel="tree")
+#' compareTo(iris[,-5], iris[,5], list(agghoo_run, standardCV_run),
+#' gmodel="knn", params=c(3, 7, 13, 17, 23, 31),
+#' CV = list(type="vfold", V=5, shuffle=T))
+#'
+#' # Random tests to check that method doesn't fail in 1D case
+#' M <- matrix(rnorm(200), ncol=2)
+#' compareTo(as.matrix(M[,-2]), M[,2], list(agghoo_run, standardCV_run), gmodel="knn")
+#' compareTo(as.matrix(M[,-2]), M[,2], list(agghoo_run, standardCV_run), gmodel="tree")
+#'
#' @export
compareTo <- function(
data, target, method_s, rseed=-1, floss=NULL, verbose=TRUE, ...
) {
+print("call llllll")
+
+browser()
+
+
+
if (rseed >= 0)
set.seed(rseed)
n <- nrow(data)
#' Set it to any value <=1 to say "no parallelism".
#' @param verbose TRUE to print task numbers and "Errors:" in the end.
#'
+#' @examples
+#' data(iris)
+#' library(mlbench)
+#' data(SynthDiabetes)
+#'
+#' # Run both agghoo and standard CV, averaging errors over N=10 runs
+#' # (possible for a single method but wouldn't make much sense...).
+#' compareMulti(SynthDiabetes[,-9], SynthDiabetes[,9],
+#' list(agghoo_run, standardCV_run), N=10, gmodel="rf")
+#'
+#' # For example to use average of squared differences.
+#' # Default is "mean(abs(y1 - y2))".
+#' loss2 <- function(y1, y2) mean((y1 - y2)^2)
+#'
+#' # In regression on artificial datasets (TODO: real data?)
+#' data <- mlbench.twonorm(300, 3)$x
+#' target <- rowSums(data)
+#' compareMulti(data, target, list(agghoo_run, standardCV_run),
+#' N=10, gmodel="tree", params=c(1, 3, 5, 7, 9), loss=loss2,
+#' CV = list(type="MC", V=12, test_size=0.3))
+#'
+#' compareMulti(data, target, list(agghoo_run, standardCV_run),
+#' N=10, floss=loss2, CV = list(type="vfold", V=10, shuffle=FALSE))
+#'
#' @export
compareMulti <- function(
data, target, method_s, N=100, nc=NA, floss=NULL, verbose=TRUE, ...
#' @inheritParams compareMulti
#' @param V_range Values of V to be tested.
#'
+#' @examples
+#' library(mlbench)
+#' data(SynthDiabetes)
+#'
+#' # Compare several values of V
+#' compareRange(SynthDiabetes[,-9], SynthDiabetes[,9],
+#' list(agghoo_run, standardCV_run), N=10, V_range=c(10, 20, 30))
+#'
#' @export
compareRange <- function(
data, target, method_s, N=100, nc=NA, floss=NULL, V_range=c(10,15,20), ...
projects / agghoo.git / commitdiff