From: Benjamin Auder Date: Thu, 22 Oct 2026 04:41:21 +0000 (+0200) Subject: update X-Git-Url: https://git.auder.net/variants/Checkered/complete_rules.html?a=commitdiff_plain;h=refs%2Fheads%2Fmain;p=agghoo.git update --- diff --git a/.gitignore b/.gitignore index ae9cf7e..a69b9f1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ -/man/ +/pkg/man/*.Rd .RData +/.unversioned/ +*.Rcheck/ +*.tar.gz diff --git a/DESCRIPTION b/DESCRIPTION deleted file mode 100644 index 140abb3..0000000 --- a/DESCRIPTION +++ /dev/null @@ -1,37 +0,0 @@ -Package: agghoo -Title: Aggregated Hold-Out Cross Validation -Date: 2022-08-30 -Version: 0.1-0 -Description: The 'agghoo' procedure is an alternative to usual cross-validation. - Instead of choosing the best model trained on V subsamples, it determines - a winner model for each subsample, and then aggregate the V outputs. - For the details, see "Aggregated hold-out" by Guillaume Maillard, - Sylvain Arlot, Matthieu Lerasle (2021) - published in Journal of Machine Learning Research 22(20):1--55. -Author: Sylvain Arlot [cph,ctb], - Benjamin Auder [aut,cre,cph], - Melina Gallopin [cph,ctb], - Matthieu Lerasle [cph,ctb], - Guillaume Maillard [cph,ctb] -Maintainer: Benjamin Auder -Depends: - R (>= 3.5.0) -Imports: - class, - parallel, - R6, - rpart, - FNN -Suggests: - roxygen2 -URL: https://git.auder.net/?p=agghoo.git -License: MIT + file LICENSE -RoxygenNote: 7.2.1 -Collate: - 'compareTo.R' - 'agghoo.R' - 'R6_AgghooCV.R' - 'R6_Model.R' - 'checks.R' - 'utils.R' - 'A_NAMESPACE.R' diff --git a/example/example.R b/example/example.R deleted file mode 100644 index 7fae2ce..0000000 --- a/example/example.R +++ /dev/null @@ -1,43 +0,0 @@ -library(agghoo) - -data(iris) #already there -library(mlbench) -data(PimaIndiansDiabetes) - -# Run only agghoo on iris dataset (split into train/test, etc). -# Default parameters: see ?agghoo and ?AgghooCV -compareTo(iris[,-5], iris[,5], agghoo_run) - -# Run both agghoo and standard CV, specifiying some parameters. -compareTo(iris[,-5], iris[,5], list(agghoo_run, standardCV_run), gmodel="tree") -compareTo(iris[,-5], iris[,5], list(agghoo_run, standardCV_run), - gmodel="knn", params=c(3, 7, 13, 17, 23, 31), - CV = list(type="vfold", V=5, shuffle=T)) - -# Run both agghoo and standard CV, averaging errors over N=10 runs -# (possible for a single method but wouldn't make much sense...). -compareMulti(PimaIndiansDiabetes[,-9], PimaIndiansDiabetes[,9], - list(agghoo_run, standardCV_run), N=10, gmodel="rf") - -# Compare several values of V -compareRange(PimaIndiansDiabetes[,-9], PimaIndiansDiabetes[,9], - list(agghoo_run, standardCV_run), N=10, V_range=c(10, 20, 30)) - -# For example to use average of squared differences. -# Default is "mean(abs(y1 - y2))". -loss2 <- function(y1, y2) mean((y1 - y2)^2) - -# In regression on artificial datasets (TODO: real data?) -data <- mlbench.twonorm(300, 3)$x -target <- rowSums(data) -compareMulti(data, target, list(agghoo_run, standardCV_run), - N=10, gmodel="tree", params=c(1, 3, 5, 7, 9), loss=loss2, - CV = list(type="MC", V=12, test_size=0.3)) - -compareMulti(data, target, list(agghoo_run, standardCV_run), - N=10, floss=loss2, CV = list(type="vfold", V=10, shuffle=F)) - -# Random tests to check that method doesn't fail in 1D case -M <- matrix(rnorm(200), ncol=2) -compareTo(as.matrix(M[,-2]), M[,2], list(agghoo_run, standardCV_run), gmodel="knn") -compareTo(as.matrix(M[,-2]), M[,2], list(agghoo_run, standardCV_run), gmodel="tree") diff --git a/pkg/DESCRIPTION b/pkg/DESCRIPTION new file mode 100644 index 0000000..d7a5196 --- /dev/null +++ b/pkg/DESCRIPTION @@ -0,0 +1,52 @@ +Encoding: UTF-8 +Package: agghoo +Title: Aggregated Hold-Out Cross Validation +Version: 0.1-1 +Description: The 'agghoo' procedure is an alternative to usual cross-validation. + Instead of choosing the best model trained on V subsamples, it determines + a winner model for each subsample, and then aggregate the V outputs. + For the details, see "Aggregated hold-out" by Guillaume Maillard, + Sylvain Arlot, Matthieu Lerasle (2021) + published in Journal of Machine Learning Research 22(20):1--55. +Authors@R: c(person(given = "Sylvain", + family = "Arlot", + role = c("cph", "ctb"), + email = "sylvain.arlot@universite-paris-saclay.fr"), + person(given = "Benjamin", + family = "Auder", + role = c("aut", "cre", "cph"), + email = "benjamin.auder@universite-paris-saclay.fr"), + person(given = "Melina", + family = "Gallopin", + role = c("cph", "ctb"), + email = "melina.gallopin@universite-paris-saclay.fr"), + person(given = "Matthieu", + family = "Lerasle", + role = c("cph", "ctb"), + email = "matthieu.lerasle@universite-paris-saclay.fr"), + person(given = "Guillaume", + family = "Maillard", + role = c("cph", "ctb"), + email = "guillaume.maillard@uni.lu")) +Depends: + R (>= 3.5.0) +Imports: + class, + parallel, + R6, + rpart, + FNN +Suggests: + mlbench, + roxygen2 +URL: https://git.auder.net/?p=agghoo.git +License: MIT + file LICENSE +Collate: + 'compareTo.R' + 'agghoo.R' + 'R6_AgghooCV.R' + 'R6_Model.R' + 'checks.R' + 'utils.R' + 'A_NAMESPACE.R' +Config/roxygen2/version: 8.1.1 diff --git a/LICENSE b/pkg/LICENSE similarity index 86% rename from LICENSE rename to pkg/LICENSE index 094ff81..f466c53 100644 --- a/LICENSE +++ b/pkg/LICENSE @@ -1,2 +1,2 @@ -YEAR: 2021-2022 +YEAR: 2021-2026 COPYRIGHT HOLDER: Sylvain Arlot, Benjamin Auder, Melina Gallopin, Matthieu Lerasle, Guillaume Maillard diff --git a/NAMESPACE b/pkg/NAMESPACE similarity index 100% rename from NAMESPACE rename to pkg/NAMESPACE diff --git a/R/A_NAMESPACE.R b/pkg/R/A_NAMESPACE.R similarity index 100% rename from R/A_NAMESPACE.R rename to pkg/R/A_NAMESPACE.R diff --git a/R/R6_AgghooCV.R b/pkg/R/R6_AgghooCV.R similarity index 100% rename from R/R6_AgghooCV.R rename to pkg/R/R6_AgghooCV.R diff --git a/R/R6_Model.R b/pkg/R/R6_Model.R similarity index 100% rename from R/R6_Model.R rename to pkg/R/R6_Model.R diff --git a/R/agghoo.R b/pkg/R/agghoo.R similarity index 100% rename from R/agghoo.R rename to pkg/R/agghoo.R diff --git a/R/checks.R b/pkg/R/checks.R similarity index 100% rename from R/checks.R rename to pkg/R/checks.R diff --git a/R/compareTo.R b/pkg/R/compareTo.R similarity index 66% rename from R/compareTo.R rename to pkg/R/compareTo.R index 0eb517c..aca6f9d 100644 --- a/R/compareTo.R +++ b/pkg/R/compareTo.R @@ -1,7 +1,25 @@ #' standardCV_core #' #' Cross-validation method, added here as an example. -#' Parameters are described in ?agghoo and ?AgghooCV +#' +#' @param data Matrix or data.frame +#' @param target Vector of targets (generally numeric or factor) +#' @param task "regression" or "classification". +#' Default: classification if target not numeric. +#' @param gmodel Generic model returning a predictive function +#' Default: tree if mixed data, knn/ppr otherwise. +#' @param params List of parameters for cross-validation (each defining a model) +#' @param loss Function assessing the error of a prediction +#' Default: error rate or mean(abs(error)). +#' @param CV List describing cross-validation to run. Slots: \cr +#' - type: 'vfold' or 'MC' for Monte-Carlo (default: MC) \cr +#' - V: number of runs (default: 10) \cr +#' - test_size: percentage of data in the test dataset, for MC +#' (irrelevant for V-fold). Default: 0.2. \cr +#' - shuffle: wether or not to shuffle data before V-fold. +#' Irrelevant for Monte-Carlo; default: TRUE \cr +#' Default (if NULL): type="MC", V=10, test_size=0.2 +#' standardCV_core <- function(data, target, task, gmodel, params, loss, CV) { n <- nrow(data) shuffle_inds <- NULL @@ -37,7 +55,9 @@ standardCV_core <- function(data, target, task, gmodel, params, loss, CV) { #' CVvoting_core #' #' "voting" cross-validation method, added here as an example. -#' Parameters are described in ?agghoo and ?AgghooCV +#' +#' @inheritParams standardCV_core +#' CVvoting_core <- function(data, target, task, gmodel, params, loss, CV) { CV <- checkCV(CV) n <- nrow(data) @@ -76,8 +96,15 @@ CVvoting_core <- function(data, target, task, gmodel, params, loss, CV) { #' standardCV_run #' #' Run and eval the standard cross-validation procedure. -#' Parameters are rather explicit except "floss", which corresponds to the -#' "final" loss function, applied to compute the error on testing dataset. +#' +#' @param dataTrain Training data +#' @param dataTest Testing data +#' @param targetTrain Training target +#' @param targetTest Testing target +#' @param floss loss function applied on the testing dataset +#' @param verbose TRUE for a verbose output +#' @param ... Arguments of [standardCV_core()] +#' standardCV_run <- function( dataTrain, dataTest, targetTrain, targetTest, floss, verbose, ... ) { @@ -100,8 +127,10 @@ standardCV_run <- function( #' CVvoting_run #' #' Run and eval the voting cross-validation procedure. -#' Parameters are rather explicit except "floss", which corresponds to the -#' "final" loss function, applied to compute the error on testing dataset. +#' +#' @inheritParams standardCV_run +#' @param ... Arguments of [CVvoting_core()] +#' CVvoting_run <- function( dataTrain, dataTest, targetTrain, targetTest, floss, verbose, ... ) { @@ -124,8 +153,10 @@ CVvoting_run <- function( #' agghoo_run #' #' Run and eval the agghoo procedure. -#' Parameters are rather explicit except "floss", which corresponds to the -#' "final" loss function, applied to compute the error on testing dataset. +#' +#' @inheritParams standardCV_run +#' @param ... Optional arguments of [agghoo()] : task, gmodel, params, loss, and CV (see [standardCV_core()]) +#' agghoo_run <- function( dataTrain, dataTest, targetTrain, targetTest, floss, verbose, ... ) { @@ -159,10 +190,34 @@ agghoo_run <- function( #' @param verbose TRUE to request methods to be verbose. #' @param ... arguments passed to method_s function(s) #' +#' @examples +#' data(iris) +#' +#' # Run only agghoo on iris dataset (split into train/test, etc). +#' # Default parameters: see ?agghoo and ?AgghooCV +#' compareTo(iris[,-5], iris[,5], agghoo_run) +#' +#' # Run both agghoo and standard CV, specifiying some parameters. +#' compareTo(iris[,-5], iris[,5], list(agghoo_run, standardCV_run), gmodel="tree") +#' compareTo(iris[,-5], iris[,5], list(agghoo_run, standardCV_run), +#' gmodel="knn", params=c(3, 7, 13, 17, 23, 31), +#' CV = list(type="vfold", V=5, shuffle=T)) +#' +#' # Random tests to check that method doesn't fail in 1D case +#' M <- matrix(rnorm(200), ncol=2) +#' compareTo(as.matrix(M[,-2]), M[,2], list(agghoo_run, standardCV_run), gmodel="knn") +#' compareTo(as.matrix(M[,-2]), M[,2], list(agghoo_run, standardCV_run), gmodel="tree") +#' #' @export compareTo <- function( data, target, method_s, rseed=-1, floss=NULL, verbose=TRUE, ... ) { +print("call llllll") + +browser() + + + if (rseed >= 0) set.seed(rseed) n <- nrow(data) @@ -199,6 +254,30 @@ compareTo <- function( #' Set it to any value <=1 to say "no parallelism". #' @param verbose TRUE to print task numbers and "Errors:" in the end. #' +#' @examples +#' data(iris) +#' library(mlbench) +#' data(SynthDiabetes) +#' +#' # Run both agghoo and standard CV, averaging errors over N=10 runs +#' # (possible for a single method but wouldn't make much sense...). +#' compareMulti(SynthDiabetes[,-9], SynthDiabetes[,9], +#' list(agghoo_run, standardCV_run), N=10, gmodel="rf") +#' +#' # For example to use average of squared differences. +#' # Default is "mean(abs(y1 - y2))". +#' loss2 <- function(y1, y2) mean((y1 - y2)^2) +#' +#' # In regression on artificial datasets (TODO: real data?) +#' data <- mlbench.twonorm(300, 3)$x +#' target <- rowSums(data) +#' compareMulti(data, target, list(agghoo_run, standardCV_run), +#' N=10, gmodel="tree", params=c(1, 3, 5, 7, 9), loss=loss2, +#' CV = list(type="MC", V=12, test_size=0.3)) +#' +#' compareMulti(data, target, list(agghoo_run, standardCV_run), +#' N=10, floss=loss2, CV = list(type="vfold", V=10, shuffle=FALSE)) +#' #' @export compareMulti <- function( data, target, method_s, N=100, nc=NA, floss=NULL, verbose=TRUE, ... @@ -230,6 +309,14 @@ compareMulti <- function( #' @inheritParams compareMulti #' @param V_range Values of V to be tested. #' +#' @examples +#' library(mlbench) +#' data(SynthDiabetes) +#' +#' # Compare several values of V +#' compareRange(SynthDiabetes[,-9], SynthDiabetes[,9], +#' list(agghoo_run, standardCV_run), N=10, V_range=c(10, 20, 30)) +#' #' @export compareRange <- function( data, target, method_s, N=100, nc=NA, floss=NULL, V_range=c(10,15,20), ... diff --git a/R/utils.R b/pkg/R/utils.R similarity index 100% rename from R/utils.R rename to pkg/R/utils.R