pkg/R/F_Neighbors.R

   1 #' Neighbors Forecaster
   2 #'
   3 #' Predict next serie as a weighted combination of curves observed on "similar" days in
   4 #' the past (and future if 'opera'=FALSE); the nature of the similarity is controlled by
   5 #' the options 'simtype' and 'local' (see below).
   6 #'
   7 #' Optional arguments:
   8 #' \itemize{
   9 #'   \item local: TRUE (default) to constrain neighbors to be "same days in same season"
  10 #'   \item simtype: 'endo' for a similarity based on the series only,<cr>
  11 #'             'exo' for a similarity based on exogenous variables only,<cr>
  12 #'             'mix' for the product of 'endo' and 'exo',<cr>
  13 #'             'none' (default) to apply a simple average: no computed weights
  14 #'   \item window: A window for similarities computations; override cross-validation
  15 #'     window estimation.
  16 #' }
  17 #' The method is summarized as follows:
  18 #' \enumerate{
  19 #'   \item Determine N (=20) recent days without missing values, and preceded by a
  20 #'     curve also without missing values.
  21 #'   \item Optimize the window parameters (if relevant) on the N chosen days.
  22 #'   \item Considering the optimized window, compute the neighbors (with locality
  23 #'     constraint or not), compute their similarities -- using a gaussian kernel if
  24 #'     simtype != "none" -- and average accordingly the "tomorrows of neigbors" to
  25 #'     obtain the final prediction.
  26 #' }
  27 #'
  28 #' @usage # NeighborsForecaster$new(pjump)
  29 #'
  30 #' @docType class
  31 #' @format R6 class, inherits Forecaster
  32 #' @aliases F_Neighbors
  33 #'
  34 NeighborsForecaster = R6::R6Class("NeighborsForecaster",
  35     inherit = Forecaster,
  36
  37     public = list(
  38         predictShape = function(data, today, memory, predict_from, horizon, ...)
  39         {
  40             # (re)initialize computed parameters
  41             private$.params <- list("weights"=NA, "indices"=NA, "window"=NA)
  42
  43             # Do not forecast on days with NAs (TODO: softer condition...)
  44             if (any(is.na(data$getSerie(today-1))) ||
  45                 (predict_from>=2 && any(is.na(data$getSerie(today)[1:(predict_from-1)]))))
  46             {
  47                 return (NA)
  48             }
  49
  50             # Get optional args
  51             local = ifelse(hasArg("local"), list(...)$local, TRUE) #same level + season?
  52             simtype = ifelse(hasArg("simtype"), list(...)$simtype, "none") #or "endo", or "exo"
  53             opera = ifelse(hasArg("opera"), list(...)$opera, FALSE) #operational mode?
  54
  55             # Determine indices of no-NAs days preceded by no-NAs yerstedays
  56             tdays = .getNoNA2(data, max(today-memory,2), ifelse(opera,today-1,data$getSize()))
  57             if (!opera)
  58                 tdays = setdiff(tdays, today) #always exclude current day
  59
  60             # Shortcut if window is known or local==TRUE && simtype==none
  61             if (hasArg("window") || (local && simtype=="none"))
  62             {
  63                 return ( private$.predictShapeAux(data, tdays, today, predict_from, horizon,
  64                     local, list(...)$window, simtype, opera, TRUE) )
  65             }
  66
  67             # Indices of similar days for cross-validation; TODO: 20 = magic number
  68             cv_days = getSimilarDaysIndices(today, data, limit=20, same_season=FALSE,
  69                 days_in=tdays, operational=opera)
  70
  71             # Optimize h : h |--> sum of prediction errors on last N "similar" days
  72             errorOnLastNdays = function(window, simtype)
  73             {
  74                 error = 0
  75                 nb_jours = 0
  76                 for (i in seq_along(cv_days))
  77                 {
  78                     # mix_strategy is never used here (simtype != "mix"), therefore left blank
  79                     prediction = private$.predictShapeAux(data, tdays, cv_days[i], predict_from,
  80                         horizon, local, window, simtype, opera, FALSE)
  81                     if (!is.na(prediction[1]))
  82                     {
  83                         nb_jours = nb_jours + 1
  84                         error = error +
  85                             mean((data$getSerie(cv_days[i])[predict_from:horizon] - prediction)^2)
  86                     }
  87                 }
  88                 return (error / nb_jours)
  89             }
  90
  91             # TODO: 7 == magic number
  92             if (simtype=="endo" || simtype=="mix")
  93             {
  94                 best_window_endo = optimize(
  95                     errorOnLastNdays, c(0,7), simtype="endo")$minimum
  96             }
  97             if (simtype=="exo" || simtype=="mix")
  98             {
  99                 best_window_exo = optimize(
 100                     errorOnLastNdays, c(0,7), simtype="exo")$minimum
 101             }
 102
 103             best_window =
 104                 if (simtype == "endo")
 105                     best_window_endo
 106                 else if (simtype == "exo")
 107                     best_window_exo
 108                 else if (simtype == "mix")
 109                     c(best_window_endo,best_window_exo)
 110                 else #none: value doesn't matter
 111                     1
 112
 113             return( private$.predictShapeAux(data, tdays, today, predict_from, horizon, local,
 114                 best_window, simtype, opera, TRUE) )
 115         }
 116     ),
 117     private = list(
 118         # Precondition: "yersteday until predict_from-1" is full (no NAs)
 119         .predictShapeAux = function(data, tdays, today, predict_from, horizon, local, window,
 120             simtype, opera, final_call)
 121         {
 122             tdays_cut = tdays[ tdays != today ]
 123             if (length(tdays_cut) == 0)
 124                 return (NA)
 125
 126             if (local)
 127             {
 128                 # limit=Inf to not censor any day (TODO: finite limit? 60?)
 129                 tdays = getSimilarDaysIndices(today, data, limit=Inf, same_season=TRUE,
 130                     days_in=tdays_cut, operational=opera)
 131 #               if (length(tdays) <= 1)
 132 #                   return (NA)
 133                 # TODO: 10 == magic number
 134                 tdays = .getConstrainedNeighbs(today, data, tdays, min_neighbs=10)
 135                 if (length(tdays) == 1)
 136                 {
 137                     if (final_call)
 138                     {
 139                         private$.params$weights <- 1
 140                         private$.params$indices <- tdays
 141                         private$.params$window <- 1
 142                     }
 143                     return ( data$getSerie(tdays[1])[predict_from:horizon] )
 144                 }
 145                 max_neighbs = 10 #TODO: 12 = arbitrary number
 146                 if (length(tdays) > max_neighbs)
 147                 {
 148                     distances2 <- .computeDistsEndo(data, today, tdays, predict_from)
 149                     ordering <- order(distances2)
 150                     tdays <- tdays[ ordering[1:max_neighbs] ]
 151                 }
 152             }
 153             else
 154                 tdays = tdays_cut #no conditioning
 155
 156             if (simtype == "endo" || simtype == "mix")
 157             {
 158                 # Compute endogen similarities using given window
 159                 window_endo = ifelse(simtype=="mix", window[1], window)
 160
 161                 # Distances from last observed day to selected days in the past
 162                 # TODO: redundant computation if local==TRUE
 163                 distances2 <- .computeDistsEndo(data, today, tdays, predict_from)
 164
 165                 simils_endo <- .computeSimils(distances2, window_endo)
 166             }
 167
 168             if (simtype == "exo" || simtype == "mix")
 169             {
 170                 # Compute exogen similarities using given window
 171                 window_exo = ifelse(simtype=="mix", window[2], window)
 172
 173                 distances2 <- .computeDistsExo(data, today, tdays)
 174
 175                 simils_exo <- .computeSimils(distances2, window_exo)
 176             }
 177
 178             similarities =
 179                 if (simtype == "exo")
 180                     simils_exo
 181                 else if (simtype == "endo")
 182                     simils_endo
 183                 else if (simtype == "mix")
 184                     simils_endo * simils_exo
 185                 else #none
 186                     rep(1, length(tdays))
 187             similarities = similarities / sum(similarities)
 188
 189             prediction = rep(0, horizon-predict_from+1)
 190             for (i in seq_along(tdays))
 191             {
 192                 prediction = prediction +
 193                     similarities[i] * data$getSerie(tdays[i])[predict_from:horizon]
 194             }
 195
 196             if (final_call)
 197             {
 198                 private$.params$weights <- similarities
 199                 private$.params$indices <- tdays
 200                 private$.params$window <-
 201                     if (simtype=="endo")
 202                         window_endo
 203                     else if (simtype=="exo")
 204                         window_exo
 205                     else if (simtype=="mix")
 206                         c(window_endo,window_exo)
 207                     else #none
 208                         1
 209             }
 210
 211             return (prediction)
 212         }
 213     )
 214 )
 215
 216 # getConstrainedNeighbs
 217 #
 218 # Get indices of neighbors of similar pollution level (among same season + day type).
 219 #
 220 # @param today Index of current day
 221 # @param data Object of class Data
 222 # @param tdays Current set of "second days" (no-NA pairs)
 223 # @param min_neighbs Minimum number of points in a neighborhood
 224 # @param max_neighbs Maximum number of points in a neighborhood
 225 #
 226 .getConstrainedNeighbs = function(today, data, tdays, min_neighbs=10)
 227 {
 228     levelToday = data$getLevelHat(today)
 229 #   levelYersteday = data$getLevel(today-1)
 230     distances = sapply(tdays, function(i) {
 231 #       sqrt((data$getLevel(i-1)-levelYersteday)^2 + (data$getLevel(i)-levelToday)^2)
 232         abs(data$getLevel(i)-levelToday)
 233     })
 234     #TODO: 1, +1, +3 : magic numbers
 235     dist_thresh = 1
 236     min_neighbs = min(min_neighbs,length(tdays))
 237     repeat
 238     {
 239         same_pollution = (distances <= dist_thresh)
 240         nb_neighbs = sum(same_pollution)
 241         if (nb_neighbs >= min_neighbs) #will eventually happen
 242             break
 243         dist_thresh = dist_thresh + ifelse(dist_thresh>1,3,1)
 244     }
 245     tdays = tdays[same_pollution]
 246 #   max_neighbs = 12
 247 #   if (nb_neighbs > max_neighbs)
 248 #   {
 249 #       # Keep only max_neighbs closest neighbors
 250 #       tdays = tdays[ order(distances[same_pollution])[1:max_neighbs] ]
 251 #   }
 252     tdays
 253 }
 254
 255 # compute similarities
 256 #
 257 # Apply the gaussian kernel on computed squared distances.
 258 #
 259 # @param distances2 Squared distances
 260 # @param window Window parameter for the kernel
 261 #
 262 .computeSimils <- function(distances2, window)
 263 {
 264     sd_dist = sd(distances2)
 265     if (sd_dist < .25 * sqrt(.Machine$double.eps))
 266     {
 267 #       warning("All computed distances are very close: stdev too small")
 268         sd_dist = 1 #mostly for tests... FIXME:
 269     }
 270     exp(-distances2/(sd_dist*window^2))
 271 }
 272
 273 .computeDistsEndo <- function(data, today, tdays, predict_from)
 274 {
 275     lastSerie = c( data$getSerie(today-1),
 276         data$getSerie(today)[if (predict_from>=2) 1:(predict_from-1) else c()] )
 277     sapply(tdays, function(i) {
 278         delta = lastSerie - c(data$getSerie(i-1),
 279             data$getSerie(i)[if (predict_from>=2) 1:(predict_from-1) else c()])
 280 #       sqrt(mean(delta^2))
 281         sqrt(sum(delta^2))
 282     })
 283 }
 284
 285 .computeDistsExo <- function(data, today, tdays)
 286 {
 287     M = matrix( ncol=1+length(tdays), nrow=1+length(data$getExo(1)) )
 288     M[,1] = c( data$getLevelHat(today), as.double(data$getExoHat(today)) )
 289     for (i in seq_along(tdays))
 290         M[,i+1] = c( data$getLevel(tdays[i]), as.double(data$getExo(tdays[i])) )
 291
 292     sigma = cov(t(M)) #NOTE: robust covariance is way too slow
 293     # TODO: 10 == magic number; more robust way == det, or always ginv()
 294     sigma_inv =
 295         if (length(tdays) > 10)
 296             solve(sigma)
 297         else
 298             MASS::ginv(sigma)
 299
 300     # Distances from last observed day to days in the past
 301     sapply(seq_along(tdays), function(i) {
 302         delta = M[,1] - M[,i+1]
 303         delta %*% sigma_inv %*% delta
 304     })
 305 }