pkg/R/F_Neighbors.R

   1 #' Neighbors Forecaster
   2 #'
   3 #' Predict next serie as a weighted combination of "futures of the past" days,
   4 #' where days in the past are chosen and weighted according to some similarity measures.
   5 #'
   6 #' The main method is \code{predictShape()}, taking arguments data, today, memory,
   7 #' predict_from, horizon respectively for the dataset (object output of
   8 #' \code{getData()}), the current index, the data depth (in days), the first predicted
   9 #' hour and the last predicted hour.
  10 #' In addition, optional arguments can be passed:
  11 #' \itemize{
  12 #'   \item local : TRUE (default) to constrain neighbors to be "same days within same
  13 #'     season"
  14 #'   \item simtype : 'endo' for a similarity based on the series only,<cr>
  15 #'             'exo' for a similarity based on exogenous variables only,<cr>
  16 #'             'mix' for the product of 'endo' and 'exo',<cr>
  17 #'             'none' (default) to apply a simple average: no computed weights
  18 #'   \item window : A window for similarities computations; override cross-validation
  19 #'     window estimation.
  20 #' }
  21 #' The method is summarized as follows:
  22 #' \enumerate{
  23 #'   \item Determine N (=20) recent days without missing values, and followed by a
  24 #'     tomorrow also without missing values.
  25 #'   \item Optimize the window parameters (if relevant) on the N chosen days.
  26 #'   \item Considering the optimized window, compute the neighbors (with locality
  27 #'     constraint or not), compute their similarities -- using a gaussian kernel if
  28 #'     simtype != "none" -- and average accordingly the "tomorrows of neigbors" to
  29 #'     obtain the final prediction.
  30 #' }
  31 #'
  32 #' @usage # NeighborsForecaster$new(pjump)
  33 #'
  34 #' @docType class
  35 #' @format R6 class, inherits Forecaster
  36 #' @aliases F_Neighbors
  37 #'
  38 NeighborsForecaster = R6::R6Class("NeighborsForecaster",
  39     inherit = Forecaster,
  40
  41     public = list(
  42         predictShape = function(data, today, memory, predict_from, horizon, ...)
  43         {
  44             # (re)initialize computed parameters
  45             private$.params <- list("weights"=NA, "indices"=NA, "window"=NA)
  46
  47             # Do not forecast on days with NAs (TODO: softer condition...)
  48             if (any(is.na(data$getSerie(today-1))) ||
  49                 (predict_from>=2 && any(is.na(data$getSerie(today)[1:(predict_from-1)]))))
  50             {
  51                 return (NA)
  52             }
  53
  54             # Get optional args
  55             local = ifelse(hasArg("local"), list(...)$local, TRUE) #same level + season?
  56             simtype = ifelse(hasArg("simtype"), list(...)$simtype, "none") #or "endo", or "exo"
  57             opera = ifelse(hasArg("opera"), list(...)$opera, FALSE) #operational mode?
  58
  59             # Determine indices of no-NAs days preceded by no-NAs yerstedays
  60             tdays = .getNoNA2(data, max(today-memory,2), ifelse(opera,today-1,data$getSize()))
  61             if (!opera)
  62                 tdays = setdiff(tdays, today) #always exclude current day
  63
  64             # Shortcut if window is known or local==TRUE && simtype==none
  65             if (hasArg("window") || (local && simtype=="none"))
  66             {
  67                 return ( private$.predictShapeAux(data, tdays, today, predict_from, horizon,
  68                     local, list(...)$window, simtype, opera, TRUE) )
  69             }
  70
  71             # Indices of similar days for cross-validation; TODO: 20 = magic number
  72             cv_days = getSimilarDaysIndices(today, data, limit=20, same_season=FALSE,
  73                 days_in=tdays, operational=opera)
  74
  75             # Optimize h : h |--> sum of prediction errors on last N "similar" days
  76             errorOnLastNdays = function(window, simtype)
  77             {
  78                 error = 0
  79                 nb_jours = 0
  80                 for (i in seq_along(cv_days))
  81                 {
  82                     # mix_strategy is never used here (simtype != "mix"), therefore left blank
  83                     prediction = private$.predictShapeAux(data, tdays, cv_days[i], predict_from,
  84                         horizon, local, window, simtype, opera, FALSE)
  85                     if (!is.na(prediction[1]))
  86                     {
  87                         nb_jours = nb_jours + 1
  88                         error = error +
  89                             mean((data$getSerie(cv_days[i])[predict_from:horizon] - prediction)^2)
  90                     }
  91                 }
  92                 return (error / nb_jours)
  93             }
  94
  95             # TODO: 7 == magic number
  96             if (simtype=="endo" || simtype=="mix")
  97             {
  98                 best_window_endo = optimize(
  99                     errorOnLastNdays, c(0,7), simtype="endo")$minimum
 100             }
 101             if (simtype=="exo" || simtype=="mix")
 102             {
 103                 best_window_exo = optimize(
 104                     errorOnLastNdays, c(0,7), simtype="exo")$minimum
 105             }
 106
 107             best_window =
 108                 if (simtype == "endo")
 109                     best_window_endo
 110                 else if (simtype == "exo")
 111                     best_window_exo
 112                 else if (simtype == "mix")
 113                     c(best_window_endo,best_window_exo)
 114                 else #none: value doesn't matter
 115                     1
 116
 117             return( private$.predictShapeAux(data, tdays, today, predict_from, horizon, local,
 118                 best_window, simtype, opera, TRUE) )
 119         }
 120     ),
 121     private = list(
 122         # Precondition: "yersteday until predict_from-1" is full (no NAs)
 123         .predictShapeAux = function(data, tdays, today, predict_from, horizon, local, window,
 124             simtype, opera, final_call)
 125         {
 126             tdays_cut = tdays[ tdays != today ]
 127             if (length(tdays_cut) == 0)
 128                 return (NA)
 129
 130             if (local)
 131             {
 132                 # limit=Inf to not censor any day (TODO: finite limit? 60?)
 133                 tdays = getSimilarDaysIndices(today, data, limit=Inf, same_season=TRUE,
 134                     days_in=tdays_cut, operational=opera)
 135 #               if (length(tdays) <= 1)
 136 #                   return (NA)
 137                 # TODO: 10 == magic number
 138                 tdays = .getConstrainedNeighbs(today, data, tdays, min_neighbs=10)
 139                 if (length(tdays) == 1)
 140                 {
 141                     if (final_call)
 142                     {
 143                         private$.params$weights <- 1
 144                         private$.params$indices <- tdays
 145                         private$.params$window <- 1
 146                     }
 147                     return ( data$getSerie(tdays[1])[predict_from:horizon] )
 148                 }
 149                 max_neighbs = 10 #TODO: 12 = arbitrary number
 150                 if (length(tdays) > max_neighbs)
 151                 {
 152                     distances2 <- .computeDistsEndo(data, today, tdays, predict_from)
 153                     ordering <- order(distances2)
 154                     tdays <- tdays[ ordering[1:max_neighbs] ]
 155
 156                     print("VVVVV")
 157                     print(sort(distances2)[1:max_neighbs])
 158                     print(integerIndexToDate(today,data))
 159                     print(lapply(tdays,function(i) integerIndexToDate(i,data)))
 160                     print(rbind(data$getSeries(tdays-1), data$getSeries(tdays)))
 161                 }
 162             }
 163             else
 164                 tdays = tdays_cut #no conditioning
 165
 166             if (simtype == "endo" || simtype == "mix")
 167             {
 168                 # Compute endogen similarities using given window
 169                 window_endo = ifelse(simtype=="mix", window[1], window)
 170
 171                 # Distances from last observed day to selected days in the past
 172                 # TODO: redundant computation if local==TRUE
 173                 distances2 <- .computeDistsEndo(data, today, tdays, predict_from)
 174
 175                 simils_endo <- .computeSimils(distances2, window_endo)
 176             }
 177
 178             if (simtype == "exo" || simtype == "mix")
 179             {
 180                 # Compute exogen similarities using given window
 181                 window_exo = ifelse(simtype=="mix", window[2], window)
 182
 183                 distances2 <- .computeDistsExo(data, today, tdays)
 184
 185                 simils_exo <- .computeSimils(distances2, window_exo)
 186             }
 187
 188             similarities =
 189                 if (simtype == "exo")
 190                     simils_exo
 191                 else if (simtype == "endo")
 192                     simils_endo
 193                 else if (simtype == "mix")
 194                     simils_endo * simils_exo
 195                 else #none
 196                     rep(1, length(tdays))
 197             similarities = similarities / sum(similarities)
 198
 199             prediction = rep(0, horizon-predict_from+1)
 200             for (i in seq_along(tdays))
 201             {
 202                 prediction = prediction +
 203                     similarities[i] * data$getSerie(tdays[i])[predict_from:horizon]
 204             }
 205
 206             if (final_call)
 207             {
 208                 private$.params$weights <- similarities
 209                 private$.params$indices <- tdays
 210                 private$.params$window <-
 211                     if (simtype=="endo")
 212                         window_endo
 213                     else if (simtype=="exo")
 214                         window_exo
 215                     else if (simtype=="mix")
 216                         c(window_endo,window_exo)
 217                     else #none
 218                         1
 219             }
 220
 221             return (prediction)
 222         }
 223     )
 224 )
 225
 226 # getConstrainedNeighbs
 227 #
 228 # Get indices of neighbors of similar pollution level (among same season + day type).
 229 #
 230 # @param today Index of current day
 231 # @param data Object of class Data
 232 # @param tdays Current set of "second days" (no-NA pairs)
 233 # @param min_neighbs Minimum number of points in a neighborhood
 234 # @param max_neighbs Maximum number of points in a neighborhood
 235 #
 236 .getConstrainedNeighbs = function(today, data, tdays, min_neighbs=10)
 237 {
 238     levelToday = data$getLevelHat(today)
 239 #   levelYersteday = data$getLevel(today-1)
 240     distances = sapply(tdays, function(i) {
 241 #       sqrt((data$getLevel(i-1)-levelYersteday)^2 + (data$getLevel(i)-levelToday)^2)
 242         abs(data$getLevel(i)-levelToday)
 243     })
 244     #TODO: 1, +1, +3 : magic numbers
 245     dist_thresh = 1
 246     min_neighbs = min(min_neighbs,length(tdays))
 247     repeat
 248     {
 249         same_pollution = (distances <= dist_thresh)
 250         nb_neighbs = sum(same_pollution)
 251         if (nb_neighbs >= min_neighbs) #will eventually happen
 252             break
 253         dist_thresh = dist_thresh + ifelse(dist_thresh>1,3,1)
 254     }
 255     tdays = tdays[same_pollution]
 256 #   max_neighbs = 12
 257 #   if (nb_neighbs > max_neighbs)
 258 #   {
 259 #       # Keep only max_neighbs closest neighbors
 260 #       tdays = tdays[ order(distances[same_pollution])[1:max_neighbs] ]
 261 #   }
 262     tdays
 263 }
 264
 265 # compute similarities
 266 #
 267 # Apply the gaussian kernel on computed squared distances.
 268 #
 269 # @param distances2 Squared distances
 270 # @param window Window parameter for the kernel
 271 #
 272 .computeSimils <- function(distances2, window)
 273 {
 274     sd_dist = sd(distances2)
 275     if (sd_dist < .25 * sqrt(.Machine$double.eps))
 276     {
 277 #       warning("All computed distances are very close: stdev too small")
 278         sd_dist = 1 #mostly for tests... FIXME:
 279     }
 280     exp(-distances2/(sd_dist*window^2))
 281 }
 282
 283 .computeDistsEndo <- function(data, today, tdays, predict_from)
 284 {
 285     lastSerie = c( data$getSerie(today-1),
 286         data$getSerie(today)[if (predict_from>=2) 1:(predict_from-1) else c()] )
 287     sapply(tdays, function(i) {
 288         delta = lastSerie - c(data$getSerie(i-1),
 289             data$getSerie(i)[if (predict_from>=2) 1:(predict_from-1) else c()])
 290 #       sqrt(mean(delta^2))
 291         sqrt(sum(delta^2))
 292     })
 293 }
 294
 295 .computeDistsExo <- function(data, today, tdays)
 296 {
 297     M = matrix( ncol=1+length(tdays), nrow=1+length(data$getExo(1)) )
 298     M[,1] = c( data$getLevelHat(today), as.double(data$getExoHat(today)) )
 299     for (i in seq_along(tdays))
 300         M[,i+1] = c( data$getLevel(tdays[i]), as.double(data$getExo(tdays[i])) )
 301
 302     sigma = cov(t(M)) #NOTE: robust covariance is way too slow
 303     # TODO: 10 == magic number; more robust way == det, or always ginv()
 304     sigma_inv =
 305         if (length(tdays) > 10)
 306             solve(sigma)
 307         else
 308             MASS::ginv(sigma)
 309
 310     # Distances from last observed day to days in the past
 311     sapply(seq_along(tdays), function(i) {
 312         delta = M[,1] - M[,i+1]
 313         delta %*% sigma_inv %*% delta
 314     })
 315 }