TODO: comprendre pkoi pas mêmes voisins
[talweg.git] / pkg / R / F_Neighbors.R
CommitLineData
25b75559 1#' Neighbors Forecaster
3d69ff21 2#'
c4c329f6
BA
3#' Predict next serie as a weighted combination of "futures of the past" days,
4#' where days in the past are chosen and weighted according to some similarity measures.
c4c329f6 5#'
102bcfda 6#' The main method is \code{predictShape()}, taking arguments data, today, memory,
d2ab47a7
BA
7#' predict_from, horizon respectively for the dataset (object output of
8#' \code{getData()}), the current index, the data depth (in days), the first predicted
9#' hour and the last predicted hour.
102bcfda
BA
10#' In addition, optional arguments can be passed:
11#' \itemize{
12#' \item local : TRUE (default) to constrain neighbors to be "same days within same
13#' season"
14#' \item simtype : 'endo' for a similarity based on the series only,<cr>
d2ab47a7 15#' 'exo' for a similarity based on exogenous variables only,<cr>
102bcfda
BA
16#' 'mix' for the product of 'endo' and 'exo',<cr>
17#' 'none' (default) to apply a simple average: no computed weights
18#' \item window : A window for similarities computations; override cross-validation
19#' window estimation.
20#' }
21#' The method is summarized as follows:
22#' \enumerate{
23#' \item Determine N (=20) recent days without missing values, and followed by a
24#' tomorrow also without missing values.
25#' \item Optimize the window parameters (if relevant) on the N chosen days.
26#' \item Considering the optimized window, compute the neighbors (with locality
27#' constraint or not), compute their similarities -- using a gaussian kernel if
28#' simtype != "none" -- and average accordingly the "tomorrows of neigbors" to
29#' obtain the final prediction.
30#' }
c4c329f6 31#'
4e821712 32#' @usage # NeighborsForecaster$new(pjump)
689aa1d3 33#'
102bcfda 34#' @docType class
c4c329f6 35#' @format R6 class, inherits Forecaster
3ddf1c12 36#' @aliases F_Neighbors
546b0cb6 37#'
25b75559 38NeighborsForecaster = R6::R6Class("NeighborsForecaster",
a66a84b5 39 inherit = Forecaster,
25b75559
BA
40
41 public = list(
d2ab47a7 42 predictShape = function(data, today, memory, predict_from, horizon, ...)
3d69ff21
BA
43 {
44 # (re)initialize computed parameters
a66a84b5 45 private$.params <- list("weights"=NA, "indices"=NA, "window"=NA)
3d69ff21 46
a5a3a294 47 # Do not forecast on days with NAs (TODO: softer condition...)
cf3bb001
BA
48 if (any(is.na(data$getSerie(today-1))) ||
49 (predict_from>=2 && any(is.na(data$getSerie(today)[1:(predict_from-1)]))))
d2ab47a7 50 {
a5a3a294 51 return (NA)
d2ab47a7 52 }
a5a3a294 53
f17665c7 54 # Get optional args
445e7bbc
BA
55 local = ifelse(hasArg("local"), list(...)$local, TRUE) #same level + season?
56 simtype = ifelse(hasArg("simtype"), list(...)$simtype, "none") #or "endo", or "exo"
638f27f4
BA
57 opera = ifelse(hasArg("opera"), list(...)$opera, FALSE) #operational mode?
58
59 # Determine indices of no-NAs days preceded by no-NAs yerstedays
60 tdays = .getNoNA2(data, max(today-memory,2), ifelse(opera,today-1,data$getSize()))
61 if (!opera)
62 tdays = setdiff(tdays, today) #always exclude current day
63
dca259e4
BA
64 # Shortcut if window is known or local==TRUE && simtype==none
65 if (hasArg("window") || (local && simtype=="none"))
a66a84b5 66 {
638f27f4
BA
67 return ( private$.predictShapeAux(data, tdays, today, predict_from, horizon,
68 local, list(...)$window, simtype, opera, TRUE) )
a66a84b5 69 }
3d69ff21 70
6774e53d 71 # Indices of similar days for cross-validation; TODO: 20 = magic number
aa059de7 72 cv_days = getSimilarDaysIndices(today, data, limit=20, same_season=FALSE,
638f27f4 73 days_in=tdays, operational=opera)
5e838b3e 74
445e7bbc 75 # Optimize h : h |--> sum of prediction errors on last N "similar" days
aa059de7 76 errorOnLastNdays = function(window, simtype)
3d69ff21
BA
77 {
78 error = 0
79 nb_jours = 0
5e838b3e 80 for (i in seq_along(cv_days))
3d69ff21 81 {
f17665c7 82 # mix_strategy is never used here (simtype != "mix"), therefore left blank
cf3bb001 83 prediction = private$.predictShapeAux(data, tdays, cv_days[i], predict_from,
638f27f4 84 horizon, local, window, simtype, opera, FALSE)
f17665c7 85 if (!is.na(prediction[1]))
3d69ff21
BA
86 {
87 nb_jours = nb_jours + 1
af3b84f4 88 error = error +
cf3bb001 89 mean((data$getSerie(cv_days[i])[predict_from:horizon] - prediction)^2)
3d69ff21
BA
90 }
91 }
92 return (error / nb_jours)
93 }
94
445e7bbc 95 # TODO: 7 == magic number
eef54517 96 if (simtype=="endo" || simtype=="mix")
af3b84f4 97 {
aa059de7
BA
98 best_window_endo = optimize(
99 errorOnLastNdays, c(0,7), simtype="endo")$minimum
af3b84f4 100 }
eef54517 101 if (simtype=="exo" || simtype=="mix")
af3b84f4 102 {
eef54517
BA
103 best_window_exo = optimize(
104 errorOnLastNdays, c(0,7), simtype="exo")$minimum
3d69ff21 105 }
eef54517
BA
106
107 best_window =
108 if (simtype == "endo")
109 best_window_endo
110 else if (simtype == "exo")
111 best_window_exo
112 else if (simtype == "mix")
113 c(best_window_endo,best_window_exo)
114 else #none: value doesn't matter
115 1
116
cf3bb001 117 return( private$.predictShapeAux(data, tdays, today, predict_from, horizon, local,
638f27f4 118 best_window, simtype, opera, TRUE) )
25b75559
BA
119 }
120 ),
121 private = list(
638f27f4 122 # Precondition: "yersteday until predict_from-1" is full (no NAs)
cf3bb001 123 .predictShapeAux = function(data, tdays, today, predict_from, horizon, local, window,
638f27f4 124 simtype, opera, final_call)
3d69ff21 125 {
638f27f4
BA
126 tdays_cut = tdays[ tdays != today ]
127 if (length(tdays_cut) == 0)
3d69ff21
BA
128 return (NA)
129
aa059de7
BA
130 if (local)
131 {
dca259e4
BA
132 # limit=Inf to not censor any day (TODO: finite limit? 60?)
133 tdays = getSimilarDaysIndices(today, data, limit=Inf, same_season=TRUE,
638f27f4
BA
134 days_in=tdays_cut, operational=opera)
135# if (length(tdays) <= 1)
136# return (NA)
137 # TODO: 10 == magic number
138 tdays = .getConstrainedNeighbs(today, data, tdays, min_neighbs=10)
cf3bb001 139 if (length(tdays) == 1)
aa059de7
BA
140 {
141 if (final_call)
142 {
143 private$.params$weights <- 1
cf3bb001 144 private$.params$indices <- tdays
aa059de7
BA
145 private$.params$window <- 1
146 }
cf3bb001 147 return ( data$getSerie(tdays[1])[predict_from:horizon] )
aa059de7 148 }
dca259e4
BA
149 max_neighbs = 12 #TODO: 12 = arbitrary number
150 if (length(tdays) > max_neighbs)
151 {
152 distances2 <- .computeDistsEndo(data, today, tdays, predict_from)
153 ordering <- order(distances2)
154 tdays <- tdays[ ordering[1:max_neighbs] ]
155 }
aa059de7
BA
156 }
157 else
cf3bb001 158 tdays = tdays_cut #no conditioning
aa059de7 159
445e7bbc 160 if (simtype == "endo" || simtype == "mix")
3d69ff21 161 {
aa059de7
BA
162 # Compute endogen similarities using given window
163 window_endo = ifelse(simtype=="mix", window[1], window)
3d69ff21 164
638f27f4 165 # Distances from last observed day to selected days in the past
dca259e4 166 # TODO: redundant computation if local==TRUE
638f27f4
BA
167 distances2 <- .computeDistsEndo(data, today, tdays, predict_from)
168
3ddf1c12 169 simils_endo <- .computeSimils(distances2, window_endo)
3d69ff21
BA
170 }
171
445e7bbc 172 if (simtype == "exo" || simtype == "mix")
3d69ff21 173 {
aa059de7 174 # Compute exogen similarities using given window
445e7bbc 175 window_exo = ifelse(simtype=="mix", window[2], window)
3d69ff21 176
638f27f4 177 distances2 <- .computeDistsExo(data, today, tdays)
3d69ff21 178
3ddf1c12 179 simils_exo <- .computeSimils(distances2, window_exo)
3d69ff21
BA
180 }
181
3d69ff21 182 similarities =
f17665c7 183 if (simtype == "exo")
3d69ff21 184 simils_exo
f17665c7
BA
185 else if (simtype == "endo")
186 simils_endo
445e7bbc 187 else if (simtype == "mix")
f17665c7 188 simils_endo * simils_exo
445e7bbc 189 else #none
cf3bb001 190 rep(1, length(tdays))
ea5c7e56 191 similarities = similarities / sum(similarities)
3d69ff21 192
d2ab47a7 193 prediction = rep(0, horizon-predict_from+1)
cf3bb001 194 for (i in seq_along(tdays))
d2ab47a7
BA
195 {
196 prediction = prediction +
cf3bb001 197 similarities[i] * data$getSerie(tdays[i])[predict_from:horizon]
d2ab47a7 198 }
99f83c9a 199
3d69ff21
BA
200 if (final_call)
201 {
a66a84b5 202 private$.params$weights <- similarities
cf3bb001 203 private$.params$indices <- tdays
a66a84b5 204 private$.params$window <-
546b0cb6 205 if (simtype=="endo")
aa059de7 206 window_endo
546b0cb6 207 else if (simtype=="exo")
aa059de7 208 window_exo
eef54517 209 else if (simtype=="mix")
aa059de7 210 c(window_endo,window_exo)
eef54517
BA
211 else #none
212 1
3d69ff21 213 }
99f83c9a 214
3d69ff21
BA
215 return (prediction)
216 }
217 )
218)
3ddf1c12 219
689aa1d3
BA
220# getConstrainedNeighbs
221#
222# Get indices of neighbors of similar pollution level (among same season + day type).
223#
224# @param today Index of current day
225# @param data Object of class Data
cf3bb001 226# @param tdays Current set of "second days" (no-NA pairs)
689aa1d3
BA
227# @param min_neighbs Minimum number of points in a neighborhood
228# @param max_neighbs Maximum number of points in a neighborhood
229#
638f27f4 230.getConstrainedNeighbs = function(today, data, tdays, min_neighbs=10)
3ddf1c12 231{
d2ab47a7 232 levelToday = data$getLevelHat(today)
638f27f4 233# levelYersteday = data$getLevel(today-1)
cf3bb001 234 distances = sapply(tdays, function(i) {
638f27f4
BA
235# sqrt((data$getLevel(i-1)-levelYersteday)^2 + (data$getLevel(i)-levelToday)^2)
236 abs(data$getLevel(i)-levelToday)
d2ab47a7
BA
237 })
238 #TODO: 1, +1, +3 : magic numbers
239 dist_thresh = 1
cf3bb001 240 min_neighbs = min(min_neighbs,length(tdays))
3ddf1c12
BA
241 repeat
242 {
243 same_pollution = (distances <= dist_thresh)
244 nb_neighbs = sum(same_pollution)
245 if (nb_neighbs >= min_neighbs) #will eventually happen
246 break
d2ab47a7 247 dist_thresh = dist_thresh + ifelse(dist_thresh>1,3,1)
3ddf1c12 248 }
cf3bb001 249 tdays = tdays[same_pollution]
638f27f4
BA
250# max_neighbs = 12
251# if (nb_neighbs > max_neighbs)
252# {
253# # Keep only max_neighbs closest neighbors
254# tdays = tdays[ order(distances[same_pollution])[1:max_neighbs] ]
255# }
cf3bb001 256 tdays
3ddf1c12
BA
257}
258
689aa1d3
BA
259# compute similarities
260#
261# Apply the gaussian kernel on computed squared distances.
262#
263# @param distances2 Squared distances
264# @param window Window parameter for the kernel
265#
3ddf1c12
BA
266.computeSimils <- function(distances2, window)
267{
268 sd_dist = sd(distances2)
269 if (sd_dist < .25 * sqrt(.Machine$double.eps))
270 {
271# warning("All computed distances are very close: stdev too small")
272 sd_dist = 1 #mostly for tests... FIXME:
273 }
274 exp(-distances2/(sd_dist*window^2))
275}
638f27f4
BA
276
277.computeDistsEndo <- function(data, today, tdays, predict_from)
278{
279 lastSerie = c( data$getSerie(today-1),
280 data$getSerie(today)[if (predict_from>=2) 1:(predict_from-1) else c()] )
281 sapply(tdays, function(i) {
282 delta = lastSerie - c(data$getSerie(i-1),
283 data$getSerie(i)[if (predict_from>=2) 1:(predict_from-1) else c()])
284 sqrt(mean(delta^2))
285 })
286}
287
288.computeDistsExo <- function(data, today, tdays)
289{
290 M = matrix( ncol=1+length(tdays), nrow=1+length(data$getExo(1)) )
291 M[,1] = c( data$getLevelHat(today), as.double(data$getExoHat(today)) )
292 for (i in seq_along(tdays))
293 M[,i+1] = c( data$getLevel(tdays[i]), as.double(data$getExo(tdays[i])) )
294
295 sigma = cov(t(M)) #NOTE: robust covariance is way too slow
296 # TODO: 10 == magic number; more robust way == det, or always ginv()
297 sigma_inv =
298 if (length(tdays) > 10)
299 solve(sigma)
300 else
301 MASS::ginv(sigma)
302
303 # Distances from last observed day to days in the past
304 sapply(seq_along(tdays), function(i) {
305 delta = M[,1] - M[,i+1]
306 delta %*% sigma_inv %*% delta
307 })
308}