data/preprocessing/convert_32kEDF.R

   1 convert_32kEDF = function(orig_csv, nb_series_per_chunk)
   2 {
   3     datetimes = #...TODO: all 3 years? year-by-year is better
   4     orig_con = file(orig_csv, open="r") #2009, 2010 or 2011
   5     ignored = readLines(orig_con, 1) #skip header
   6     serie_length = length(datetimes) #around 365*24*2 = 17520
   7     sep = if (year==2009) "," else if (year==2010) ";" else ";"
   8
   9 scan(orig_con, character(), sep=",", nlines=1, quiet=TRUE)
  10     library(sqldf, quietly=TRUE)
  11     ids = read.csv.sql(file_csv, header = TRUE, sep = ","
  12         sql = "select * from file_csv group by FK_CCU_ID")
  13     index = 0
  14     repeat
  15     {
  16         if (index+1 >= length(ids))
  17             break
  18         request = "select CPP_DATE_PUISSANCE,CPP_PUISSANCE_BRUTE where FK_CCU_ID in ("
  19         for (id in ids[index + seq_len(nb_series_per_chunk)])
  20             request = paste(request, id, ",", sep="")
  21         request = paste(request, ") order by FK_CCU_ID,CPP_DATE_PUISSANCE", sep="")
  22         series_chunk = read.csv.sql(file_csv, header = TRUE, sep = ",", sql = request)
  23
  24         index = index + 17520