## File : 00_convertir-donnnes_2009.r
## Description : Converts flat EDF's 32K data into a full data matrix 
##               layout [individuals, variables]. Rownames are EDF's ids.
##               We process the original flat file sequentially by lines
##               to avoid exceding the available RAM memory (and so avoiding
##               swaping which is a computational burden).


rm(list = ls())

# setwd("~/ownCloud/projects/2014_EDF-Orsay-Lyon2/codes")


## 1. Read auxiliar data files ####

identifiants <- read.table("identifs.txt")[ ,1]
dates0       <- read.table("datesall.txt")[, 1]
dates        <- dates0[grep("2009", dates0)]
rm(dates0)

n <- length(identifiants)
p <- length(dates)

blocks <- c(rep(6000, 3), 7011)  # We'll process 1000 x p lines at each
                                  # iteration of the reading loop

## 2. Process the large flat file ####
## We want to check that every time step recorded for each id.

con <- file("~/tmp/data/2009.csv")  # Establish a connection to the file
open(con, "r")                      # Open the connection
rien <- readLines(con = con, n = 1); rm(rien) # Discard 1st line

for(b in seq_along(blocks)){      # Reading loop
  nb <- blocks[b]
  actual <- readLines(con = con, n = nb * length(dates))
  auxmat <- matrix(unlist(strsplit(actual, ",")), ncol = 4, byrow = TRUE)
  rm(actual)
  auxdf  <- data.frame(id   = as.integer(auxmat[, 1]),
                       date = auxmat[, 2],
                       val  = as.numeric(
                         ifelse(auxmat[,3] == "", auxmat[, 4], auxmat[, 3])))
  rm(auxmat) # free up some space
  
  tab <- table(auxdf$id)
  idtt <- NULL
  for(tt in as.integer(names(which(tab < p)))) {  # id with less than p records!
    print(tt)
    idtt <- c(idtt, which(auxdf$id == tt))
  }
  
  idmat         <- matrix(auxdf$id[-idtt], ncol = p, byrow = TRUE)
  alldatesperid <- apply(idmat[-idtt, ], 1, sd) == 0
  valmat        <- matrix(auxdf$val[-idtt], ncol = p, byrow = TRUE)
  
  # store separatelly full records from partial records
  write.table(file = paste0("~/tmp/2009_full_", b, ".txt"), valmat,
              row.names = idmat[, 1], col.names = FALSE) 
  write.table(file = paste0("~/tmp/2009_partial_", b, ".txt"), auxdf[idtt, ])
}

close(con)                      # close connection to the file

rm(auxdf, idmat, valmat, alldatesperid, b,  # clean up some memory
   idtt, blocks, tab, tt, con)


## 3. Complete partial records ####
## After analysis, partial records are only 119 clients from which one only
## time step (01JAN2009:00:00:00) is lacking.

df_partial <- NULL
for(f in list.files("~/tmp/", "2009_partial_*")) 
  df_partial <- rbind(df_partial, read.table(paste0('~/tmp/', f)))

tab <- table(df_partial$id)
id_incomp <- as.integer(names(which(tab < p))) # Incomplete records

df_partial_full <- rbind(df_partial, 
                         data.frame(id   = id_incomp,
                                    date = "01JAN2009:00:00:00",  
                                    val  = NA))

rm(df_partial)

# tab2 <- table(df_partial_full$id)  # Check that df_partial_full is full
# head(sort(tab2))


## 4. Reorder the lines to get the data matrix ####
## As we paste chunks of partial records and impute some time steps,
## the original order of the data is broken. We fix it by reordering
## the ids and then the data.

idx_ordered <- order(df_partial_full$id)             # order ids
df_partial_full2 <- df_partial_full[idx_ordered, ] 
rm(df_partial_full)

# Order data values following the correct dates (as the date is a factor
# we need to seek for each value: this is computationnaly innefficient).

valmat  <- matrix(df_partial_full2$val,  ncol = p, byrow = TRUE)
datemat <- matrix(df_partial_full2$date, ncol = p, byrow = TRUE)
idmat   <- matrix(df_partial_full2$id,   ncol = p, byrow = TRUE)

# Use this for as a check by running it twice. On the second run no
# printing should be done (because records should be ordered).
for(line in 1:nrow(datemat)) {
  if(any(datemat[line, ] != dates)) { # TRUE is line is not ordered
    cat(sprintf("\nline %i is not ordered", line))
    
    neworder         <- match(dates, datemat[line, ])
    valmat[line , ]  <- valmat[ line, neworder]
    datemat[line , ] <- datemat[line, neworder]
  } 
}


## 5. Write on disk the full data matrix of partial records ####

write.table(file = "~/tmp/2009_full_Z.txt", valmat, 
            row.names = idmat[, 1], col.names = FALSE) 
rm(list = ls())
gc()

## 6. Compile data files in BASH ####

# cat 2009_full*.txt > 2009_full.txt
# rm 2009_full_*.txt 2009_partial_*.txt


## A. data.table & reshape2 #### 
## When large RAM memory is available, one could use this code to process
## everything in memory.

#library(data.table)
#library(reshape2)

#dt <- fread(input  = "~/tmp/data/2009_chunk.csv")

#dt[, charge := ifelse(is.na(CPP_PUISSANCE_CORRIGEE), 
#                      CPP_PUISSANCE_BRUTE,
#                      CPP_PUISSANCE_CORRIGEE), ]
#dt[, CPP_PUISSANCE_CORRIGEE := NULL]
#dt[, CPP_PUISSANCE_BRUTE := NULL]

#dt2 <- dcast.data.table(data = dt, CPP_DATE_PUISSANCE + FK_CCU_ID ~ charge)


## Z. Probably stuff to be deleted 

# searchpos <- function(row) {
#   str  <- strsplit(row, ",")
#   
#   auxmat <- matrix(unlist(str), ncol = 4, byrow = TRUE); rm(str)
#   
#   auxdf  <- data.frame(id   = as.integer(auxmat[, 1]),
#                        date = auxmat[, 2],
#                        val  = as.numeric(
#                          ifelse(auxmat[,3] == "", auxmat[, 4], auxmat[, 3]))
#   )
#   rm(auxmat)
#   
#   idmat <- matrix(auxdf$id, ncol = length(dates), byrow = TRUE)
#   alldatesperid <- apply(idmat, 1, sd) == 0
#   
#   
#   #  lines <- match(auxdf$id, identifiants)
#   #  cols  <- match(auxdf$date, dates)
#   
#   return(cbind(lines, cols, auxdf$val))
# }