#' @param data_ascii Either a matrix or CSV file, with items in rows
#' @param indices Indices of the lines to retrieve
#' @param data_bin_file Name of binary file on output (\code{binarize})
#' @param data_ascii Either a matrix or CSV file, with items in rows
#' @param indices Indices of the lines to retrieve
#' @param data_bin_file Name of binary file on output (\code{binarize})
-#' or intput (\code{getDataInFile})
-#' @param nb_per_chunk Number of lines to process in one batch
+#' or input (\code{getDataInFile})
+#' @param nb_per_chunk Number of lines to process in one batch (big.matrix or connection)
#' @inheritParams claws
#' @param getData Function to retrieve data chunks
#' @param transform Transformation function to apply on data chunks
#' @inheritParams claws
#' @param getData Function to retrieve data chunks
#' @param transform Transformation function to apply on data chunks
binarize = function(data_ascii, data_bin_file, nb_per_chunk,
sep=",", nbytes=4, endian=.Platform$endian)
{
binarize = function(data_ascii, data_bin_file, nb_per_chunk,
sep=",", nbytes=4, endian=.Platform$endian)
{
if (is.character(data_ascii))
data_ascii = file(data_ascii, open="r")
else if (methods::is(data_ascii,"connection") && !isOpen(data_ascii))
open(data_ascii)
is_matrix = !methods::is(data_ascii,"connection")
if (is.character(data_ascii))
data_ascii = file(data_ascii, open="r")
else if (methods::is(data_ascii,"connection") && !isOpen(data_ascii))
open(data_ascii)
is_matrix = !methods::is(data_ascii,"connection")
data_bin = file(data_bin_file, open=ifelse(first_write,"wb","ab"))
data_bin = file(data_bin_file, open=ifelse(first_write,"wb","ab"))
writeBin(0L, data_bin, size=8, endian=endian)
writeBin(0L, data_bin, size=8, endian=endian)
data_line = scan(data_ascii, double(), sep=sep, nlines=1, quiet=TRUE)
writeBin(data_line, data_bin, size=nbytes, endian=endian)
data_length = length(data_line)
data_line = scan(data_ascii, double(), sep=sep, nlines=1, quiet=TRUE)
writeBin(data_line, data_bin, size=nbytes, endian=endian)
data_length = length(data_line)
- if (index <= nrow(data_ascii))
- as.double(t(data_ascii[index:min(nrow(data_ascii),index+nb_per_chunk-1),]))
+ if (index <= ncol(data_ascii))
+ as.double(data_ascii[,index:min(ncol(data_ascii),index+nb_per_chunk-1)])
data_chunk = scan(data_ascii, double(), sep=sep, nlines=nb_per_chunk, quiet=TRUE)
data_chunk = scan(data_ascii, double(), sep=sep, nlines=nb_per_chunk, quiet=TRUE)
writeBin(data_chunk, data_bin, size=nbytes, endian=endian)
}
writeBin(data_chunk, data_bin, size=nbytes, endian=endian)
}
binarizeTransform = function(getData, transform, data_bin_file, nb_per_chunk,
nbytes=4, endian=.Platform$endian)
{
binarizeTransform = function(getData, transform, data_bin_file, nb_per_chunk,
nbytes=4, endian=.Platform$endian)
{
binarize(transformed_chunk, data_bin_file, nb_per_chunk, ",", nbytes, endian)
binarize(transformed_chunk, data_bin_file, nb_per_chunk, ",", nbytes, endian)
}
#' @rdname de_serialize
#' @export
getDataInFile = function(indices, data_bin_file, nbytes=4, endian=.Platform$endian)
{
}
#' @rdname de_serialize
#' @export
getDataInFile = function(indices, data_bin_file, nbytes=4, endian=.Platform$endian)
{
- data_bin = file(data_bin_file, "rb")
- data_size = file.info(data_bin_file)$size
+ data_bin = file(data_bin_file, "rb") #source binary file
+
+ data_size = file.info(data_bin_file)$size #number of bytes in the file
+ # data_length: length of a vector in the binary file (first element, 8 bytes)
data_length = readBin(data_bin, "integer", n=1, size=8, endian=endian)
data_length = readBin(data_bin, "integer", n=1, size=8, endian=endian)
- #Ou t(sapply(...)) (+ rapide ?)
- data_ascii = do.call( rbind, lapply( indices, function(i) {
+
+ # Seek all 'indices' columns in the binary file, using data_length and nbytes
+ # to compute the offset ( index i at 8 + i*data_length*nbytes )
+ data_ascii = do.call( cbind, lapply( indices, function(i) {
- if (offset > data_size)
- return (vector("double",0))
- ignored = seek(data_bin, offset)
+ if (offset >= data_size)
+ return (NULL)
+ ignored = seek(data_bin, offset) #position cursor at computed offset
readBin(data_bin, "double", n=data_length, size=nbytes, endian=endian)
} ) )
close(data_bin)
readBin(data_bin, "double", n=data_length, size=nbytes, endian=endian)
} ) )
close(data_bin)