X-Git-Url: https://git.auder.net/?p=epclust.git;a=blobdiff_plain;f=epclust%2FR%2Fde_serialize.R;h=eba6772c69b8052add0567a13d7d2c6e3bcee6a1;hp=f28038b773b5ab995be33a70eb7271822177ca8c;hb=282342bafdc9ff65c5df98c6e2304d63b33b9fb2;hpb=3c5a4b0880db63367a474a568e1322b3999932fe diff --git a/epclust/R/de_serialize.R b/epclust/R/de_serialize.R index f28038b..eba6772 100644 --- a/epclust/R/de_serialize.R +++ b/epclust/R/de_serialize.R @@ -27,35 +27,35 @@ NULL #' @rdname de_serialize #' @export -binarize = function(data_ascii, data_bin_file, nb_per_chunk, +binarize <- function(data_ascii, data_bin_file, nb_per_chunk, sep=",", nbytes=4, endian=.Platform$endian) { # data_ascii can be of two types: [big.]matrix, or connection if (is.character(data_ascii)) - data_ascii = file(data_ascii, open="r") + data_ascii <- file(data_ascii, open="r") else if (methods::is(data_ascii,"connection") && !isOpen(data_ascii)) open(data_ascii) - is_matrix = !methods::is(data_ascii,"connection") + is_matrix <- !methods::is(data_ascii,"connection") # At first call, the length of a stored row is written. So it's important to determine # if the serialization process already started. - first_write = (!file.exists(data_bin_file) || file.info(data_bin_file)$size == 0) + first_write <- (!file.exists(data_bin_file) || file.info(data_bin_file)$size == 0) # Open the binary file for writing (or 'append' if already exists) - data_bin = file(data_bin_file, open=ifelse(first_write,"wb","ab")) + data_bin <- file(data_bin_file, open=ifelse(first_write,"wb","ab")) if (first_write) { # Write data length on first call: number of items always on 8 bytes writeBin(0L, data_bin, size=8, endian=endian) if (is_matrix) - data_length = nrow(data_ascii) + data_length <- nrow(data_ascii) else #connection { # Read the first line to know data length, and write it then - data_line = scan(data_ascii, double(), sep=sep, nlines=1, quiet=TRUE) + data_line <- scan(data_ascii, double(), sep=sep, nlines=1, quiet=TRUE) writeBin(data_line, data_bin, size=nbytes, endian=endian) - data_length = length(data_line) + data_length <- length(data_line) } } @@ -63,21 +63,21 @@ binarize = function(data_ascii, data_bin_file, nb_per_chunk, { # Data is processed by chunks; although this may not be so useful for (normal) matrix # input, it could for a file-backed big.matrix. It's easier to follow a unified pattern. - index = 1 + index <- 1 } repeat { if (is_matrix) { - data_chunk = + data_chunk <- if (index <= ncol(data_ascii)) as.double(data_ascii[,index:min(ncol(data_ascii),index+nb_per_chunk-1)]) else double(0) - index = index + nb_per_chunk + index <- index + nb_per_chunk } else #connection - data_chunk = scan(data_ascii, double(), sep=sep, nlines=nb_per_chunk, quiet=TRUE) + data_chunk <- scan(data_ascii, double(), sep=sep, nlines=nb_per_chunk, quiet=TRUE) # Data size is unknown in the case of a connection if (length(data_chunk)==0) @@ -89,8 +89,8 @@ binarize = function(data_ascii, data_bin_file, nb_per_chunk, if (first_write) { - # Write data_length, = (file_size-1) / (nbytes*nbWritten) at offset 0 in data_bin - ignored = seek(data_bin, 0) + # Write data_length, == (file_size-1) / (nbytes*nbWritten) at offset 0 in data_bin + ignored <- seek(data_bin, 0) writeBin(data_length, data_bin, size=8, endian=endian) } close(data_bin) @@ -101,47 +101,47 @@ binarize = function(data_ascii, data_bin_file, nb_per_chunk, #' @rdname de_serialize #' @export -binarizeTransform = function(getData, transform, data_bin_file, nb_per_chunk, +binarizeTransform <- function(getData, transform, data_bin_file, nb_per_chunk, nbytes=4, endian=.Platform$endian) { - nb_items = 0 #side-effect: store the number of transformed items - index = 1 + nb_items <- 0 #side-effect: store the number of transformed items + index <- 1 repeat { # Retrieve a chunk of data in a binary file (generally obtained by binarize()) - data_chunk = getData((index-1)+seq_len(nb_per_chunk)) + data_chunk <- getData((index-1)+seq_len(nb_per_chunk)) if (is.null(data_chunk)) break # Apply transformation on the current chunk (by columns) - transformed_chunk = transform(data_chunk) + transformed_chunk <- transform(data_chunk) # Save the result in binary format binarize(transformed_chunk, data_bin_file, nb_per_chunk, ",", nbytes, endian) - index = index + nb_per_chunk - nb_items = nb_items + ncol(data_chunk) + index <- index + nb_per_chunk + nb_items <- nb_items + ncol(data_chunk) } nb_items #number of transformed items } #' @rdname de_serialize #' @export -getDataInFile = function(indices, data_bin_file, nbytes=4, endian=.Platform$endian) +getDataInFile <- function(indices, data_bin_file, nbytes=4, endian=.Platform$endian) { - data_bin = file(data_bin_file, "rb") #source binary file + data_bin <- file(data_bin_file, "rb") #source binary file - data_size = file.info(data_bin_file)$size #number of bytes in the file + data_size <- file.info(data_bin_file)$size #number of bytes in the file # data_length: length of a vector in the binary file (first element, 8 bytes) - data_length = readBin(data_bin, "integer", n=1, size=8, endian=endian) + data_length <- readBin(data_bin, "integer", n=1, size=8, endian=endian) # Seek all 'indices' columns in the binary file, using data_length and nbytes # to compute the offset ( index i at 8 + i*data_length*nbytes ) - data_ascii = do.call( cbind, lapply( indices, function(i) { - offset = 8+(i-1)*data_length*nbytes + data_ascii <- do.call( cbind, lapply( indices, function(i) { + offset <- 8+(i-1)*data_length*nbytes if (offset >= data_size) return (NULL) - ignored = seek(data_bin, offset) #position cursor at computed offset + ignored <- seek(data_bin, offset) #position cursor at computed offset readBin(data_bin, "double", n=data_length, size=nbytes, endian=endian) } ) ) close(data_bin)