option to run sequentially. various fixes. R CMD check OK

[epclust.git] / epclust / R / de_serialize.R

diff --git a/epclust/R/de_serialize.R b/epclust/R/de_serialize.R
index 8dde258..b6684d2 100644 (file)
--- a/epclust/R/de_serialize.R
+++ b/epclust/R/de_serialize.R
@@ -1,12 +1,15 @@
 #' @name de_serialize
 #' @rdname de_serialize
-#' @aliases binarize getDataInFile
+#' @aliases binarize binarizeTransform getDataInFile
 #'
-#' @title (De)Serialization of a matrix
+#' @title (De)Serialization of a [big]matrix or data stream
 #'
 #' @description \code{binarize()} serializes a matrix or CSV file with minimal overhead,
 #'   into a binary file. \code{getDataInFile()} achieves the inverse task: she retrieves
-#'   (ASCII) data rows from indices in the binary file
+#'   (ASCII) data rows from indices in the binary file. Finally,
+#'   \code{binarizeTransform()} serialize transformations of all data chunks; to use it,
+#'   a data-retrieval function must be provided, thus \code{binarize} will most likely be
+#'   used first (and then a function defined to seek in generated binary file)
 #'
 #' @param data_ascii Either a matrix or CSV file, with items in rows
 #' @param indices Indices of the lines to retrieve
@@ -14,9 +17,12 @@
 #'   or intput (\code{getDataInFile})
 #' @param nb_per_chunk Number of lines to process in one batch
 #' @inheritParams claws
+#' @param getData Function to retrieve data chunks
+#' @param transform Transformation function to apply on data chunks
 #'
 #' @return For \code{getDataInFile()}, the matrix with rows corresponding to the
-#'   requested indices
+#'   requested indices. \code{binarizeTransform} returns the number of processed lines.
+#'   \code{binarize} is designed to serialize in several calls, thus returns nothing.
 NULL
 
 #' @rdname de_serialize
@@ -28,6 +34,7 @@ binarize = function(data_ascii, data_bin_file, nb_per_chunk,
                data_ascii = file(data_ascii, open="r")
        else if (methods::is(data_ascii,"connection") && !isOpen(data_ascii))
                open(data_ascii)
+       is_matrix = !methods::is(data_ascii,"connection")
 
        first_write = (!file.exists(data_bin_file) || file.info(data_bin_file)$size == 0)
        data_bin = file(data_bin_file, open=ifelse(first_write,"wb","ab"))
@@ -37,9 +44,9 @@ binarize = function(data_ascii, data_bin_file, nb_per_chunk,
        {
                #number of items always on 8 bytes
                writeBin(0L, data_bin, size=8, endian=endian)
-               if (is.matrix(data_ascii))
+               if ( is_matrix )
                        data_length = ncol(data_ascii)
-               else #if (is(data, "connection"))
+               else #connection
                {
                        data_line = scan(data_ascii, double(), sep=sep, nlines=1, quiet=TRUE)
                        writeBin(data_line, data_bin, size=nbytes, endian=endian)
@@ -47,18 +54,17 @@ binarize = function(data_ascii, data_bin_file, nb_per_chunk,
                }
        }
 
-       if (is.matrix(data_ascii))
+       if (is_matrix)
                index = 1
        repeat
        {
-               if (is.matrix(data_ascii))
+               if ( is_matrix )
                {
-                       range = index:min(nrow(data_ascii),index+nb_per_chunk)
                        data_chunk =
-                               if (range[1] <= nrow(data_ascii))
-                                       as.double(t(data_ascii[range,]))
+                               if (index <= nrow(data_ascii))
+                                       as.double(t(data_ascii[index:min(nrow(data_ascii),index+nb_per_chunk-1),]))
                                else
-                                       integer(0)
+                                       double(0)
                        index = index + nb_per_chunk
                }
                else
@@ -70,16 +76,36 @@ binarize = function(data_ascii, data_bin_file, nb_per_chunk,
 
        if (first_write)
        {
-               #ecrire file_size-1 / (nbytes*nbWritten) en 0 dans bin_data ! ignored == file_size
+               # Write data_length, = (file_size-1) / (nbytes*nbWritten) at offset 0 in data_bin
                ignored = seek(data_bin, 0)
                writeBin(data_length, data_bin, size=8, endian=endian)
        }
        close(data_bin)
 
-       if (methods::is(data_ascii,"connection"))
+       if ( ! is_matrix )
                close(data_ascii)
 }
 
+#' @rdname de_serialize
+#' @export
+binarizeTransform = function(getData, transform, data_bin_file, nb_per_chunk,
+       nbytes=4, endian=.Platform$endian)
+{
+       nb_items = 0
+       index = 1
+       repeat
+       {
+               data_chunk = getData((index-1)+seq_len(nb_per_chunk))
+               if (is.null(data_chunk))
+                       break
+               transformed_chunk = transform(data_chunk)
+               binarize(transformed_chunk, data_bin_file, nb_per_chunk, ",", nbytes, endian)
+               index = index + nb_per_chunk
+               nb_items = nb_items + nrow(data_chunk)
+       }
+       nb_items
+}
+
 #' @rdname de_serialize
 #' @export
 getDataInFile = function(indices, data_bin_file, nbytes=4, endian=.Platform$endian)