X-Git-Url: https://git.auder.net/?a=blobdiff_plain;f=epclust%2FR%2Fde_serialize.R;h=b6684d2abfbf35b6e99ccfd5d529e2b8f3d50688;hb=492cd9e74a79cbcc0ecde55fa3071a44b7e463dc;hp=682db53c18a2c129af70431b5d9dc85b0c192331;hpb=56857861dc15088cf58e7438968fe5714b22168e;p=epclust.git

diff --git a/epclust/R/de_serialize.R b/epclust/R/de_serialize.R
index 682db53..b6684d2 100644
--- a/epclust/R/de_serialize.R
+++ b/epclust/R/de_serialize.R
@@ -1,11 +1,40 @@
-#data: matrix of double or connection
-serialize = function(data_ascii, data_bin_file, nb_per_chunk,
+#' @name de_serialize
+#' @rdname de_serialize
+#' @aliases binarize binarizeTransform getDataInFile
+#'
+#' @title (De)Serialization of a [big]matrix or data stream
+#'
+#' @description \code{binarize()} serializes a matrix or CSV file with minimal overhead,
+#'   into a binary file. \code{getDataInFile()} achieves the inverse task: she retrieves
+#'   (ASCII) data rows from indices in the binary file. Finally,
+#'   \code{binarizeTransform()} serialize transformations of all data chunks; to use it,
+#'   a data-retrieval function must be provided, thus \code{binarize} will most likely be
+#'   used first (and then a function defined to seek in generated binary file)
+#'
+#' @param data_ascii Either a matrix or CSV file, with items in rows
+#' @param indices Indices of the lines to retrieve
+#' @param data_bin_file Name of binary file on output (\code{binarize})
+#'   or intput (\code{getDataInFile})
+#' @param nb_per_chunk Number of lines to process in one batch
+#' @inheritParams claws
+#' @param getData Function to retrieve data chunks
+#' @param transform Transformation function to apply on data chunks
+#'
+#' @return For \code{getDataInFile()}, the matrix with rows corresponding to the
+#'   requested indices. \code{binarizeTransform} returns the number of processed lines.
+#'   \code{binarize} is designed to serialize in several calls, thus returns nothing.
+NULL
+
+#' @rdname de_serialize
+#' @export
+binarize = function(data_ascii, data_bin_file, nb_per_chunk,
 	sep=",", nbytes=4, endian=.Platform$endian)
 {
 	if (is.character(data_ascii))
 		data_ascii = file(data_ascii, open="r")
-	else if (is(data_ascii,"connection") && !isOpen(data_ascii))
+	else if (methods::is(data_ascii,"connection") && !isOpen(data_ascii))
 		open(data_ascii)
+	is_matrix = !methods::is(data_ascii,"connection")
 
 	first_write = (!file.exists(data_bin_file) || file.info(data_bin_file)$size == 0)
 	data_bin = file(data_bin_file, open=ifelse(first_write,"wb","ab"))
@@ -14,60 +43,84 @@ serialize = function(data_ascii, data_bin_file, nb_per_chunk,
 	if (first_write)
 	{
 		#number of items always on 8 bytes
-		writeBin(0L, data_bin, size=8) #,endian="little")
-		if (is.matrix(data_ascii))
+		writeBin(0L, data_bin, size=8, endian=endian)
+		if ( is_matrix )
 			data_length = ncol(data_ascii)
-		else #if (is(data, "connection"))
+		else #connection
 		{
 			data_line = scan(data_ascii, double(), sep=sep, nlines=1, quiet=TRUE)
-			writeBin(data_line, data_bin, size=nbytes)
+			writeBin(data_line, data_bin, size=nbytes, endian=endian)
 			data_length = length(data_line)
 		}
 	}
 
-	if (is.matrix(data_ascii))
+	if (is_matrix)
 		index = 1
 	repeat
 	{
-		if (is.matrix(data_ascii))
+		if ( is_matrix )
 		{
-			range = index:min(nrow(data_ascii),index+nb_per_chunk)
 			data_chunk =
-				if (range[1] <= nrow(data_ascii))
-					as.double(t(data_ascii[range,]))
+				if (index <= nrow(data_ascii))
+					as.double(t(data_ascii[index:min(nrow(data_ascii),index+nb_per_chunk-1),]))
 				else
-					integer(0)
+					double(0)
 			index = index + nb_per_chunk
 		}
 		else
 			data_chunk = scan(data_ascii, double(), sep=sep, nlines=nb_per_chunk, quiet=TRUE)
 		if (length(data_chunk)==0)
 			break
-		writeBin(data_chunk, data_bin, size=nbytes)
+		writeBin(data_chunk, data_bin, size=nbytes, endian=endian)
 	}
 
 	if (first_write)
 	{
-		#ecrire file_size-1 / (nbytes*nbWritten) en 0 dans bin_data ! ignored == file_size
+		# Write data_length, = (file_size-1) / (nbytes*nbWritten) at offset 0 in data_bin
 		ignored = seek(data_bin, 0)
-		writeBin(data_length, data_bin, size=8)
+		writeBin(data_length, data_bin, size=8, endian=endian)
 	}
 	close(data_bin)
 
-	if (is(data_ascii,"connection"))
+	if ( ! is_matrix )
 		close(data_ascii)
 }
 
-#read in binary file, always same structure
+#' @rdname de_serialize
+#' @export
+binarizeTransform = function(getData, transform, data_bin_file, nb_per_chunk,
+	nbytes=4, endian=.Platform$endian)
+{
+	nb_items = 0
+	index = 1
+	repeat
+	{
+		data_chunk = getData((index-1)+seq_len(nb_per_chunk))
+		if (is.null(data_chunk))
+			break
+		transformed_chunk = transform(data_chunk)
+		binarize(transformed_chunk, data_bin_file, nb_per_chunk, ",", nbytes, endian)
+		index = index + nb_per_chunk
+		nb_items = nb_items + nrow(data_chunk)
+	}
+	nb_items
+}
+
+#' @rdname de_serialize
+#' @export
 getDataInFile = function(indices, data_bin_file, nbytes=4, endian=.Platform$endian)
 {
 	data_bin = file(data_bin_file, "rb")
-	data_length = readBin(data_bin, "integer", 1, 8, endian)
+	data_size = file.info(data_bin_file)$size
+	data_length = readBin(data_bin, "integer", n=1, size=8, endian=endian)
 	#Ou t(sapply(...)) (+ rapide ?)
 	data_ascii = do.call( rbind, lapply( indices, function(i) {
-		ignored = seek(data_bin, 8+((i-1)*data_length*nbytes))
-		readBin(data_bin, "double", n=data_length, size=nbytes)
+		offset = 8+(i-1)*data_length*nbytes
+		if (offset > data_size)
+			return (vector("double",0))
+		ignored = seek(data_bin, offset)
+		readBin(data_bin, "double", n=data_length, size=nbytes, endian=endian)
 	} ) )
 	close(data_bin)
-	data_ascii
+	if (ncol(data_ascii)>0) data_ascii else NULL
 }