ignore generated binary files while debugging

[epclust.git] / epclust / R / de_serialize.R

#' @name de_serialize
#' @rdname de_serialize
#' @aliases binarize getDataInFile
#'
#' @title (De)Serialization of a matrix
#'
#' @description \code{binarize()} serializes a matrix or CSV file with minimal overhead,
#'   into a binary file. \code{getDataInFile()} achieves the inverse task: she retrieves
#'   (ASCII) data rows from indices in the binary file
#'
#' @param data_ascii Either a matrix or CSV file, with items in rows
#' @param indices Indices of the lines to retrieve
#' @param data_bin_file Name of binary file on output (\code{binarize})
#'   or intput (\code{getDataInFile})
#' @param nb_per_chunk Number of lines to process in one batch
#' @inheritParams claws
#'
#' @return For \code{getDataInFile()}, the matrix with rows corresponding to the
#'   requested indices
NULL

#' @rdname de_serialize
#' @export
binarize = function(data_ascii, data_bin_file, nb_per_chunk,
	sep=",", nbytes=4, endian=.Platform$endian)
{
	if (is.character(data_ascii))
		data_ascii = file(data_ascii, open="r")
	else if (methods::is(data_ascii,"connection") && !isOpen(data_ascii))
		open(data_ascii)

	first_write = (!file.exists(data_bin_file) || file.info(data_bin_file)$size == 0)
	data_bin = file(data_bin_file, open=ifelse(first_write,"wb","ab"))

	#write data length on first call
	if (first_write)
	{
		#number of items always on 8 bytes
		writeBin(0L, data_bin, size=8, endian=endian)
		if (is.matrix(data_ascii))
			data_length = ncol(data_ascii)
		else #if (is(data, "connection"))
		{
			data_line = scan(data_ascii, double(), sep=sep, nlines=1, quiet=TRUE)
			writeBin(data_line, data_bin, size=nbytes, endian=endian)
			data_length = length(data_line)
		}
	}

	if (is.matrix(data_ascii))
		index = 1
	repeat
	{
		if (is.matrix(data_ascii))
		{
			range = index:min(nrow(data_ascii),index+nb_per_chunk)
			data_chunk =
				if (range[1] <= nrow(data_ascii))
					as.double(t(data_ascii[range,]))
				else
					integer(0)
			index = index + nb_per_chunk
		}
		else
			data_chunk = scan(data_ascii, double(), sep=sep, nlines=nb_per_chunk, quiet=TRUE)
		if (length(data_chunk)==0)
			break
		writeBin(data_chunk, data_bin, size=nbytes, endian=endian)
	}

	if (first_write)
	{
		#ecrire file_size-1 / (nbytes*nbWritten) en 0 dans bin_data ! ignored == file_size
		ignored = seek(data_bin, 0)
		writeBin(data_length, data_bin, size=8, endian=endian)
	}
	close(data_bin)

	if (methods::is(data_ascii,"connection"))
		close(data_ascii)
}

#' @rdname de_serialize
#' @export
getDataInFile = function(indices, data_bin_file, nbytes=4, endian=.Platform$endian)
{
	data_bin = file(data_bin_file, "rb")
	data_size = file.info(data_bin_file)$size
	data_length = readBin(data_bin, "integer", n=1, size=8, endian=endian)
	#Ou t(sapply(...)) (+ rapide ?)
	data_ascii = do.call( rbind, lapply( indices, function(i) {
		offset = 8+(i-1)*data_length*nbytes
		if (offset > data_size)
			return (vector("double",0))
		ignored = seek(data_bin, offset)
		readBin(data_bin, "double", n=data_length, size=nbytes, endian=endian)
	} ) )
	close(data_bin)
	if (ncol(data_ascii)>0) data_ascii else NULL
}