+context("de_serialize")
-#data: matrix of double or connection
-serialize = function(data, file, type, nb_per_chunk)
+test_that("serialization + getDataInFile retrieve original data / from matrix",
{
- bin_data = file(file, "ab")
- #write data length on first call
- nbytes = ifelse(type=="double",8,4)
- first_write = FALSE
- if (file.info(file)$size == 0)
+ data_bin_file = ".epclust_test_m.bin"
+ unlink(data_bin_file)
+
+ # Dataset 200 cols / 30 rows
+ data_ascii = matrix(runif(200*30,-10,10),nrow=30)
+ nbytes = 4 #lead to a precision of 1e-7 / 1e-8
+ endian = "little"
+
+ # Simulate serialization in one single call
+ binarize(data_ascii, data_bin_file, 500, ",", nbytes, endian)
+ expect_equal(file.info(data_bin_file)$size, length(data_ascii)*nbytes+8)
+ for (indices in list(c(1,3,5), 3:13, c(5,20,50), c(75,130:135), 196:200))
{
- #number of items always on 8 bytes
- writeBin(0L, bin_data, size=8) #,endian="little")
- first_write = TRUE
+ data_lines = getDataInFile(indices, data_bin_file, nbytes, endian)
+ expect_equal(data_lines, data_ascii[,indices], tolerance=1e-6)
}
- if (is.matrix(data))
+ unlink(data_bin_file)
+
+ # Serialization in several calls (last call complete, next call NULL)
+ for (i in 1:20)
+ binarize(data_ascii[,((i-1)*10+1):(i*10)], data_bin_file, 20, ",", nbytes, endian)
+ expect_equal(file.info(data_bin_file)$size, length(data_ascii)*nbytes+8)
+ for (indices in list(c(1,3,5), 3:13, c(5,20,50), c(75,130:135), 196:200))
{
- writeBin(t(data), bin_data, size=nbytes)
- data_length = ncol(data)
+ data_lines = getDataInFile(indices, data_bin_file, nbytes, endian)
+ expect_equal(data_lines, data_ascii[,indices], tolerance=1e-6)
}
- else #if (is(data, "connection"))
+ unlink(data_bin_file)
+})
+
+test_that("serialization + transform + getDataInFile retrieve original transforms",
+{
+ data_bin_file = ".epclust_test_t.bin"
+ unlink(data_bin_file)
+
+ # Dataset 200 cols / 30 rows
+ data_ascii = matrix(runif(200*30,-10,10),nrow=30)
+ nbytes = 8
+ endian = "little"
+
+ binarize(data_ascii, data_bin_file, 500, ",", nbytes, endian)
+ # Serialize transformation (just compute range) into a new binary file
+ trans_bin_file = ".epclust_test_t_trans.bin"
+ unlink(trans_bin_file)
+ getSeries = function(inds) getDataInFile(inds, data_bin_file, nbytes, endian)
+ binarizeTransform(getSeries, function(series) apply(series, 2, range),
+ trans_bin_file, 250, nbytes, endian)
+ unlink(data_bin_file)
+ expect_equal(file.info(trans_bin_file)$size, 2*ncol(data_ascii)*nbytes+8)
+ for (indices in list(c(1,3,5), 3:13, c(5,20,50), c(75,130:135), 196:200))
{
- if (first_write)
- {
- data_line = scan(data, double(), sep=",", nlines=1)
- writeBin(data_line, bin_data, size=nbytes)
- data_length = length(data_line)
- }
- repeat
- {
- data_chunk = scan(data, double(), sep=",", nlines=nb_per_chunk)
- if (length(data_chunk)==0)
- break
- writeBin(data_chunk, bin_data, size=nbytes)
- }
+ trans_cols = getDataInFile(indices, trans_bin_file, nbytes, endian)
+ expect_equal(trans_cols, apply(data_ascii[,indices],2,range), tolerance=1e-6)
}
- if (first_write)
+ unlink(trans_bin_file)
+})
+
+test_that("serialization + getDataInFile retrieve original data / from connection",
+{
+ data_bin_file = ".epclust_test_c.bin"
+ unlink(data_bin_file)
+
+ # Dataset 300 cols / 50 rows
+ data_csv = system.file("testdata","de_serialize.csv",package="epclust")
+ nbytes = 8
+ endian = "big"
+ data_ascii = t( as.matrix(read.table(data_csv, sep=";", header=FALSE)) ) #for ref
+
+ # Simulate serialization in one single call
+ binarize(data_csv, data_bin_file, 350, ";", nbytes, endian)
+ expect_equal(file.info(data_bin_file)$size, 300*50*8+8)
+ for (indices in list(c(1,3,5), 3:13, c(5,20,50), c(75,130:135), 196:200))
{
- #ecrire file_size-1 / (nbytes*nbWritten) en 0 dans bin_data ! ignored == file_size
- ignored = seek(bin_data, 0)
- writeBin(data_length, bin_data, size=8)
+ data_cols = getDataInFile(indices,data_bin_file,nbytes,endian)
+ #HACK: rows naming required to match (ref) data structure
+ rownames(data_cols) <- paste("V",seq_len(nrow(data_ascii)), sep="")
+ expect_equal(data_cols, data_ascii[,indices])
}
- close(bin_data)
-}
+ unlink(data_bin_file)
-#TODO: read in binary file, always same structure
-getDataFromFile(indices, file, type)
-{
- bin_data = file(file, "rb")
- nbytes = ifelse(type=="double",8,4)
- data_length = readBin(bin_data,"double",1,nbytes) #,endian="little")
- t(sapply(indices, function(i) readBin(bin_data,"double",n=data_length,size=nbytes)))
-}
+ # Serialization in several calls / chunks of 29 --> 29*10 + 10, incomplete last
+ data_con = file(data_csv, "r")
+ binarize(data_con, data_bin_file, 29, ";", nbytes, endian)
+ expect_equal(file.info(data_bin_file)$size, 300*50*8+8)
+ for (indices in list(c(1,3,5), 3:13, c(5,20,50), c(75,130:135), 196:200))
+ {
+ data_cols = getDataInFile(indices,data_bin_file,nbytes,endian)
+ rownames(data_cols) <- paste("V",seq_len(nrow(data_ascii)), sep="")
+ expect_equal(data_cols, data_ascii[,indices])
+ }
+ unlink(data_bin_file)
+ #close(data_con) --> done in binarize()
+})