X-Git-Url: https://git.auder.net/?a=blobdiff_plain;f=epclust%2Ftests%2Ftestthat%2Ftest.de_serialize.R;h=25f2c2bf21f2061ee6aa2968cc6bf3b7b1866f5d;hb=37c82bbafbffc19e8b47a521952bac58f189e9ea;hp=41c57e412b631899eb4f2a53ec28f9bc1150cfb8;hpb=62deb4244895a20a35397dfb062f0b9fe94c5012;p=epclust.git diff --git a/epclust/tests/testthat/test.de_serialize.R b/epclust/tests/testthat/test.de_serialize.R index 41c57e4..25f2c2b 100644 --- a/epclust/tests/testthat/test.de_serialize.R +++ b/epclust/tests/testthat/test.de_serialize.R @@ -1,52 +1,95 @@ +context("de_serialize") -#data: matrix of double or connection -serialize = function(data, file, type, nb_per_chunk) +test_that("serialization + getDataInFile retrieve original data / from matrix", { - bin_data = file(file, "ab") - #write data length on first call - nbytes = ifelse(type=="double",8,4) - first_write = FALSE - if (file.info(file)$size == 0) + data_bin_file = ".epclust_test_m.bin" + unlink(data_bin_file) + + #dataset 200 cols / 30 rows + data_ascii = matrix(runif(200*30,-10,10),nrow=30) + nbytes = 4 #lead to a precision of 1e-7 / 1e-8 + endian = "little" + + #Simulate serialization in one single call + binarize(data_ascii, data_bin_file, 500, ",", nbytes, endian) + expect_equal(file.info(data_bin_file)$size, length(data_ascii)*nbytes+8) + for (indices in list(c(1,3,5), 3:13, c(5,20,50), c(75,130:135), 196:200)) { - #number of items always on 8 bytes - writeBin(0L, bin_data, size=8) #,endian="little") - first_write = TRUE + data_lines = getDataInFile(indices, data_bin_file, nbytes, endian) + expect_equal(data_lines, data_ascii[,indices], tolerance=1e-6) } - if (is.matrix(data)) + unlink(data_bin_file) + + #...in several calls (last call complete, next call NULL) + for (i in 1:20) + binarize(data_ascii[((i-1)*10+1):(i*10),], data_bin_file, 20, ",", nbytes, endian) + expect_equal(file.info(data_bin_file)$size, length(data_ascii)*nbytes+8) + for (indices in list(c(1,3,5), 3:13, c(5,20,50), c(75,130:135), 196:200)) { - writeBin(t(data), bin_data, size=nbytes) - data_length = ncol(data) + data_lines = getDataInFile(indices, data_bin_file, nbytes, endian) + expect_equal(data_lines, data_ascii[,indices], tolerance=1e-6) } - else #if (is(data, "connection")) + unlink(data_bin_file) +}) + +test_that("serialization + transform + getDataInFile retrieve original transforms", +{ + data_bin_file = ".epclust_test_t.bin" + unlink(data_bin_file) + + #dataset 200 cols / 30 rows + data_ascii = matrix(runif(200*30,-10,10),nrow=30) + nbytes = 8 + endian = "little" + + binarize(data_ascii, data_bin_file, 500, ",", nbytes, endian) + # Serialize transformation (just compute range) into a new binary file + trans_bin_file = ".epclust_test_t_trans.bin" + unlink(trans_bin_file) + getSeries = function(inds) getDataInFile(inds, data_bin_file, nbytes, endian) + binarizeTransform(getSeries, function(series) apply(series, 2, range), + trans_bin_file, 250, nbytes, endian) + unlink(data_bin_file) + expect_equal(file.info(trans_bin_file)$size, 2*nrow(data_ascii)*nbytes+8) + for (indices in list(c(1,3,5), 3:13, c(5,20,50), c(75,130:135), 196:200)) { - if (first_write) - { - data_line = scan(data, double(), sep=",", nlines=1) - writeBin(data_line, bin_data, size=nbytes) - data_length = length(data_line) - } - repeat - { - data_chunk = scan(data, double(), sep=",", nlines=nb_per_chunk) - if (length(data_chunk)==0) - break - writeBin(data_chunk, bin_data, size=nbytes) - } + trans_cols = getDataInFile(indices, trans_bin_file, nbytes, endian) + expect_equal(trans_cols, apply(data_ascii[indices,],2,range), tolerance=1e-6) } - if (first_write) + unlink(trans_bin_file) +}) + +test_that("serialization + getDataInFile retrieve original data / from connection", +{ + data_bin_file = ".epclust_test_c.bin" + unlink(data_bin_file) + + #dataset 300 cols / 50 rows + data_csv = system.file("testdata","de_serialize.csv",package="epclust") + nbytes = 8 + endian = "big" + data_ascii = as.matrix(read.csv(data_csv, sep=";", header=FALSE)) #for ref + + #Simulate serialization in one single call + binarize(data_csv, data_bin_file, 350, ";", nbytes, endian) + expect_equal(file.info(data_bin_file)$size, 300*50*8+8) + for (indices in list(c(1,3,5), 3:13, c(5,20,50), c(75,130:135), 196:200)) { - #ecrire file_size-1 / (nbytes*nbWritten) en 0 dans bin_data ! ignored == file_size - ignored = seek(bin_data, 0) - writeBin(data_length, bin_data, size=8) + #HACK: as.matrix(as.data.frame( )) required to match (ref) data structure + data_cols = as.matrix(as.data.frame( getDataInFile(indices,data_bin_file,nbytes,endian) )) + expect_equal(data_cols, data_ascii[,indices]) } - close(bin_data) -} + unlink(data_bin_file) -#TODO: read in binary file, always same structure -getDataFromFile(indices, file, type) -{ - bin_data = file(file, "rb") - nbytes = ifelse(type=="double",8,4) - data_length = readBin(bin_data,"double",1,nbytes) #,endian="little") - t(sapply(indices, function(i) readBin(bin_data,"double",n=data_length,size=nbytes))) -} + #...in several calls / chunks of 29 --> 29*10 + 10, incomplete last + data_con = file(data_csv, "r") + binarize(data_con, data_bin_file, 29, ";", nbytes, endian) + expect_equal(file.info(data_bin_file)$size, 300*50*8+8) + for (indices in list(c(1,3,5), 3:13, c(5,20,50), c(75,130:135), 196:200)) + { + data_cols = as.matrix(as.data.frame( getDataInFile(indices,data_bin_file,nbytes,endian) )) + expect_equal(data_cols, data_ascii[,indices]) + } + unlink(data_bin_file) + #close(data_con) --> done in binarize() +})