From: Benjamin Auder Date: Thu, 9 Mar 2017 14:56:37 +0000 (+0100) Subject: save intermediate X-Git-Url: https://git.auder.net/%7B%7B%20asset%28%27mixstore/images/scripts/doc/pieces/%7B%7B?a=commitdiff_plain;h=6ad3f3fd0ec4f3cd1fd5de4a287c1893293e5bcc;p=epclust.git save intermediate --- diff --git a/TODO b/TODO index 4f865b0..52b139d 100644 --- a/TODO +++ b/TODO @@ -59,3 +59,9 @@ plot manifold 2D distances WER / fenetre tempo forme des courbes / medoids / gain en prevision: clust puis full --> enercast + +réduire taille 17519 trop long ? + +synchrone : sum +cwt : trim R part +// : clever by rows retenir cwt... diff --git a/epclust/DESCRIPTION b/epclust/DESCRIPTION index 300f86e..1c6c51d 100644 --- a/epclust/DESCRIPTION +++ b/epclust/DESCRIPTION @@ -36,9 +36,9 @@ Suggests: License: MIT + file LICENSE RoxygenNote: 6.0.1 Collate: - 'RcppExports.R' 'main.R' 'clustering.R' 'de_serialize.R' - 'a_NAMESPACE.R' + 'A_NAMESPACE.R' + 'RcppExports.R' 'plot.R' diff --git a/epclust/R/a_NAMESPACE.R b/epclust/R/A_NAMESPACE.R similarity index 100% rename from epclust/R/a_NAMESPACE.R rename to epclust/R/A_NAMESPACE.R diff --git a/epclust/R/clustering.R b/epclust/R/clustering.R index 7e06c43..c226786 100644 --- a/epclust/R/clustering.R +++ b/epclust/R/clustering.R @@ -121,35 +121,29 @@ computeSynchrones = function(medoids, getRefSeries, computeSynchronesChunk = function(indices) { - ref_series = getRefSeries(indices) - nb_series = nrow(ref_series) - if (parll) { require("bigmemory", quietly=TRUE) - require("synchronicity", quietly=TRUE) + requireNamespace("synchronicity", quietly=TRUE) require("epclust", quietly=TRUE) synchrones <- bigmemory::attach.big.matrix(synchrones_desc) + counts <- bigmemory::attach.big.matrix(counts_desc) medoids <- bigmemory::attach.big.matrix(medoids_desc) m <- synchronicity::attach.mutex(m_desc) } + ref_series = getRefSeries(indices) + nb_series = nrow(ref_series) + #get medoids indices for this chunk of series mi = computeMedoidsIndices(medoids@address, ref_series) -# #R-equivalent, requiring a matrix (thus potentially breaking "fit-in-memory" hope) -# mat_meds = medoids[,] -# mi = rep(NA,nb_series) -# for (i in 1:nb_series) -# mi[i] <- which.min( rowSums( sweep(mat_meds, 2, ref_series[i,], '-')^2 ) ) -# rm(mat_meds); gc() for (i in seq_len(nb_series)) { if (parll) synchronicity::lock(m) - synchrones[mi[i],] = synchrones[mi[i],] + ref_series[i,] -#TODO: remove counts - counts[mi[i],1] = counts[mi[i],1] + 1 + synchrones[ mi[i], ] = synchrones[ mi[i], ] + ref_series[i,] + counts[ mi[i] ] = counts[ mi[i] ] + 1 #TODO: remove counts? if (parll) synchronicity::unlock(m) } @@ -168,12 +162,11 @@ computeSynchrones = function(medoids, getRefSeries, m <- synchronicity::boost.mutex() m_desc <- synchronicity::describe(m) synchrones_desc = bigmemory::describe(synchrones) + counts_desc = bigmemory::describe(counts) medoids_desc = bigmemory::describe(medoids) - cl = parallel::makeCluster(ncores_clust) - parallel::clusterExport(cl, - varlist=c("synchrones_desc","counts","verbose","m_desc","medoids_desc","getRefSeries"), - envir=environment()) + parallel::clusterExport(cl, varlist=c("synchrones_desc","counts_desc","counts", + "verbose","m_desc","medoids_desc","getRefSeries"), envir=environment()) } indices_workers = .spreadIndices(seq_len(nb_ref_curves), nb_series_per_chunk) @@ -215,6 +208,15 @@ computeWerDists = function(synchrones, ncores_clust=1,verbose=FALSE,parll=TRUE) if (verbose) cat(paste("--- Compute WER dists\n", sep="")) + + + +#TODO: serializer les CWT, les récupérer via getDataInFile +#--> OK, faut juste stocker comme séries simples de taille delta*ncol (53*17519) + + + + n <- nrow(synchrones) delta <- ncol(synchrones) #TODO: automatic tune of all these parameters ? (for other users) @@ -232,16 +234,25 @@ computeWerDists = function(synchrones, ncores_clust=1,verbose=FALSE,parll=TRUE) totnoct = noctave + as.integer(s0log/nvoice) + 1 Xwer_dist <- bigmemory::big.matrix(nrow=n, ncol=n, type="double") - fcoefs = rep(1/3, 3) #moving average on 3 values # Generate n(n-1)/2 pairs for WER distances computations +# pairs = list() +# V = seq_len(n) +# for (i in 1:n) +# { +# V = V[-1] +# pairs = c(pairs, lapply(V, function(v) c(i,v))) +# } + # Generate "smart" pairs for WER distances computations pairs = list() - V = seq_len(n) - for (i in 1:n) + F = floor(2*n/3) + for (i in 1:F) + pairs = c(pairs, lapply((i+1):n, function(v) c(i,v))) + V = (F+1):n + for (i in (F+1):(n-1)) { V = V[-1] - pairs = c(pairs, lapply(V, function(v) c(i,v))) - } + pairs = c(pairs, # Distance between rows i and j computeDistancesIJ = function(pair) @@ -264,21 +275,21 @@ computeWerDists = function(synchrones, ncores_clust=1,verbose=FALSE,parll=TRUE) sqres <- sweep(ts.cwt,2,sqs,'*') sqres / max(Mod(sqres)) } -#browser() + i = pair[1] ; j = pair[2] if (verbose && j==i+1) cat(paste(" Distances (",i,",",j,"), (",i,",",j+1,") ...\n", sep="")) -print(system.time( { cwt_i <- computeCWT(i) - cwt_j <- computeCWT(j) } )) + cwt_i <- computeCWT(i) + cwt_j <- computeCWT(j) -print(system.time( { +#print(system.time( { num <- epclustFilter(Mod(cwt_i * Conj(cwt_j))) WX <- epclustFilter(Mod(cwt_i * Conj(cwt_i))) WY <- epclustFilter(Mod(cwt_j * Conj(cwt_j))) wer2 <- sum(colSums(num)^2) / sum(colSums(WX) * colSums(WY)) Xwer_dist[i,j] <- sqrt(delta * ncol(cwt_i) * max(1 - wer2, 0.)) #FIXME: wer2 should be < 1 Xwer_dist[j,i] <- Xwer_dist[i,j] -} ) ) +#} ) ) Xwer_dist[i,i] = 0. } @@ -291,7 +302,7 @@ print(system.time( { parallel::clusterExport(cl, varlist=c("synchrones_desc","Xwer_dist_desc","totnoct", "nvoice","w0","s0log","noctave","s0","verbose"), envir=environment()) } -browser() + ignored <- if (parll) parallel::parLapply(cl, pairs, computeDistancesIJ) @@ -300,7 +311,7 @@ browser() if (parll) parallel::stopCluster(cl) -#browser() + Xwer_dist[n,n] = 0. distances <- Xwer_dist[,] rm(Xwer_dist) ; gc() diff --git a/epclust/tests/testthat/test.Filter.R b/epclust/tests/testthat/test.Filter.R new file mode 100644 index 0000000..d94a5ac --- /dev/null +++ b/epclust/tests/testthat/test.Filter.R @@ -0,0 +1,8 @@ +TODO: test computeMedoids + filter +# #R-equivalent, requiring a matrix (thus potentially breaking "fit-in-memory" hope) +# mat_meds = medoids[,] +# mi = rep(NA,nb_series) +# for (i in 1:nb_series) +# mi[i] <- which.min( rowSums( sweep(mat_meds, 2, ref_series[i,], '-')^2 ) ) +# rm(mat_meds); gc() + diff --git a/epclust/tests/testthat/test.computeMedoidsIndices.R b/epclust/tests/testthat/test.computeMedoidsIndices.R new file mode 100644 index 0000000..be5b2b4 --- /dev/null +++ b/epclust/tests/testthat/test.computeMedoidsIndices.R @@ -0,0 +1,42 @@ +context("computeMedoidsIndices") + +test_that("serialization + getDataInFile retrieve original data / from matrix", +{ + data_bin_file = "/tmp/epclust_test_m.bin" + unlink(data_bin_file) + + #dataset 200 lignes / 30 columns + data_ascii = matrix(runif(200*30,-10,10),ncol=30) + nbytes = 4 #lead to a precision of 1e-7 / 1e-8 + endian = "little" + + #Simulate serialization in one single call + binarize(data_ascii, data_bin_file, 500, ",", nbytes, endian) + expect_equal(file.info(data_bin_file)$size, length(data_ascii)*nbytes+8) + for (indices in list(c(1,3,5), 3:13, c(5,20,50), c(75,130:135), 196:200)) + { + data_lines = getDataInFile(indices, data_bin_file, nbytes, endian) + expect_equal(data_lines, data_ascii[indices,], tolerance=1e-6) + } + unlink(data_bin_file) + + #...in several calls (last call complete, next call NULL) + for (i in 1:20) + binarize(data_ascii[((i-1)*10+1):(i*10),], data_bin_file, 20, ",", nbytes, endian) + expect_equal(file.info(data_bin_file)$size, length(data_ascii)*nbytes+8) + for (indices in list(c(1,3,5), 3:13, c(5,20,50), c(75,130:135), 196:200)) + { + data_lines = getDataInFile(indices, data_bin_file, nbytes, endian) + expect_equal(data_lines, data_ascii[indices,], tolerance=1e-6) + } + unlink(data_bin_file) +}) + +TODO: test computeMedoids + filter +# #R-equivalent, requiring a matrix (thus potentially breaking "fit-in-memory" hope) +# mat_meds = medoids[,] +# mi = rep(NA,nb_series) +# for (i in 1:nb_series) +# mi[i] <- which.min( rowSums( sweep(mat_meds, 2, ref_series[i,], '-')^2 ) ) +# rm(mat_meds); gc() +