From f802edd91668c00e68e194c986e7443b34c642bc Mon Sep 17 00:00:00 2001 From: benjamin Date: Thu, 4 Oct 2018 14:30:08 +0200 Subject: [PATCH] adapt block size in save_parallel and load_parallel --- source/lac/scalapack.cc | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/source/lac/scalapack.cc b/source/lac/scalapack.cc index 8067646a20..5d301e3b36 100644 --- a/source/lac/scalapack.cc +++ b/source/lac/scalapack.cc @@ -2739,7 +2739,22 @@ ScaLAPACKMatrix::save_parallel( 1, n_mpi_processes); - const int MB = n_rows, NB = std::ceil((double)n_columns / n_mpi_processes); + const int MB = n_rows; + /* + * If the ratio n_columns/n_mpi_processes is smaller than the column block + * size of the original matrix, the redistribution and saving of the matrix + * requires a significant amount of MPI communication. Therefore, it is better + * to set a minimum value for the block size NB, causing only + * ceil(n_columns/NB) processes being actively involved in saving the matrix. + * Example: A 2*10^9 x 400 matrix is distributed on a 80 x 5 process grid + * using block size 32. Instead of distributing the matrix on a 1 x 400 + * process grid with a row block size of 2*10^9 and a column block size of 1, + * the minimum value for NB yields that only ceil(400/32)=13 processes will be + * writing the matrix to disk. + */ + const int NB = std::max((int)std::ceil((double)n_columns / n_mpi_processes), + column_block_size); + ScaLAPACKMatrix tmp(n_rows, n_columns, column_grid, MB, NB); copy_to(tmp); @@ -3155,7 +3170,10 @@ ScaLAPACKMatrix::load_parallel(const char *filename) 1, n_mpi_processes); - const int MB = n_rows, NB = std::ceil((double)n_columns / n_mpi_processes); + const int MB = n_rows; + // for the choice of NB see explanation in save_parallel() + const int NB = std::max((int)std::ceil((double)n_columns / n_mpi_processes), + column_block_size); ScaLAPACKMatrix tmp(n_rows, n_columns, column_grid, MB, NB); // get pointer to data held by the process -- 2.39.5