From 640cab7c4d286701e489e2a6ee362c7a8c373f04 Mon Sep 17 00:00:00 2001 From: Benjamin Brands Date: Mon, 29 Jan 2018 21:17:46 +0100 Subject: [PATCH] separated tests for copying sub-matrices --- include/deal.II/lac/scalapack.h | 24 +++-- tests/scalapack/scalapack_07.cc | 15 --- tests/scalapack/scalapack_11.cc | 91 +++++++++++++++++++ tests/scalapack/scalapack_11.mpirun=1.output | 36 ++++++++ tests/scalapack/scalapack_11.mpirun=11.output | 36 ++++++++ tests/scalapack/scalapack_11.mpirun=9.output | 36 ++++++++ 6 files changed, 213 insertions(+), 25 deletions(-) create mode 100644 tests/scalapack/scalapack_11.cc create mode 100644 tests/scalapack/scalapack_11.mpirun=1.output create mode 100644 tests/scalapack/scalapack_11.mpirun=11.output create mode 100644 tests/scalapack/scalapack_11.mpirun=9.output diff --git a/include/deal.II/lac/scalapack.h b/include/deal.II/lac/scalapack.h index dbfd1c7abc..42468c81fc 100644 --- a/include/deal.II/lac/scalapack.h +++ b/include/deal.II/lac/scalapack.h @@ -169,14 +169,14 @@ public: * Copy a submatrix (subset) of the distributed matrix A to a submatrix of the distributed matrix @p B. * * - The global row and column index of the first element of the submatrix A is provided by @p offset_A - * with row index=offset_A.first and column index=offset_A.second + * with row index=offset_A.first and column index=offset_A.second. * - * - The global row and column index of the first element of the submatrix B is provided by @p offset_B. - * with row index=offset_B.first and column index=offset_B.second + * - The global row and column index of the first element of the submatrix B is provided by @p offset_B + * with row index=offset_B.first and column index=offset_B.second. * * - The dimension of the submatrix to be copied is given by @p submatrix_size - * with number of rows=submatrix_size.first and number of columns=submatrix_size.second - * . + * with number of rows=submatrix_size.first and number of columns=submatrix_size.second. + * * * If it is necessary to copy complete matrices with an identical block-cyclic distribution, * use copy_to(ScaLAPACKMatrix &dest) with only one argument to avoid communication. @@ -191,8 +191,10 @@ public: /** * Stores the distributed matrix in @p filename using HDF5. + * In case that deal.II was built without HDF5 + * a call to this function will cause an exception to be thrown. * - * If HDF5 was build with MPI, parallel I/O is used to save the matrix. + * If HDF5 was built with MPI, parallel I/O is used to save the matrix. * Otherwise, just one process will do the output. This means that * internally the distributed matrix is copied to one process, which * does the output. Therefore, the matrix has to fit into the memory @@ -202,6 +204,8 @@ public: /** * Loads the distributed matrix from file @p filename using HDF5. + * In case that deal.II was built without HDF5 + * a call to this function will cause an exception to be thrown. * * The matrix must have the same dimensions as the matrix stored in the file. * @@ -290,12 +294,12 @@ public: * $\min \Vert B - A*X\Vert$. * * Upon exit the rows 0 to N-1 of $B$ contain the least square solution vectors. The residual sum of squares - * for each column is given by the sum of squares of elements N to M-1 in that column + * for each column is given by the sum of squares of elements N to M-1 in that column. * * - 2. If transpose==false and $M < N$: find minimum norm solutions of underdetermined systems * $A * X = B$. * - * Upon exit the columns of $B$ contain the minimum norm solution vectors + * Upon exit the columns of $B$ contain the minimum norm solution vectors. * * - 3. If transpose==true and $M \geq N$: find minimum norm solutions of underdetermined system * $ A^\top X = B$. @@ -306,8 +310,8 @@ public: * $\min \Vert B - A^\top X\Vert$. * * Upon exit the rows 0 to M-1 contain the least square solution vectors. The residual sum of squares - * for each column is given by the sum of squares of elements M to N-1 in that column - * . + * for each column is given by the sum of squares of elements M to N-1 in that column. + * * If transpose==false then $B \in \mathbb{R}^{M \times N_{\rm RHS}}$, * otherwise $B \in \mathbb{R}^{N \times N_{\rm RHS}}}$. * The matrices $A$ and $B$ must have an identical block cyclic distribution for rows and columns. diff --git a/tests/scalapack/scalapack_07.cc b/tests/scalapack/scalapack_07.cc index 83dc712be2..5eae020904 100644 --- a/tests/scalapack/scalapack_07.cc +++ b/tests/scalapack/scalapack_07.cc @@ -17,7 +17,6 @@ #include "../lapack/create_matrix.h" // test copying of distributed ScaLAPACKMatrices using ScaLAPACK routine p_gemr2d -// test copying submatrices of distributed ScaLAPACKMatrices using ScaLAPACK routine p_gemr2d #include #include @@ -90,20 +89,6 @@ void test(const unsigned int block_size_i, const unsigned int block_size_j) AssertThrow(test_2d.frobenius_norm() < 1e-12,ExcInternalError()); AssertThrow(test_1d.frobenius_norm() < 1e-12,ExcInternalError()); AssertThrow(test_one.frobenius_norm() < 1e-12,ExcInternalError()); - - //copying submatrices - unsigned int sub_size=100; - std::pair offset_A = std::make_pair(49,99); - std::pair offset_B = std::make_pair(4,7); - std::pair submatrix_size = std::make_pair(sub_size,sub_size); - ScaLAPACKMatrix scalapack_matrix_dest(sub_size+offset_B.first,sub_size+offset_B.second,grid_2d,block_size_j,block_size_i); - scalapack_matrix_2d.copy_to(scalapack_matrix_dest,offset_A,offset_B,submatrix_size); - FullMatrix dest (sub_size+offset_B.first,sub_size+offset_B.second); - scalapack_matrix_dest.copy_to(dest); - for (unsigned int i=0; i +#include +#include +#include +#include + +#include + +#include +#include + + +template +void test(const unsigned int block_size_i, const unsigned int block_size_j) +{ + MPI_Comm mpi_communicator(MPI_COMM_WORLD); + const unsigned int n_mpi_processes(Utilities::MPI::n_mpi_processes(mpi_communicator)); + const unsigned int this_mpi_process(Utilities::MPI::this_mpi_process(mpi_communicator)); + + ConditionalOStream pcout (std::cout, (this_mpi_process ==0)); + + const unsigned int size = 500; + //create FullMatrix and fill it + FullMatrix full(size); + unsigned int count=0; + for (unsigned int i=0; i grid = std::make_shared(mpi_communicator,size,size,block_size_i,block_size_i); + + ScaLAPACKMatrix scalapack_matrix (size, size, grid, block_size_i, block_size_i); + + pcout << "2D grid matrix: dim=" << scalapack_matrix.m() << "x" << scalapack_matrix.n() + << "; blocks=" << block_size_i << "x" << block_size_i + << "; grid=" << grid->get_process_grid_rows() << "x" << grid->get_process_grid_columns() << std::endl << std::endl; + + scalapack_matrix = full; + unsigned int sub_size=100; + std::pair offset_A = std::make_pair(49,99); + std::pair offset_B = std::make_pair(4,7); + std::pair submatrix_size = std::make_pair(sub_size,sub_size); + ScaLAPACKMatrix scalapack_matrix_dest(sub_size+offset_B.first,sub_size+offset_B.second,grid,block_size_j,block_size_i); + scalapack_matrix.copy_to(scalapack_matrix_dest,offset_A,offset_B,submatrix_size); + FullMatrix dest (sub_size+offset_B.first,sub_size+offset_B.second); + scalapack_matrix_dest.copy_to(dest); + + for (unsigned int i=0; i blocks_i = {{16,32,64}}; + const std::vector blocks_j = {{16,32,64}}; + + for (const auto &s : blocks_i) + for (const auto &b : blocks_j) + test(s,b); + + for (const auto &s : blocks_i) + for (const auto &b : blocks_j) + test(s,b); +} diff --git a/tests/scalapack/scalapack_11.mpirun=1.output b/tests/scalapack/scalapack_11.mpirun=1.output new file mode 100644 index 0000000000..b483cc9d3e --- /dev/null +++ b/tests/scalapack/scalapack_11.mpirun=1.output @@ -0,0 +1,36 @@ +2D grid matrix: dim=500x500; blocks=16x16; grid=1x1 + +2D grid matrix: dim=500x500; blocks=16x16; grid=1x1 + +2D grid matrix: dim=500x500; blocks=16x16; grid=1x1 + +2D grid matrix: dim=500x500; blocks=32x32; grid=1x1 + +2D grid matrix: dim=500x500; blocks=32x32; grid=1x1 + +2D grid matrix: dim=500x500; blocks=32x32; grid=1x1 + +2D grid matrix: dim=500x500; blocks=64x64; grid=1x1 + +2D grid matrix: dim=500x500; blocks=64x64; grid=1x1 + +2D grid matrix: dim=500x500; blocks=64x64; grid=1x1 + +2D grid matrix: dim=500x500; blocks=16x16; grid=1x1 + +2D grid matrix: dim=500x500; blocks=16x16; grid=1x1 + +2D grid matrix: dim=500x500; blocks=16x16; grid=1x1 + +2D grid matrix: dim=500x500; blocks=32x32; grid=1x1 + +2D grid matrix: dim=500x500; blocks=32x32; grid=1x1 + +2D grid matrix: dim=500x500; blocks=32x32; grid=1x1 + +2D grid matrix: dim=500x500; blocks=64x64; grid=1x1 + +2D grid matrix: dim=500x500; blocks=64x64; grid=1x1 + +2D grid matrix: dim=500x500; blocks=64x64; grid=1x1 + diff --git a/tests/scalapack/scalapack_11.mpirun=11.output b/tests/scalapack/scalapack_11.mpirun=11.output new file mode 100644 index 0000000000..73f104ff40 --- /dev/null +++ b/tests/scalapack/scalapack_11.mpirun=11.output @@ -0,0 +1,36 @@ +2D grid matrix: dim=500x500; blocks=16x16; grid=3x3 + +2D grid matrix: dim=500x500; blocks=16x16; grid=3x3 + +2D grid matrix: dim=500x500; blocks=16x16; grid=3x3 + +2D grid matrix: dim=500x500; blocks=32x32; grid=3x3 + +2D grid matrix: dim=500x500; blocks=32x32; grid=3x3 + +2D grid matrix: dim=500x500; blocks=32x32; grid=3x3 + +2D grid matrix: dim=500x500; blocks=64x64; grid=3x3 + +2D grid matrix: dim=500x500; blocks=64x64; grid=3x3 + +2D grid matrix: dim=500x500; blocks=64x64; grid=3x3 + +2D grid matrix: dim=500x500; blocks=16x16; grid=3x3 + +2D grid matrix: dim=500x500; blocks=16x16; grid=3x3 + +2D grid matrix: dim=500x500; blocks=16x16; grid=3x3 + +2D grid matrix: dim=500x500; blocks=32x32; grid=3x3 + +2D grid matrix: dim=500x500; blocks=32x32; grid=3x3 + +2D grid matrix: dim=500x500; blocks=32x32; grid=3x3 + +2D grid matrix: dim=500x500; blocks=64x64; grid=3x3 + +2D grid matrix: dim=500x500; blocks=64x64; grid=3x3 + +2D grid matrix: dim=500x500; blocks=64x64; grid=3x3 + diff --git a/tests/scalapack/scalapack_11.mpirun=9.output b/tests/scalapack/scalapack_11.mpirun=9.output new file mode 100644 index 0000000000..73f104ff40 --- /dev/null +++ b/tests/scalapack/scalapack_11.mpirun=9.output @@ -0,0 +1,36 @@ +2D grid matrix: dim=500x500; blocks=16x16; grid=3x3 + +2D grid matrix: dim=500x500; blocks=16x16; grid=3x3 + +2D grid matrix: dim=500x500; blocks=16x16; grid=3x3 + +2D grid matrix: dim=500x500; blocks=32x32; grid=3x3 + +2D grid matrix: dim=500x500; blocks=32x32; grid=3x3 + +2D grid matrix: dim=500x500; blocks=32x32; grid=3x3 + +2D grid matrix: dim=500x500; blocks=64x64; grid=3x3 + +2D grid matrix: dim=500x500; blocks=64x64; grid=3x3 + +2D grid matrix: dim=500x500; blocks=64x64; grid=3x3 + +2D grid matrix: dim=500x500; blocks=16x16; grid=3x3 + +2D grid matrix: dim=500x500; blocks=16x16; grid=3x3 + +2D grid matrix: dim=500x500; blocks=16x16; grid=3x3 + +2D grid matrix: dim=500x500; blocks=32x32; grid=3x3 + +2D grid matrix: dim=500x500; blocks=32x32; grid=3x3 + +2D grid matrix: dim=500x500; blocks=32x32; grid=3x3 + +2D grid matrix: dim=500x500; blocks=64x64; grid=3x3 + +2D grid matrix: dim=500x500; blocks=64x64; grid=3x3 + +2D grid matrix: dim=500x500; blocks=64x64; grid=3x3 + -- 2.39.5