From 1d881e3608c79be2497cd760df8c385d26647a12 Mon Sep 17 00:00:00 2001 From: Peter Munch Date: Tue, 3 Nov 2020 14:44:23 +0100 Subject: [PATCH] Introduce Utilities::MPI::mpi_processes_within_communicator --- include/deal.II/base/mpi.h | 8 ++++++ .../matrix_free/matrix_free.templates.h | 21 ++------------ source/base/mpi.cc | 28 +++++++++++++++++++ 3 files changed, 38 insertions(+), 19 deletions(-) diff --git a/include/deal.II/base/mpi.h b/include/deal.II/base/mpi.h index 19ae43725f..f04914626c 100644 --- a/include/deal.II/base/mpi.h +++ b/include/deal.II/base/mpi.h @@ -152,6 +152,14 @@ namespace Utilities unsigned int this_mpi_process(const MPI_Comm &mpi_communicator); + /** + * Return a vector of the ranks (within @p comm_large) of a subset of + * processes specified by @p comm_small. + */ + const std::vector + mpi_processes_within_communicator(const MPI_Comm &comm_large, + const MPI_Comm &comm_small); + /** * Consider an unstructured communication pattern where every process in * an MPI universe wants to send some data to a subset of the other diff --git a/include/deal.II/matrix_free/matrix_free.templates.h b/include/deal.II/matrix_free/matrix_free.templates.h index 6933925da7..f7b58e7526 100644 --- a/include/deal.II/matrix_free/matrix_free.templates.h +++ b/include/deal.II/matrix_free/matrix_free.templates.h @@ -1495,26 +1495,9 @@ MatrixFree::initialize_indices( if (is_non_buffering_sm_supported) { - // TODO: move into Utilities::MPI - const auto procs_of_sm = [](const MPI_Comm &comm, - const MPI_Comm &comm_shared) { - if (Utilities::MPI::job_supports_mpi() == false) - return std::vector{0}; - - const unsigned int rank = Utilities::MPI::this_mpi_process(comm); - const unsigned int size = - Utilities::MPI::n_mpi_processes(comm_shared); - - std::vector ranks(size); - MPI_Allgather( - &rank, 1, MPI_UNSIGNED, ranks.data(), 1, MPI_UNSIGNED, comm_shared); - - return ranks; - }; - // gather the ranks of the shared-memory domain - const std::vector sm_procs = - procs_of_sm(task_info.communicator, communicator_sm); + const auto sm_procs = Utilities::MPI::mpi_processes_within_communicator( + task_info.communicator, communicator_sm); // get my rank within the shared-memory domain const unsigned int my_sm_pid = std::distance( diff --git a/source/base/mpi.cc b/source/base/mpi.cc index 269aea2e59..41b0a37c39 100644 --- a/source/base/mpi.cc +++ b/source/base/mpi.cc @@ -135,6 +135,26 @@ namespace Utilities } + + const std::vector + mpi_processes_within_communicator(const MPI_Comm &comm_large, + const MPI_Comm &comm_small) + { + if (Utilities::MPI::job_supports_mpi() == false) + return std::vector{0}; + + const unsigned int rank = Utilities::MPI::this_mpi_process(comm_large); + const unsigned int size = Utilities::MPI::n_mpi_processes(comm_small); + + std::vector ranks(size); + MPI_Allgather( + &rank, 1, MPI_UNSIGNED, ranks.data(), 1, MPI_UNSIGNED, comm_small); + + return ranks; + } + + + MPI_Comm duplicate_communicator(const MPI_Comm &mpi_communicator) { @@ -686,6 +706,14 @@ namespace Utilities + const std::vector + mpi_processes_within_communicator(const MPI_Comm &, const MPI_Comm &) + { + return std::vector{0}; + } + + + std::vector create_ascending_partitioning(const MPI_Comm & /*comm*/, const IndexSet::size_type local_size) -- 2.39.5