From 8d7326576c70785eaaaf48b3258a5aa290eef6ed Mon Sep 17 00:00:00 2001 From: Timo Heister Date: Sat, 12 Feb 2022 10:10:24 -0500 Subject: [PATCH] MPI: move mpi_type_id() out of internal Move mpi_type_id() into Utilities::MPI because it is useful for projects using deal.II. Also introduce a fake template for doxygen. --- include/deal.II/base/mpi.h | 157 +++++++++++++++++- include/deal.II/base/mpi.templates.h | 140 +--------------- .../mpi_noncontiguous_partitioner.templates.h | 4 +- .../mg_transfer_global_coarsening.templates.h | 7 +- source/base/partitioner.cc | 13 +- source/base/process_grid.cc | 11 +- .../repartitioning_policy_tools.cc | 17 +- source/distributed/tria_base.cc | 27 ++- source/lac/scalapack.cc | 4 +- source/matrix_free/vector_data_exchange.cc | 25 +-- tests/mpi/renumber_cuthill_mckee.cc | 6 +- tests/mpi/renumber_cuthill_mckee_02.cc | 6 +- 12 files changed, 213 insertions(+), 204 deletions(-) diff --git a/include/deal.II/base/mpi.h b/include/deal.II/base/mpi.h index 6c99d8867c..bd5c60ed22 100644 --- a/include/deal.II/base/mpi.h +++ b/include/deal.II/base/mpi.h @@ -139,6 +139,22 @@ namespace Utilities */ namespace MPI { +#ifdef DOXYGEN + /** + * Given a pointer to an object of class T, return the matching + * `MPI_Datatype` to be used for MPI communication. + * + * As an example, passing an `int*` to this function returns `MPI_INT`. + * + * @note In reality, these functions are not template functions templated + * on the parameter T, but free standing inline function overloads. This + * templated version only exists so that it shows up in the documentation. + */ + template + MPI_Datatype + mpi_type_id(const T *); +#endif + /** * Return the number of MPI processes there exist in the given * @ref GlossMPICommunicator "communicator" @@ -1279,17 +1295,154 @@ namespace Utilities compute_set_union(const std::set &set, const MPI_Comm &comm); + #ifndef DOXYGEN - // declaration for an internal function that lives in mpi.templates.h + + /* --------------------------- inline functions ------------------------- */ + +# ifdef DEAL_II_WITH_MPI + inline MPI_Datatype + mpi_type_id(const bool *) + { +# if DEAL_II_MPI_VERSION_GTE(2, 2) + return MPI_CXX_BOOL; +# else + return MPI_C_BOOL; +# endif + } + + + + inline MPI_Datatype + mpi_type_id(const char *) + { + return MPI_CHAR; + } + + + + inline MPI_Datatype + mpi_type_id(const signed char *) + { + return MPI_SIGNED_CHAR; + } + + + + inline MPI_Datatype + mpi_type_id(const short *) + { + return MPI_SHORT; + } + + + + inline MPI_Datatype + mpi_type_id(const int *) + { + return MPI_INT; + } + + + + inline MPI_Datatype + mpi_type_id(const long int *) + { + return MPI_LONG; + } + + + + inline MPI_Datatype + mpi_type_id(const unsigned char *) + { + return MPI_UNSIGNED_CHAR; + } + + + + inline MPI_Datatype + mpi_type_id(const unsigned short *) + { + return MPI_UNSIGNED_SHORT; + } + + + + inline MPI_Datatype + mpi_type_id(const unsigned int *) + { + return MPI_UNSIGNED; + } + + + + inline MPI_Datatype + mpi_type_id(const unsigned long int *) + { + return MPI_UNSIGNED_LONG; + } + + + + inline MPI_Datatype + mpi_type_id(const unsigned long long int *) + { + return MPI_UNSIGNED_LONG_LONG; + } + + + + inline MPI_Datatype + mpi_type_id(const float *) + { + return MPI_FLOAT; + } + + + + inline MPI_Datatype + mpi_type_id(const double *) + { + return MPI_DOUBLE; + } + + + + inline MPI_Datatype + mpi_type_id(const long double *) + { + return MPI_LONG_DOUBLE; + } + + + + inline MPI_Datatype + mpi_type_id(const std::complex *) + { + return MPI_COMPLEX; + } + + + + inline MPI_Datatype + mpi_type_id(const std::complex *) + { + return MPI_DOUBLE_COMPLEX; + } +# endif + + namespace internal { + // declaration for an internal function that lives in mpi.templates.h template void all_reduce(const MPI_Op & mpi_op, const ArrayView &values, const MPI_Comm & mpi_communicator, const ArrayView & output); - } + } // namespace internal diff --git a/include/deal.II/base/mpi.templates.h b/include/deal.II/base/mpi.templates.h index fe5d640253..ab5b455752 100644 --- a/include/deal.II/base/mpi.templates.h +++ b/include/deal.II/base/mpi.templates.h @@ -39,142 +39,6 @@ namespace Utilities { namespace internal { -#ifdef DEAL_II_WITH_MPI - /** - * Return the corresponding MPI data type id for the argument given. - */ - inline MPI_Datatype - mpi_type_id(const bool *) - { -# if DEAL_II_MPI_VERSION_GTE(2, 2) - return MPI_CXX_BOOL; -# else - return MPI_C_BOOL; -# endif - } - - - - inline MPI_Datatype - mpi_type_id(const char *) - { - return MPI_CHAR; - } - - - - inline MPI_Datatype - mpi_type_id(const signed char *) - { - return MPI_SIGNED_CHAR; - } - - - - inline MPI_Datatype - mpi_type_id(const short *) - { - return MPI_SHORT; - } - - - - inline MPI_Datatype - mpi_type_id(const int *) - { - return MPI_INT; - } - - - - inline MPI_Datatype - mpi_type_id(const long int *) - { - return MPI_LONG; - } - - - - inline MPI_Datatype - mpi_type_id(const unsigned char *) - { - return MPI_UNSIGNED_CHAR; - } - - - - inline MPI_Datatype - mpi_type_id(const unsigned short *) - { - return MPI_UNSIGNED_SHORT; - } - - - - inline MPI_Datatype - mpi_type_id(const unsigned int *) - { - return MPI_UNSIGNED; - } - - - - inline MPI_Datatype - mpi_type_id(const unsigned long int *) - { - return MPI_UNSIGNED_LONG; - } - - - - inline MPI_Datatype - mpi_type_id(const unsigned long long int *) - { - return MPI_UNSIGNED_LONG_LONG; - } - - - - inline MPI_Datatype - mpi_type_id(const float *) - { - return MPI_FLOAT; - } - - - - inline MPI_Datatype - mpi_type_id(const double *) - { - return MPI_DOUBLE; - } - - - - inline MPI_Datatype - mpi_type_id(const long double *) - { - return MPI_LONG_DOUBLE; - } - - - - inline MPI_Datatype - mpi_type_id(const std::complex *) - { - return MPI_COMPLEX; - } - - - - inline MPI_Datatype - mpi_type_id(const std::complex *) - { - return MPI_DOUBLE_COMPLEX; - } -#endif - - template void all_reduce(const MPI_Op & mpi_op, @@ -221,7 +85,7 @@ namespace Utilities MPI_IN_PLACE, static_cast(output.data()), static_cast(values.size()), - internal::mpi_type_id(values.data()), + mpi_type_id(values.data()), mpi_op, mpi_communicator); AssertThrowMPI(ierr); @@ -261,7 +125,7 @@ namespace Utilities MPI_IN_PLACE, static_cast(output.data()), static_cast(values.size() * 2), - internal::mpi_type_id(static_cast(nullptr)), + mpi_type_id(static_cast(nullptr)), mpi_op, mpi_communicator); AssertThrowMPI(ierr); diff --git a/include/deal.II/base/mpi_noncontiguous_partitioner.templates.h b/include/deal.II/base/mpi_noncontiguous_partitioner.templates.h index a12f00b0fa..cb4b6f286c 100644 --- a/include/deal.II/base/mpi_noncontiguous_partitioner.templates.h +++ b/include/deal.II/base/mpi_noncontiguous_partitioner.templates.h @@ -109,7 +109,7 @@ namespace Utilities const int ierr = MPI_Irecv(buffers.data() + recv_ptr[i], recv_ptr[i + 1] - recv_ptr[i], - Utilities::MPI::internal::mpi_type_id(buffers.data()), + Utilities::MPI::mpi_type_id(buffers.data()), recv_ranks[i], tag, communicator, @@ -138,7 +138,7 @@ namespace Utilities const int ierr = MPI_Isend(buffers.data() + send_ptr[i], send_ptr[i + 1] - send_ptr[i], - Utilities::MPI::internal::mpi_type_id(buffers.data()), + Utilities::MPI::mpi_type_id(buffers.data()), send_ranks[i], tag, communicator, diff --git a/include/deal.II/multigrid/mg_transfer_global_coarsening.templates.h b/include/deal.II/multigrid/mg_transfer_global_coarsening.templates.h index 576bced433..38aad10c13 100644 --- a/include/deal.II/multigrid/mg_transfer_global_coarsening.templates.h +++ b/include/deal.II/multigrid/mg_transfer_global_coarsening.templates.h @@ -572,7 +572,7 @@ namespace internal const auto ierr_1 = MPI_Isend( buffer.data(), buffer.size(), - Utilities::MPI::internal::mpi_type_id(buffer.data()), + Utilities::MPI::mpi_type_id(buffer.data()), i.first, Utilities::MPI::internal::Tags::fine_dof_handler_view_reinit, communicator, @@ -641,8 +641,7 @@ namespace internal int message_length; const int ierr_2 = MPI_Get_count(&status, - Utilities::MPI::internal::mpi_type_id( - buffer.data()), + Utilities::MPI::mpi_type_id(buffer.data()), &message_length); AssertThrowMPI(ierr_2); @@ -651,7 +650,7 @@ namespace internal const int ierr_3 = MPI_Recv( buffer.data(), buffer.size(), - Utilities::MPI::internal::mpi_type_id(buffer.data()), + Utilities::MPI::mpi_type_id(buffer.data()), status.MPI_SOURCE, Utilities::MPI::internal::Tags::fine_dof_handler_view_reinit, communicator, diff --git a/source/base/partitioner.cc b/source/base/partitioner.cc index 4d325cc905..f1b87c5d7e 100644 --- a/source/base/partitioner.cc +++ b/source/base/partitioner.cc @@ -76,13 +76,12 @@ namespace Utilities types::global_dof_index prefix_sum = 0; #ifdef DEAL_II_WITH_MPI - const int ierr = - MPI_Exscan(&local_size, - &prefix_sum, - 1, - Utilities::MPI::internal::mpi_type_id(&prefix_sum), - MPI_SUM, - communicator); + const int ierr = MPI_Exscan(&local_size, + &prefix_sum, + 1, + Utilities::MPI::mpi_type_id(&prefix_sum), + MPI_SUM, + communicator); AssertThrowMPI(ierr); #endif diff --git a/source/base/process_grid.cc b/source/base/process_grid.cc index 18b7905bbc..692c5898dd 100644 --- a/source/base/process_grid.cc +++ b/source/base/process_grid.cc @@ -247,12 +247,11 @@ namespace Utilities Assert(count > 0, ExcInternalError()); if (mpi_communicator_inactive_with_root != MPI_COMM_NULL) { - const int ierr = - MPI_Bcast(value, - count, - Utilities::MPI::internal::mpi_type_id(value), - 0 /*from root*/, - mpi_communicator_inactive_with_root); + const int ierr = MPI_Bcast(value, + count, + Utilities::MPI::mpi_type_id(value), + 0 /*from root*/, + mpi_communicator_inactive_with_root); AssertThrowMPI(ierr); } } diff --git a/source/distributed/repartitioning_policy_tools.cc b/source/distributed/repartitioning_policy_tools.cc index 135c8abbb6..e4a7e0c8d4 100644 --- a/source/distributed/repartitioning_policy_tools.cc +++ b/source/distributed/repartitioning_policy_tools.cc @@ -89,7 +89,7 @@ namespace RepartitioningPolicyTools const int ierr = MPI_Exscan(&process_has_active_locally_owned_cells, &offset, 1, - Utilities::MPI::internal::mpi_type_id( + Utilities::MPI::mpi_type_id( &process_has_active_locally_owned_cells), MPI_SUM, comm); @@ -308,13 +308,12 @@ namespace RepartitioningPolicyTools // determine partial sum of weights of this process uint64_t process_local_weight_offset = 0; - int ierr = - MPI_Exscan(&process_local_weight, - &process_local_weight_offset, - 1, - Utilities::MPI::internal::mpi_type_id(&process_local_weight), - MPI_SUM, - tria->get_communicator()); + int ierr = MPI_Exscan(&process_local_weight, + &process_local_weight_offset, + 1, + Utilities::MPI::mpi_type_id(&process_local_weight), + MPI_SUM, + tria->get_communicator()); AssertThrowMPI(ierr); // total weight of all processes @@ -322,7 +321,7 @@ namespace RepartitioningPolicyTools ierr = MPI_Bcast(&total_weight, 1, - Utilities::MPI::internal::mpi_type_id(&total_weight), + Utilities::MPI::mpi_type_id(&total_weight), n_subdomains - 1, mpi_communicator); AssertThrowMPI(ierr); diff --git a/source/distributed/tria_base.cc b/source/distributed/tria_base.cc index 94a82a4c88..18736bd4f2 100644 --- a/source/distributed/tria_base.cc +++ b/source/distributed/tria_base.cc @@ -427,7 +427,7 @@ namespace parallel MPI_Exscan(&n_locally_owned_cells, &cell_index, 1, - Utilities::MPI::internal::mpi_type_id(&n_locally_owned_cells), + Utilities::MPI::mpi_type_id(&n_locally_owned_cells), MPI_SUM, this->mpi_communicator); AssertThrowMPI(ierr); @@ -493,13 +493,13 @@ namespace parallel std::vector cell_index( this->n_global_levels(), 0); - int ierr = MPI_Exscan(n_locally_owned_cells.data(), - cell_index.data(), - this->n_global_levels(), - Utilities::MPI::internal::mpi_type_id( - n_locally_owned_cells.data()), - MPI_SUM, - this->mpi_communicator); + int ierr = + MPI_Exscan(n_locally_owned_cells.data(), + cell_index.data(), + this->n_global_levels(), + Utilities::MPI::mpi_type_id(n_locally_owned_cells.data()), + MPI_SUM, + this->mpi_communicator); AssertThrowMPI(ierr); // 3) determine global number of "active" cells on each level @@ -509,12 +509,11 @@ namespace parallel for (unsigned int l = 0; l < this->n_global_levels(); ++l) n_cells_level[l] = n_locally_owned_cells[l] + cell_index[l]; - ierr = - MPI_Bcast(n_cells_level.data(), - this->n_global_levels(), - Utilities::MPI::internal::mpi_type_id(n_cells_level.data()), - this->n_subdomains - 1, - this->mpi_communicator); + ierr = MPI_Bcast(n_cells_level.data(), + this->n_global_levels(), + Utilities::MPI::mpi_type_id(n_cells_level.data()), + this->n_subdomains - 1, + this->mpi_communicator); AssertThrowMPI(ierr); // 4) give global indices to locally-owned cells on level and mark diff --git a/source/lac/scalapack.cc b/source/lac/scalapack.cc index 3ac33789b6..84465baeb8 100644 --- a/source/lac/scalapack.cc +++ b/source/lac/scalapack.cc @@ -184,14 +184,14 @@ ScaLAPACKMatrix::ScaLAPACKMatrix( } int ierr = MPI_Bcast(&n_rows, 1, - Utilities::MPI::internal::mpi_type_id(&n_rows), + Utilities::MPI::mpi_type_id(&n_rows), 0 /*from root*/, process_grid->mpi_communicator); AssertThrowMPI(ierr); ierr = MPI_Bcast(&n_columns, 1, - Utilities::MPI::internal::mpi_type_id(&n_columns), + Utilities::MPI::mpi_type_id(&n_columns), 0 /*from root*/, process_grid->mpi_communicator); AssertThrowMPI(ierr); diff --git a/source/matrix_free/vector_data_exchange.cc b/source/matrix_free/vector_data_exchange.cc index c8adede572..afb8ab83bb 100644 --- a/source/matrix_free/vector_data_exchange.cc +++ b/source/matrix_free/vector_data_exchange.cc @@ -896,7 +896,7 @@ namespace internal const int ierr = MPI_Irecv(buffer.data() + ghost_targets_data[i][1] + offset, ghost_targets_data[i][2], - Utilities::MPI::internal::mpi_type_id(buffer.data()), + Utilities::MPI::mpi_type_id(buffer.data()), ghost_targets_data[i][0], communication_channel + 1, comm, @@ -920,7 +920,7 @@ namespace internal const int ierr = MPI_Isend(temporary_storage.data() + import_targets_data[i][1], import_targets_data[i][2], - Utilities::MPI::internal::mpi_type_id(data_this.data()), + Utilities::MPI::mpi_type_id(data_this.data()), import_targets_data[i][0], communication_channel + 1, comm, @@ -1173,7 +1173,7 @@ namespace internal const int ierr = MPI_Isend(buffer.data() + ghost_targets_data[i][1], ghost_targets_data[i][2], - Utilities::MPI::internal::mpi_type_id(buffer.data()), + Utilities::MPI::mpi_type_id(buffer.data()), ghost_targets_data[i][0], communication_channel + 0, comm, @@ -1184,15 +1184,16 @@ namespace internal for (unsigned int i = 0; i < import_targets_data.size(); ++i) { - const int ierr = MPI_Irecv( - temporary_storage.data() + import_targets_data[i][1], - import_targets_data[i][2], - Utilities::MPI::internal::mpi_type_id(temporary_storage.data()), - import_targets_data[i][0], - communication_channel + 0, - comm, - requests.data() + sm_ghost_ranks.size() + sm_import_ranks.size() + - ghost_targets_data.size() + i); + const int ierr = + MPI_Irecv(temporary_storage.data() + import_targets_data[i][1], + import_targets_data[i][2], + Utilities::MPI::mpi_type_id(temporary_storage.data()), + import_targets_data[i][0], + communication_channel + 0, + comm, + requests.data() + sm_ghost_ranks.size() + + sm_import_ranks.size() + ghost_targets_data.size() + + i); AssertThrowMPI(ierr); } #endif diff --git a/tests/mpi/renumber_cuthill_mckee.cc b/tests/mpi/renumber_cuthill_mckee.cc index 83c506f7c0..750c664a85 100644 --- a/tests/mpi/renumber_cuthill_mckee.cc +++ b/tests/mpi/renumber_cuthill_mckee.cc @@ -119,16 +119,14 @@ test() if (myid == i) MPI_Send(&renumbering[0], renumbering.size(), - Utilities::MPI::internal::mpi_type_id( - &complete_renumbering[0]), + Utilities::MPI::mpi_type_id(&complete_renumbering[0]), 0, i, MPI_COMM_WORLD); else if (myid == 0) MPI_Recv(&complete_renumbering[offset], dofs_per_proc[i].n_elements(), - Utilities::MPI::internal::mpi_type_id( - &complete_renumbering[0]), + Utilities::MPI::mpi_type_id(&complete_renumbering[0]), i, i, MPI_COMM_WORLD, diff --git a/tests/mpi/renumber_cuthill_mckee_02.cc b/tests/mpi/renumber_cuthill_mckee_02.cc index 1403590989..5e2a5b215d 100644 --- a/tests/mpi/renumber_cuthill_mckee_02.cc +++ b/tests/mpi/renumber_cuthill_mckee_02.cc @@ -85,16 +85,14 @@ test() if (myid == i) MPI_Send(&renumbering[0], renumbering.size(), - Utilities::MPI::internal::mpi_type_id( - &complete_renumbering[0]), + Utilities::MPI::mpi_type_id(&complete_renumbering[0]), 0, i, MPI_COMM_WORLD); else if (myid == 0) MPI_Recv(&complete_renumbering[offset], locally_owned_dofs_per_processor[i].n_elements(), - Utilities::MPI::internal::mpi_type_id( - &complete_renumbering[0]), + Utilities::MPI::mpi_type_id(&complete_renumbering[0]), i, i, MPI_COMM_WORLD, -- 2.39.5