From 87ad5f9caa869c35ad3d31bf8b12a0be71d5123b Mon Sep 17 00:00:00 2001 From: Wolfgang Bangerth Date: Wed, 11 May 2022 13:03:49 -0600 Subject: [PATCH] Replace std::vector by RequestType. Same for AnswerType. --- .../base/mpi_compute_index_owner_internal.h | 10 +- .../deal.II/base/mpi_consensus_algorithms.h | 332 ++++++++---------- .../base/mpi_consensus_algorithms.templates.h | 218 +++++------- .../fe/fe_tools_extrapolate.templates.h | 12 +- .../lac/affine_constraints.templates.h | 10 +- .../matrix_free/matrix_free.templates.h | 15 +- .../mg_transfer_global_coarsening.templates.h | 15 +- source/base/mpi.cc | 9 +- .../base/mpi_compute_index_owner_internal.cc | 5 +- source/base/mpi_consensus_algorithms.cc | 85 +++-- source/base/mpi_noncontiguous_partitioner.cc | 5 +- source/base/partitioner.cc | 5 +- .../repartitioning_policy_tools.cc | 5 +- source/grid/grid_tools.cc | 3 +- source/grid/tria_description.cc | 12 +- source/matrix_free/vector_data_exchange.cc | 5 +- source/multigrid/mg_tools.cc | 5 +- tests/base/consensus_algorithm_01.cc | 12 +- 18 files changed, 360 insertions(+), 403 deletions(-) diff --git a/include/deal.II/base/mpi_compute_index_owner_internal.h b/include/deal.II/base/mpi_compute_index_owner_internal.h index b8d68b78ce..1b7ce89dbc 100644 --- a/include/deal.II/base/mpi_compute_index_owner_internal.h +++ b/include/deal.II/base/mpi_compute_index_owner_internal.h @@ -77,8 +77,9 @@ namespace Utilities */ class DictionaryPayLoad : public ConsensusAlgorithms::Process< - std::pair, - unsigned int> + std::vector< + std::pair>, + std::vector> { public: /** @@ -282,8 +283,9 @@ namespace Utilities */ class ConsensusAlgorithmsPayload : public ConsensusAlgorithms::Process< - std::pair, - unsigned int> + std::vector< + std::pair>, + std::vector> { public: /** diff --git a/include/deal.II/base/mpi_consensus_algorithms.h b/include/deal.II/base/mpi_consensus_algorithms.h index d01ac63e97..18dbbc6fcb 100644 --- a/include/deal.II/base/mpi_consensus_algorithms.h +++ b/include/deal.II/base/mpi_consensus_algorithms.h @@ -155,8 +155,7 @@ namespace Utilities * @note The buffer is empty. Before using it, you have to set its size. */ virtual void - create_request(const unsigned int other_rank, - std::vector &send_buffer); + create_request(const unsigned int other_rank, RequestType &send_buffer); /** * Prepare the buffer where the payload of the answer of the request to @@ -171,9 +170,9 @@ namespace Utilities * its size. */ virtual void - answer_request(const unsigned int other_rank, - const std::vector &buffer_recv, - std::vector & request_buffer); + answer_request(const unsigned int other_rank, + const RequestType &buffer_recv, + AnswerType & request_buffer); /** * Process the payload of the answer of the request to the process with @@ -183,8 +182,8 @@ namespace Utilities * @param[in] recv_buffer data to be sent part of the request (optional) */ virtual void - read_answer(const unsigned int other_rank, - const std::vector &recv_buffer); + read_answer(const unsigned int other_rank, + const AnswerType & recv_buffer); }; @@ -281,16 +280,14 @@ namespace Utilities * is to be performed. */ virtual std::vector - run(const std::vector &targets, - const std::function(const unsigned int)> - & create_request, - const std::function( - const unsigned int, - const std::vector &)> &answer_request, - const std::function &)> - & process_answer, - const MPI_Comm &comm) = 0; + run( + const std::vector & targets, + const std::function &create_request, + const std::function &answer_request, + const std::function + & process_answer, + const MPI_Comm &comm) = 0; private: /** @@ -363,23 +360,21 @@ namespace Utilities * @copydoc Interface::run() */ virtual std::vector - run(const std::vector &targets, - const std::function(const unsigned int)> - & create_request, - const std::function( - const unsigned int, - const std::vector &)> &answer_request, - const std::function &)> - & process_answer, - const MPI_Comm &comm) override; + run( + const std::vector & targets, + const std::function &create_request, + const std::function &answer_request, + const std::function + & process_answer, + const MPI_Comm &comm) override; private: #ifdef DEAL_II_WITH_MPI /** * Buffers for sending requests. */ - std::vector> send_buffers; + std::vector> send_buffers; /** * Requests for sending requests. @@ -393,7 +388,7 @@ namespace Utilities * resized and consequently its elements (the pointers) are moved * around. */ - std::vector>> request_buffers; + std::vector>> request_buffers; /** * Requests for sending answers to requests. @@ -421,8 +416,7 @@ namespace Utilities */ bool all_locally_originated_receives_are_completed( - const std::function &)> + const std::function & process_answer, const MPI_Comm &comm); @@ -448,10 +442,9 @@ namespace Utilities */ void maybe_answer_one_request( - const std::function( - const unsigned int, - const std::vector &)> &answer_request, - const MPI_Comm & comm); + const std::function &answer_request, + const MPI_Comm & comm); /** * Start to send all requests via ISend and post IRecvs for the incoming @@ -459,10 +452,9 @@ namespace Utilities */ void start_communication( - const std::vector &targets, - const std::function(const unsigned int)> - & create_request, - const MPI_Comm &comm); + const std::vector & targets, + const std::function &create_request, + const MPI_Comm & comm); /** * After all rank has received all answers, the MPI data structures can @@ -489,14 +481,11 @@ namespace Utilities */ template std::vector - nbx(const std::vector &targets, - const std::function(const unsigned int)> - & create_request, - const std::function( - const unsigned int, - const std::vector &)> &answer_request, - const std::function &)> + nbx(const std::vector & targets, + const std::function &create_request, + const std::function &answer_request, + const std::function & process_answer, const MPI_Comm &comm); @@ -563,28 +552,26 @@ namespace Utilities * @copydoc Interface::run() */ virtual std::vector - run(const std::vector &targets, - const std::function(const unsigned int)> - & create_request, - const std::function( - const unsigned int, - const std::vector &)> &answer_request, - const std::function &)> - & process_answer, - const MPI_Comm &comm) override; + run( + const std::vector & targets, + const std::function &create_request, + const std::function &answer_request, + const std::function + & process_answer, + const MPI_Comm &comm) override; private: #ifdef DEAL_II_WITH_MPI /** * Buffers for sending requests. */ - std::vector> send_buffers; + std::vector> send_buffers; /** * Buffers for receiving answers to requests. */ - std::vector> recv_buffers; + std::vector> recv_buffers; /** * MPI request objects for sending request messages. @@ -594,7 +581,7 @@ namespace Utilities /** * Buffers for sending answers to requests. */ - std::vector> requests_buffers; + std::vector> requests_buffers; /** * Requests for sending answers to requests. @@ -612,21 +599,20 @@ namespace Utilities */ unsigned int start_communication( - const std::vector &targets, - const std::function(const unsigned int)> - & create_request, - const MPI_Comm &comm); + const std::vector & targets, + const std::function &create_request, + const MPI_Comm & comm); /** * The `index`th request message from another rank has been received: * process the request and send an answer. */ void - answer_one_request(const unsigned int index, - const std::function( - const unsigned int, - const std::vector &)> &answer_request, - const MPI_Comm & comm); + answer_one_request( + const unsigned int index, + const std::function &answer_request, + const MPI_Comm & comm); /** * Receive and process all of the incoming responses to the @@ -635,8 +621,7 @@ namespace Utilities void process_incoming_answers( const unsigned int n_targets, - const std::function &)> + const std::function & process_answer, const MPI_Comm &comm); @@ -678,14 +663,11 @@ namespace Utilities */ template std::vector - pex(const std::vector &targets, - const std::function(const unsigned int)> - & create_request, - const std::function( - const unsigned int, - const std::vector &)> &answer_request, - const std::function &)> + pex(const std::vector & targets, + const std::function &create_request, + const std::function &answer_request, + const std::function & process_answer, const MPI_Comm &comm); @@ -725,16 +707,14 @@ namespace Utilities * @copydoc Interface::run() */ virtual std::vector - run(const std::vector &targets, - const std::function(const unsigned int)> - & create_request, - const std::function( - const unsigned int, - const std::vector &)> &answer_request, - const std::function &)> - & process_answer, - const MPI_Comm &comm) override; + run( + const std::vector & targets, + const std::function &create_request, + const std::function &answer_request, + const std::function + & process_answer, + const MPI_Comm &comm) override; }; @@ -751,14 +731,11 @@ namespace Utilities */ template std::vector - pex(const std::vector &targets, - const std::function(const unsigned int)> - & create_request, - const std::function( - const unsigned int, - const std::vector &)> &answer_request, - const std::function &)> + pex(const std::vector & targets, + const std::function &create_request, + const std::function &answer_request, + const std::function & process_answer, const MPI_Comm &comm); @@ -815,16 +792,14 @@ namespace Utilities * @note The function call is delegated to another ConsensusAlgorithms::Interface implementation. */ virtual std::vector - run(const std::vector &targets, - const std::function(const unsigned int)> - & create_request, - const std::function( - const unsigned int, - const std::vector &)> &answer_request, - const std::function &)> - & process_answer, - const MPI_Comm &comm) override; + run( + const std::vector & targets, + const std::function &create_request, + const std::function &answer_request, + const std::function + & process_answer, + const MPI_Comm &comm) override; private: // Pointer to the actual ConsensusAlgorithms::Interface implementation. @@ -849,16 +824,14 @@ namespace Utilities */ template std::vector - selector(const std::vector &targets, - const std::function(const unsigned int)> - & create_request, - const std::function( - const unsigned int, - const std::vector &)> &answer_request, - const std::function &)> - & process_answer, - const MPI_Comm &comm); + selector( + const std::vector & targets, + const std::function &create_request, + const std::function + &answer_request, + const std::function + & process_answer, + const MPI_Comm &comm); @@ -884,15 +857,12 @@ namespace Utilities AnonymousProcess( const std::function()> &function_compute_targets, + const std::function + & function_create_request = {}, const std::function &)> - &function_create_request = {}, - const std::function &, - std::vector &)> - &function_answer_request = {}, - const std::function &)> + const RequestType &, + AnswerType &)> &function_answer_request = {}, + const std::function &function_read_answer = {}); /** @@ -905,34 +875,33 @@ namespace Utilities * @copydoc Process::create_request() */ void - create_request(const unsigned int other_rank, - std::vector &send_buffer) override; + create_request(const unsigned int other_rank, + RequestType & send_buffer) override; /** * @copydoc Process::answer_request() */ void - answer_request(const unsigned int other_rank, - const std::vector &buffer_recv, - std::vector & request_buffer) override; + answer_request(const unsigned int other_rank, + const RequestType &buffer_recv, + AnswerType & request_buffer) override; /** * @copydoc Process::read_answer() */ void - read_answer(const unsigned int other_rank, - const std::vector &recv_buffer) override; + read_answer(const unsigned int other_rank, + const AnswerType & recv_buffer) override; private: const std::function()> function_compute_targets; - const std::function &)> + const std::function function_create_request; - const std::function &, - std::vector &)> + const std::function< + void(const unsigned int, const RequestType &, AnswerType &)> function_answer_request; - const std::function &)> + const std::function function_read_answer; }; @@ -942,14 +911,11 @@ namespace Utilities template std::vector - nbx(const std::vector &targets, - const std::function(const unsigned int)> - & create_request, - const std::function( - const unsigned int, - const std::vector &)> &answer_request, - const std::function &)> + nbx(const std::vector & targets, + const std::function &create_request, + const std::function &answer_request, + const std::function & process_answer, const MPI_Comm &comm) { @@ -961,14 +927,11 @@ namespace Utilities template std::vector - pex(const std::vector &targets, - const std::function(const unsigned int)> - & create_request, - const std::function( - const unsigned int, - const std::vector &)> &answer_request, - const std::function &)> + pex(const std::vector & targets, + const std::function &create_request, + const std::function &answer_request, + const std::function & process_answer, const MPI_Comm &comm) { @@ -980,16 +943,14 @@ namespace Utilities template std::vector - serial(const std::vector &targets, - const std::function(const unsigned int)> - & create_request, - const std::function( - const unsigned int, - const std::vector &)> &answer_request, - const std::function &)> - & process_answer, - const MPI_Comm &comm) + serial( + const std::vector & targets, + const std::function &create_request, + const std::function + &answer_request, + const std::function + & process_answer, + const MPI_Comm &comm) { return Serial().run( targets, create_request, answer_request, process_answer, comm); @@ -999,16 +960,14 @@ namespace Utilities template std::vector - selector(const std::vector &targets, - const std::function(const unsigned int)> - & create_request, - const std::function( - const unsigned int, - const std::vector &)> &answer_request, - const std::function &)> - & process_answer, - const MPI_Comm &comm) + selector( + const std::vector & targets, + const std::function &create_request, + const std::function + &answer_request, + const std::function + & process_answer, + const MPI_Comm &comm) { return Selector().run( targets, create_request, answer_request, process_answer, comm); @@ -1020,15 +979,12 @@ namespace Utilities AnonymousProcess::AnonymousProcess( const std::function()> &function_compute_targets, + const std::function + & function_create_request, const std::function &)> - &function_create_request, - const std::function &, - std::vector &)> - &function_answer_request, - const std::function &)> + const RequestType &, + AnswerType &)> &function_answer_request, + const std::function &function_read_answer) : function_compute_targets(function_compute_targets) , function_create_request(function_create_request) @@ -1050,8 +1006,8 @@ namespace Utilities template void AnonymousProcess::create_request( - const unsigned int other_rank, - std::vector &send_buffer) + const unsigned int other_rank, + RequestType & send_buffer) { if (function_create_request) function_create_request(other_rank, send_buffer); @@ -1062,9 +1018,9 @@ namespace Utilities template void AnonymousProcess::answer_request( - const unsigned int other_rank, - const std::vector &buffer_recv, - std::vector & request_buffer) + const unsigned int other_rank, + const RequestType &buffer_recv, + AnswerType & request_buffer) { if (function_answer_request) function_answer_request(other_rank, buffer_recv, request_buffer); @@ -1075,8 +1031,8 @@ namespace Utilities template void AnonymousProcess::read_answer( - const unsigned int other_rank, - const std::vector &recv_buffer) + const unsigned int other_rank, + const AnswerType & recv_buffer) { if (function_read_answer) function_read_answer(other_rank, recv_buffer); diff --git a/include/deal.II/base/mpi_consensus_algorithms.templates.h b/include/deal.II/base/mpi_consensus_algorithms.templates.h index 1ebfa7d8a6..3fab0c987f 100644 --- a/include/deal.II/base/mpi_consensus_algorithms.templates.h +++ b/include/deal.II/base/mpi_consensus_algorithms.templates.h @@ -74,10 +74,9 @@ namespace Utilities template void - Process::answer_request( - const unsigned int, - const std::vector &, - std::vector &) + Process::answer_request(const unsigned int, + const RequestType &, + AnswerType &) { // nothing to do } @@ -86,9 +85,8 @@ namespace Utilities template void - Process::create_request( - const unsigned int, - std::vector &) + Process::create_request(const unsigned int, + RequestType &) { // nothing to do } @@ -97,9 +95,8 @@ namespace Utilities template void - Process::read_answer( - const unsigned int, - const std::vector &) + Process::read_answer(const unsigned int, + const AnswerType &) { // nothing to do } @@ -150,20 +147,18 @@ namespace Utilities process.compute_targets(), /* create_request: */ [&process](const unsigned int target) { - std::vector request; + RequestType request; process.create_request(target, request); return request; }, /* answer_request: */ - [&process](const unsigned int source, - const std::vector &request) { - std::vector answer; + [&process](const unsigned int source, const RequestType &request) { + AnswerType answer; process.answer_request(source, request, answer); return answer; }, /* process_answer: */ - [&process](const unsigned int target, - const std::vector &answer) { + [&process](const unsigned int target, const AnswerType &answer) { process.read_answer(target, answer); }, comm); @@ -183,14 +178,11 @@ namespace Utilities template std::vector NBX::run( - const std::vector &targets, - const std::function(const unsigned int)> - & create_request, - const std::function( - const unsigned int, - const std::vector &)> &answer_request, - const std::function &)> + const std::vector & targets, + const std::function &create_request, + const std::function + &answer_request, + const std::function & process_answer, const MPI_Comm &comm) { @@ -246,10 +238,9 @@ namespace Utilities template void NBX::start_communication( - const std::vector &targets, - const std::function(const unsigned int)> - & create_request, - const MPI_Comm &comm) + const std::vector & targets, + const std::function &create_request, + const MPI_Comm & comm) { #ifdef DEAL_II_WITH_MPI // 1) @@ -270,13 +261,14 @@ namespace Utilities AssertIndexRange(rank, Utilities::MPI::n_mpi_processes(comm)); auto &send_buffer = send_buffers[index]; - send_buffer = (create_request ? create_request(rank) : - std::vector()); + send_buffer = + (create_request ? Utilities::pack(create_request(rank)) : + std::vector()); // Post a request to send data auto ierr = MPI_Isend(send_buffer.data(), - send_buffer.size() * sizeof(RequestType), - MPI_BYTE, + send_buffer.size(), + MPI_CHAR, rank, tag_request, comm, @@ -301,8 +293,7 @@ namespace Utilities bool NBX:: all_locally_originated_receives_are_completed( - const std::function &)> + const std::function & process_answer, const MPI_Comm &comm) { @@ -344,31 +335,28 @@ namespace Utilities int message_size; { const int ierr = - MPI_Get_count(&status, MPI_BYTE, &message_size); + MPI_Get_count(&status, MPI_CHAR, &message_size); AssertThrowMPI(ierr); } - Assert(message_size % sizeof(AnswerType) == 0, - ExcInternalError()); - std::vector recv_buffer(message_size / - sizeof(AnswerType)); + std::vector recv_buffer(message_size); { const int tag_deliver = Utilities::MPI::internal::Tags:: consensus_algorithm_nbx_process_deliver; - const int ierr = - MPI_Recv(recv_buffer.data(), - recv_buffer.size() * sizeof(AnswerType), - MPI_BYTE, - target, - tag_deliver, - comm, - MPI_STATUS_IGNORE); + const int ierr = MPI_Recv(recv_buffer.data(), + recv_buffer.size(), + MPI_CHAR, + target, + tag_deliver, + comm, + MPI_STATUS_IGNORE); AssertThrowMPI(ierr); } if (process_answer) - process_answer(target, recv_buffer); + process_answer(target, + Utilities::unpack(recv_buffer)); // Finally, remove this rank from the list of outstanding // targets: @@ -399,10 +387,9 @@ namespace Utilities template void NBX::maybe_answer_one_request( - const std::function( - const unsigned int, - const std::vector &)> &answer_request, - const MPI_Comm & comm) + const std::function + & answer_request, + const MPI_Comm &comm) { #ifdef DEAL_II_WITH_MPI @@ -437,17 +424,14 @@ namespace Utilities // get size of incoming message int number_amount; - auto ierr = MPI_Get_count(&status, MPI_BYTE, &number_amount); + auto ierr = MPI_Get_count(&status, MPI_CHAR, &number_amount); AssertThrowMPI(ierr); // allocate memory for incoming message - Assert(number_amount % sizeof(RequestType) == 0, - ExcInternalError()); - std::vector buffer_recv(number_amount / - sizeof(RequestType)); + std::vector buffer_recv(number_amount); ierr = MPI_Recv(buffer_recv.data(), number_amount, - MPI_BYTE, + MPI_CHAR, other_rank, tag_request, comm, @@ -456,17 +440,18 @@ namespace Utilities // Allocate memory for an answer message to the current request, // and ask the 'process' object to produce an answer: - request_buffers.emplace_back( - std::make_unique>()); + request_buffers.emplace_back(std::make_unique>()); auto &request_buffer = *request_buffers.back(); if (answer_request) - request_buffer = answer_request(other_rank, buffer_recv); + request_buffer = Utilities::pack( + answer_request(other_rank, + Utilities::unpack(buffer_recv))); // Then initiate sending the answer back to the requester. request_requests.emplace_back(std::make_unique()); ierr = MPI_Isend(request_buffer.data(), - request_buffer.size() * sizeof(AnswerType), - MPI_BYTE, + request_buffer.size(), + MPI_CHAR, other_rank, tag_deliver, comm, @@ -564,14 +549,11 @@ namespace Utilities template std::vector PEX::run( - const std::vector &targets, - const std::function(const unsigned int)> - & create_request, - const std::function( - const unsigned int, - const std::vector &)> &answer_request, - const std::function &)> + const std::vector & targets, + const std::function &create_request, + const std::function + &answer_request, + const std::function & process_answer, const MPI_Comm &comm) { @@ -608,10 +590,9 @@ namespace Utilities template unsigned int PEX::start_communication( - const std::vector &targets, - const std::function(const unsigned int)> - & create_request, - const MPI_Comm &comm) + const std::vector & targets, + const std::function &create_request, + const MPI_Comm & comm) { #ifdef DEAL_II_WITH_MPI const int tag_request = Utilities::MPI::internal::Tags:: @@ -641,13 +622,13 @@ namespace Utilities // pack data which should be sent auto &send_buffer = send_buffers[i]; - send_buffer = (create_request ? create_request(rank) : - std::vector()); + if (create_request) + send_buffer = Utilities::pack(create_request(rank)); // start to send data auto ierr = MPI_Isend(send_buffer.data(), - send_buffer.size() * sizeof(RequestType), - MPI_BYTE, + send_buffer.size(), + MPI_CHAR, rank, tag_request, comm, @@ -669,11 +650,10 @@ namespace Utilities template void PEX::answer_one_request( - const unsigned int index, - const std::function( - const unsigned int, - const std::vector &)> &answer_request, - const MPI_Comm & comm) + const unsigned int index, + const std::function + & answer_request, + const MPI_Comm &comm) { #ifdef DEAL_II_WITH_MPI const int tag_request = Utilities::MPI::internal::Tags:: @@ -698,18 +678,15 @@ namespace Utilities "received. This algorithm does not expect this to happen.")); requesting_processes.insert(other_rank); - std::vector buffer_recv; - // Actually get the incoming message: int number_amount; - ierr = MPI_Get_count(&status, MPI_BYTE, &number_amount); + ierr = MPI_Get_count(&status, MPI_CHAR, &number_amount); AssertThrowMPI(ierr); - Assert(number_amount % sizeof(RequestType) == 0, ExcInternalError()); - buffer_recv.resize(number_amount / sizeof(RequestType)); + std::vector buffer_recv(number_amount); ierr = MPI_Recv(buffer_recv.data(), number_amount, - MPI_BYTE, + MPI_CHAR, other_rank, tag_request, comm, @@ -720,12 +697,14 @@ namespace Utilities // the answer and post a send for it. auto &request_buffer = requests_buffers[index]; request_buffer = - (answer_request ? answer_request(other_rank, buffer_recv) : - std::vector()); + (answer_request ? + Utilities::pack(answer_request( + other_rank, Utilities::unpack(buffer_recv))) : + std::vector()); ierr = MPI_Isend(request_buffer.data(), - request_buffer.size() * sizeof(AnswerType), - MPI_BYTE, + request_buffer.size(), + MPI_CHAR, other_rank, tag_deliver, comm, @@ -744,8 +723,7 @@ namespace Utilities void PEX::process_incoming_answers( const unsigned int n_targets, - const std::function &)> + const std::function & process_answer, const MPI_Comm &comm) { @@ -770,20 +748,18 @@ namespace Utilities const auto other_rank = status.MPI_SOURCE; int message_size; { - const int ierr = MPI_Get_count(&status, MPI_BYTE, &message_size); + const int ierr = MPI_Get_count(&status, MPI_CHAR, &message_size); AssertThrowMPI(ierr); } - Assert(message_size % sizeof(AnswerType) == 0, ExcInternalError()); - std::vector recv_buffer(message_size / - sizeof(AnswerType)); + std::vector recv_buffer(message_size); // Now actually receive the answer. Because the MPI_Probe // above blocks until we have a message, we know that the // following MPI_Recv call will immediately succeed. { const int ierr = MPI_Recv(recv_buffer.data(), - recv_buffer.size() * sizeof(AnswerType), - MPI_BYTE, + recv_buffer.size(), + MPI_CHAR, other_rank, tag_deliver, comm, @@ -792,7 +768,8 @@ namespace Utilities } if (process_answer) - process_answer(other_rank, recv_buffer); + process_answer(other_rank, + Utilities::unpack(recv_buffer)); } #else (void)n_targets; @@ -843,14 +820,11 @@ namespace Utilities template std::vector Serial::run( - const std::vector &targets, - const std::function(const unsigned int)> - & create_request, - const std::function( - const unsigned int, - const std::vector &)> &answer_request, - const std::function &)> + const std::vector & targets, + const std::function &create_request, + const std::function + &answer_request, + const std::function & process_answer, const MPI_Comm &comm) { @@ -874,11 +848,10 @@ namespace Utilities // Since the caller indicates that there is a target, and since we // know that it is the current process, let the process send // something to itself. - const std::vector request = - (create_request ? create_request(0) : std::vector()); - const std::vector answer = - (answer_request ? answer_request(0, request) : - std::vector()); + const RequestType request = + (create_request ? create_request(0) : RequestType()); + const AnswerType answer = + (answer_request ? answer_request(0, request) : AnswerType()); if (process_answer) process_answer(0, answer); @@ -901,14 +874,11 @@ namespace Utilities template std::vector Selector::run( - const std::vector &targets, - const std::function(const unsigned int)> - & create_request, - const std::function( - const unsigned int, - const std::vector &)> &answer_request, - const std::function &)> + const std::vector & targets, + const std::function &create_request, + const std::function + &answer_request, + const std::function & process_answer, const MPI_Comm &comm) { diff --git a/include/deal.II/fe/fe_tools_extrapolate.templates.h b/include/deal.II/fe/fe_tools_extrapolate.templates.h index 3168ffe92e..c883d07ad2 100644 --- a/include/deal.II/fe/fe_tools_extrapolate.templates.h +++ b/include/deal.II/fe/fe_tools_extrapolate.templates.h @@ -1177,11 +1177,13 @@ namespace FETools Assert(answer.size() == 0, ExcInternalError()); }; - Utilities::MPI::ConsensusAlgorithms::selector(destinations, - create_request, - answer_request, - read_answer, - communicator); + Utilities::MPI::ConsensusAlgorithms::selector, + std::vector>( + destinations, + create_request, + answer_request, + read_answer, + communicator); } diff --git a/include/deal.II/lac/affine_constraints.templates.h b/include/deal.II/lac/affine_constraints.templates.h index aa39f8fe46..a8425c37ff 100644 --- a/include/deal.II/lac/affine_constraints.templates.h +++ b/include/deal.II/lac/affine_constraints.templates.h @@ -261,8 +261,8 @@ namespace internal true); Utilities::MPI::ConsensusAlgorithms::Selector< - std::pair, - unsigned int>(constrained_indices_process, mpi_communicator) + std::vector>, + std::vector>(constrained_indices_process, mpi_communicator) .run(); // step 2: collect all locally owned constraints @@ -388,8 +388,10 @@ namespace internal true); Utilities::MPI::ConsensusAlgorithms::Selector< - std::pair, - unsigned int>(locally_relevant_dofs_process, mpi_communicator) + std::vector< + std::pair>, + std::vector>(locally_relevant_dofs_process, + mpi_communicator) .run(); const auto locally_relevant_dofs_by_ranks = diff --git a/include/deal.II/matrix_free/matrix_free.templates.h b/include/deal.II/matrix_free/matrix_free.templates.h index 46ce68fa0a..de78cc0428 100644 --- a/include/deal.II/matrix_free/matrix_free.templates.h +++ b/include/deal.II/matrix_free/matrix_free.templates.h @@ -2170,13 +2170,14 @@ MatrixFree::initialize_indices( } }; - Utilities::MPI::ConsensusAlgorithms:: - Selector() - .run(targets, - create_request, - answer_request, - process_answer, - communicator_sm); + Utilities::MPI::ConsensusAlgorithms::Selector< + std::vector, + std::vector>() + .run(targets, + create_request, + answer_request, + process_answer, + communicator_sm); return cells; }(); diff --git a/include/deal.II/multigrid/mg_transfer_global_coarsening.templates.h b/include/deal.II/multigrid/mg_transfer_global_coarsening.templates.h index 70ab1f48b4..78d62799c1 100644 --- a/include/deal.II/multigrid/mg_transfer_global_coarsening.templates.h +++ b/include/deal.II/multigrid/mg_transfer_global_coarsening.templates.h @@ -567,8 +567,9 @@ namespace internal false); Utilities::MPI::ConsensusAlgorithms::Selector< - std::pair, - unsigned int> + std::vector< + std::pair>, + std::vector> consensus_algorithm(process, communicator); consensus_algorithm.run(); } @@ -593,8 +594,9 @@ namespace internal true); Utilities::MPI::ConsensusAlgorithms::Selector< - std::pair, - unsigned int> + std::vector< + std::pair>, + std::vector> consensus_algorithm(process, communicator); consensus_algorithm.run(); @@ -3086,8 +3088,9 @@ MGTwoLevelTransfer>::reinit( false); Utilities::MPI::ConsensusAlgorithms::Selector< - std::pair, - unsigned int> + std::vector< + std::pair>, + std::vector> consensus_algorithm(process, communicator); consensus_algorithm.run(); diff --git a/source/base/mpi.cc b/source/base/mpi.cc index d5f7eef4c7..29cef1bd59 100644 --- a/source/base/mpi.cc +++ b/source/base/mpi.cc @@ -381,7 +381,7 @@ namespace Utilities if (Utilities::MPI::min((my_destinations_are_unique ? 1 : 0), mpi_comm) == 1) { - return ConsensusAlgorithms::nbx( + return ConsensusAlgorithms::nbx, std::vector>( destinations, {}, {}, {}, mpi_comm); } @@ -477,7 +477,7 @@ namespace Utilities if (Utilities::MPI::min((my_destinations_are_unique ? 1 : 0), mpi_comm) == 1) { - return ConsensusAlgorithms::nbx( + return ConsensusAlgorithms::nbx, std::vector>( destinations, {}, {}, {}, mpi_comm) .size(); } @@ -1098,8 +1098,9 @@ namespace Utilities // partition (i.e. in the dictionary). This process returns the actual // owner of the index. ConsensusAlgorithms::Selector< - std::pair, - unsigned int> + std::vector< + std::pair>, + std::vector> consensus_algorithm(process, comm); consensus_algorithm.run(); diff --git a/source/base/mpi_compute_index_owner_internal.cc b/source/base/mpi_compute_index_owner_internal.cc index 71f03410f2..e3aa23f2c8 100644 --- a/source/base/mpi_compute_index_owner_internal.cc +++ b/source/base/mpi_compute_index_owner_internal.cc @@ -408,8 +408,9 @@ namespace Utilities actually_owning_rank_list); ConsensusAlgorithms::Selector< - std::pair, - unsigned int> + std::vector< + std::pair>, + std::vector> consensus_algo(temp, comm); consensus_algo.run(); } diff --git a/source/base/mpi_consensus_algorithms.cc b/source/base/mpi_consensus_algorithms.cc index c03d41141a..e383c5f654 100644 --- a/source/base/mpi_consensus_algorithms.cc +++ b/source/base/mpi_consensus_algorithms.cc @@ -15,6 +15,9 @@ #include +#include + + DEAL_II_NAMESPACE_OPEN namespace Utilities @@ -23,68 +26,78 @@ namespace Utilities { namespace ConsensusAlgorithms { - template class Process; + template class Process, + std::vector>; - template class Interface; + template class Interface, + std::vector>; - template class NBX; + template class NBX, std::vector>; - template class PEX; + template class PEX, std::vector>; - template class Serial; + template class Serial, + std::vector>; - template class Selector; + template class Selector, + std::vector>; - template class Process< - std::pair, - unsigned int>; + template class Process>, + std::vector>; - template class Interface< - std::pair, - unsigned int>; + template class Interface>, + std::vector>; - template class Selector< - std::pair, - unsigned int>; + template class Selector>, + std::vector>; - template class NBX< - std::pair, - unsigned int>; + template class NBX>, + std::vector>; - template class Serial< - std::pair, - unsigned int>; + template class Serial>, + std::vector>; - template class PEX< - std::pair, - unsigned int>; + template class PEX>, + std::vector>; #ifdef DEAL_II_WITH_64BIT_INDICES - template class Process; + template class Process, + std::vector>; - template class Interface; + template class Interface, + std::vector>; - template class NBX; + template class NBX, + std::vector>; - template class Serial; + template class Serial, + std::vector>; - template class PEX; + template class PEX, + std::vector>; - template class Selector; + template class Selector, + std::vector>; #endif - template class Process; + template class Process, std::vector>; - template class Interface; + template class Interface, std::vector>; - template class NBX; + template class NBX, std::vector>; - template class PEX; + template class PEX, std::vector>; - template class Serial; + template class Serial, std::vector>; - template class Selector; + template class Selector, std::vector>; } // namespace ConsensusAlgorithms } // end of namespace MPI diff --git a/source/base/mpi_noncontiguous_partitioner.cc b/source/base/mpi_noncontiguous_partitioner.cc index 8a40b54fba..4298da1df0 100644 --- a/source/base/mpi_noncontiguous_partitioner.cc +++ b/source/base/mpi_noncontiguous_partitioner.cc @@ -115,8 +115,9 @@ namespace Utilities true); Utilities::MPI::ConsensusAlgorithms::Selector< - std::pair, - unsigned int> + std::vector< + std::pair>, + std::vector> consensus_algorithm(process, communicator); consensus_algorithm.run(); diff --git a/source/base/partitioner.cc b/source/base/partitioner.cc index c83ba56042..10e42fd043 100644 --- a/source/base/partitioner.cc +++ b/source/base/partitioner.cc @@ -281,8 +281,9 @@ namespace Utilities // in the static partition (i.e. in the dictionary). This process // returns the actual owner of the index. ConsensusAlgorithms::Selector< - std::pair, - unsigned int> + std::vector< + std::pair>, + std::vector> consensus_algorithm(process, communicator); consensus_algorithm.run(); diff --git a/source/distributed/repartitioning_policy_tools.cc b/source/distributed/repartitioning_policy_tools.cc index 16e054e066..e6781c69e9 100644 --- a/source/distributed/repartitioning_policy_tools.cc +++ b/source/distributed/repartitioning_policy_tools.cc @@ -156,8 +156,9 @@ namespace RepartitioningPolicyTools false); Utilities::MPI::ConsensusAlgorithms::Selector< - std::pair, - unsigned int> + std::vector< + std::pair>, + std::vector> consensus_algorithm(process, communicator); consensus_algorithm.run(); } diff --git a/source/grid/grid_tools.cc b/source/grid/grid_tools.cc index 79b2957816..1ddaf4030a 100644 --- a/source/grid/grid_tools.cc +++ b/source/grid/grid_tools.cc @@ -6080,7 +6080,8 @@ namespace GridTools } }; - Utilities::MPI::ConsensusAlgorithms::selector( + Utilities::MPI::ConsensusAlgorithms::selector, + std::vector>( potential_owners_ranks, create_request, answer_request, diff --git a/source/grid/tria_description.cc b/source/grid/tria_description.cc index f7ce80c9f4..aea46c970a 100644 --- a/source/grid/tria_description.cc +++ b/source/grid/tria_description.cc @@ -143,12 +143,12 @@ namespace TriangulationDescription }; - dealii::Utilities::MPI::ConsensusAlgorithms::selector( - relevant_processes, - create_request, - answer_request, - process_answer, - comm); + dealii::Utilities::MPI::ConsensusAlgorithms:: + selector, std::vector>(relevant_processes, + create_request, + answer_request, + process_answer, + comm); } /** diff --git a/source/matrix_free/vector_data_exchange.cc b/source/matrix_free/vector_data_exchange.cc index e652a11fb7..f4b81fba22 100644 --- a/source/matrix_free/vector_data_exchange.cc +++ b/source/matrix_free/vector_data_exchange.cc @@ -441,8 +441,9 @@ namespace internal /*track_index_requests = */ true); Utilities::MPI::ConsensusAlgorithms::Selector< - std::pair, - unsigned int> + std::vector< + std::pair>, + std::vector> consensus_algorithm(process, comm); consensus_algorithm.run(); diff --git a/source/multigrid/mg_tools.cc b/source/multigrid/mg_tools.cc index 7b2bc36083..e1a9c19405 100644 --- a/source/multigrid/mg_tools.cc +++ b/source/multigrid/mg_tools.cc @@ -1823,8 +1823,9 @@ namespace MGTools false); Utilities::MPI::ConsensusAlgorithms::Selector< - std::pair, - unsigned int> + std::vector< + std::pair>, + std::vector> consensus_algorithm; consensus_algorithm.run(process, communicator); diff --git a/tests/base/consensus_algorithm_01.cc b/tests/base/consensus_algorithm_01.cc index 7f91866e2a..bd3e994c9f 100644 --- a/tests/base/consensus_algorithm_01.cc +++ b/tests/base/consensus_algorithm_01.cc @@ -27,24 +27,24 @@ test(const MPI_Comm &comm) const unsigned int my_rank = dealii::Utilities::MPI::this_mpi_process(comm); const unsigned int n_rank = dealii::Utilities::MPI::n_mpi_processes(comm); - using T1 = unsigned int; - using T2 = unsigned int; + using T1 = std::vector; + using T2 = std::vector; const auto sources = dealii::Utilities::MPI::ConsensusAlgorithms::selector( /* target_processes: */ std::vector{(my_rank + 1) % n_rank}, /* create_request: */ - [my_rank](const unsigned int) { return std::vector({my_rank}); }, + [my_rank](const unsigned int) { return T1({my_rank}); }, /* answer_request: */ - [my_rank](const unsigned int other_rank, const std::vector &request) { + [my_rank](const unsigned int other_rank, const T1 &request) { AssertDimension(other_rank, request.front()); deallog << "ConsensusAlgorithmProcess::answer_request() passed!" << std::endl; - return std::vector({my_rank}); + return T2({my_rank}); }, /* process_answer: */ - [](const unsigned int other_rank, const std::vector &answer) { + [](const unsigned int other_rank, const T2 &answer) { AssertDimension(other_rank, answer.front()); deallog << "ConsensusAlgorithmProcess::function_read_answer() passed!" << std::endl; -- 2.39.5