From 80eaa12bbf28b0da67907c3eb604be0b170f1122 Mon Sep 17 00:00:00 2001 From: Wolfgang Bangerth Date: Fri, 13 May 2022 16:31:04 -0600 Subject: [PATCH] Avoid double packing/unpacking for CA algorithms. --- .../fe/fe_tools_extrapolate.templates.h | 61 ++++++++++--------- source/grid/grid_tools.cc | 50 +++++++-------- source/grid/tria_description.cc | 34 ++++------- source/multigrid/mg_tools.cc | 1 + 4 files changed, 66 insertions(+), 80 deletions(-) diff --git a/include/deal.II/fe/fe_tools_extrapolate.templates.h b/include/deal.II/fe/fe_tools_extrapolate.templates.h index c883d07ad2..fc92f2a2a6 100644 --- a/include/deal.II/fe/fe_tools_extrapolate.templates.h +++ b/include/deal.II/fe/fe_tools_extrapolate.templates.h @@ -20,6 +20,7 @@ #include #include +#include #include #include @@ -1140,45 +1141,45 @@ namespace FETools // front which processes (and from how many processes) we have to // expect information from. const auto create_request = - [&cells_to_send](const types::subdomain_id other_rank) { - std::vector cells_for_this_destination; - for (const auto &cell : cells_to_send) - if (cell.receiver == other_rank) - cells_for_this_destination.emplace_back(cell); - - return Utilities::pack(cells_for_this_destination, false); - }; + [&cells_to_send]( + const types::subdomain_id other_rank) -> std::vector { + std::vector cells_for_this_destination; + for (const auto &cell : cells_to_send) + if (cell.receiver == other_rank) + cells_for_this_destination.emplace_back(cell); + + return cells_for_this_destination; + }; const auto answer_request = - [&received_cells](const unsigned int other_rank, - const std::vector &request) { - // We got a message from 'other_rank', so let us decode the - // message in the same way as we have assembled it above. - // Note that the cells just received do not contain - // information where they came from, and we have to add that - // ourselves for later use. - for (CellData &cell_data : - Utilities::unpack>(request, false)) - { - cell_data.receiver = other_rank; - received_cells.emplace_back(std::move(cell_data)); - } + [&received_cells](const unsigned int other_rank, + const std::vector &request) -> int { + // We got a message from 'other_rank', so let us decode the + // message in the same way as we have assembled it above. + // Note that the cells just received do not contain + // information where they came from, and we have to add that + // ourselves for later use. + for (CellData cell_data : request) + { + cell_data.receiver = other_rank; + received_cells.emplace_back(std::move(cell_data)); + } - // Nothing left to do here, we don't actually need to provide an - // answer: - return std::vector(); - }; + // Nothing left to do here, we don't actually need to provide an + // answer: + return 0; + }; const auto read_answer = [](const unsigned int /*other_rank*/, - const std::vector &answer) { + const int &answer) { // We don't put anything into the answers, so nothing should - // have been coming out at this end either: + // have been coming out at this end either that differs from + // the default-sent zero integer: (void)answer; - Assert(answer.size() == 0, ExcInternalError()); + Assert(answer == 0, ExcInternalError()); }; - Utilities::MPI::ConsensusAlgorithms::selector, - std::vector>( + Utilities::MPI::ConsensusAlgorithms::selector, int>( destinations, create_request, answer_request, diff --git a/source/grid/grid_tools.cc b/source/grid/grid_tools.cc index 1ddaf4030a..85ba1aa885 100644 --- a/source/grid/grid_tools.cc +++ b/source/grid/grid_tools.cc @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -58,6 +59,7 @@ #include + DEAL_II_DISABLE_EXTRA_DIAGNOSTICS #include #include @@ -5985,6 +5987,9 @@ namespace GridTools "The marked_vertices vector has to be either empty or its size has " "to equal the number of vertices of the triangulation.")); + using RequestType = std::vector>>; + using AnswerType = std::vector; + // In the case that a marked_vertices vector has been given and none // of its entries is true, we know that this process does not own // any of the incoming points (and it will not send any data) so @@ -5997,36 +6002,31 @@ namespace GridTools const auto create_request = [&](const unsigned int other_rank) { const auto other_rank_index = translate(other_rank); - std::vector>> temp; - temp.reserve(potential_owners_ptrs[other_rank_index + 1] - - potential_owners_ptrs[other_rank_index]); + RequestType request; + request.reserve(potential_owners_ptrs[other_rank_index + 1] - + potential_owners_ptrs[other_rank_index]); for (unsigned int i = potential_owners_ptrs[other_rank_index]; i < potential_owners_ptrs[other_rank_index + 1]; ++i) - temp.emplace_back(potential_owners_indices[i], - points[potential_owners_indices[i]]); + request.emplace_back(potential_owners_indices[i], + points[potential_owners_indices[i]]); - return Utilities::pack(temp, false); + return request; }; const auto answer_request = - [&](const unsigned int & other_rank, - const std::vector &request) -> std::vector { - const auto recv_buffer_unpacked = Utilities::unpack< - std::vector>>>(request, - false); - - std::vector request_buffer_temp( - recv_buffer_unpacked.size(), 0); + [&](const unsigned int &other_rank, + const RequestType & request) -> AnswerType { + AnswerType answer(request.size(), 0); if (has_relevant_vertices) { cell_hint = cache.get_triangulation().begin_active(); - for (unsigned int i = 0; i < recv_buffer_unpacked.size(); ++i) + for (unsigned int i = 0; i < request.size(); ++i) { - const auto &index_and_point = recv_buffer_unpacked[i]; + const auto &index_and_point = request[i]; const auto cells_and_reference_positions = find_all_locally_owned_active_cells_around_point( @@ -6051,27 +6051,24 @@ namespace GridTools numbers::invalid_unsigned_int); } - request_buffer_temp[i] = cells_and_reference_positions.size(); + answer[i] = cells_and_reference_positions.size(); } } if (perform_handshake) - return Utilities::pack(request_buffer_temp, false); + return answer; else return {}; }; - const auto process_answer = [&](const unsigned int other_rank, - const std::vector &answer) { + const auto process_answer = [&](const unsigned int other_rank, + const AnswerType & answer) { if (perform_handshake) { - const auto recv_buffer_unpacked = - Utilities::unpack>(answer, false); - const auto other_rank_index = translate(other_rank); - for (unsigned int i = 0; i < recv_buffer_unpacked.size(); ++i) - for (unsigned int j = 0; j < recv_buffer_unpacked[i]; ++j) + for (unsigned int i = 0; i < answer.size(); ++i) + for (unsigned int j = 0; j < answer[i]; ++j) recv_components.emplace_back( other_rank, potential_owners_indices @@ -6080,8 +6077,7 @@ namespace GridTools } }; - Utilities::MPI::ConsensusAlgorithms::selector, - std::vector>( + Utilities::MPI::ConsensusAlgorithms::selector( potential_owners_ranks, create_request, answer_request, diff --git a/source/grid/tria_description.cc b/source/grid/tria_description.cc index aea46c970a..c58ef335eb 100644 --- a/source/grid/tria_description.cc +++ b/source/grid/tria_description.cc @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -123,32 +124,19 @@ namespace TriangulationDescription const auto other_rank_index = std::distance(relevant_processes.begin(), ptr); - return dealii::Utilities::pack(description_temp[other_rank_index], - false); - }; - - const auto answer_request = [&](const unsigned int, - const std::vector &request) { - this->merge( - dealii::Utilities::unpack>(request, - false), - vertices_have_unique_ids); - return std::vector(); - }; - - const auto process_answer = [](const unsigned int, - const std::vector &answer) { - (void)answer; - Assert(answer.size() == 0, ExcInternalError()); + return description_temp[other_rank_index]; }; + const auto answer_request = + [&](const unsigned int, + const DescriptionTemp &request) { + this->merge(request, vertices_have_unique_ids); + return char{}; + }; - dealii::Utilities::MPI::ConsensusAlgorithms:: - selector, std::vector>(relevant_processes, - create_request, - answer_request, - process_answer, - comm); + dealii::Utilities::MPI::ConsensusAlgorithms::selector< + DescriptionTemp, + char>(relevant_processes, create_request, answer_request, {}, comm); } /** diff --git a/source/multigrid/mg_tools.cc b/source/multigrid/mg_tools.cc index e1a9c19405..b139e56a1d 100644 --- a/source/multigrid/mg_tools.cc +++ b/source/multigrid/mg_tools.cc @@ -16,6 +16,7 @@ #include #include #include +#include #include #include -- 2.39.5