From 621a7fe54b8242a7605882f2c60a87e34c087b34 Mon Sep 17 00:00:00 2001 From: Peter Munch Date: Sat, 5 Apr 2025 22:32:05 +0200 Subject: [PATCH] NoncontiguousPartitioner: allow duplicate ghost indices --- .../base/mpi_noncontiguous_partitioner.h | 19 ++++++++++++ .../mpi_noncontiguous_partitioner.templates.h | 19 ++++++++---- source/base/mpi_noncontiguous_partitioner.cc | 30 ++++++++++++++++--- .../base/mpi_noncontiguous_partitioner_03.cc | 10 +++++++ ...ncontiguous_partitioner_03.mpirun=2.output | 4 +++ 5 files changed, 73 insertions(+), 9 deletions(-) diff --git a/include/deal.II/base/mpi_noncontiguous_partitioner.h b/include/deal.II/base/mpi_noncontiguous_partitioner.h index 0072d787c0..b721166c1d 100644 --- a/include/deal.II/base/mpi_noncontiguous_partitioner.h +++ b/include/deal.II/base/mpi_noncontiguous_partitioner.h @@ -320,10 +320,29 @@ namespace Utilities * Local index of each entry in recv_buffer within the destination * vector. * + * In the case that an entry is more than once requested by the + * same rank, local index of each entry in recv_buffer within + * recv_indices_duplicates_ptr. + * * @note Together with `recv_ptr` this forms a CRS data structure. */ std::vector recv_indices; + /** + * In the case that an entry is more than once requested by the + * same rank, offset of each index in recv_indices_duplicates. + */ + std::vector recv_indices_duplicates_ptr; + + /** + * Local index of each entry within the destination vector in the + * case that an entry is more than once requested by the same rank. + * + * @note Together with `recv_indices_duplicates_ptr` + * this forms a CRS data structure. + */ + std::vector recv_indices_duplicates; + /** * Buffer containing the values sorted by rank for sending and receiving. * diff --git a/include/deal.II/base/mpi_noncontiguous_partitioner.templates.h b/include/deal.II/base/mpi_noncontiguous_partitioner.templates.h index 3d5bd94027..cfc0dd0416 100644 --- a/include/deal.II/base/mpi_noncontiguous_partitioner.templates.h +++ b/include/deal.II/base/mpi_noncontiguous_partitioner.templates.h @@ -206,12 +206,21 @@ namespace Utilities AssertThrowMPI(ierr); AssertIndexRange(i + 1, recv_ptr.size()); - for (types::global_dof_index j = recv_ptr[i], c = 0; - j < recv_ptr[i + 1]; - ++j, ++c) + for (types::global_dof_index j = recv_ptr[i]; j < recv_ptr[i + 1]; + ++j) for (unsigned int comp = 0; comp < n_components_to_be_used; ++comp) - dst[recv_indices[j] * n_components_to_be_used + comp] = - buffers[(recv_ptr[i] + c) * n_components_to_be_used + comp]; + { + const auto &value = buffers[j * n_components_to_be_used + comp]; + + if (recv_indices_duplicates_ptr.empty()) + dst[recv_indices[j] * n_components_to_be_used + comp] = value; + else + for (auto k = recv_indices_duplicates_ptr[recv_indices[j]]; + k < recv_indices_duplicates_ptr[recv_indices[j] + 1]; + ++k) + dst[recv_indices_duplicates[k] * n_components_to_be_used + + comp] = value; + } } // wait that all data packages have been sent diff --git a/source/base/mpi_noncontiguous_partitioner.cc b/source/base/mpi_noncontiguous_partitioner.cc index 9effc1f79f..8eb8da23cd 100644 --- a/source/base/mpi_noncontiguous_partitioner.cc +++ b/source/base/mpi_noncontiguous_partitioner.cc @@ -216,15 +216,37 @@ namespace Utilities } { - std::vector temp_map_recv( + std::vector> temp_map_recv( index_set_want.n_elements()); for (types::global_dof_index i = 0; i < indices_want.size(); ++i) if (indices_want[i] != numbers::invalid_dof_index) - temp_map_recv[index_set_want.index_within_set(indices_want[i])] = i; + temp_map_recv[index_set_want.index_within_set(indices_want[i])] + .push_back(i); - for (auto &i : recv_indices) - i = temp_map_recv[i]; + const bool use_fast_path = + std::all_of(temp_map_recv.begin(), + temp_map_recv.end(), + [&](const auto &x) { return x.size() == 1; }); + + if (use_fast_path) + { + for (auto &i : recv_indices) + i = temp_map_recv[i][0]; + } + else + { + recv_indices_duplicates_ptr = {0}; + + for (const auto &indices : temp_map_recv) + { + for (const auto &index : indices) + recv_indices_duplicates.emplace_back(index); + + recv_indices_duplicates_ptr.push_back( + recv_indices_duplicates.size()); + } + } } } } // namespace MPI diff --git a/tests/base/mpi_noncontiguous_partitioner_03.cc b/tests/base/mpi_noncontiguous_partitioner_03.cc index c93783b666..e8b758a421 100644 --- a/tests/base/mpi_noncontiguous_partitioner_03.cc +++ b/tests/base/mpi_noncontiguous_partitioner_03.cc @@ -87,4 +87,14 @@ main(int argc, char *argv[]) test(comm, {4, 5, 6, 7}, {0, 1, 2, 3}); deallog.pop(); } + + { + deallog.push("duplicates"); + + if (rank == 0) + test(comm, {0, 1, 2, 3}, {4, 4, 5, 6, 7, 6}); + else + test(comm, {4, 5, 6, 7}, {0, 1, 2, 1, 1, 3}); + deallog.pop(); + } } diff --git a/tests/base/mpi_noncontiguous_partitioner_03.mpirun=2.output b/tests/base/mpi_noncontiguous_partitioner_03.mpirun=2.output index bfef1b6851..5b76439aa6 100644 --- a/tests/base/mpi_noncontiguous_partitioner_03.mpirun=2.output +++ b/tests/base/mpi_noncontiguous_partitioner_03.mpirun=2.output @@ -5,6 +5,8 @@ DEAL:0:padding-src::0 1 2 3 4 DEAL:0:padding-src::100 101 102 103 DEAL:0:padding-dst::0 1 2 3 DEAL:0:padding-dst::100 101 0 102 103 +DEAL:0:duplicates::0 1 2 3 +DEAL:0:duplicates::100 100 101 102 103 102 DEAL:1:padding-non::100 101 102 103 DEAL:1:padding-non::0 1 2 3 @@ -12,4 +14,6 @@ DEAL:1:padding-src::100 101 102 103 DEAL:1:padding-src::0 1 3 4 DEAL:1:padding-dst::100 101 102 103 DEAL:1:padding-dst::0 1 2 3 +DEAL:1:duplicates::100 101 102 103 +DEAL:1:duplicates::0 1 2 1 1 3 -- 2.39.5