From 4234e6777fed126fe6302828a16ca1890fec4d9f Mon Sep 17 00:00:00 2001 From: Peter Munch Date: Sun, 22 Mar 2020 15:41:04 +0100 Subject: [PATCH] Take care of alignment (by 64 bytes) --- include/deal.II/lac/la_sm_vector.templates.h | 45 ++++++++++++++++---- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/include/deal.II/lac/la_sm_vector.templates.h b/include/deal.II/lac/la_sm_vector.templates.h index 44dfbea92f..ce89fef8b7 100644 --- a/include/deal.II/lac/la_sm_vector.templates.h +++ b/include/deal.II/lac/la_sm_vector.templates.h @@ -56,25 +56,29 @@ namespace LinearAlgebra allocated_size = new_alloc_size; + const unsigned int size_sm = + Utilities::MPI::n_mpi_processes(comm_shared); + const unsigned int rank_sm = + Utilities::MPI::this_mpi_process(comm_shared); + data.values_win = new MPI_Win; Number *data_this = (Number *)malloc(0); - data.others.resize(Utilities::MPI::n_mpi_processes(comm_shared)); - + data.others.resize(size_sm); MPI_Info info; MPI_Info_create(&info); MPI_Info_set(info, "alloc_shared_noncontig", "true"); - MPI_Win_allocate_shared(new_alloc_size * sizeof(Number), + const std::size_t align_by = 64; + + MPI_Win_allocate_shared(new_alloc_size * sizeof(Number) + align_by, sizeof(Number), info, comm_shared, data_this, data.values_win); - for (unsigned int i = 0; - i < Utilities::MPI::n_mpi_processes(comm_shared); - i++) + for (unsigned int i = 0; i < size_sm; i++) { int disp_unit; MPI_Aint ssize; @@ -82,9 +86,32 @@ namespace LinearAlgebra *data.values_win, i, &ssize, &disp_unit, &data.others[i]); } - data.values = { - data.others[Utilities::MPI::this_mpi_process(comm_shared)], - [&data](Number *&) { MPI_Win_free(data.values_win); }}; + Number * ptr_unaligned = data.others[rank_sm]; + Number * ptr_aligned = ptr_unaligned; + std::size_t s = new_alloc_size * sizeof(Number) + align_by; + + AssertThrow(std::align(align_by, + new_alloc_size * sizeof(Number), + reinterpret_cast(ptr_aligned), + s) != nullptr, + ExcNotImplemented()); + + unsigned int n_align_local = ptr_aligned - ptr_unaligned; + std::vector n_align_sm(size_sm); + + MPI_Allgather(&n_align_local, + 1, + MPI_UNSIGNED, + n_align_sm.data(), + 1, + MPI_UNSIGNED, + comm_shared); + + for (unsigned int i = 0; i < size_sm; i++) + data.others[i] += n_align_sm[i]; + + data.values = {ptr_aligned, + [&data](Number *&) { MPI_Win_free(data.values_win); }}; } }; } // namespace internal -- 2.39.5