From ee5ea51fc36c6f00878e6130545fbffb4afcdb3d Mon Sep 17 00:00:00 2001 From: Peter Munch Date: Mon, 23 Mar 2020 11:06:00 +0100 Subject: [PATCH] Use dealii partitioner at less places --- include/deal.II/lac/la_sm_partitioner.h | 32 +++++++++-- include/deal.II/lac/la_sm_vector.templates.h | 59 ++++++++++++-------- 2 files changed, 64 insertions(+), 27 deletions(-) diff --git a/include/deal.II/lac/la_sm_partitioner.h b/include/deal.II/lac/la_sm_partitioner.h index 7973662c90..4bca93cc5d 100644 --- a/include/deal.II/lac/la_sm_partitioner.h +++ b/include/deal.II/lac/la_sm_partitioner.h @@ -70,6 +70,9 @@ namespace LinearAlgebra reinit(const IndexSet &is_locally_owned, const IndexSet &is_locally_ghost) { this->n_local_elements = is_locally_owned.n_elements(); + this->n_ghost_elements = is_locally_ghost.n_elements(); + + this->n_mpi_processes_ = Utilities::MPI::n_mpi_processes(comm); std::vector sm_ranks( Utilities::MPI::n_mpi_processes(comm_sm)); @@ -444,31 +447,52 @@ namespace LinearAlgebra compress_finish(data_this, data_others, buffer); } + std::size_t + local_size() const + { + return n_local_elements; + } + + std::size_t + n_ghost_indices() const + { + return n_ghost_elements; + } + + std::size_t + n_mpi_processes() const + { + return n_mpi_processes_; + } + private: const MPI_Comm &comm; const MPI_Comm &comm_sm; + unsigned int n_mpi_processes_; + unsigned int n_local_elements; + unsigned int n_ghost_elements; std::vector recv_remote_ranks; std::vector recv_remote_ptr = {0}; - mutable std::vector recv_remote_req; + mutable std::vector recv_remote_req; // TODO: move std::vector recv_sm_ranks; std::vector recv_sm_ptr = {0}; - mutable std::vector recv_sm_req; + mutable std::vector recv_sm_req; // TODO: move std::vector recv_sm_indices; std::vector recv_sm_offset; std::vector send_remote_ranks; std::vector send_remote_ptr = {0}; std::vector send_remote_indices; - mutable std::vector send_remote_req; + mutable std::vector send_remote_req; // TODO: move std::vector send_sm_ranks; std::vector send_sm_ptr = {0}; std::vector send_sm_indices; - mutable std::vector send_sm_req; + mutable std::vector send_sm_req; // TODO: move std::vector send_sm_offset; }; diff --git a/include/deal.II/lac/la_sm_vector.templates.h b/include/deal.II/lac/la_sm_vector.templates.h index 796cfff466..28975551b3 100644 --- a/include/deal.II/lac/la_sm_vector.templates.h +++ b/include/deal.II/lac/la_sm_vector.templates.h @@ -192,7 +192,7 @@ namespace LinearAlgebra partitioner = v.partitioner; partitioner_sm = v.partitioner_sm; const size_type new_allocated_size = - partitioner->local_size() + partitioner->n_ghost_indices(); + partitioner_sm->local_size() + partitioner_sm->n_ghost_indices(); resize_val(new_allocated_size, partitioner_sm->get_sm_mpi_communicator()); } @@ -253,7 +253,8 @@ namespace LinearAlgebra // set vector size and allocate memory const size_type new_allocated_size = - this->partitioner->local_size() + this->partitioner->n_ghost_indices(); + this->partitioner_sm->local_size() + + this->partitioner_sm->n_ghost_indices(); resize_val(new_allocated_size, partitioner_sm->get_sm_mpi_communicator()); // initialize to zero @@ -295,7 +296,10 @@ namespace LinearAlgebra { dealii::internal::VectorOperations:: functions::copy( - thread_loop_partitioner, partitioner->local_size(), v.data, data); + thread_loop_partitioner, + partitioner_sm->local_size(), + v.data, + data); } } @@ -431,7 +435,7 @@ namespace LinearAlgebra thread_loop_partitioner = c.thread_loop_partitioner; - const size_type this_size = partitioner->local_size(); + const size_type this_size = partitioner_sm->local_size(); if (this_size > 0) { dealii::internal::VectorOperations:: @@ -500,8 +504,8 @@ namespace LinearAlgebra Vector::zero_out_ghosts() const { if (data.values != nullptr) - std::fill_n(data.values.get() + partitioner->local_size(), - partitioner->n_ghost_indices(), + std::fill_n(data.values.get() + partitioner_sm->local_size(), + partitioner_sm->n_ghost_indices(), Number()); vector_is_ghosted = false; @@ -652,7 +656,7 @@ namespace LinearAlgebra dealii::internal::VectorOperations:: functions::subtract_vector( - thread_loop_partitioner, partitioner->local_size(), v.data, data); + thread_loop_partitioner, partitioner_sm->local_size(), v.data, data); if (vector_is_ghosted) update_ghost_values(); @@ -693,7 +697,11 @@ namespace LinearAlgebra dealii::internal::VectorOperations:: functions::add_av( - thread_loop_partitioner, partitioner->local_size(), a, v.data, data); + thread_loop_partitioner, + partitioner_sm->local_size(), + a, + v.data, + data); } @@ -772,7 +780,7 @@ namespace LinearAlgebra dealii::internal::VectorOperations:: functions::sadd_xav( thread_loop_partitioner, - partitioner->local_size(), + partitioner_sm->local_size(), x, a, v.data, @@ -876,7 +884,11 @@ namespace LinearAlgebra dealii::internal::VectorOperations:: functions::equ_au( - thread_loop_partitioner, partitioner->local_size(), a, v.data, data); + thread_loop_partitioner, + partitioner_sm->local_size(), + a, + v.data, + data); if (vector_is_ghosted) @@ -920,11 +932,12 @@ namespace LinearAlgebra if (PointerComparison::equal(this, &v)) return norm_sqr_local(); - AssertDimension(partitioner->local_size(), v.partitioner->local_size()); + AssertDimension(partitioner_sm->local_size(), + v.partitioner_sm->local_size()); return dealii::internal::VectorOperations:: functions::dot( - thread_loop_partitioner, partitioner->local_size(), v.data, data); + thread_loop_partitioner, partitioner_sm->local_size(), v.data, data); } @@ -940,9 +953,9 @@ namespace LinearAlgebra const VectorType &v = dynamic_cast(vv); Number local_result = inner_product_local(v); - if (partitioner->n_mpi_processes() > 1) + if (partitioner_sm->n_mpi_processes() > 1) return Utilities::MPI::sum(local_result, - partitioner->get_mpi_communicator()); + partitioner_sm->get_mpi_communicator()); else return local_result; } @@ -958,7 +971,7 @@ namespace LinearAlgebra dealii::internal::VectorOperations:: functions::norm_2( - thread_loop_partitioner, partitioner->local_size(), sum, data); + thread_loop_partitioner, partitioner_sm->local_size(), sum, data); AssertIsFinite(sum); @@ -1012,9 +1025,9 @@ namespace LinearAlgebra Vector::norm_sqr() const { real_type local_result = norm_sqr_local(); - if (partitioner->n_mpi_processes() > 1) + if (partitioner_sm->n_mpi_processes() > 1) return Utilities::MPI::sum(local_result, - partitioner->get_mpi_communicator()); + partitioner_sm->get_mpi_communicator()); else return local_result; } @@ -1058,7 +1071,7 @@ namespace LinearAlgebra { real_type max = 0.; - const size_type local_size = partitioner->local_size(); + const size_type local_size = partitioner_sm->local_size(); internal::la_parallel_vector_templates_functions< Number, MemorySpaceType>::linfty_norm_local(data, local_size, max); @@ -1073,9 +1086,9 @@ namespace LinearAlgebra Vector::linfty_norm() const { const real_type local_result = linfty_norm_local(); - if (partitioner->n_mpi_processes() > 1) + if (partitioner_sm->n_mpi_processes() > 1) return Utilities::MPI::max(local_result, - partitioner->get_mpi_communicator()); + partitioner_sm->get_mpi_communicator()); else return local_result; } @@ -1089,7 +1102,7 @@ namespace LinearAlgebra const Vector &v, const Vector &w) { - const size_type vec_size = partitioner->local_size(); + const size_type vec_size = partitioner_sm->local_size(); AssertDimension(vec_size, v.local_size()); AssertDimension(vec_size, w.local_size()); @@ -1121,9 +1134,9 @@ namespace LinearAlgebra const VectorType &w = dynamic_cast(ww); Number local_result = add_and_dot_local(a, v, w); - if (partitioner->n_mpi_processes() > 1) + if (partitioner_sm->n_mpi_processes() > 1) return Utilities::MPI::sum(local_result, - partitioner->get_mpi_communicator()); + partitioner_sm->get_mpi_communicator()); else return local_result; } -- 2.39.5