From: Peter Munch Date: Sat, 21 Mar 2020 12:48:22 +0000 (+0100) Subject: Add MemorySpaceData and start to implement resize_val X-Git-Url: https://gitweb.dealii.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=10eb5f96d8abd40188047e2f5b3cf53c607ef82d;p=dealii.git Add MemorySpaceData and start to implement resize_val --- diff --git a/include/deal.II/lac/la_sm_vector.h b/include/deal.II/lac/la_sm_vector.h index 0888c831f0..748b73db31 100644 --- a/include/deal.II/lac/la_sm_vector.h +++ b/include/deal.II/lac/la_sm_vector.h @@ -75,6 +75,32 @@ namespace LinearAlgebra { namespace SharedMPI { + template + struct MemorySpaceData + { + void + copy_to(Number *begin, std::size_t n_elements) + { + Assert(false, ExcNotImplemented()); + (void)begin; + (void)n_elements; + } + + void + copy_from(Number *begin, std::size_t n_elements) + { + Assert(false, ExcNotImplemented()); + (void)begin; + (void)n_elements; + } + + std::unique_ptr> values; + MPI_Win *values_win = nullptr; + + std::unique_ptr values_dev; + }; + + template class Vector : public ::dealii::LinearAlgebra::VectorSpaceVector, public Subscriptor @@ -451,13 +477,10 @@ namespace LinearAlgebra size_type allocated_size; - mutable ::dealii::MemorySpace::MemorySpaceData data; - - mutable std::shared_ptr<::dealii::parallel::internal::TBBPartitioner> - thread_loop_partitioner; + mutable MemorySpaceData data; - mutable ::dealii::MemorySpace::MemorySpaceData - import_data; + // needed? + mutable MemorySpaceData import_data; mutable bool vector_is_ghosted; @@ -487,92 +510,26 @@ namespace LinearAlgebra struct Policy { static inline typename Vector::iterator - begin(::dealii::MemorySpace::MemorySpaceData &) + begin(MemorySpaceData &) { Assert(false, ExcNotImplemented()); return nullptr; } static inline typename Vector::const_iterator - begin( - const ::dealii::MemorySpace::MemorySpaceData &) + begin(const MemorySpaceData &) { Assert(false, ExcNotImplemented()); return nullptr; } static inline Number * - get_values( - ::dealii::MemorySpace::MemorySpaceData &) + get_values(MemorySpaceData &) { Assert(false, ExcNotImplemented()); return nullptr; } }; - - - - template - struct Policy - { - static inline - typename Vector::iterator - begin(::dealii::MemorySpace:: - MemorySpaceData &data) - { - Assert(false, ExcNotImplemented()); - return data.values.get(); - } - - static inline - typename Vector::const_iterator - begin(const ::dealii::MemorySpace:: - MemorySpaceData &data) - { - Assert(false, ExcNotImplemented()); - return data.values.get(); - } - - static inline Number * - get_values(::dealii::MemorySpace:: - MemorySpaceData &data) - { - Assert(false, ExcNotImplemented()); - return data.values.get(); - } - }; - - - - template - struct Policy - { - static inline - typename Vector::iterator - begin(::dealii::MemorySpace:: - MemorySpaceData &data) - { - Assert(false, ExcNotImplemented()); - return data.values_dev.get(); - } - - static inline - typename Vector::const_iterator - begin(const ::dealii::MemorySpace:: - MemorySpaceData &data) - { - Assert(false, ExcNotImplemented()); - return data.values_dev.get(); - } - - static inline Number * - get_values(::dealii::MemorySpace:: - MemorySpaceData &data) - { - Assert(false, ExcNotImplemented()); - return data.values_dev.get(); - } - }; } // namespace internal diff --git a/include/deal.II/lac/la_sm_vector.templates.h b/include/deal.II/lac/la_sm_vector.templates.h index 6049b90438..fd4ca5de3b 100644 --- a/include/deal.II/lac/la_sm_vector.templates.h +++ b/include/deal.II/lac/la_sm_vector.templates.h @@ -38,6 +38,61 @@ namespace LinearAlgebra { namespace SharedMPI { + namespace internal + { + template + struct la_parallel_vector_templates_functions + { + using size_type = types::global_dof_index; + + static void + resize_val(const types::global_dof_index new_alloc_size, + types::global_dof_index & allocated_size, + MemorySpaceData & data) + { + Assert(allocated_size == 0, ExcNotImplemented()); + Assert(data.values == nullptr, ExcNotImplemented()); + + allocated_size = new_alloc_size; + + // TODO + std::vector data_others; + MPI_Comm comm_shared; + + data.values_win = new MPI_Win; + Number *data_this = (Number *)malloc(0); + data_others.resize(Utilities::MPI::n_mpi_processes(comm_shared)); + + + MPI_Info info; + MPI_Info_create(&info); + MPI_Info_set(info, "alloc_shared_noncontig", "true"); + + MPI_Win_allocate_shared(new_alloc_size * sizeof(Number), + sizeof(Number), + info, + comm_shared, + data_this, + data.values_win); + + for (unsigned int i = 0; + i < Utilities::MPI::n_mpi_processes(comm_shared); + i++) + { + int disp_unit; + MPI_Aint ssize; + MPI_Win_shared_query( + *data.values_win, i, &ssize, &disp_unit, &data_others[i]); + } + + data.values = { + data_others[Utilities::MPI::this_mpi_process(comm_shared)], + [&data](Number *&) { MPI_Win_free(data.values_win); }}; + } + }; + } // namespace internal + + template void Vector::clear_mpi_requests() @@ -51,8 +106,9 @@ namespace LinearAlgebra void Vector::resize_val(const size_type new_alloc_size) { - Assert(false, ExcNotImplemented()); - (void)new_alloc_size; + internal::la_parallel_vector_templates_functions< + Number, + MemorySpaceType>::resize_val(new_alloc_size, allocated_size, data); } @@ -116,8 +172,26 @@ namespace LinearAlgebra Vector::reinit( const std::shared_ptr &partitioner_in) { - Assert(false, ExcNotImplemented()); - (void)partitioner_in; + clear_mpi_requests(); + partitioner = partitioner_in; + + // set vector size and allocate memory + const size_type new_allocated_size = + partitioner->local_size() + partitioner->n_ghost_indices(); + resize_val(new_allocated_size); + + // initialize to zero + this->operator=(Number()); + + + // do not reallocate import_data directly, but only upon request. It + // is only used as temporary storage for compress() and + // update_ghost_values, and we might have vectors where we never + // call these methods and hence do not need to have the storage. + import_data.values.reset(); + import_data.values_dev.reset(); + + vector_is_ghosted = false; }