From 30e9a8fd98ac30ed32f17ca84277bd36c7964f3c Mon Sep 17 00:00:00 2001 From: Daniel Arndt Date: Thu, 1 Dec 2022 22:57:21 +0000 Subject: [PATCH] Use original memory_space_data design --- include/deal.II/base/memory_space_data.h | 214 +++++++---------------- 1 file changed, 67 insertions(+), 147 deletions(-) diff --git a/include/deal.II/base/memory_space_data.h b/include/deal.II/base/memory_space_data.h index dc9e6d12ea..77bbc00de2 100644 --- a/include/deal.II/base/memory_space_data.h +++ b/include/deal.II/base/memory_space_data.h @@ -23,9 +23,7 @@ #include #include -DEAL_II_DISABLE_EXTRA_DIAGNOSTICS #include -DEAL_II_ENABLE_EXTRA_DIAGNOSTICS #include #include @@ -37,42 +35,37 @@ DEAL_II_NAMESPACE_OPEN namespace MemorySpace { /** - * Data structure + * Structure which stores data on the host or the device depending on the + * template parameter @p MemorySpace. The data is copied into the structure + * which then owns the data and will release the memory when the destructor is + * called. */ template struct MemorySpaceData { - MemorySpaceData() - { - static_assert(std::is_same::value || - std::is_same::value, - "MemorySpace should be Host or Device"); - } + MemorySpaceData(); /** - * Copy the active data (values for Host and values for Device) to @p begin. + * Copy the active data (values for Host and values_dev for Device) to @p begin. * If the data is on the device it is moved to the host. */ void - copy_to(T *begin, std::size_t n_elements) - { - (void)begin; - (void)n_elements; - } + copy_to(T *begin, const std::size_t n_elements); /** * Copy the data in @p begin to the active data of the structure (values for - * Host and values for Device). The pointer @p begin must be on the host. + * Host and values_dev for Device). The pointer @p begin must be on the host. */ void - copy_from(T *begin, std::size_t n_elements) - { - (void)begin; - (void)n_elements; - } + copy_from(const T *begin, const std::size_t n_elements); /** - * Pointer to data on the device. + * Kokkos View to a host buffer used for MPI communication + */ + Kokkos::View values_host_buffer; + + /** + * Kokkos View to the data on the device */ Kokkos::View values; @@ -83,12 +76,12 @@ namespace MemorySpace // The pointer is shared so that MemorySpaceData can be copied and // MemorySpaceData::values can be used in Kokkos::parallel_for. This // pointer owns the data when using shared memory with MPI. In this case, - // the (host) Kokkos::View is non-owning. When shared memory with MPI is not - // used, the pointer is not used. + // the Kokkos::View in non-owning. When shared memory with MPI is not used, + // the pointer is not used. std::shared_ptr values_sm_ptr; /** - * Pointers to the data of the processes sharing the same memory. Not used for MemorySpace::Device. + * Pointers to the data of the processes sharing the same memory. */ std::vector> values_sm; }; @@ -100,146 +93,73 @@ namespace MemorySpace */ template inline void - swap(MemorySpaceData &, - MemorySpaceData &) - { - static_assert(std::is_same::value || - std::is_same::value, - "MemorySpace should be Host or Device"); - } + swap(MemorySpaceData &u, MemorySpaceData &v); -#ifndef DOXYGEN - template - struct MemorySpaceData - { - using MemorySpace = Host; +#ifndef DOXYGEN - MemorySpaceData() - : values((dealii::Impl::ensure_kokkos_initialized(), + template + MemorySpaceData::MemorySpaceData() + : values_host_buffer((dealii::Impl::ensure_kokkos_initialized(), Kokkos::View("host data", 0))) - {} - - void - copy_to(T *begin, std::size_t n_elements) - { - Assert(n_elements <= values.extent(0), - ExcMessage("n_elements greater than the size of values.")); - using ExecutionSpace = typename MemorySpace::kokkos_space::execution_space; - Kokkos:: - View> - begin_view(begin, n_elements); - Kokkos::deep_copy( - ExecutionSpace{}, - begin_view, - Kokkos::subview(values, Kokkos::make_pair(std::size_t(0), n_elements))); - ExecutionSpace{}.fence(); - } - - void - copy_from(T *begin, std::size_t n_elements) - { - Assert(n_elements <= values.extent(0), - ExcMessage("n_elements greater than the size of values.")); - using ExecutionSpace = typename MemorySpace::kokkos_space::execution_space; - Kokkos::View> - begin_view(begin, n_elements); - Kokkos::deep_copy( - ExecutionSpace{}, - Kokkos::subview(values, Kokkos::make_pair(std::size_t(0), n_elements)), - begin_view); - ExecutionSpace{}.fence(); - } - - // unused - Kokkos::View values_host_buffer; - - Kokkos::View values; - - std::shared_ptr values_sm_ptr; - - std::vector> values_sm; - }; + , values(Kokkos::View( + "memoryspace data", + 0)) + {} - template - inline void - swap(MemorySpaceData &u, MemorySpaceData &v) + template + void + MemorySpaceData::copy_to(T * begin, + const std::size_t n_elements) { - std::swap(u.values, v.values); - std::swap(u.values_sm_ptr, v.values_sm_ptr); + Assert(n_elements <= values.extent(0), + ExcMessage("n_elements greater than the size of values.")); + using ExecutionSpace = typename MemorySpace::kokkos_space::execution_space; + Kokkos:: + View> + begin_view(begin, n_elements); + Kokkos::deep_copy( + ExecutionSpace{}, + begin_view, + Kokkos::subview(values, Kokkos::make_pair(std::size_t(0), n_elements))); + ExecutionSpace{}.fence(); } - template - struct MemorySpaceData + template + void + MemorySpaceData::copy_from(const T * begin, + const std::size_t n_elements) { - using MemorySpace = Device; - - MemorySpaceData() - : values_host_buffer((dealii::Impl::ensure_kokkos_initialized(), - Kokkos::View("host data", 0))) - , values(Kokkos::View( - "memoryspace data", - 0)) - {} - - void - copy_to(T *begin, std::size_t n_elements) - { - Assert(n_elements <= values.extent(0), - ExcMessage("n_elements greater than the size of values.")); - using ExecutionSpace = typename MemorySpace::kokkos_space::execution_space; - Kokkos:: - View> - begin_view(begin, n_elements); - Kokkos::deep_copy( - ExecutionSpace{}, - begin_view, - Kokkos::subview(values, Kokkos::make_pair(std::size_t(0), n_elements))); - ExecutionSpace{}.fence(); - } - - void - copy_from(T *begin, std::size_t n_elements) - { - Assert(n_elements <= values.extent(0), - ExcMessage("n_elements greater than the size of values.")); - using ExecutionSpace = typename MemorySpace::kokkos_space::execution_space; - Kokkos::View> - begin_view(begin, n_elements); - Kokkos::deep_copy( - ExecutionSpace{}, - Kokkos::subview(values, Kokkos::make_pair(std::size_t(0), n_elements)), - begin_view); - ExecutionSpace{}.fence(); - } - - Kokkos::View values_host_buffer; - - Kokkos::View values; - - // unused - std::shared_ptr values_sm_ptr; - - // unused - std::vector> values_sm; - }; + Assert(n_elements <= values.extent(0), + ExcMessage("n_elements greater than the size of values.")); + using ExecutionSpace = typename MemorySpace::kokkos_space::execution_space; + Kokkos::View> + begin_view(begin, n_elements); + Kokkos::deep_copy( + ExecutionSpace{}, + Kokkos::subview(values, Kokkos::make_pair(std::size_t(0), n_elements)), + begin_view); + ExecutionSpace{}.fence(); + } - template + /** + * Swap function similar to std::swap. + */ + template inline void - swap(MemorySpaceData &u, MemorySpaceData &v) + swap(MemorySpaceData &u, MemorySpaceData &v) { - std::swap(u.values, v.values); - std::swap(u.values_host_buffer, v.values_host_buffer); + std::swap(u.values_host_buffer, v.values_host_buffer); + std::swap(u.values, v.values); + std::swap(u.values_sm_ptr, v.values_sm_ptr); } #endif -- 2.39.5