From c9730f9e2dd85619bbde23c759d12281f5a329a1 Mon Sep 17 00:00:00 2001 From: Bruno Turcksin Date: Sat, 1 Jul 2023 20:44:40 +0000 Subject: [PATCH] Use pinned memory when MPI is not GPU aware --- include/deal.II/base/memory_space_data.h | 9 +++++++++ include/deal.II/lac/la_parallel_vector.templates.h | 12 ++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/include/deal.II/base/memory_space_data.h b/include/deal.II/base/memory_space_data.h index 82fd3472b1..932e076483 100644 --- a/include/deal.II/base/memory_space_data.h +++ b/include/deal.II/base/memory_space_data.h @@ -64,7 +64,11 @@ namespace MemorySpace * Kokkos View owning a host buffer used for MPI communication. */ // FIXME Should we move this somewhere else? +#if KOKKOS_VERSION < 40000 Kokkos::View values_host_buffer; +#else + Kokkos::View values_host_buffer; +#endif /** * Kokkos View owning the data on the @ref GlossDevice "device" (unless @p values_sm_ptr is used). @@ -105,7 +109,12 @@ namespace MemorySpace MemorySpaceData::MemorySpaceData() : values_host_buffer( (dealii::internal::ensure_kokkos_initialized(), +# if KOKKOS_VERSION < 40000 Kokkos::View("host buffer", 0))) +# else + Kokkos::View("host pinned buffer", + 0))) +# endif , values(Kokkos::View( "memoryspace data", 0)) diff --git a/include/deal.II/lac/la_parallel_vector.templates.h b/include/deal.II/lac/la_parallel_vector.templates.h index 848aa0446a..2b9a2c0529 100644 --- a/include/deal.II/lac/la_parallel_vector.templates.h +++ b/include/deal.II/lac/la_parallel_vector.templates.h @@ -963,8 +963,6 @@ namespace LinearAlgebra Kokkos::pair range( partitioner->locally_owned_size(), partitioner->locally_owned_size() + partitioner->n_ghost_indices()); - if (data.values_host_buffer.size() > 0) - Kokkos::deep_copy(Kokkos::subview(data.values_host_buffer, range), 0); if (data.values.size() > 0) Kokkos::deep_copy(Kokkos::subview(data.values, range), 0); @@ -1015,8 +1013,13 @@ namespace LinearAlgebra // uses a view of the array and thus we need the data on the host to // outlive the scope of the function. data.values_host_buffer = +# if KOKKOS_VERSION < 40000 Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, data.values); +# else + Kokkos::create_mirror_view_and_copy(Kokkos::SharedHostPinnedSpace{}, + data.values); +# endif partitioner->import_from_ghosted_array_start( operation, communication_channel, @@ -1158,8 +1161,13 @@ namespace LinearAlgebra // uses a view of the array and thus we need the data on the host to // outlive the scope of the function. data.values_host_buffer = +# if KOKKOS_VERSION < 40000 Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, data.values); +# else + Kokkos::create_mirror_view_and_copy(Kokkos::SharedHostPinnedSpace{}, + data.values); +# endif partitioner->export_to_ghosted_array_start( communication_channel, -- 2.39.5