From: Bruno Turcksin <bruno.turcksin@gmail.com>
Date: Sat, 1 Jul 2023 20:44:40 +0000 (+0000)
Subject: Use pinned memory when MPI is not GPU aware
X-Git-Tag: relicensing~815^2
X-Git-Url: https://gitweb.dealii.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=c9730f9e2dd85619bbde23c759d12281f5a329a1;p=dealii.git

Use pinned memory when MPI is not GPU aware
---

diff --git a/include/deal.II/base/memory_space_data.h b/include/deal.II/base/memory_space_data.h
index 82fd3472b1..932e076483 100644
--- a/include/deal.II/base/memory_space_data.h
+++ b/include/deal.II/base/memory_space_data.h
@@ -64,7 +64,11 @@ namespace MemorySpace
      * Kokkos View owning a host buffer used for MPI communication.
      */
     // FIXME Should we move this somewhere else?
+#if KOKKOS_VERSION < 40000
     Kokkos::View<T *, Kokkos::HostSpace> values_host_buffer;
+#else
+    Kokkos::View<T *, Kokkos::SharedHostPinnedSpace> values_host_buffer;
+#endif
 
     /**
      * Kokkos View owning the data on the @ref GlossDevice "device" (unless @p values_sm_ptr is used).
@@ -105,7 +109,12 @@ namespace MemorySpace
   MemorySpaceData<T, MemorySpace>::MemorySpaceData()
     : values_host_buffer(
         (dealii::internal::ensure_kokkos_initialized(),
+#  if KOKKOS_VERSION < 40000
          Kokkos::View<T *, Kokkos::HostSpace>("host buffer", 0)))
+#  else
+         Kokkos::View<T *, Kokkos::SharedHostPinnedSpace>("host pinned buffer",
+                                                          0)))
+#  endif
     , values(Kokkos::View<T *, typename MemorySpace::kokkos_space>(
         "memoryspace data",
         0))
diff --git a/include/deal.II/lac/la_parallel_vector.templates.h b/include/deal.II/lac/la_parallel_vector.templates.h
index 848aa0446a..2b9a2c0529 100644
--- a/include/deal.II/lac/la_parallel_vector.templates.h
+++ b/include/deal.II/lac/la_parallel_vector.templates.h
@@ -963,8 +963,6 @@ namespace LinearAlgebra
       Kokkos::pair<size_type, size_type> range(
         partitioner->locally_owned_size(),
         partitioner->locally_owned_size() + partitioner->n_ghost_indices());
-      if (data.values_host_buffer.size() > 0)
-        Kokkos::deep_copy(Kokkos::subview(data.values_host_buffer, range), 0);
       if (data.values.size() > 0)
         Kokkos::deep_copy(Kokkos::subview(data.values, range), 0);
 
@@ -1015,8 +1013,13 @@ namespace LinearAlgebra
           // uses a view of the array and thus we need the data on the host to
           // outlive the scope of the function.
           data.values_host_buffer =
+#    if KOKKOS_VERSION < 40000
             Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{},
                                                 data.values);
+#    else
+            Kokkos::create_mirror_view_and_copy(Kokkos::SharedHostPinnedSpace{},
+                                                data.values);
+#    endif
           partitioner->import_from_ghosted_array_start(
             operation,
             communication_channel,
@@ -1158,8 +1161,13 @@ namespace LinearAlgebra
           // uses a view of the array and thus we need the data on the host to
           // outlive the scope of the function.
           data.values_host_buffer =
+#    if KOKKOS_VERSION < 40000
             Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{},
                                                 data.values);
+#    else
+            Kokkos::create_mirror_view_and_copy(Kokkos::SharedHostPinnedSpace{},
+                                                data.values);
+#    endif
 
           partitioner->export_to_ghosted_array_start<Number, MemorySpace::Host>(
             communication_channel,