From 800459dd3ade7c6022d170508ffec604ffdb4131 Mon Sep 17 00:00:00 2001
From: Martin Kronbichler <kronbichler@lnm.mw.tum.de>
Date: Tue, 1 Oct 2019 14:55:49 +0200
Subject: [PATCH] Synchronize CUDA device in MPI ghost exchange

---
 include/deal.II/base/partitioner.templates.h | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/include/deal.II/base/partitioner.templates.h b/include/deal.II/base/partitioner.templates.h
index 167e01c45c..da662e10de 100644
--- a/include/deal.II/base/partitioner.templates.h
+++ b/include/deal.II/base/partitioner.templates.h
@@ -126,6 +126,7 @@ namespace Utilities
                   import_indices_plain_dev[i].first.get(),
                   locally_owned_array.data(),
                   chunk_size);
+              cudaDeviceSynchronize();
             }
           else
 #    endif
@@ -400,6 +401,11 @@ namespace Utilities
             ExcMessage("Index overflow: Maximum message size in MPI is 2GB. "
                        "The number of ghost entries times the size of 'Number' "
                        "exceeds this value. This is not supported."));
+#    if defined(DEAL_II_COMPILER_CUDA_AWARE) && \
+      defined(DEAL_II_MPI_WITH_CUDA_SUPPORT)
+          if (std::is_same<MemorySpaceType, MemorySpace::CUDA>::value)
+            cudaDeviceSynchronize();
+#    endif
           const int ierr =
             MPI_Isend(ghost_array_ptr,
                       ghost_targets_data[i].second * sizeof(Number),
-- 
2.39.5