From: Martin Kronbichler <kronbichler@lnm.mw.tum.de>
Date: Tue, 1 Oct 2019 12:55:49 +0000 (+0200)
Subject: Synchronize CUDA device in MPI ghost exchange
X-Git-Tag: v9.2.0-rc1~998^2
X-Git-Url: https://gitweb.dealii.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=800459dd3ade7c6022d170508ffec604ffdb4131;p=dealii.git

Synchronize CUDA device in MPI ghost exchange
---

diff --git a/include/deal.II/base/partitioner.templates.h b/include/deal.II/base/partitioner.templates.h
index 167e01c45c..da662e10de 100644
--- a/include/deal.II/base/partitioner.templates.h
+++ b/include/deal.II/base/partitioner.templates.h
@@ -126,6 +126,7 @@ namespace Utilities
                   import_indices_plain_dev[i].first.get(),
                   locally_owned_array.data(),
                   chunk_size);
+              cudaDeviceSynchronize();
             }
           else
 #    endif
@@ -400,6 +401,11 @@ namespace Utilities
             ExcMessage("Index overflow: Maximum message size in MPI is 2GB. "
                        "The number of ghost entries times the size of 'Number' "
                        "exceeds this value. This is not supported."));
+#    if defined(DEAL_II_COMPILER_CUDA_AWARE) && \
+      defined(DEAL_II_MPI_WITH_CUDA_SUPPORT)
+          if (std::is_same<MemorySpaceType, MemorySpace::CUDA>::value)
+            cudaDeviceSynchronize();
+#    endif
           const int ierr =
             MPI_Isend(ghost_array_ptr,
                       ghost_targets_data[i].second * sizeof(Number),