From 2db7e8fe6c0b80b85085c5cb5a2f5a667e36c8e0 Mon Sep 17 00:00:00 2001
From: Daniel Arndt <daniel.arndt@iwr.uni-heidelberg.de>
Date: Thu, 16 Aug 2018 10:18:30 +0200
Subject: [PATCH] Introduce DEAL_II_COMPILER_CUDA_AWARE

---
 doc/doxygen/options.dox.in                           |  1 +
 include/deal.II/base/config.h.in                     |  7 +++++++
 include/deal.II/base/cuda.h                          | 12 ++++++------
 include/deal.II/base/numbers.h                       |  2 +-
 include/deal.II/lac/cuda_atomic.h                    |  2 +-
 include/deal.II/lac/cuda_kernels.h                   |  2 +-
 include/deal.II/lac/cuda_kernels.templates.h         |  2 +-
 include/deal.II/lac/cuda_solver_direct.h             |  2 +-
 include/deal.II/lac/cuda_sparse_matrix.h             |  2 +-
 include/deal.II/matrix_free/cuda_fe_evaluation.h     |  2 +-
 .../matrix_free/cuda_hanging_nodes_internal.h        |  2 +-
 include/deal.II/matrix_free/cuda_matrix_free.h       |  2 +-
 .../deal.II/matrix_free/cuda_matrix_free.templates.h |  2 +-
 .../matrix_free/cuda_tensor_product_kernels.h        |  2 +-
 14 files changed, 25 insertions(+), 17 deletions(-)

diff --git a/doc/doxygen/options.dox.in b/doc/doxygen/options.dox.in
index e94a58259d..6f2038a253 100644
--- a/doc/doxygen/options.dox.in
+++ b/doc/doxygen/options.dox.in
@@ -176,6 +176,7 @@ PREDEFINED             = DOXYGEN=1 \
                          DEAL_II_WITH_CXX14=1 \
                          DEAL_II_WITH_CXX17=1 \
                          DEAL_II_WITH_CUDA=1 \
+                         DEAL_II_COMPILER_CUDA_AWARE=1 \
                          DEAL_II_WITH_GSL=1 \
                          DEAL_II_WITH_GMSH=1 \
                          DEAL_II_WITH_HDF5=1 \
diff --git a/include/deal.II/base/config.h.in b/include/deal.II/base/config.h.in
index 2aaa907e5b..c0666e98a1 100644
--- a/include/deal.II/base/config.h.in
+++ b/include/deal.II/base/config.h.in
@@ -96,6 +96,13 @@
 #cmakedefine DEAL_II_RESTRICT @DEAL_II_RESTRICT@
 #cmakedefine DEAL_II_COMPILER_HAS_DIAGNOSTIC_PRAGMA
 
+/*
+ * A variable to tell if the compiler used in the current compilation process
+ * understands CUDA code.
+ */
+#if defined(DEAL_II_WITH_CUDA) && defined(__CUDACC__)
+#  define DEAL_II_COMPILER_CUDA_AWARE
+#endif
 
 /***********************************************************************
  * CPU features:
diff --git a/include/deal.II/base/cuda.h b/include/deal.II/base/cuda.h
index fa26cacf2a..9a4d57dd47 100644
--- a/include/deal.II/base/cuda.h
+++ b/include/deal.II/base/cuda.h
@@ -20,7 +20,7 @@
 
 #include <deal.II/base/exceptions.h>
 
-#if defined(DEAL_II_WITH_CUDA) && defined(__CUDACC__)
+#ifdef DEAL_II_COMPILER_CUDA_AWARE
 #  include <cusolverDn.h>
 #  include <cusolverSp.h>
 #  include <cusparse.h>
@@ -58,7 +58,7 @@ namespace Utilities
        */
       ~Handle();
 
-#if defined(DEAL_II_WITH_CUDA) && defined(__CUDACC__)
+#ifdef DEAL_II_COMPILER_CUDA_AWARE
       cusolverDnHandle_t cusolver_dn_handle;
 
       cusolverSpHandle_t cusolver_sp_handle;
@@ -77,7 +77,7 @@ namespace Utilities
     inline void
     malloc(T *&pointer, const unsigned int n_elements)
     {
-#if defined(DEAL_II_WITH_CUDA) && defined(__CUDACC__)
+#ifdef DEAL_II_COMPILER_CUDA_AWARE
       cudaError_t cuda_error_code =
         cudaMalloc(&pointer, n_elements * sizeof(T));
       AssertCuda(cuda_error_code);
@@ -94,7 +94,7 @@ namespace Utilities
     inline void
     free(T *&pointer)
     {
-#if defined(DEAL_II_WITH_CUDA) && defined(__CUDACC__)
+#ifdef DEAL_II_COMPILER_CUDA_AWARE
       cudaError_t cuda_error_code = cudaFree(pointer);
       AssertCuda(cuda_error_code);
       pointer = nullptr;
@@ -110,7 +110,7 @@ namespace Utilities
     inline void
     copy_to_host(const T *pointer_dev, std::vector<T> &vector_host)
     {
-#if defined(DEAL_II_WITH_CUDA) && defined(__CUDACC__)
+#ifdef DEAL_II_COMPILER_CUDA_AWARE
       cudaError_t cuda_error_code = cudaMemcpy(vector_host.data(),
                                                pointer_dev,
                                                vector_host.size() * sizeof(T),
@@ -130,7 +130,7 @@ namespace Utilities
     inline void
     copy_to_dev(const std::vector<T> &vector_host, T *pointer_dev)
     {
-#if defined(DEAL_II_WITH_CUDA) && defined(__CUDACC__)
+#ifdef DEAL_II_COMPILER_CUDA_AWARE
       cudaError_t cuda_error_code = cudaMemcpy(pointer_dev,
                                                vector_host.data(),
                                                vector_host.size() * sizeof(T),
diff --git a/include/deal.II/base/numbers.h b/include/deal.II/base/numbers.h
index 1bc05e6345..7c5e9a2712 100644
--- a/include/deal.II/base/numbers.h
+++ b/include/deal.II/base/numbers.h
@@ -25,7 +25,7 @@
 #include <complex>
 #include <cstdlib>
 
-#if defined(DEAL_II_WITH_CUDA) && defined(__CUDACC__)
+#ifdef DEAL_II_COMPILER_CUDA_AWARE
 #  define DEAL_II_CUDA_HOST_DEV __host__ __device__
 #else
 #  define DEAL_II_CUDA_HOST_DEV
diff --git a/include/deal.II/lac/cuda_atomic.h b/include/deal.II/lac/cuda_atomic.h
index e717b633b9..7a6cc60496 100644
--- a/include/deal.II/lac/cuda_atomic.h
+++ b/include/deal.II/lac/cuda_atomic.h
@@ -18,7 +18,7 @@
 
 #include <deal.II/base/config.h>
 
-#if defined(DEAL_II_WITH_CUDA) && defined(__CUDACC__)
+#ifdef DEAL_II_COMPILER_CUDA_AWARE
 
 DEAL_II_NAMESPACE_OPEN
 
diff --git a/include/deal.II/lac/cuda_kernels.h b/include/deal.II/lac/cuda_kernels.h
index 367df1c081..8e525463c4 100644
--- a/include/deal.II/lac/cuda_kernels.h
+++ b/include/deal.II/lac/cuda_kernels.h
@@ -18,7 +18,7 @@
 
 #include <deal.II/base/config.h>
 
-#if defined(DEAL_II_WITH_CUDA) && defined(__CUDACC__)
+#ifdef DEAL_II_COMPILER_CUDA_AWARE
 
 
 #  include <deal.II/base/cuda_size.h>
diff --git a/include/deal.II/lac/cuda_kernels.templates.h b/include/deal.II/lac/cuda_kernels.templates.h
index 81d8ef9bcf..3a22883453 100644
--- a/include/deal.II/lac/cuda_kernels.templates.h
+++ b/include/deal.II/lac/cuda_kernels.templates.h
@@ -18,7 +18,7 @@
 
 #include <deal.II/lac/cuda_kernels.h>
 
-#if defined(DEAL_II_WITH_CUDA) && defined(__CUDACC__)
+#ifdef DEAL_II_COMPILER_CUDA_AWARE
 
 DEAL_II_NAMESPACE_OPEN
 
diff --git a/include/deal.II/lac/cuda_solver_direct.h b/include/deal.II/lac/cuda_solver_direct.h
index 493a70e9f1..e53eef8acc 100644
--- a/include/deal.II/lac/cuda_solver_direct.h
+++ b/include/deal.II/lac/cuda_solver_direct.h
@@ -18,7 +18,7 @@
 
 #include <deal.II/base/config.h>
 
-#if defined(DEAL_II_WITH_CUDA) && defined(__CUDACC__)
+#ifdef DEAL_II_COMPILER_CUDA_AWARE
 #  include <deal.II/base/cuda.h>
 
 #  include <deal.II/lac/cuda_sparse_matrix.h>
diff --git a/include/deal.II/lac/cuda_sparse_matrix.h b/include/deal.II/lac/cuda_sparse_matrix.h
index 0be2bf0e82..51cf57d2fb 100644
--- a/include/deal.II/lac/cuda_sparse_matrix.h
+++ b/include/deal.II/lac/cuda_sparse_matrix.h
@@ -20,7 +20,7 @@
 
 #include <deal.II/base/subscriptor.h>
 
-#if defined(DEAL_II_WITH_CUDA) && defined(__CUDACC__)
+#ifdef DEAL_II_COMPILER_CUDA_AWARE
 #  include <deal.II/base/cuda.h>
 
 #  include <deal.II/lac/cuda_vector.h>
diff --git a/include/deal.II/matrix_free/cuda_fe_evaluation.h b/include/deal.II/matrix_free/cuda_fe_evaluation.h
index 2f7c46a766..b097cfc5a7 100644
--- a/include/deal.II/matrix_free/cuda_fe_evaluation.h
+++ b/include/deal.II/matrix_free/cuda_fe_evaluation.h
@@ -18,7 +18,7 @@
 
 #include <deal.II/base/config.h>
 
-#if defined(DEAL_II_WITH_CUDA) && defined(__CUDACC__)
+#ifdef DEAL_II_COMPILER_CUDA_AWARE
 
 #  include <deal.II/base/tensor.h>
 #  include <deal.II/base/utilities.h>
diff --git a/include/deal.II/matrix_free/cuda_hanging_nodes_internal.h b/include/deal.II/matrix_free/cuda_hanging_nodes_internal.h
index f9ee32cc69..946af3f1c8 100644
--- a/include/deal.II/matrix_free/cuda_hanging_nodes_internal.h
+++ b/include/deal.II/matrix_free/cuda_hanging_nodes_internal.h
@@ -18,7 +18,7 @@
 
 #include <deal.II/base/config.h>
 
-#if defined(DEAL_II_WITH_CUDA) && defined(__CUDACC__)
+#ifdef DEAL_II_COMPILER_CUDA_AWARE
 
 #  include <deal.II/base/utilities.h>
 
diff --git a/include/deal.II/matrix_free/cuda_matrix_free.h b/include/deal.II/matrix_free/cuda_matrix_free.h
index 0bc0dacf57..3ebe12d846 100644
--- a/include/deal.II/matrix_free/cuda_matrix_free.h
+++ b/include/deal.II/matrix_free/cuda_matrix_free.h
@@ -19,7 +19,7 @@
 
 #include <deal.II/base/config.h>
 
-#if defined(DEAL_II_WITH_CUDA) && defined(__CUDACC__)
+#ifdef DEAL_II_COMPILER_CUDA_AWARE
 
 #  include <deal.II/base/quadrature.h>
 #  include <deal.II/base/tensor.h>
diff --git a/include/deal.II/matrix_free/cuda_matrix_free.templates.h b/include/deal.II/matrix_free/cuda_matrix_free.templates.h
index a227dff179..d8cc558025 100644
--- a/include/deal.II/matrix_free/cuda_matrix_free.templates.h
+++ b/include/deal.II/matrix_free/cuda_matrix_free.templates.h
@@ -19,7 +19,7 @@
 
 #include <deal.II/matrix_free/cuda_matrix_free.h>
 
-#if defined(DEAL_II_WITH_CUDA) && defined(__CUDACC__)
+#ifdef DEAL_II_COMPILER_CUDA_AWARE
 
 #  include <deal.II/base/cuda_size.h>
 #  include <deal.II/base/graph_coloring.h>
diff --git a/include/deal.II/matrix_free/cuda_tensor_product_kernels.h b/include/deal.II/matrix_free/cuda_tensor_product_kernels.h
index e1232ad56d..81c8fff6ab 100644
--- a/include/deal.II/matrix_free/cuda_tensor_product_kernels.h
+++ b/include/deal.II/matrix_free/cuda_tensor_product_kernels.h
@@ -19,7 +19,7 @@
 
 #include <deal.II/base/config.h>
 
-#if defined(DEAL_II_WITH_CUDA) && defined(__CUDACC__)
+#ifdef DEAL_II_COMPILER_CUDA_AWARE
 
 DEAL_II_NAMESPACE_OPEN
 
-- 
2.39.5