From 4437ebfbd5800b584a89f9f5e78c473bdbf64801 Mon Sep 17 00:00:00 2001
From: Bruno Turcksin <bruno.turcksin@gmail.com>
Date: Fri, 12 Nov 2021 21:25:37 +0000
Subject: [PATCH] Fix bug in build system when using Kokkos

---
 cmake/configure/configure_2_trilinos.cmake | 14 ++++++++++++++
 cmake/modules/FindKOKKOS.cmake             | 17 +++++++++++++----
 2 files changed, 27 insertions(+), 4 deletions(-)

diff --git a/cmake/configure/configure_2_trilinos.cmake b/cmake/configure/configure_2_trilinos.cmake
index fd415de1a1..8a9a4a0796 100644
--- a/cmake/configure/configure_2_trilinos.cmake
+++ b/cmake/configure/configure_2_trilinos.cmake
@@ -177,6 +177,20 @@ MACRO(FEATURE_TRILINOS_FIND_EXTERNAL var)
     ENDIF()
 
     IF(DEAL_II_TRILINOS_WITH_KOKKOS)
+      CHECK_SYMBOL_EXISTS(
+        "KOKKOS_ENABLE_CUDA"
+        "Kokkos_Macros.hpp"
+        DEAL_II_KOKKOS_CUDA_EXISTS
+        )
+      IF(DEAL_II_KOKKOS_CUDA_EXISTS)
+        # We need to disable SIMD vectorization for CUDA device code.
+        # Otherwise, nvcc compilers from version 9 on will emit an error message like:
+        # "[...] contains a vector, which is not supported in device code". We
+        # would like to set the variable in check_01_cpu_feature but at that point
+        # we don't know if CUDA support is enabled in Kokkos
+        SET(DEAL_II_VECTORIZATION_WIDTH_IN_BITS 0)
+      ENDIF()
+
       CHECK_SYMBOL_EXISTS(
         "KOKKOS_ENABLE_CUDA_LAMBDA"
         "Kokkos_Macros.hpp"
diff --git a/cmake/modules/FindKOKKOS.cmake b/cmake/modules/FindKOKKOS.cmake
index 6c39f3c7d9..7c992628f8 100644
--- a/cmake/modules/FindKOKKOS.cmake
+++ b/cmake/modules/FindKOKKOS.cmake
@@ -44,10 +44,18 @@ ELSE()
       STRING(REGEX REPLACE "\\$<\\$<LINK_LANGUAGE:CXX>:([^>]*)>" "\\1" KOKKOS_EXTRA_LD_FLAGS "${KOKKOS_EXTRA_LD_FLAGS_FULL}")
       GET_PROPERTY(KOKKOS_COMPILE_FLAGS_FULL TARGET Kokkos::kokkoscore PROPERTY INTERFACE_COMPILE_OPTIONS)
       STRING(REGEX REPLACE "\\$<\\$<COMPILE_LANGUAGE:CXX>:([^>]*)>" "\\1" KOKKOS_COMPILE_FLAGS "${KOKKOS_COMPILE_FLAGS_FULL}")
-      # In serial the flag is empty but ADD_FLAGS does not support adding an empty
-      # flag
-      IF(KOKKOS_COMPILE_FLAGS)
-        ADD_FLAGS(DEAL_II_CXX_FLAGS ${KOKKOS_COMPILE_FLAGS})
+
+      # We need to disable SIMD vectorization for CUDA device code.
+      # Otherwise, nvcc compilers from version 9 on will emit an error message like:
+      # "[...] contains a vector, which is not supported in device code". We
+      # would like to set the variable in check_01_cpu_feature but at that point
+      # we don't know if CUDA support is enabled in Kokkos
+      IF(Kokkos_ENABLE_CUDA)
+        SET(DEAL_II_VECTORIZATION_WIDTH_IN_BITS 0)
+        IF(DEAL_II_WITH_KOKKOS_BACKEND)
+          # Require lambda support to use Kokkos as a backend
+          KOKKOS_CHECK(OPTIONS CUDA_LAMBDA)
+        ENDIF()
       ENDIF()
 
       DEAL_II_FIND_LIBRARY(KOKKOS_CORE_LIBRARY
@@ -72,6 +80,7 @@ ELSE()
     LIBRARIES REQUIRED KOKKOS_CORE_LIBRARY KOKKOS_CONTAINERS_LIBRARY
     INCLUDE_DIRS REQUIRED KOKKOS_INSTALL_INCLUDE_DIR
     USER_INCLUDE_DIRS REQUIRED KOKKOS_INSTALL_INCLUDE_DIR
+    CXX_FLAGS OPTIONAL KOKKOS_COMPILE_FLAGS
     LINKER_FLAGS OPTIONAL KOKKOS_EXTRA_LD_FLAGS
     CLEAR KOKKOS_DIR KOKKOS_CORE_LIBRARY KOKKOS_CONTAINERS_LIBRARY
     )
-- 
2.39.5