From 2d61594acb4c68803a67236f155bb3bbc002efef Mon Sep 17 00:00:00 2001 From: Daniel Arndt Date: Thu, 27 Oct 2022 22:01:04 +0000 Subject: [PATCH] Move CUDA compilation to Kokkos with nvcc_wrapper --- .github/workflows/linux.yml | 42 ++++++++- cmake/config/Config.cmake.in | 22 ----- .../cuda_compute_capability.cu | 30 ------ cmake/configure/configure_10_cuda.cmake | 94 ------------------- cmake/macros/macro_deal_ii_add_library.cmake | 26 ----- cmake/macros/macro_deal_ii_add_test.cmake | 2 + ..._deal_ii_initialize_cached_variables.cmake | 16 ---- .../macro_deal_ii_insource_setup_target.cmake | 25 ----- .../macro_deal_ii_invoke_autopilot.cmake | 8 -- cmake/macros/macro_deal_ii_pickup_tests.cmake | 5 - cmake/macros/macro_deal_ii_setup_target.cmake | 25 ----- cmake/modules/FindKOKKOS.cmake | 6 +- cmake/setup_cached_variables.cmake | 20 ---- cmake/setup_write_config.cmake | 7 -- examples/step-64/{step-64.cu => step-64.cc} | 0 include/deal.II/base/config.h.in | 2 + include/deal.II/base/tensor.h | 22 +---- source/CMakeLists.txt | 22 ----- source/base/CMakeLists.txt | 2 + source/lac/CMakeLists.txt | 9 ++ source/matrix_free/CMakeLists.txt | 2 + tests/setup_testsubproject.cmake | 4 +- 22 files changed, 66 insertions(+), 325 deletions(-) delete mode 100644 cmake/configure/CUDAComputeCapability/cuda_compute_capability.cu rename examples/step-64/{step-64.cu => step-64.cc} (100%) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 49950dd842..cdb129cf15 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -76,6 +76,20 @@ jobs: run: | mpicc -v cmake --version + - uses: actions/checkout@v2 + with: + repository: kokkos/kokkos + ref: 3.7.00 + path: kokkos + - name: install kokkos + working-directory: kokkos + run: | + mkdir build + cd build + cmake -D BUILD_SHARED_LIBS=ON \ + -D CMAKE_INSTALL_PREFIX=${GITHUB_WORKSPACE}/kokkos-install \ + .. + make install - name: configure run: | mkdir build @@ -83,13 +97,15 @@ jobs: cmake -D CMAKE_BUILD_TYPE=Debug \ -D DEAL_II_CXX_FLAGS='-Werror -std=c++17' \ -D DEAL_II_EARLY_DEPRECATIONS=ON \ + -D DEAL_II_WITH_KOKKOS="ON" \ -D DEAL_II_WITH_CGAL="ON" \ -D DEAL_II_WITH_MPI="ON" \ - -D DEAL_II_WITH_TRILINOS="ON" \ + -D DEAL_II_WITH_TRILINOS="OFF" \ -D DEAL_II_WITH_PETSC="ON" \ -D DEAL_II_WITH_METIS="ON" \ -D DEAL_II_WITH_HDF5="ON" \ -D DEAL_II_COMPONENT_EXAMPLES="OFF" \ + -D KOKKOS_DIR=${GITHUB_WORKSPACE}/kokkos-install \ .. - name: archive uses: actions/upload-artifact@v3 @@ -126,6 +142,9 @@ jobs: name: linux debug cuda-10 runs-on: [ubuntu-18.04] + env: + CUDA_ROOT: /usr/local/cuda + steps: - uses: actions/checkout@v3 - name: modules @@ -148,13 +167,34 @@ jobs: run: | mpicc -v cmake --version + - uses: actions/checkout@v2 + with: + repository: kokkos/kokkos + ref: 3.7.00 + path: kokkos + - name: install kokkos + working-directory: kokkos + run: | + mkdir build + cd build + cmake -D BUILD_SHARED_LIBS=ON \ + -D CMAKE_CXX_COMPILER=${GITHUB_WORKSPACE}/kokkos/bin/nvcc_wrapper \ + -D CMAKE_INSTALL_PREFIX=${GITHUB_WORKSPACE}/kokkos-install \ + -D Kokkos_ENABLE_CUDA=ON \ + -D Kokkos_ENABLE_CUDA_LAMBDA=ON \ + -D Kokkos_ARCH_VOLTA70=ON \ + .. + make install - name: configure run: | cmake -D CMAKE_BUILD_TYPE=Debug \ + -D CMAKE_CXX_COMPILER=${GITHUB_WORKSPACE}/kokkos/bin/nvcc_wrapper \ -D DEAL_II_CUDA_FLAGS='-arch=sm_70' \ -D DEAL_II_CXX_FLAGS='-Werror -Wno-non-template-friend' \ -D DEAL_II_EARLY_DEPRECATIONS=ON \ -D DEAL_II_WITH_CUDA="ON" \ + -D DEAL_II_WITH_KOKKOS="ON" \ + -D KOKKOS_DIR=${GITHUB_WORKSPACE}/kokkos-install \ -D DEAL_II_WITH_MPI="ON" \ -D DEAL_II_WITH_P4EST="ON" \ -D DEAL_II_COMPONENT_EXAMPLES="ON" \ diff --git a/cmake/config/Config.cmake.in b/cmake/config/Config.cmake.in index a3904c7f49..ec0e07e119 100644 --- a/cmake/config/Config.cmake.in +++ b/cmake/config/Config.cmake.in @@ -149,11 +149,6 @@ SET(DEAL_II_MPIEXEC_NUMPROC_FLAG "@MPIEXEC_NUMPROC_FLAG@") SET(DEAL_II_MPIEXEC_PREFLAGS "@MPIEXEC_PREFLAGS@") SET(DEAL_II_MPIEXEC_POSTFLAGS "@MPIEXEC_POSTFLAGS@") -# -# CUDA specific setup: -# -SET(CMAKE_CUDA_ARCHITECTURES "@CMAKE_CUDA_ARCHITECTURES@") - # # Build a static executable: # @@ -178,23 +173,6 @@ SET(DEAL_II_LIBRARIES_RELEASE "@CONFIG_LIBRARIES_RELEASE@") SET(DEAL_II_LIBRARIES "@CONFIG_LIBRARIES@") -# -# Information about CUDA configuration -# - -SET(DEAL_II_CUDA_TOOLKIT_ROOT_DIR "@CUDA_TOOLKIT_ROOT_DIR@") -SET(DEAL_II_CUDA_COMPILER "@CMAKE_CUDA_COMPILER@") - -# used for all cuda targets: -SET(DEAL_II_CUDA_FLAGS "@DEAL_II_CUDA_FLAGS@") - -# _additionally_ used for debug targets: -SET(DEAL_II_CUDA_FLAGS_DEBUG "@DEAL_II_CUDA_FLAGS_DEBUG@") - -# _additionally_ used for release targets: -SET(DEAL_II_CUDA_FLAGS_RELEASE "@DEAL_II_CUDA_FLAGS_RELEASE@") - - # # Information about library targets and feature configuration # diff --git a/cmake/configure/CUDAComputeCapability/cuda_compute_capability.cu b/cmake/configure/CUDAComputeCapability/cuda_compute_capability.cu deleted file mode 100644 index 23225d4632..0000000000 --- a/cmake/configure/CUDAComputeCapability/cuda_compute_capability.cu +++ /dev/null @@ -1,30 +0,0 @@ -// --------------------------------------------------------------------- -// -// Copyright (C) 2019 by the deal.II authors -// -// This file is part of the deal.II library. -// -// The deal.II library is free software; you can use it, redistribute -// it, and/or modify it under the terms of the GNU Lesser General -// Public License as published by the Free Software Foundation; either -// version 2.1 of the License, or (at your option) any later version. -// The full text of the license can be found in the file LICENSE.md at -// the top level directory of deal.II. -// -// --------------------------------------------------------------------- - -#include - -int main() -{ - cudaDeviceProp device_properties; - const cudaError_t error = cudaGetDeviceProperties(&device_properties, - /*device*/0); - if( error != cudaSuccess) - { - std::cout << "CUDA error: " << cudaGetErrorString(error) << '\n'; - return error; - } - std::cout << device_properties.major << device_properties.minor; - return 0; -} diff --git a/cmake/configure/configure_10_cuda.cmake b/cmake/configure/configure_10_cuda.cmake index 4b1b78602b..53cec07703 100644 --- a/cmake/configure/configure_10_cuda.cmake +++ b/cmake/configure/configure_10_cuda.cmake @@ -86,113 +86,19 @@ MACRO(FEATURE_CUDA_FIND_EXTERNAL var) _cuda_ensure_feature_off(10 17 14) - IF("${DEAL_II_CUDA_FLAGS_SAVED}" MATCHES "-arch[ ]*sm_([0-9]*)") - SET(CUDA_COMPUTE_CAPABILITY "${CMAKE_MATCH_1}") - ELSEIF("${DEAL_II_CUDA_FLAGS_SAVED}" MATCHES "-arch=sm_([0-9]*)") - SET(CUDA_COMPUTE_CAPABILITY "${CMAKE_MATCH_1}") - ELSEIF(DEAL_II_ALLOW_PLATFORM_INTROSPECTION) - # - # Try to autodetect the CUDA Compute Capability by asking the device - # - SET(_binary_test_dir ${CMAKE_CURRENT_BINARY_DIR}/cmake/configure/CUDAComputeCapabilityWorkdir) - FILE(REMOVE_RECURSE ${_binary_test_dir}) - FILE(MAKE_DIRECTORY ${_binary_test_dir}) - - EXECUTE_PROCESS( - COMMAND ${CUDA_NVCC_EXECUTABLE} - -ccbin=${CMAKE_CXX_COMPILER} - ${CMAKE_CURRENT_SOURCE_DIR}/cmake/configure/CUDAComputeCapability/cuda_compute_capability.cu - -o cuda_compute_capability - WORKING_DIRECTORY ${_binary_test_dir} - OUTPUT_QUIET - ERROR_QUIET - ) - EXECUTE_PROCESS(COMMAND ${_binary_test_dir}/cuda_compute_capability - RESULT_VARIABLE _result - OUTPUT_VARIABLE CUDA_COMPUTE_CAPABILITY) - IF(${_result} EQUAL 0) - ADD_FLAGS(DEAL_II_CUDA_FLAGS "-arch=sm_${CUDA_COMPUTE_CAPABILITY}") - MESSAGE(STATUS "Detected CUDA Compute Capability ${CUDA_COMPUTE_CAPABILITY}") - ELSE() - MESSAGE(STATUS "Couldn't detect CUDA Compute Capability! " - "The error message was: ${CUDA_COMPUTE_CAPABILITY}") - SET(CUDA_ADDITIONAL_ERROR_STRING - ${CUDA_ADDITIONAL_ERROR_STRING} - "Couldn't detect CUDA Compute Capability! " - "The error message was: ${CUDA_COMPUTE_CAPABILITY}\n" - "Please check the return value of ${_binary_test_dir}/cuda_compute_capability.\n" - "If you want to disable the autodetection, set the compute capability to be used manually." - ) - SET(${var} FALSE) - ENDIF() - ELSE() - # - # Assume a cuda compute capability of 35 - # - SET(CUDA_COMPUTE_CAPABILITY "35") - ADD_FLAGS(DEAL_II_CUDA_FLAGS "-arch=sm_35") - ENDIF() - - IF("${CUDA_COMPUTE_CAPABILITY}" LESS "35") - MESSAGE(STATUS "Too low CUDA Compute Capability specified -- deal.II requires at least 3.5 ") - SET(CUDA_ADDITIONAL_ERROR_STRING - ${CUDA_ADDITIONAL_ERROR_STRING} - "Too low CUDA Compute Capability specified: ${CUDA_COMPUTE_CAPABILITY}\n" - "deal.II requires at least Compute Capability 3.5\n" - "which is used as default if nothing is specified." - ) - SET(${var} FALSE) - ENDIF() - # cuSOLVER requires OpenMP FIND_PACKAGE(OpenMP REQUIRED) SET(DEAL_II_LINKER_FLAGS "${DEAL_II_LINKER_FLAGS} ${OpenMP_CXX_FLAGS}") - - ADD_FLAGS(DEAL_II_CUDA_FLAGS_DEBUG "-G") ENDIF() ENDMACRO() MACRO(FEATURE_CUDA_CONFIGURE_EXTERNAL) - - # - # Ensure that we enable CMake-internal CUDA support with the right - # compiler: - # - SET(CMAKE_CUDA_COMPILER "${CUDA_NVCC_EXECUTABLE}") - IF(NOT CMAKE_CUDA_HOST_COMPILER) - SET(CMAKE_CUDA_HOST_COMPILER "${CMAKE_CXX_COMPILER}") - ENDIF() - ENABLE_LANGUAGE(CUDA) - - MARK_AS_ADVANCED(CMAKE_CUDA_HOST_COMPILER) - - # - # Work around a cmake 3.10 bug, see https://gitlab.kitware.com/cmake/cmake/issues/17797 - # because make does not support rsp link commands - # - SET(CMAKE_CUDA_USE_RESPONSE_FILE_FOR_INCLUDES 0) - SET(CMAKE_CUDA_USE_RESPONSE_FILE_FOR_LIBRARIES 0) - SET(CMAKE_CUDA_USE_RESPONSE_FILE_FOR_OBJECTS 0) - - # - # Disable CUDA_ARCHITECTURES target properties - # - IF(CMAKE_VERSION VERSION_GREATER_EQUAL 3.18) - SET(CMAKE_CUDA_ARCHITECTURES OFF) - ENDIF() - # We cannot use -pedantic as compiler flags. nvcc generates code that # produces a lot of warnings when pedantic is enabled. So filter out the # flag: # STRING(REPLACE "-pedantic" "" DEAL_II_CXX_FLAGS "${DEAL_II_CXX_FLAGS}") - - # - # Export definitions: - # - STRING(SUBSTRING "${CUDA_COMPUTE_CAPABILITY}" 0 1 CUDA_COMPUTE_CAPABILITY_MAJOR) - STRING(SUBSTRING "${CUDA_COMPUTE_CAPABILITY}" 1 1 CUDA_COMPUTE_CAPABILITY_MINOR) ENDMACRO() diff --git a/cmake/macros/macro_deal_ii_add_library.cmake b/cmake/macros/macro_deal_ii_add_library.cmake index 7fd749bea4..002c6e607e 100644 --- a/cmake/macros/macro_deal_ii_add_library.cmake +++ b/cmake/macros/macro_deal_ii_add_library.cmake @@ -59,32 +59,6 @@ MACRO(DEAL_II_ADD_LIBRARY _library) PUBLIC ${DEAL_II_DEFINITIONS} ${DEAL_II_DEFINITIONS_${_build}} ) - IF(DEAL_II_WITH_CUDA) - # - # Add cxx compiler and cuda compilation flags to cuda source files: - # - - SET(_cuda_flags "${DEAL_II_CUDA_FLAGS} ${DEAL_II_CUDA_FLAGS_${_build}}") - SEPARATE_ARGUMENTS(_cuda_flags) - - # - # Workaround: cuda will split every compiler option with a comma - # (','), so remove all compiler flags that contain a comma: - # - STRING(REGEX REPLACE "[^ ]*,[^ ]*" "" _cxx_flags - "${DEAL_II_CXX_FLAGS} ${DEAL_II_CXX_FLAGS_${_build}}" - ) - - TARGET_COMPILE_OPTIONS(${_library}_${_build_lowercase} PUBLIC - $<$:-Xcompiler ${_cxx_flags}> - $<$:${_cuda_flags}> - ) - - SET_TARGET_PROPERTIES(${_library}_${_build_lowercase} PROPERTIES - CUDA_SEPARABLE_COMPILATION FALSE - ) - ENDIF() - ENDIF() SET_PROPERTY(GLOBAL APPEND PROPERTY DEAL_II_OBJECTS_${_build} diff --git a/cmake/macros/macro_deal_ii_add_test.cmake b/cmake/macros/macro_deal_ii_add_test.cmake index 1cd35c2e20..00e68df10c 100644 --- a/cmake/macros/macro_deal_ii_add_test.cmake +++ b/cmake/macros/macro_deal_ii_add_test.cmake @@ -414,6 +414,8 @@ FUNCTION(DEAL_II_ADD_TEST _category _test_name _comparison_file) ${CMAKE_CURRENT_BINARY_DIR}/${_target_short}/interrupt_guard.cc ) + set_source_files_properties(${_source_file} PROPERTIES LANGUAGE CXX) + SET_TARGET_PROPERTIES(${_target} PROPERTIES OUTPUT_NAME ${_target_short}) DEAL_II_SETUP_TARGET(${_target} ${_build}) diff --git a/cmake/macros/macro_deal_ii_initialize_cached_variables.cmake b/cmake/macros/macro_deal_ii_initialize_cached_variables.cmake index 53c7bdbdfa..37c4356d02 100644 --- a/cmake/macros/macro_deal_ii_initialize_cached_variables.cmake +++ b/cmake/macros/macro_deal_ii_initialize_cached_variables.cmake @@ -95,22 +95,6 @@ MACRO(DEAL_II_INITIALIZE_CACHED_VARIABLES) ) - IF(DEAL_II_WITH_CUDA) - SET(CMAKE_CUDA_COMPILER ${DEAL_II_CUDA_COMPILER} CACHE STRING - "CUDA Compiler.") - SET(CMAKE_CUDA_HOST_COMPILER ${DEAL_II_CXX_COMPILER} CACHE STRING - "CXX Compiler.") - SET(CMAKE_CUDA_FLAGS "" CACHE STRING - "Flags used by the compiler during all build types." - ) - SET(CMAKE_CUDA_FLAGS_DEBUG "" CACHE STRING - "Flags used by the compiler during debug builds." - ) - SET(CMAKE_CUDA_FLAGS_RELEASE "" CACHE STRING - "Flags used by the compiler during release builds." - ) - ENDIF() - MARK_AS_ADVANCED(CMAKE_INSTALL_PREFIX) ENDMACRO() diff --git a/cmake/macros/macro_deal_ii_insource_setup_target.cmake b/cmake/macros/macro_deal_ii_insource_setup_target.cmake index 0854250ec0..7819afdc00 100644 --- a/cmake/macros/macro_deal_ii_insource_setup_target.cmake +++ b/cmake/macros/macro_deal_ii_insource_setup_target.cmake @@ -60,31 +60,6 @@ MACRO(DEAL_II_INSOURCE_SETUP_TARGET _target _build) PUBLIC ${DEAL_II_DEFINITIONS} ${DEAL_II_DEFINITIONS_${_build}} ) - IF(DEAL_II_WITH_CUDA) - # - # Add cxx compiler and cuda compilation flags to cuda source files: - # - - SET(_cuda_flags "${DEAL_II_CUDA_FLAGS} ${DEAL_II_CUDA_FLAGS_${_build}}") - SEPARATE_ARGUMENTS(_cuda_flags) - - # - # Workaround: cuda will split every compiler option with a comma - # (','), so remove all compiler flags that contain a comma: - # - STRING(REGEX REPLACE "[^ ]*,[^ ]*" "" _cxx_flags - "${DEAL_II_CXX_FLAGS} ${DEAL_II_CXX_FLAGS_${_build}}" - ) - - TARGET_COMPILE_OPTIONS(${_target} PUBLIC - $<$:-Xcompiler ${_cxx_flags}> - $<$:${_cuda_flags}> - ) - - SET_TARGET_PROPERTIES(${_target} PROPERTIES - CUDA_SEPARABLE_COMPILATION FALSE - ) - ENDIF() ENDIF() GET_PROPERTY(_type TARGET ${_target} PROPERTY TYPE) diff --git a/cmake/macros/macro_deal_ii_invoke_autopilot.cmake b/cmake/macros/macro_deal_ii_invoke_autopilot.cmake index 62e5f853a9..3ccfdfcbbc 100644 --- a/cmake/macros/macro_deal_ii_invoke_autopilot.cmake +++ b/cmake/macros/macro_deal_ii_invoke_autopilot.cmake @@ -44,14 +44,6 @@ MACRO(DEAL_II_INVOKE_AUTOPILOT) SET(_make_command " $ make") ENDIF() - # Make sure we can treat CUDA targets if available - IF(DEAL_II_WITH_CUDA) - IF(NOT CMAKE_CUDA_HOST_COMPILER) - SET(CMAKE_CUDA_HOST_COMPILER "${CMAKE_CXX_COMPILER}") - ENDIF() - ENABLE_LANGUAGE(CUDA) - ENDIF() - # Define and setup a compilation target: ADD_EXECUTABLE(${TARGET} ${TARGET_SRC}) DEAL_II_SETUP_TARGET(${TARGET}) diff --git a/cmake/macros/macro_deal_ii_pickup_tests.cmake b/cmake/macros/macro_deal_ii_pickup_tests.cmake index fc21612d92..c322129234 100644 --- a/cmake/macros/macro_deal_ii_pickup_tests.cmake +++ b/cmake/macros/macro_deal_ii_pickup_tests.cmake @@ -116,11 +116,6 @@ MACRO(DEAL_II_PICKUP_TESTS) ENDIF() ENDIF() - IF(DEAL_II_WITH_CUDA) - FIND_PACKAGE(CUDA) - SET(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} -std=c++14 -arch=sm_35 -Xcompiler ${OpenMP_CXX_FLAGS}) - ENDIF() - FIND_PACKAGE(Perl REQUIRED) FIND_PROGRAM(NUMDIFF_EXECUTABLE diff --git a/cmake/macros/macro_deal_ii_setup_target.cmake b/cmake/macros/macro_deal_ii_setup_target.cmake index b712629666..ac2eda2902 100644 --- a/cmake/macros/macro_deal_ii_setup_target.cmake +++ b/cmake/macros/macro_deal_ii_setup_target.cmake @@ -134,31 +134,6 @@ MACRO(DEAL_II_SETUP_TARGET _target) PUBLIC ${DEAL_II_USER_DEFINITIONS} ${DEAL_II_USER_DEFINITIONS_${_build}} ) - IF(DEAL_II_WITH_CUDA) - # - # Add cxx compiler and cuda compilation flags to cuda source files: - # - - SET(_cuda_flags "${DEAL_II_CUDA_FLAGS} ${DEAL_II_CUDA_FLAGS_${_build}}") - SEPARATE_ARGUMENTS(_cuda_flags) - - # - # Workaround: cuda will split every compiler option with a comma - # (','), so remove all compiler flags that contain a comma: - # - STRING(REGEX REPLACE "[^ ]*,[^ ]*" "" _cxx_flags - "${DEAL_II_CXX_FLAGS} ${DEAL_II_CXX_FLAGS_${_build}}" - ) - - TARGET_COMPILE_OPTIONS(${_target} PUBLIC - $<$:-Xcompiler ${_cxx_flags}> - $<$:${_cuda_flags}> - ) - - SET_TARGET_PROPERTIES(${_target} PROPERTIES - CUDA_SEPARABLE_COMPILATION FALSE - ) - ENDIF() ENDIF() # diff --git a/cmake/modules/FindKOKKOS.cmake b/cmake/modules/FindKOKKOS.cmake index 49928cb21d..44c8715e89 100644 --- a/cmake/modules/FindKOKKOS.cmake +++ b/cmake/modules/FindKOKKOS.cmake @@ -54,10 +54,8 @@ ELSE() # we don't know if CUDA support is enabled in Kokkos IF(Kokkos_ENABLE_CUDA) SET(DEAL_II_VECTORIZATION_WIDTH_IN_BITS 0) - IF(DEAL_II_WITH_KOKKOS_BACKEND) - # Require lambda support to use Kokkos as a backend - KOKKOS_CHECK(OPTIONS CUDA_LAMBDA) - ENDIF() + # Require lambda support to use Kokkos as a backend + KOKKOS_CHECK(OPTIONS CUDA_LAMBDA) ENDIF() DEAL_II_FIND_LIBRARY(KOKKOS_CORE_LIBRARY diff --git a/cmake/setup_cached_variables.cmake b/cmake/setup_cached_variables.cmake index 739f0f7261..279a13ff76 100644 --- a/cmake/setup_cached_variables.cmake +++ b/cmake/setup_cached_variables.cmake @@ -229,12 +229,6 @@ SET(DEAL_II_REMOVED_FLAGS CMAKE_Fortran_FLAGS_DEBUG CMAKE_Fortran_FLAGS_MINSIZEREL CMAKE_Fortran_FLAGS_RELWITHDEBINFO - CUDA_NVCC_FLAGS - CMAKE_CUDA_FLAGS - CMAKE_CUDA_FLAGS_RELEASE - CMAKE_CUDA_FLAGS_DEBUG - CMAKE_CUDA_FLAGS_MINSIZEREL - CMAKE_CUDA_FLAGS_RELWITHDEBINFO CMAKE_SHARED_LINKER_FLAGS CMAKE_SHARED_LINKER_FLAGS_DEBUG CMAKE_SHARED_LINKER_FLAGS_MINSIZEREL @@ -256,9 +250,6 @@ ENDFOREACH() # SET(DEAL_II_USED_FLAGS - DEAL_II_CUDA_FLAGS - DEAL_II_CUDA_FLAGS_DEBUG - DEAL_II_CUDA_FLAGS_RELEASE DEAL_II_CXX_FLAGS DEAL_II_CXX_FLAGS_DEBUG DEAL_II_CXX_FLAGS_RELEASE @@ -289,7 +280,6 @@ ENDFOREACH() # FOREACH(_flag - CUDA_FLAGS CUDA_FLAGS_RELEASE CUDA_FLAGS_DEBUG CXX_FLAGS CXX_FLAGS_RELEASE CXX_FLAGS_DEBUG ) IF(NOT "${CMAKE_${_flag}}" STREQUAL "") @@ -309,13 +299,6 @@ FOREACH(_flag LINKER_FLAGS LINKER_FLAGS_DEBUG LINKER_FLAGS_RELEASE) ENDIF() ENDFOREACH() -IF(NOT "${CUDA_NVCC_FLAGS}}" STREQUAL "") - MESSAGE(STATUS - "Prepending \${CUDA_NVCC_FLAGS} to \${DEAL_II_CUDA_FLAGS}" - ) - SET(DEAL_II_CUDA_FLAGS "${CUDA_NVCC_FLAGS} ${DEAL_II_CUDA_FLAGS}") -ENDIF() - # @@ -357,8 +340,6 @@ SET(DEAL_II_CXX_FLAGS_SAVED "$ENV{CXXFLAGS} ${DEAL_II_CXX_FLAGS_SAVED}") STRING(STRIP "${DEAL_II_CXX_FLAGS_SAVED}" DEAL_II_CXX_FLAGS_SAVED) SET(DEAL_II_LINKER_FLAGS_SAVED "$ENV{LDFLAGS} ${DEAL_II_LINKER_FLAGS_SAVED}") STRING(STRIP "${DEAL_II_LINKER_FLAGS_SAVED}" DEAL_II_LINKER_FLAGS_SAVED) -SET(DEAL_II_CUDA_FLAGS_SAVED "$ENV{NVCCFLAGS} ${DEAL_II_CUDA_FLAGS_SAVED}") -STRING(STRIP "${DEAL_II_CUDA_FLAGS_SAVED}" DEAL_II_CUDA_FLAGS_SAVED) UNSET(ENV{CXXFLAGS}) UNSET(ENV{LDFLAGS}) UNSET(ENV{NVCCFLAGS}) @@ -370,7 +351,6 @@ UNSET(ENV{NVCCFLAGS}) # # ######################################################################## - OPTION(DEAL_II_WITH_64BIT_INDICES "If set to ON, then use 64-bit data types to represent global degree of freedom indices. The default is to OFF. You only want to set this to ON if you will solve problems with more than 2^31 (approximately 2 billion) unknowns. If set to ON, you also need to ensure that both Trilinos and/or PETSc support 64-bit indices." OFF diff --git a/cmake/setup_write_config.cmake b/cmake/setup_write_config.cmake index e8bbb9e12b..979cbb031e 100644 --- a/cmake/setup_write_config.cmake +++ b/cmake/setup_write_config.cmake @@ -224,13 +224,6 @@ FOREACH(_feature ${_deal_ii_features_sorted}) IF(_feature MATCHES "THREADS" AND DEFINED TBB_VERSION) _detailed("# TBB_VERSION = ${TBB_VERSION}\n") ENDIF() - IF(_feature MATCHES "CUDA" AND DEFINED CUDA_COMPUTE_CAPABILITY) - _detailed("# CMAKE_CUDA_COMPILER = ${CMAKE_CUDA_COMPILER}\n") - _detailed("# CUDA_COMPUTE_CAPABILITY = ${CUDA_COMPUTE_CAPABILITY_MAJOR}.${CUDA_COMPUTE_CAPABILITY_MINOR}\n") - _detailed("# DEAL_II_CUDA_FLAGS = ${DEAL_II_CUDA_FLAGS}\n") - _detailed("# DEAL_II_CUDA_FLAGS_RELEASE = ${DEAL_II_CUDA_FLAGS_RELEASE}\n") - _detailed("# DEAL_II_CUDA_FLAGS_DEBUG = ${DEAL_II_CUDA_FLAGS_DEBUG}\n") - ENDIF() # diff --git a/examples/step-64/step-64.cu b/examples/step-64/step-64.cc similarity index 100% rename from examples/step-64/step-64.cu rename to examples/step-64/step-64.cc diff --git a/include/deal.II/base/config.h.in b/include/deal.II/base/config.h.in index a201f301de..c6dfc71539 100644 --- a/include/deal.II/base/config.h.in +++ b/include/deal.II/base/config.h.in @@ -577,3 +577,5 @@ static_assert( "deal.II."); #endif + + diff --git a/include/deal.II/base/tensor.h b/include/deal.II/base/tensor.h index 8bb4f15da5..43e80e82d9 100644 --- a/include/deal.II/base/tensor.h +++ b/include/deal.II/base/tensor.h @@ -1328,7 +1328,10 @@ constexpr DEAL_II_ALWAYS_INLINE DEAL_II_CUDA_HOST_DEV Tensor::Tensor( const ArrayView &initializer) { + // We cannot use Assert in a CUDA kernel +# ifndef __CUDA_ARCH__ AssertDimension(initializer.size(), n_independent_components); +# endif for (unsigned int i = 0; i < n_independent_components; ++i) (*this)[unrolled_to_component_indices(i)] = initializer[i]; @@ -1385,7 +1388,6 @@ Tensor::Tensor(Tensor &&other) noexcept } # endif - namespace internal { namespace TensorSubscriptor @@ -1404,24 +1406,10 @@ namespace internal return values[i]; } - // The variables within this struct will be referenced in the next function. - // It is a workaround that allows returning a reference to a static variable - // while allowing constexpr evaluation of the function. - // It has to be defined outside the function because constexpr functions - // cannot define static variables - template - struct Uninitialized - { - static ArrayElementType value; - }; - - template - Type Uninitialized::value; - template constexpr inline DEAL_II_ALWAYS_INLINE DEAL_II_CUDA_HOST_DEV ArrayElementType & - subscript(ArrayElementType *, + subscript(ArrayElementType *dummy, const unsigned int, std::integral_constant) { @@ -1432,7 +1420,7 @@ namespace internal ExcMessage( "Cannot access elements of an object of type Tensor.")); # endif - return Uninitialized::value; + return *dummy; } } // namespace TensorSubscriptor } // namespace internal diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 9c3314943a..ee539b26b9 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -84,20 +84,6 @@ FOREACH(build ${DEAL_II_BUILD_TYPES}) GET_PROPERTY(_objects GLOBAL PROPERTY DEAL_II_OBJECTS_${build}) - # - # FIXME: Somehow the cuda object files lose the "generated" property. - # This is an ugly hack - # - FOREACH(_object ${_objects}) - IF("${_object}" MATCHES "cu.o$") - SET_SOURCE_FILES_PROPERTIES("${_object}" - PROPERTIES - EXTERNAL_OBJECT TRUE - GENERATED TRUE - ) - ENDIF() - ENDFOREACH() - ADD_LIBRARY(${DEAL_II_BASE_NAME}${DEAL_II_${build}_SUFFIX} dummy.cc # Workaround for a bug in the Xcode generator ${_objects} @@ -122,14 +108,6 @@ FOREACH(build ${DEAL_II_BUILD_TYPES}) RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DEAL_II_EXECUTABLE_RELDIR}" ) - IF(DEAL_II_WITH_CUDA) - # We need to use the CUDA linker to avoid confusing nvlink with sonaming. - SET_TARGET_PROPERTIES(${DEAL_II_BASE_NAME}${DEAL_II_${build}_SUFFIX} - PROPERTIES - LINKER_LANGUAGE "CUDA" - ) - ENDIF() - IF(CMAKE_SYSTEM_NAME MATCHES "Darwin") SET_TARGET_PROPERTIES(${DEAL_II_BASE_NAME}${DEAL_II_${build}_SUFFIX} PROPERTIES diff --git a/source/base/CMakeLists.txt b/source/base/CMakeLists.txt index 24d7cb4dd2..a174c93ac6 100644 --- a/source/base/CMakeLists.txt +++ b/source/base/CMakeLists.txt @@ -117,6 +117,8 @@ IF(DEAL_II_WITH_CUDA) cuda.cu partitioner.cu ) + set_source_files_properties(cuda.cu PROPERTIES LANGUAGE CXX) + set_source_files_properties(partitioner.cu PROPERTIES LANGUAGE CXX) ENDIF() # determined by profiling diff --git a/source/lac/CMakeLists.txt b/source/lac/CMakeLists.txt index fda0205828..342e0a234a 100644 --- a/source/lac/CMakeLists.txt +++ b/source/lac/CMakeLists.txt @@ -67,6 +67,8 @@ IF(DEAL_II_WITH_CUDA) vector_memory.cu affine_constraints.cu ) + set_source_files_properties(vector_memory.cu PROPERTIES LANGUAGE CXX) + set_source_files_properties(affine_constraints.cu PROPERTIES LANGUAGE CXX) ENDIF() SET(_inst @@ -171,6 +173,13 @@ IF(DEAL_II_WITH_CUDA) la_parallel_vector.cu read_write_vector.cu ) + set_source_files_properties(cuda_kernels.cu PROPERTIES LANGUAGE CXX) + set_source_files_properties(cuda_precondition.cu PROPERTIES LANGUAGE CXX) + set_source_files_properties(cuda_solver_direct.cu PROPERTIES LANGUAGE CXX) + set_source_files_properties(cuda_sparse_matrix.cu PROPERTIES LANGUAGE CXX) + set_source_files_properties(cuda_vector.cu PROPERTIES LANGUAGE CXX) + set_source_files_properties(la_parallel_vector.cu PROPERTIES LANGUAGE CXX) + set_source_files_properties(read_write_vector.cu PROPERTIES LANGUAGE CXX) ENDIF() # determined by profiling diff --git a/source/matrix_free/CMakeLists.txt b/source/matrix_free/CMakeLists.txt index 6154a09d43..f4fde2f6ff 100644 --- a/source/matrix_free/CMakeLists.txt +++ b/source/matrix_free/CMakeLists.txt @@ -62,6 +62,8 @@ IF(DEAL_II_WITH_CUDA) SET(_inst ${_inst} ) + + set_source_files_properties(cuda_matrix_free.cu PROPERTIES LANGUAGE CXX) ENDIF() DEAL_II_ADD_LIBRARY(obj_matrix_free OBJECT ${_src} ${_header} ${_inst}) diff --git a/tests/setup_testsubproject.cmake b/tests/setup_testsubproject.cmake index d131b95ed9..30ecc17103 100644 --- a/tests/setup_testsubproject.cmake +++ b/tests/setup_testsubproject.cmake @@ -12,9 +12,7 @@ FOREACH(_var SET(${_var} "${${_var}}" CACHE STRING "" FORCE) ENDFOREACH() -IF(DEAL_II_WITH_CUDA) - PROJECT(TESTSUITE CXX CUDA) -ENDIF() +PROJECT(TESTSUITE CXX) # # A custom target that does absolutely nothing. It is used in the main -- 2.39.5