From 087372cf5a2cad14d16acac8ece08a9973f4d3ba Mon Sep 17 00:00:00 2001 From: Daniel Arndt Date: Fri, 25 Nov 2022 10:50:26 -0500 Subject: [PATCH] Introduce DEAL_II_HOST_DEV_ALWAYS_INLINE --- include/deal.II/base/numbers.h | 7 +++-- include/deal.II/base/tensor.h | 56 +++++++++++++++++----------------- 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/include/deal.II/base/numbers.h b/include/deal.II/base/numbers.h index d53070eb1e..c57e03068a 100644 --- a/include/deal.II/base/numbers.h +++ b/include/deal.II/base/numbers.h @@ -34,6 +34,7 @@ #define DEAL_II_HOST_DEV KOKKOS_FUNCTION #define DEAL_II_CUDA_HOST_DEV DEAL_II_HOST_DEV +#define DEAL_II_HOST_DEV_ALWAYS_INLINE KOKKOS_FORCEINLINE_FUNCTION // Forward-declare the automatic differentiation types so we can add prototypes // for our own wrappers. @@ -728,8 +729,8 @@ namespace internal template struct NumberType { - static constexpr DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV const T & - value(const T &t) + static constexpr DEAL_II_HOST_DEV_ALWAYS_INLINE const T & + value(const T &t) { return t; } @@ -743,7 +744,7 @@ namespace internal // Type T is constructible from F. template - static constexpr DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV T + static constexpr DEAL_II_HOST_DEV_ALWAYS_INLINE T value(const F &f, std::enable_if_t::type, typename std::decay::type>::value && diff --git a/include/deal.II/base/tensor.h b/include/deal.II/base/tensor.h index 2232675e72..7531e840cc 100644 --- a/include/deal.II/base/tensor.h +++ b/include/deal.II/base/tensor.h @@ -547,7 +547,7 @@ public: * * @note This function can also be used in CUDA device code. */ - constexpr DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV + constexpr DEAL_II_HOST_DEV_ALWAYS_INLINE Tensor(); /** @@ -955,7 +955,7 @@ namespace internal template -constexpr DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV +constexpr DEAL_II_HOST_DEV_ALWAYS_INLINE Tensor<0, dim, Number>::Tensor() // Some auto-differentiable numbers need explicit // zero initialization such as adtl::adouble. @@ -966,7 +966,7 @@ Tensor<0, dim, Number>::Tensor() template template -constexpr DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV +constexpr DEAL_II_HOST_DEV_ALWAYS_INLINE Tensor<0, dim, Number>::Tensor(const OtherNumber &initializer) : value(internal::NumberType::value(initializer)) {} @@ -975,7 +975,7 @@ Tensor<0, dim, Number>::Tensor(const OtherNumber &initializer) template template -constexpr DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV +constexpr DEAL_II_HOST_DEV_ALWAYS_INLINE Tensor<0, dim, Number>::Tensor(const Tensor<0, dim, OtherNumber> &p) : Tensor{p.value} {} @@ -983,7 +983,7 @@ Tensor<0, dim, Number>::Tensor(const Tensor<0, dim, OtherNumber> &p) # ifdef DEAL_II_DELETED_MOVE_CONSTRUCTOR_BUG template -constexpr DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV +constexpr DEAL_II_HOST_DEV_ALWAYS_INLINE Tensor<0, dim, Number>::Tensor(const Tensor<0, dim, Number> &other) : value{other.value} {} @@ -991,7 +991,7 @@ Tensor<0, dim, Number>::Tensor(const Tensor<0, dim, Number> &other) template -constexpr DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV +constexpr DEAL_II_HOST_DEV_ALWAYS_INLINE Tensor<0, dim, Number>::Tensor(Tensor<0, dim, Number> &&other) noexcept : value{std::move(other.value)} {} @@ -1035,7 +1035,7 @@ Tensor<0, dim, Number>::end_raw() const template -constexpr inline DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV +constexpr inline DEAL_II_HOST_DEV_ALWAYS_INLINE Tensor<0, dim, Number>::operator Number &() { // We cannot use Assert inside a CUDA kernel @@ -1063,7 +1063,7 @@ constexpr inline DEAL_II_ALWAYS_INLINE template template -constexpr inline DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV Tensor<0, dim, Number> & +constexpr inline DEAL_II_HOST_DEV_ALWAYS_INLINE Tensor<0, dim, Number> & Tensor<0, dim, Number>::operator=(const Tensor<0, dim, OtherNumber> &p) { value = internal::NumberType::value(p.value); @@ -1073,7 +1073,7 @@ Tensor<0, dim, Number>::operator=(const Tensor<0, dim, OtherNumber> &p) # if defined(__INTEL_COMPILER) || defined(DEAL_II_DELETED_MOVE_CONSTRUCTOR_BUG) template -constexpr inline DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV Tensor<0, dim, Number> & +constexpr inline DEAL_II_HOST_DEV_ALWAYS_INLINE Tensor<0, dim, Number> & Tensor<0, dim, Number>::operator=(const Tensor<0, dim, Number> &p) { value = p.value; @@ -1083,7 +1083,7 @@ Tensor<0, dim, Number>::operator=(const Tensor<0, dim, Number> &p) # ifdef DEAL_II_DELETED_MOVE_CONSTRUCTOR_BUG template -constexpr DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV Tensor<0, dim, Number> & +constexpr DEAL_II_HOST_DEV_ALWAYS_INLINE Tensor<0, dim, Number> & Tensor<0, dim, Number>::operator=(Tensor<0, dim, Number> &&other) noexcept { value = std::move(other.value); @@ -1095,7 +1095,7 @@ Tensor<0, dim, Number>::operator=(Tensor<0, dim, Number> &&other) noexcept template template -constexpr inline DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV Tensor<0, dim, Number> & +constexpr inline DEAL_II_HOST_DEV_ALWAYS_INLINE Tensor<0, dim, Number> & Tensor<0, dim, Number>::operator=(const OtherNumber &d) { value = internal::NumberType::value(d); @@ -1131,7 +1131,7 @@ Tensor<0, dim, Number>::operator!=(const Tensor<0, dim, OtherNumber> &p) const template template -constexpr inline DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV Tensor<0, dim, Number> & +constexpr inline DEAL_II_HOST_DEV_ALWAYS_INLINE Tensor<0, dim, Number> & Tensor<0, dim, Number>::operator+=(const Tensor<0, dim, OtherNumber> &p) { value += p.value; @@ -1141,7 +1141,7 @@ Tensor<0, dim, Number>::operator+=(const Tensor<0, dim, OtherNumber> &p) template template -constexpr inline DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV Tensor<0, dim, Number> & +constexpr inline DEAL_II_HOST_DEV_ALWAYS_INLINE Tensor<0, dim, Number> & Tensor<0, dim, Number>::operator-=(const Tensor<0, dim, OtherNumber> &p) { value -= p.value; @@ -1155,7 +1155,7 @@ namespace internal namespace ComplexWorkaround { template - constexpr inline DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV void + constexpr inline DEAL_II_HOST_DEV_ALWAYS_INLINE void multiply_assign_scalar(Number &val, const OtherNumber &s) { val *= s; @@ -1163,7 +1163,7 @@ namespace internal # ifdef __CUDA_ARCH__ template - constexpr inline DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV void + constexpr inline DEAL_II_HOST_DEV_ALWAYS_INLINE void multiply_assign_scalar(std::complex &, const OtherNumber &) { printf("This function is not implemented for std::complex!\n"); @@ -1176,7 +1176,7 @@ namespace internal template template -constexpr inline DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV Tensor<0, dim, Number> & +constexpr inline DEAL_II_HOST_DEV_ALWAYS_INLINE Tensor<0, dim, Number> & Tensor<0, dim, Number>::operator*=(const OtherNumber &s) { internal::ComplexWorkaround::multiply_assign_scalar(value, s); @@ -1196,7 +1196,7 @@ Tensor<0, dim, Number>::operator/=(const OtherNumber &s) template -constexpr DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV Tensor<0, dim, Number> +constexpr DEAL_II_HOST_DEV_ALWAYS_INLINE Tensor<0, dim, Number> Tensor<0, dim, Number>::operator-() const { return -value; @@ -1285,7 +1285,7 @@ constexpr unsigned int Tensor<0, dim, Number>::n_independent_components; template template -constexpr DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV +constexpr DEAL_II_HOST_DEV_ALWAYS_INLINE Tensor::Tensor(const ArrayLike &initializer, std::index_sequence) : values{Tensor(initializer[indices])...} @@ -1297,7 +1297,7 @@ Tensor::Tensor(const ArrayLike &initializer, template -constexpr DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV +constexpr DEAL_II_HOST_DEV_ALWAYS_INLINE Tensor::Tensor() // We would like to use =default, but this causes compile errors with some // MSVC versions and internal compiler errors with -O1 in gcc 5.4. @@ -1307,7 +1307,7 @@ Tensor::Tensor() template -constexpr DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV +constexpr DEAL_II_HOST_DEV_ALWAYS_INLINE Tensor::Tensor(const array_type &initializer) : Tensor(initializer, std::make_index_sequence{}) {} @@ -1316,7 +1316,7 @@ Tensor::Tensor(const array_type &initializer) template template -constexpr DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV +constexpr DEAL_II_HOST_DEV_ALWAYS_INLINE Tensor::Tensor( const ArrayView &initializer) { @@ -1333,7 +1333,7 @@ Tensor::Tensor( template template -constexpr DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV +constexpr DEAL_II_HOST_DEV_ALWAYS_INLINE Tensor::Tensor( const Tensor &initializer) : Tensor(initializer, std::make_index_sequence{}) @@ -1385,7 +1385,7 @@ namespace internal namespace TensorSubscriptor { template - constexpr inline DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV ArrayElementType & + constexpr inline DEAL_II_HOST_DEV_ALWAYS_INLINE ArrayElementType & subscript(ArrayElementType * values, const unsigned int i, std::integral_constant) @@ -1398,7 +1398,7 @@ namespace internal } template - constexpr inline DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV ArrayElementType & + constexpr inline DEAL_II_HOST_DEV_ALWAYS_INLINE ArrayElementType & subscript(ArrayElementType *dummy, const unsigned int, std::integral_constant) @@ -1417,7 +1417,7 @@ namespace internal template -constexpr inline DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV +constexpr inline DEAL_II_HOST_DEV_ALWAYS_INLINE typename Tensor::value_type & Tensor::operator[](const unsigned int i) { @@ -1712,7 +1712,7 @@ Tensor::norm() const template -constexpr inline DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV +constexpr inline DEAL_II_HOST_DEV_ALWAYS_INLINE typename numbers::NumberTraits::real_type Tensor::norm_square() const { @@ -2005,7 +2005,7 @@ DEAL_II_HOST_DEV constexpr DEAL_II_ALWAYS_INLINE * @relatesalso Tensor */ template -constexpr DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV +constexpr DEAL_II_HOST_DEV_ALWAYS_INLINE Tensor<0, dim, typename ProductType::type> operator+(const Tensor<0, dim, Number> & p, const Tensor<0, dim, OtherNumber> &q) @@ -2022,7 +2022,7 @@ constexpr DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV * @relatesalso Tensor */ template -constexpr DEAL_II_ALWAYS_INLINE DEAL_II_HOST_DEV +constexpr DEAL_II_HOST_DEV_ALWAYS_INLINE Tensor<0, dim, typename ProductType::type> operator-(const Tensor<0, dim, Number> & p, const Tensor<0, dim, OtherNumber> &q) -- 2.39.5