From 7e05984520a1835161b51e88647e2499ea12ee52 Mon Sep 17 00:00:00 2001 From: Matthias Maier Date: Wed, 4 Sep 2019 17:13:25 -0500 Subject: [PATCH] suggested changes --- include/deal.II/base/vectorization.h | 124 ++++++++++++++++----------- 1 file changed, 75 insertions(+), 49 deletions(-) diff --git a/include/deal.II/base/vectorization.h b/include/deal.II/base/vectorization.h index 543efe5ed8..281b47b474 100644 --- a/include/deal.II/base/vectorization.h +++ b/include/deal.II/base/vectorization.h @@ -3968,7 +3968,7 @@ operator<<(std::ostream &out, const VectorizedArray &p) /** * TODO */ -enum SIMDComparison : int +enum class SIMDComparison : int { equal = _CMP_EQ_OQ, not_equal = _CMP_NEQ_OQ, @@ -3990,26 +3990,25 @@ compare_and_apply_mask(const Number &left, const Number &false_value) { bool mask; - switch (predicate) { case SIMDComparison::equal: - mask = left == right; + mask = (left == right); break; case SIMDComparison::not_equal: - mask = left != right; + mask = (left != right); break; case SIMDComparison::less_than: - mask = left < right; + mask = (left < right); break; case SIMDComparison::less_than_or_equal: - mask = left <= right; + mask = (left <= right); break; case SIMDComparison::greater_than: - mask = left > right; + mask = (left > right); break; case SIMDComparison::greater_than_or_equal: - mask = left >= right; + mask = (left >= right); break; } @@ -4027,31 +4026,12 @@ compare_and_apply_mask(const VectorizedArray &left, const VectorizedArray &true_value, const VectorizedArray &false_value) { - bool mask; - - switch (predicate) - { - case SIMDComparison::equal: - mask = left.data == right.data; - break; - case SIMDComparison::not_equal: - mask = left.data != right.data; - break; - case SIMDComparison::less_than: - mask = left.data < right.data; - break; - case SIMDComparison::less_than_or_equal: - mask = left.data <= right.data; - break; - case SIMDComparison::greater_than: - mask = left.data > right.data; - break; - case SIMDComparison::greater_than_or_equal: - mask = left.data >= right.data; - break; - } - - return mask ? true_value.data : false_value.data; + VectorizedArray result; + result.data = compare_and_apply_mask(left.data, + right.data, + true_value.data, + false_value.data); + return result; } //@} @@ -4059,33 +4039,33 @@ compare_and_apply_mask(const VectorizedArray &left, #ifndef DOXYGEN # if DEAL_II_COMPILER_VECTORIZATION_LEVEL >= 3 && defined(__AVX512F__) -template +template DEAL_II_ALWAYS_INLINE inline VectorizedArray compare_and_apply_mask(const VectorizedArray &left, const VectorizedArray &right, const VectorizedArray &true_values, const VectorizedArray &false_values) { - const auto mask = _mm512_cmp_ps(left.data, right.data, predicate); + const __mmask16 mask = + _mm512_cmp_ps(left.data, right.data, static_cast(predicate)); VectorizedArray result; - result.data = _mm512_or_ps(_mm512_and_ps(mask, true_values.data), - _mm512_andnot_ps(mask, false_values.data)); + result.data = _mm512_mask_mov_ps(false_values.data, mask, true_values.data); return result; } -template +template DEAL_II_ALWAYS_INLINE inline VectorizedArray compare_and_apply_mask(const VectorizedArray &left, const VectorizedArray &right, const VectorizedArray &true_values, const VectorizedArray &false_values) { - const auto mask = _mm512_cmp_pd(left.data, right.data, predicate); + const __mmask16 mask = + _mm512_cmp_pd(left.data, right.data, static_cast(predicate)); VectorizedArray result; - result.data = _mm256_or_ps(_mm512_and_ps(mask, true_values.data), - _mm512_andnot_ps(mask, false_values.data)); + result.data = _mm512_mask_mov_pd(false_values.data, mask, true_values.data); return result; } @@ -4093,14 +4073,15 @@ compare_and_apply_mask(const VectorizedArray &left, # if DEAL_II_COMPILER_VECTORIZATION_LEVEL >= 2 && defined(__AVX__) -template +template DEAL_II_ALWAYS_INLINE inline VectorizedArray compare_and_apply_mask(const VectorizedArray &left, const VectorizedArray &right, const VectorizedArray &true_values, const VectorizedArray &false_values) { - const auto mask = _mm256_cmp_ps(left.data, right.data, predicate); + const auto mask = + _mm256_cmp_ps(left.data, right.data, static_cast(predicate)); VectorizedArray result; result.data = _mm256_or_ps(_mm256_and_ps(mask, true_values.data), @@ -4109,14 +4090,15 @@ compare_and_apply_mask(const VectorizedArray &left, } -template +template DEAL_II_ALWAYS_INLINE inline VectorizedArray compare_and_apply_mask(const VectorizedArray &left, const VectorizedArray &right, const VectorizedArray &true_values, const VectorizedArray &false_values) { - const auto mask = _mm256_cmp_pd(left.data, right.data, predicate); + const auto mask = + _mm256_cmp_pd(left.data, right.data, static_cast(predicate)); VectorizedArray result; result.data = _mm256_or_ps(_mm256_and_ps(mask, true_values.data), @@ -4128,34 +4110,78 @@ compare_and_apply_mask(const VectorizedArray &left, # if DEAL_II_COMPILER_VECTORIZATION_LEVEL >= 1 && defined(__SSE2__) -template +template DEAL_II_ALWAYS_INLINE inline VectorizedArray compare_and_apply_mask(const VectorizedArray &left, const VectorizedArray &right, const VectorizedArray &true_values, const VectorizedArray &false_values) { - const auto mask = _mm_cmp_ps(left.data, right.data, predicate); + __m128 mask; + switch (predicate) + { + case SIMDComparison::equal: + mask = _mm_cmpeq_ps(left.data, right.data); + break; + case SIMDComparison::not_equal: + mask = _mm_cmpneq_ps(left.data, right.data); + break; + case SIMDComparison::less_than: + mask = _mm_cmplt_ps(left.data, right.data); + break; + case SIMDComparison::less_than_or_equal: + mask = _mm_cmple_ps(left.data, right.data); + break; + case SIMDComparison::greater_than: + mask = _mm_cmpgt_ps(left.data, right.data); + break; + case SIMDComparison::greater_than_or_equal: + mask = _mm_cmpge_ps(left.data, right.data); + break; + } VectorizedArray result; result.data = _mm_or_ps(_mm_and_ps(mask, true_values.data), _mm_andnot_ps(mask, false_values.data)); + return result; } -template +template DEAL_II_ALWAYS_INLINE inline VectorizedArray compare_and_apply_mask(const VectorizedArray &left, const VectorizedArray &right, const VectorizedArray &true_values, const VectorizedArray &false_values) { - const auto mask = _mm_cmp_pd(left.data, right.data, predicate); + __m128d mask; + switch (predicate) + { + case SIMDComparison::equal: + mask = _mm_cmpeq_pd(left.data, right.data); + break; + case SIMDComparison::not_equal: + mask = _mm_cmpneq_pd(left.data, right.data); + break; + case SIMDComparison::less_than: + mask = _mm_cmplt_pd(left.data, right.data); + break; + case SIMDComparison::less_than_or_equal: + mask = _mm_cmple_pd(left.data, right.data); + break; + case SIMDComparison::greater_than: + mask = _mm_cmpgt_pd(left.data, right.data); + break; + case SIMDComparison::greater_than_or_equal: + mask = _mm_cmpge_pd(left.data, right.data); + break; + } VectorizedArray result; result.data = _mm_or_ps(_mm_and_ps(mask, true_values.data), _mm_andnot_ps(mask, false_values.data)); + return result; } -- 2.39.5