From 7928722c09aac3bb4f4c6b62117915ba81619ab1 Mon Sep 17 00:00:00 2001 From: Peter Munch Date: Mon, 17 Oct 2022 16:16:15 +0200 Subject: [PATCH] VectorizedArray: mixed load/store --- include/deal.II/base/vectorization.h | 58 +++++++++++++++++++--- tests/base/vectorization_04.cc | 59 +++++++++++++++++++---- tests/base/vectorization_04.output | 10 ++-- tests/base/vectorization_04.output.avx | 23 +++++++++ tests/base/vectorization_04.output.avx512 | 29 +++++++++++ tests/base/vectorization_04.output.sse | 17 +++++++ 6 files changed, 176 insertions(+), 20 deletions(-) create mode 100644 tests/base/vectorization_04.output.avx create mode 100644 tests/base/vectorization_04.output.avx512 create mode 100644 tests/base/vectorization_04.output.sse diff --git a/include/deal.II/base/vectorization.h b/include/deal.II/base/vectorization.h index 6c9c43575f..fd29a24f26 100644 --- a/include/deal.II/base/vectorization.h +++ b/include/deal.II/base/vectorization.h @@ -537,9 +537,9 @@ public: * of bytes in the vectorized array, as opposed to casting a double address * to VectorizedArray*. */ - DEAL_II_ALWAYS_INLINE - void - load(const Number *ptr) + template + DEAL_II_ALWAYS_INLINE void + load(const OtherNumber *ptr) { data = *ptr; } @@ -550,9 +550,9 @@ public: * aligned by the amount of bytes in the vectorized array, as opposed to * casting a double address to VectorizedArray*. */ - DEAL_II_ALWAYS_INLINE - void - store(Number *ptr) const + template + DEAL_II_ALWAYS_INLINE void + store(OtherNumber *ptr) const { *ptr = data; } @@ -1096,6 +1096,13 @@ public: data = _mm512_loadu_pd(ptr); } + DEAL_II_ALWAYS_INLINE + void + load(const float *ptr) + { + data = _mm512_cvtps_pd(_mm256_loadu_ps(ptr)); + } + /** * Write the content of the calling class into memory in form of @p * size() to the given address. The memory need not be aligned by @@ -1109,6 +1116,13 @@ public: _mm512_storeu_pd(ptr, data); } + DEAL_II_ALWAYS_INLINE + void + store(float *ptr) const + { + _mm256_storeu_ps(ptr, _mm512_cvtpd_ps(data)); + } + /** * @copydoc VectorizedArray::streaming_store() * @note Memory must be aligned by 64 bytes. @@ -2322,6 +2336,13 @@ public: data = _mm256_loadu_pd(ptr); } + DEAL_II_ALWAYS_INLINE + void + load(const float *ptr) + { + data = _mm256_cvtps_pd(_mm_loadu_ps(ptr)); + } + /** * Write the content of the calling class into memory in form of @p * size() to the given address. The memory need not be aligned by @@ -2335,6 +2356,13 @@ public: _mm256_storeu_pd(ptr, data); } + DEAL_II_ALWAYS_INLINE + void + store(float *ptr) const + { + _mm_storeu_ps(ptr, _mm256_cvtpd_ps(data)); + } + /** * @copydoc VectorizedArray::streaming_store() * @note Memory must be aligned by 32 bytes. @@ -3400,6 +3428,15 @@ public: data = _mm_loadu_pd(ptr); } + DEAL_II_ALWAYS_INLINE + void + load(const float *ptr) + { + DEAL_II_OPENMP_SIMD_PRAGMA + for (unsigned int i = 0; i < 2; ++i) + data[i] = ptr[i]; + } + /** * Write the content of the calling class into memory in form of @p * size() to the given address. The memory need not be aligned by @@ -3413,6 +3450,15 @@ public: _mm_storeu_pd(ptr, data); } + DEAL_II_ALWAYS_INLINE + void + store(float *ptr) const + { + DEAL_II_OPENMP_SIMD_PRAGMA + for (unsigned int i = 0; i < 2; ++i) + ptr[i] = data[i]; + } + /** * @copydoc VectorizedArray::streaming_store() * @note Memory must be aligned by 16 bytes. diff --git a/tests/base/vectorization_04.cc b/tests/base/vectorization_04.cc index c7db3a7bd8..6fdc13b8c3 100644 --- a/tests/base/vectorization_04.cc +++ b/tests/base/vectorization_04.cc @@ -21,15 +21,15 @@ #include "../tests.h" -template +template void test() { - const unsigned int n_vectors = VectorizedArray::size(); + const unsigned int n_vectors = VectorizedArrayType::size(); std::vector values(n_vectors * 5); for (unsigned int i = 0; i < values.size(); ++i) values[i] = i; - AlignedVector> copied(4); + AlignedVector copied(4); // test load operation for all possible values of alignment for (unsigned int shift = 0; shift < n_vectors; ++shift) @@ -49,7 +49,7 @@ test() { for (unsigned int i = 0; i < 4; ++i) { - VectorizedArray tmp; + VectorizedArrayType tmp; tmp.load(&values[i * n_vectors]); tmp.store(&stored[i * n_vectors + shift]); } @@ -66,20 +66,59 @@ main() { initlog(); - deallog.push("double"); - test(); + deallog.push("double <-> VectorizedArray"); + test>(); deallog.pop(); - deallog.push("float"); - test(); + deallog.push("float <-> VectorizedArray"); + test>(); deallog.pop(); + deallog.push("float <-> VectorizedArray"); + test>(); + deallog.pop(); + +#if DEAL_II_VECTORIZATION_WIDTH_IN_BITS >= 128 + deallog.push("double <-> VectorizedArray"); + test>(); + deallog.pop(); + deallog.push("float <-> VectorizedArray"); + test>(); + deallog.pop(); + deallog.push("float <-> VectorizedArray"); + test>(); + deallog.pop(); +#endif + +#if DEAL_II_VECTORIZATION_WIDTH_IN_BITS >= 256 + deallog.push("double <-> VectorizedArray"); + test>(); + deallog.pop(); + deallog.push("float <-> VectorizedArray"); + test>(); + deallog.pop(); + deallog.push("float <-> VectorizedArray"); + test>(); + deallog.pop(); +#endif + +#if DEAL_II_VECTORIZATION_WIDTH_IN_BITS >= 512 + deallog.push("double <-> VectorizedArray"); + test>(); + deallog.pop(); + deallog.push("float <-> VectorizedArray"); + test>(); + deallog.pop(); + deallog.push("float <-> VectorizedArray"); + test>(); + deallog.pop(); +#endif // test long double and unsigned int: in these cases, the default path of // VectorizedArray is taken no matter what was done for double or float deallog.push("long double"); - test(); + test>(); deallog.pop(); deallog.push("unsigned int"); - test(); + test>(); deallog.pop(); } diff --git a/tests/base/vectorization_04.output b/tests/base/vectorization_04.output index 057015edea..9c7840305b 100644 --- a/tests/base/vectorization_04.output +++ b/tests/base/vectorization_04.output @@ -1,8 +1,10 @@ -DEAL:double::load OK -DEAL:double::store OK -DEAL:float::load OK -DEAL:float::store OK +DEAL:double <-> VectorizedArray::load OK +DEAL:double <-> VectorizedArray::store OK +DEAL:float <-> VectorizedArray::load OK +DEAL:float <-> VectorizedArray::store OK +DEAL:float <-> VectorizedArray::load OK +DEAL:float <-> VectorizedArray::store OK DEAL:long double::load OK DEAL:long double::store OK DEAL:unsigned int::load OK diff --git a/tests/base/vectorization_04.output.avx b/tests/base/vectorization_04.output.avx new file mode 100644 index 0000000000..d3b3722b80 --- /dev/null +++ b/tests/base/vectorization_04.output.avx @@ -0,0 +1,23 @@ + +DEAL:double <-> VectorizedArray::load OK +DEAL:double <-> VectorizedArray::store OK +DEAL:float <-> VectorizedArray::load OK +DEAL:float <-> VectorizedArray::store OK +DEAL:float <-> VectorizedArray::load OK +DEAL:float <-> VectorizedArray::store OK +DEAL:double <-> VectorizedArray::load OK +DEAL:double <-> VectorizedArray::store OK +DEAL:float <-> VectorizedArray::load OK +DEAL:float <-> VectorizedArray::store OK +DEAL:float <-> VectorizedArray::load OK +DEAL:float <-> VectorizedArray::store OK +DEAL:double <-> VectorizedArray::load OK +DEAL:double <-> VectorizedArray::store OK +DEAL:float <-> VectorizedArray::load OK +DEAL:float <-> VectorizedArray::store OK +DEAL:float <-> VectorizedArray::load OK +DEAL:float <-> VectorizedArray::store OK +DEAL:long double::load OK +DEAL:long double::store OK +DEAL:unsigned int::load OK +DEAL:unsigned int::store OK diff --git a/tests/base/vectorization_04.output.avx512 b/tests/base/vectorization_04.output.avx512 new file mode 100644 index 0000000000..ad7debc259 --- /dev/null +++ b/tests/base/vectorization_04.output.avx512 @@ -0,0 +1,29 @@ + +DEAL:double <-> VectorizedArray::load OK +DEAL:double <-> VectorizedArray::store OK +DEAL:float <-> VectorizedArray::load OK +DEAL:float <-> VectorizedArray::store OK +DEAL:float <-> VectorizedArray::load OK +DEAL:float <-> VectorizedArray::store OK +DEAL:double <-> VectorizedArray::load OK +DEAL:double <-> VectorizedArray::store OK +DEAL:float <-> VectorizedArray::load OK +DEAL:float <-> VectorizedArray::store OK +DEAL:float <-> VectorizedArray::load OK +DEAL:float <-> VectorizedArray::store OK +DEAL:double <-> VectorizedArray::load OK +DEAL:double <-> VectorizedArray::store OK +DEAL:float <-> VectorizedArray::load OK +DEAL:float <-> VectorizedArray::store OK +DEAL:float <-> VectorizedArray::load OK +DEAL:float <-> VectorizedArray::store OK +DEAL:double <-> VectorizedArray::load OK +DEAL:double <-> VectorizedArray::store OK +DEAL:float <-> VectorizedArray::load OK +DEAL:float <-> VectorizedArray::store OK +DEAL:float <-> VectorizedArray::load OK +DEAL:float <-> VectorizedArray::store OK +DEAL:long double::load OK +DEAL:long double::store OK +DEAL:unsigned int::load OK +DEAL:unsigned int::store OK diff --git a/tests/base/vectorization_04.output.sse b/tests/base/vectorization_04.output.sse new file mode 100644 index 0000000000..6f5a83cc5c --- /dev/null +++ b/tests/base/vectorization_04.output.sse @@ -0,0 +1,17 @@ + +DEAL:double <-> VectorizedArray::load OK +DEAL:double <-> VectorizedArray::store OK +DEAL:float <-> VectorizedArray::load OK +DEAL:float <-> VectorizedArray::store OK +DEAL:float <-> VectorizedArray::load OK +DEAL:float <-> VectorizedArray::store OK +DEAL:double <-> VectorizedArray::load OK +DEAL:double <-> VectorizedArray::store OK +DEAL:float <-> VectorizedArray::load OK +DEAL:float <-> VectorizedArray::store OK +DEAL:float <-> VectorizedArray::load OK +DEAL:float <-> VectorizedArray::store OK +DEAL:long double::load OK +DEAL:long double::store OK +DEAL:unsigned int::load OK +DEAL:unsigned int::store OK -- 2.39.5