From e087153140316d48c09e28b4912ee28c33dd225d Mon Sep 17 00:00:00 2001 From: Martin Kronbichler Date: Fri, 17 Oct 2014 10:21:56 +0200 Subject: [PATCH] Improved AVX-512 vectorization code path The original version of the AVX-512 vectorization (DEAL_II_VECTORIZATION_LEVEL = 3) was not properly tested. When I did some tests with gcc, it turned out that I had to fix a few issues. Now it compiles fine and the generated assembler code looks good, but there is no hardware out to test it yet. I also improved some code comments. --- include/deal.II/base/vectorization.h | 254 +++++++++++++-------------- 1 file changed, 121 insertions(+), 133 deletions(-) diff --git a/include/deal.II/base/vectorization.h b/include/deal.II/base/vectorization.h index c7247c1343..d0993978b9 100644 --- a/include/deal.II/base/vectorization.h +++ b/include/deal.II/base/vectorization.h @@ -69,7 +69,7 @@ DEAL_II_NAMESPACE_OPEN // for safety, also check that __AVX512F__ is defined in case the user manually // set some conflicting compile flags which prevent compilation -#if DEAL_II_COMPILER_VECTORIZATION_LEVEL == 3 && defined(__AVX512F__) +#if DEAL_II_COMPILER_VECTORIZATION_LEVEL >= 3 && defined(__AVX512F__) /** * Specialization of VectorizedArray class for double and AVX-512. @@ -89,7 +89,7 @@ public: VectorizedArray & operator = (const double x) { - data = _mm512_set_pd(x, x, x, x, x, x, x, x); + data = _mm512_set1_pd(x); return *this; } @@ -120,7 +120,9 @@ public: operator += (const VectorizedArray &vec) { // if the compiler supports vector arithmetics, we can simply use += - // operator on the given data type. Otherwise, we need to use the built-in + // operator on the given data type. this allows the compiler to combine + // additions with multiplication (fused multiply-add) if those + // instructions are available. Otherwise, we need to use the built-in // intrinsic command for __m512d #ifdef DEAL_II_COMPILER_USE_VECTOR_ARITHMETICS data += vec.data; @@ -220,10 +222,17 @@ private: { // to compute the absolute value, perform bitwise andnot with -0. This // will leave all value and exponent bits unchanged but force the sign - // value to +. - __m256d mask = _mm512_set_pd (-0., -0., -0., -0., -0., -0., -0., -0.); + // value to +. As opposed to AVX and SSE2, there is no andnot operation on + // double data types in AVX-512. Thus, need to separately work with + // 256-bit data fields. + __m256d mask = _mm256_set1_pd (-0.); + __m256d in[2]; + in[0] = *(reinterpret_cast<__m256d *>(&data)); + in[0] = _m256_andnot_pd(mask, in[0]); + in[1] = *(reinterpret_cast<__m256d *>(&data)+1); + in[1] = _m256_andnot_pd(mask, in[1]); VectorizedArray res; - res.data = _mm512_andnot_pd(mask, data); + res.data = _mm512_loadu_pd(reinterpret_cast(&in[0])); return res; } @@ -284,7 +293,7 @@ public: VectorizedArray & operator = (const float x) { - data = _mm512_set_ps(x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x); + data = _mm512_set1_ps(x); return *this; } @@ -314,6 +323,11 @@ public: VectorizedArray & operator += (const VectorizedArray &vec) { + // if the compiler supports vector arithmetics, we can simply use += + // operator on the given data type. this allows the compiler to combine + // additions with multiplication (fused multiply-add) if those + // instructions are available. Otherwise, we need to use the built-in + // intrinsic command for __m512d #ifdef DEAL_II_COMPILER_USE_VECTOR_ARITHMETICS data += vec.data; #else @@ -411,14 +425,19 @@ private: VectorizedArray get_abs () const { - // to compute the absolute value, perform - // bitwise andnot with -0. This will leave all - // value and exponent bits unchanged but force - // the sign value to +. - __m256 mask = _mm512_setzero_ps (-0.f, -0.f, -0.f, -0.f, -0.f, -0.f, -0.f, -0.f, - -0.f, -0.f, -0.f, -0.f, -0.f, -0.f, -0.f, -0.f); + // to compute the absolute value, perform bitwise andnot with -0. This + // will leave all value and exponent bits unchanged but force the sign + // value to +. As opposed to AVX and SSE, there is no andnot operation on + // double data types in AVX-512. Thus, need to separately work with + // 256-bit data fields. + __m256 mask = _mm256_set1_ps (-0.f); + __m256 in[2]; + in[0] = *(reinterpret_cast<__m256 *>(&data)); + in[0] = _m256_andnot_ps(mask, in[0]); + in[1] = *(reinterpret_cast<__m256 *>(&data)+1); + in[1] = _m256_andnot_ps(mask, in[1]); VectorizedArray res; - res.data = _mm512_andnot_ps(mask, data); + res.data = _mm512_loadu_ps(reinterpret_cast(&in[0])); return res; } @@ -461,7 +480,7 @@ private: -#elif DEAL_II_COMPILER_VECTORIZATION_LEVEL == 2 && defined(__AVX__) +#elif DEAL_II_COMPILER_VECTORIZATION_LEVEL >= 2 && defined(__AVX__) /** * Specialization of VectorizedArray class for double and AVX. @@ -471,14 +490,12 @@ class VectorizedArray { public: /** - * This gives the number of vectors collected - * in this class. + * This gives the number of vectors collected in this class. */ static const unsigned int n_array_elements = 4; /** - * This function can be used to set all data - * fields to a given scalar. + * This function can be used to set all data fields to a given scalar. */ VectorizedArray & operator = (const double x) @@ -513,11 +530,11 @@ public: VectorizedArray & operator += (const VectorizedArray &vec) { - // if the compiler supports vector - // arithmetics, we can simply use += operator - // on the given data type. Otherwise, we need - // to use the built-in intrinsic command for - // __m256d + // if the compiler supports vector arithmetics, we can simply use += + // operator on the given data type. this allows the compiler to combine + // additions with multiplication (fused multiply-add) if those + // instructions are available. Otherwise, we need to use the built-in + // intrinsic command for __m256d #ifdef DEAL_II_COMPILER_USE_VECTOR_ARITHMETICS data += vec.data; #else @@ -589,16 +606,15 @@ public: } /** - * Actual data field. Since this class - * represents a POD data type, it remains - * public. + * Actual data field. Since this class represents a POD data type, it + * remains public. */ __m256d data; private: /** - * Returns the square root of this field. Not - * for use in user code. Use sqrt(x) instead. + * Returns the square root of this field. Not for use in user code. Use + * sqrt(x) instead. */ VectorizedArray get_sqrt () const @@ -609,17 +625,15 @@ private: } /** - * Returns the absolute value of this - * field. Not for use in user code. Use + * Returns the absolute value of this field. Not for use in user code. Use * abs(x) instead. */ VectorizedArray get_abs () const { - // to compute the absolute value, perform - // bitwise andnot with -0. This will leave all - // value and exponent bits unchanged but force - // the sign value to +. + // to compute the absolute value, perform bitwise andnot with -0. This + // will leave all value and exponent bits unchanged but force the sign + // value to +. __m256d mask = _mm256_set1_pd (-0.); VectorizedArray res; res.data = _mm256_andnot_pd(mask, data); @@ -627,9 +641,8 @@ private: } /** - * Returns the component-wise maximum of this - * field and another one. Not for use in user - * code. Use max(x,y) instead. + * Returns the component-wise maximum of this field and another one. Not for + * use in user code. Use max(x,y) instead. */ VectorizedArray get_max (const VectorizedArray &other) const @@ -640,9 +653,8 @@ private: } /** - * Returns the component-wise minimum of this - * field and another one. Not for use in user - * code. Use min(x,y) instead. + * Returns the component-wise minimum of this field and another one. Not for + * use in user code. Use min(x,y) instead. */ VectorizedArray get_min (const VectorizedArray &other) const @@ -675,14 +687,12 @@ class VectorizedArray { public: /** - * This gives the number of vectors collected - * in this class. + * This gives the number of vectors collected in this class. */ static const unsigned int n_array_elements = 8; /** - * This function can be used to set all data - * fields to a given scalar. + * This function can be used to set all data fields to a given scalar. */ VectorizedArray & operator = (const float x) @@ -717,6 +727,11 @@ public: VectorizedArray & operator += (const VectorizedArray &vec) { + // if the compiler supports vector arithmetics, we can simply use += + // operator on the given data type. this allows the compiler to combine + // additions with multiplication (fused multiply-add) if those + // instructions are available. Otherwise, we need to use the built-in + // intrinsic command for __m256d #ifdef DEAL_II_COMPILER_USE_VECTOR_ARITHMETICS data += vec.data; #else @@ -788,17 +803,16 @@ public: } /** - * Actual data field. Since this class - * represents a POD data type, it remains - * public. + * Actual data field. Since this class represents a POD data type, it + * remains public. */ __m256 data; private: /** - * Returns the square root of this field. Not - * for use in user code. Use sqrt(x) instead. + * Returns the square root of this field. Not for use in user code. Use + * sqrt(x) instead. */ VectorizedArray get_sqrt () const @@ -807,18 +821,17 @@ private: res.data = _mm256_sqrt_ps(data); return res; } + /** - * Returns the absolute value of this - * field. Not for use in user code. Use + * Returns the absolute value of this field. Not for use in user code. Use * abs(x) instead. */ VectorizedArray get_abs () const { - // to compute the absolute value, perform - // bitwise andnot with -0. This will leave all - // value and exponent bits unchanged but force - // the sign value to +. + // to compute the absolute value, perform bitwise andnot with -0. This + // will leave all value and exponent bits unchanged but force the sign + // value to +. __m256 mask = _mm256_set1_ps (-0.f); VectorizedArray res; res.data = _mm256_andnot_ps(mask, data); @@ -826,9 +839,8 @@ private: } /** - * Returns the component-wise maximum of this - * field and another one. Not for use in user - * code. Use max(x,y) instead. + * Returns the component-wise maximum of this field and another one. Not for + * use in user code. Use max(x,y) instead. */ VectorizedArray get_max (const VectorizedArray &other) const @@ -839,9 +851,8 @@ private: } /** - * Returns the component-wise minimum of this - * field and another one. Not for use in user - * code. Use min(x,y) instead. + * Returns the component-wise minimum of this field and another one. Not for + * use in user code. Use min(x,y) instead. */ VectorizedArray get_min (const VectorizedArray &other) const @@ -881,16 +892,13 @@ class VectorizedArray { public: /** - * This gives the number of vectors collected - * in this class. + * This gives the number of vectors collected in this class. */ static const unsigned int n_array_elements = 2; /** - * This function can be used to set all data - * fields to a given scalar. + * This function can be used to set all data fields to a given scalar. */ - VectorizedArray & operator = (const double x) { @@ -995,16 +1003,15 @@ public: } /** - * Actual data field. Since this class - * represents a POD data type, it remains - * public. + * Actual data field. Since this class represents a POD data type, it + * remains public. */ __m128d data; private: /** - * Returns the square root of this field. Not - * for use in user code. Use sqrt(x) instead. + * Returns the square root of this field. Not for use in user code. Use + * sqrt(x) instead. */ VectorizedArray get_sqrt () const @@ -1015,9 +1022,8 @@ private: } /** - * Returns the absolute value of this - * field. Not for use in user code. Use abs(x) - * instead. + * Returns the absolute value of this field. Not for use in user code. Use + * abs(x) instead. */ VectorizedArray get_abs () const @@ -1033,9 +1039,8 @@ private: } /** - * Returns the component-wise maximum of this - * field and another one. Not for use in user - * code. Use max(x,y) instead. + * Returns the component-wise maximum of this field and another one. Not for + * use in user code. Use max(x,y) instead. */ VectorizedArray get_max (const VectorizedArray &other) const @@ -1046,9 +1051,8 @@ private: } /** - * Returns the component-wise minimum of this - * field and another one. Not for use in user - * code. Use min(x,y) instead. + * Returns the component-wise minimum of this field and another one. Not for + * use in user code. Use min(x,y) instead. */ VectorizedArray get_min (const VectorizedArray &other) const @@ -1081,14 +1085,12 @@ class VectorizedArray { public: /** - * This gives the number of vectors collected - * in this class. + * This gives the number of vectors collected in this class. */ static const unsigned int n_array_elements = 4; /** - * This function can be used to set all data - * fields to a given scalar. + * This function can be used to set all data fields to a given scalar. */ VectorizedArray & @@ -1195,16 +1197,15 @@ public: } /** - * Actual data field. Since this class - * represents a POD data type, it remains - * public. + * Actual data field. Since this class represents a POD data type, it + * remains public. */ __m128 data; private: /** - * Returns the square root of this field. Not - * for use in user code. Use sqrt(x) instead. + * Returns the square root of this field. Not for use in user code. Use + * sqrt(x) instead. */ VectorizedArray get_sqrt () const @@ -1215,17 +1216,15 @@ private: } /** - * Returns the absolute value of this - * field. Not for use in user code. Use + * Returns the absolute value of this field. Not for use in user code. Use * abs(x) instead. */ VectorizedArray get_abs () const { - // to compute the absolute value, perform - // bitwise andnot with -0. This will leave all - // value and exponent bits unchanged but force - // the sign value to +. + // to compute the absolute value, perform bitwise andnot with -0. This + // will leave all value and exponent bits unchanged but force the sign + // value to +. __m128 mask = _mm_set1_ps (-0.f); VectorizedArray res; res.data = _mm_andnot_ps(mask, data); @@ -1233,9 +1232,8 @@ private: } /** - * Returns the component-wise maximum of this - * field and another one. Not for use in user - * code. Use max(x,y) instead. + * Returns the component-wise maximum of this field and another one. Not for + * use in user code. Use max(x,y) instead. */ VectorizedArray get_max (const VectorizedArray &other) const @@ -1246,9 +1244,8 @@ private: } /** - * Returns the component-wise minimum of this - * field and another one. Not for use in user - * code. Use min(x,y) instead. + * Returns the component-wise minimum of this field and another one. Not for + * use in user code. Use min(x,y) instead. */ VectorizedArray get_min (const VectorizedArray &other) const @@ -1294,7 +1291,8 @@ private: * when compiling deal.II on 64-bit operating systems. On Intel Sandy Bridge * processors and newer or AMD Bulldozer processors and newer, four doubles * and eight floats are used when deal.II is configured e.g. using gcc with - * --with-cpu=native or --with-cpu=corei7-avx. + * --with-cpu=native or --with-cpu=corei7-avx. On compilations with AVX-512 + * support, eight doubles and sixteen floats are used. * * This behavior of this class is made similar to the basic data types * double and float. The definition of a vectorized array does not @@ -1318,7 +1316,8 @@ private: * VectorizedArray(), the compiler will take care of data * alignment automatically. However, when allocating a long vector of * VectorizedArray data, one needs to respect these rules. Use the - * class AlignedVector for this purpose. It is a class very similar to + * class AlignedVector or data containers based on AlignedVector (such as + * Table) for this purpose. It is a class very similar to * std::vector otherwise but always makes sure that data is correctly * aligned. * @@ -1329,21 +1328,16 @@ class VectorizedArray { public: /** - * This gives the number of vectors collected - * in this class. + * This gives the number of vectors collected in this class. */ static const unsigned int n_array_elements = 1; - // POD means that there should be no - // user-defined constructors, destructors and - // copy functions (the standard is somewhat - // relaxed in C++2011, though). + // POD means that there should be no user-defined constructors, destructors + // and copy functions (the standard is somewhat relaxed in C++2011, though). /** - * This function assigns a scalar to this - * class. + * This function assigns a scalar to this class. */ - VectorizedArray & operator = (const Number scalar) { @@ -1352,8 +1346,7 @@ public: } /** - * Access operator (only valid with component - * 0) + * Access operator (only valid with component 0) */ Number & operator [] (const unsigned int comp) @@ -1363,8 +1356,7 @@ public: } /** - * Constant access operator (only valid with - * component 0) + * Constant access operator (only valid with component 0) */ const Number & operator [] (const unsigned int comp) const @@ -1436,16 +1428,15 @@ public: } /** - * Actual data field. Since this class - * represents a POD data type, it is declared - * public. + * Actual data field. Since this class represents a POD data type, it is + * declared public. */ Number data; private: /** - * Returns the square root of this field. Not - * for use in user code. Use sqrt(x) instead. + * Returns the square root of this field. Not for use in user code. Use + * sqrt(x) instead. */ VectorizedArray get_sqrt () const @@ -1456,8 +1447,7 @@ private: } /** - * Returns the absolute value of this - * field. Not for use in user code. Use + * Returns the absolute value of this field. Not for use in user code. Use * abs(x) instead. */ VectorizedArray @@ -1469,9 +1459,8 @@ private: } /** - * Returns the component-wise maximum of this - * field and another one. Not for use in user - * code. Use max(x,y) instead. + * Returns the component-wise maximum of this field and another one. Not for + * use in user code. Use max(x,y) instead. */ VectorizedArray get_max (const VectorizedArray &other) const @@ -1482,9 +1471,8 @@ private: } /** - * Returns the component-wise minimum of this - * field and another one. Not for use in user - * code. Use min(x,y) instead. + * Returns the component-wise minimum of this field and another one. Not for + * use in user code. Use min(x,y) instead. */ VectorizedArray get_min (const VectorizedArray &other) const @@ -1909,10 +1897,10 @@ namespace std sin (const ::dealii::VectorizedArray &x) { // put values in an array and later read in that array with an unaligned - // read. This should safe some instructions as compared to directly + // read. This should save some instructions as compared to directly // setting the individual elements and also circumvents a compiler - // optimization bug in gcc-4.6 (see also deal.II developers list from - // April 2014, topic "matrix_free/step-48 Test"). + // optimization bug in gcc-4.6 with SSE2 (see also deal.II developers list + // from April 2014, topic "matrix_free/step-48 Test"). Number values[::dealii::VectorizedArray::n_array_elements]; for (unsigned int i=0; i::n_array_elements; ++i) values[i] = std::sin(x[i]); -- 2.39.5