From: Peter Munch <peterrmuench@gmail.com>
Date: Mon, 17 Oct 2022 14:16:15 +0000 (+0200)
Subject: VectorizedArray: mixed load/store
X-Git-Tag: v9.5.0-rc1~896^2
X-Git-Url: https://gitweb.dealii.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=7928722c09aac3bb4f4c6b62117915ba81619ab1;p=dealii.git

VectorizedArray: mixed load/store
---

diff --git a/include/deal.II/base/vectorization.h b/include/deal.II/base/vectorization.h
index 6c9c43575f..fd29a24f26 100644
--- a/include/deal.II/base/vectorization.h
+++ b/include/deal.II/base/vectorization.h
@@ -537,9 +537,9 @@ public:
    * of bytes in the vectorized array, as opposed to casting a double address
    * to VectorizedArray<double>*.
    */
-  DEAL_II_ALWAYS_INLINE
-  void
-  load(const Number *ptr)
+  template <typename OtherNumber>
+  DEAL_II_ALWAYS_INLINE void
+  load(const OtherNumber *ptr)
   {
     data = *ptr;
   }
@@ -550,9 +550,9 @@ public:
    * aligned by the amount of bytes in the vectorized array, as opposed to
    * casting a double address to VectorizedArray<double>*.
    */
-  DEAL_II_ALWAYS_INLINE
-  void
-  store(Number *ptr) const
+  template <typename OtherNumber>
+  DEAL_II_ALWAYS_INLINE void
+  store(OtherNumber *ptr) const
   {
     *ptr = data;
   }
@@ -1096,6 +1096,13 @@ public:
     data = _mm512_loadu_pd(ptr);
   }
 
+  DEAL_II_ALWAYS_INLINE
+  void
+  load(const float *ptr)
+  {
+    data = _mm512_cvtps_pd(_mm256_loadu_ps(ptr));
+  }
+
   /**
    * Write the content of the calling class into memory in form of @p
    * size() to the given address. The memory need not be aligned by
@@ -1109,6 +1116,13 @@ public:
     _mm512_storeu_pd(ptr, data);
   }
 
+  DEAL_II_ALWAYS_INLINE
+  void
+  store(float *ptr) const
+  {
+    _mm256_storeu_ps(ptr, _mm512_cvtpd_ps(data));
+  }
+
   /**
    * @copydoc VectorizedArray<Number>::streaming_store()
    * @note Memory must be aligned by 64 bytes.
@@ -2322,6 +2336,13 @@ public:
     data = _mm256_loadu_pd(ptr);
   }
 
+  DEAL_II_ALWAYS_INLINE
+  void
+  load(const float *ptr)
+  {
+    data = _mm256_cvtps_pd(_mm_loadu_ps(ptr));
+  }
+
   /**
    * Write the content of the calling class into memory in form of @p
    * size() to the given address. The memory need not be aligned by
@@ -2335,6 +2356,13 @@ public:
     _mm256_storeu_pd(ptr, data);
   }
 
+  DEAL_II_ALWAYS_INLINE
+  void
+  store(float *ptr) const
+  {
+    _mm_storeu_ps(ptr, _mm256_cvtpd_ps(data));
+  }
+
   /**
    * @copydoc VectorizedArray<Number>::streaming_store()
    * @note Memory must be aligned by 32 bytes.
@@ -3400,6 +3428,15 @@ public:
     data = _mm_loadu_pd(ptr);
   }
 
+  DEAL_II_ALWAYS_INLINE
+  void
+  load(const float *ptr)
+  {
+    DEAL_II_OPENMP_SIMD_PRAGMA
+    for (unsigned int i = 0; i < 2; ++i)
+      data[i] = ptr[i];
+  }
+
   /**
    * Write the content of the calling class into memory in form of @p
    * size() to the given address. The memory need not be aligned by
@@ -3413,6 +3450,15 @@ public:
     _mm_storeu_pd(ptr, data);
   }
 
+  DEAL_II_ALWAYS_INLINE
+  void
+  store(float *ptr) const
+  {
+    DEAL_II_OPENMP_SIMD_PRAGMA
+    for (unsigned int i = 0; i < 2; ++i)
+      ptr[i] = data[i];
+  }
+
   /**
    * @copydoc VectorizedArray<Number>::streaming_store()
    * @note Memory must be aligned by 16 bytes.
diff --git a/tests/base/vectorization_04.cc b/tests/base/vectorization_04.cc
index c7db3a7bd8..6fdc13b8c3 100644
--- a/tests/base/vectorization_04.cc
+++ b/tests/base/vectorization_04.cc
@@ -21,15 +21,15 @@
 
 #include "../tests.h"
 
-template <typename Number>
+template <typename Number, typename VectorizedArrayType>
 void
 test()
 {
-  const unsigned int  n_vectors = VectorizedArray<Number>::size();
+  const unsigned int  n_vectors = VectorizedArrayType::size();
   std::vector<Number> values(n_vectors * 5);
   for (unsigned int i = 0; i < values.size(); ++i)
     values[i] = i;
-  AlignedVector<VectorizedArray<Number>> copied(4);
+  AlignedVector<VectorizedArrayType> copied(4);
 
   // test load operation for all possible values of alignment
   for (unsigned int shift = 0; shift < n_vectors; ++shift)
@@ -49,7 +49,7 @@ test()
     {
       for (unsigned int i = 0; i < 4; ++i)
         {
-          VectorizedArray<Number> tmp;
+          VectorizedArrayType tmp;
           tmp.load(&values[i * n_vectors]);
           tmp.store(&stored[i * n_vectors + shift]);
         }
@@ -66,20 +66,59 @@ main()
 {
   initlog();
 
-  deallog.push("double");
-  test<double>();
+  deallog.push("double <-> VectorizedArray<double, 1>");
+  test<double, VectorizedArray<double, 1>>();
   deallog.pop();
-  deallog.push("float");
-  test<float>();
+  deallog.push("float <-> VectorizedArray<float, 1>");
+  test<float, VectorizedArray<float, 1>>();
   deallog.pop();
+  deallog.push("float <-> VectorizedArray<double, 1>");
+  test<float, VectorizedArray<double, 1>>();
+  deallog.pop();
+
+#if DEAL_II_VECTORIZATION_WIDTH_IN_BITS >= 128
+  deallog.push("double <-> VectorizedArray<double, 2>");
+  test<double, VectorizedArray<double, 2>>();
+  deallog.pop();
+  deallog.push("float <-> VectorizedArray<float, 4>");
+  test<float, VectorizedArray<float, 4>>();
+  deallog.pop();
+  deallog.push("float <-> VectorizedArray<double, 2>");
+  test<float, VectorizedArray<double, 2>>();
+  deallog.pop();
+#endif
+
+#if DEAL_II_VECTORIZATION_WIDTH_IN_BITS >= 256
+  deallog.push("double <-> VectorizedArray<double, 4>");
+  test<double, VectorizedArray<double, 4>>();
+  deallog.pop();
+  deallog.push("float <-> VectorizedArray<float, 8>");
+  test<float, VectorizedArray<float, 8>>();
+  deallog.pop();
+  deallog.push("float <-> VectorizedArray<double, 4>");
+  test<float, VectorizedArray<double, 4>>();
+  deallog.pop();
+#endif
+
+#if DEAL_II_VECTORIZATION_WIDTH_IN_BITS >= 512
+  deallog.push("double <-> VectorizedArray<double, 8>");
+  test<double, VectorizedArray<double, 8>>();
+  deallog.pop();
+  deallog.push("float <-> VectorizedArray<float, 16>");
+  test<float, VectorizedArray<float, 16>>();
+  deallog.pop();
+  deallog.push("float <-> VectorizedArray<double, 8>");
+  test<float, VectorizedArray<double, 8>>();
+  deallog.pop();
+#endif
 
   // test long double and unsigned int: in these cases, the default path of
   // VectorizedArray is taken no matter what was done for double or float
   deallog.push("long double");
-  test<long double>();
+  test<long double, VectorizedArray<long double>>();
   deallog.pop();
 
   deallog.push("unsigned int");
-  test<unsigned int>();
+  test<unsigned int, VectorizedArray<unsigned int>>();
   deallog.pop();
 }
diff --git a/tests/base/vectorization_04.output b/tests/base/vectorization_04.output
index 057015edea..9c7840305b 100644
--- a/tests/base/vectorization_04.output
+++ b/tests/base/vectorization_04.output
@@ -1,8 +1,10 @@
 
-DEAL:double::load OK
-DEAL:double::store OK
-DEAL:float::load OK
-DEAL:float::store OK
+DEAL:double <-> VectorizedArray<double, 1>::load OK
+DEAL:double <-> VectorizedArray<double, 1>::store OK
+DEAL:float <-> VectorizedArray<float, 1>::load OK
+DEAL:float <-> VectorizedArray<float, 1>::store OK
+DEAL:float <-> VectorizedArray<double, 1>::load OK
+DEAL:float <-> VectorizedArray<double, 1>::store OK
 DEAL:long double::load OK
 DEAL:long double::store OK
 DEAL:unsigned int::load OK
diff --git a/tests/base/vectorization_04.output.avx b/tests/base/vectorization_04.output.avx
new file mode 100644
index 0000000000..d3b3722b80
--- /dev/null
+++ b/tests/base/vectorization_04.output.avx
@@ -0,0 +1,23 @@
+
+DEAL:double <-> VectorizedArray<double, 1>::load OK
+DEAL:double <-> VectorizedArray<double, 1>::store OK
+DEAL:float <-> VectorizedArray<float, 1>::load OK
+DEAL:float <-> VectorizedArray<float, 1>::store OK
+DEAL:float <-> VectorizedArray<double, 1>::load OK
+DEAL:float <-> VectorizedArray<double, 1>::store OK
+DEAL:double <-> VectorizedArray<double, 2>::load OK
+DEAL:double <-> VectorizedArray<double, 2>::store OK
+DEAL:float <-> VectorizedArray<float, 4>::load OK
+DEAL:float <-> VectorizedArray<float, 4>::store OK
+DEAL:float <-> VectorizedArray<double, 2>::load OK
+DEAL:float <-> VectorizedArray<double, 2>::store OK
+DEAL:double <-> VectorizedArray<double, 4>::load OK
+DEAL:double <-> VectorizedArray<double, 4>::store OK
+DEAL:float <-> VectorizedArray<float, 8>::load OK
+DEAL:float <-> VectorizedArray<float, 8>::store OK
+DEAL:float <-> VectorizedArray<double, 4>::load OK
+DEAL:float <-> VectorizedArray<double, 4>::store OK
+DEAL:long double::load OK
+DEAL:long double::store OK
+DEAL:unsigned int::load OK
+DEAL:unsigned int::store OK
diff --git a/tests/base/vectorization_04.output.avx512 b/tests/base/vectorization_04.output.avx512
new file mode 100644
index 0000000000..ad7debc259
--- /dev/null
+++ b/tests/base/vectorization_04.output.avx512
@@ -0,0 +1,29 @@
+
+DEAL:double <-> VectorizedArray<double, 1>::load OK
+DEAL:double <-> VectorizedArray<double, 1>::store OK
+DEAL:float <-> VectorizedArray<float, 1>::load OK
+DEAL:float <-> VectorizedArray<float, 1>::store OK
+DEAL:float <-> VectorizedArray<double, 1>::load OK
+DEAL:float <-> VectorizedArray<double, 1>::store OK
+DEAL:double <-> VectorizedArray<double, 2>::load OK
+DEAL:double <-> VectorizedArray<double, 2>::store OK
+DEAL:float <-> VectorizedArray<float, 4>::load OK
+DEAL:float <-> VectorizedArray<float, 4>::store OK
+DEAL:float <-> VectorizedArray<double, 2>::load OK
+DEAL:float <-> VectorizedArray<double, 2>::store OK
+DEAL:double <-> VectorizedArray<double, 4>::load OK
+DEAL:double <-> VectorizedArray<double, 4>::store OK
+DEAL:float <-> VectorizedArray<float, 8>::load OK
+DEAL:float <-> VectorizedArray<float, 8>::store OK
+DEAL:float <-> VectorizedArray<double, 4>::load OK
+DEAL:float <-> VectorizedArray<double, 4>::store OK
+DEAL:double <-> VectorizedArray<double, 8>::load OK
+DEAL:double <-> VectorizedArray<double, 8>::store OK
+DEAL:float <-> VectorizedArray<float, 16>::load OK
+DEAL:float <-> VectorizedArray<float, 16>::store OK
+DEAL:float <-> VectorizedArray<double, 8>::load OK
+DEAL:float <-> VectorizedArray<double, 8>::store OK
+DEAL:long double::load OK
+DEAL:long double::store OK
+DEAL:unsigned int::load OK
+DEAL:unsigned int::store OK
diff --git a/tests/base/vectorization_04.output.sse b/tests/base/vectorization_04.output.sse
new file mode 100644
index 0000000000..6f5a83cc5c
--- /dev/null
+++ b/tests/base/vectorization_04.output.sse
@@ -0,0 +1,17 @@
+
+DEAL:double <-> VectorizedArray<double, 1>::load OK
+DEAL:double <-> VectorizedArray<double, 1>::store OK
+DEAL:float <-> VectorizedArray<float, 1>::load OK
+DEAL:float <-> VectorizedArray<float, 1>::store OK
+DEAL:float <-> VectorizedArray<double, 1>::load OK
+DEAL:float <-> VectorizedArray<double, 1>::store OK
+DEAL:double <-> VectorizedArray<double, 2>::load OK
+DEAL:double <-> VectorizedArray<double, 2>::store OK
+DEAL:float <-> VectorizedArray<float, 4>::load OK
+DEAL:float <-> VectorizedArray<float, 4>::store OK
+DEAL:float <-> VectorizedArray<double, 2>::load OK
+DEAL:float <-> VectorizedArray<double, 2>::store OK
+DEAL:long double::load OK
+DEAL:long double::store OK
+DEAL:unsigned int::load OK
+DEAL:unsigned int::store OK