VectorizedArray: mixed load/store

diff --git a/include/deal.II/base/vectorization.h b/include/deal.II/base/vectorization.h
index 6c9c43575fe88b23a14584844451db6ad59be02f..fd29a24f26b3e608780599bbc8d6411f95092934 100644 (file)
--- a/include/deal.II/base/vectorization.h
+++ b/include/deal.II/base/vectorization.h
@@ -537,9 +537,9 @@ public:
    * of bytes in the vectorized array, as opposed to casting a double address
    * to VectorizedArray<double>*.
    */
-  DEAL_II_ALWAYS_INLINE
-  void
-  load(const Number *ptr)
+  template <typename OtherNumber>
+  DEAL_II_ALWAYS_INLINE void
+  load(const OtherNumber *ptr)
   {
     data = *ptr;
   }
@@ -550,9 +550,9 @@ public:
    * aligned by the amount of bytes in the vectorized array, as opposed to
    * casting a double address to VectorizedArray<double>*.
    */
-  DEAL_II_ALWAYS_INLINE
-  void
-  store(Number *ptr) const
+  template <typename OtherNumber>
+  DEAL_II_ALWAYS_INLINE void
+  store(OtherNumber *ptr) const
   {
     *ptr = data;
   }
@@ -1096,6 +1096,13 @@ public:
     data = _mm512_loadu_pd(ptr);
   }
 
+  DEAL_II_ALWAYS_INLINE
+  void
+  load(const float *ptr)
+  {
+    data = _mm512_cvtps_pd(_mm256_loadu_ps(ptr));
+  }
+
   /**
    * Write the content of the calling class into memory in form of @p
    * size() to the given address. The memory need not be aligned by
@@ -1109,6 +1116,13 @@ public:
     _mm512_storeu_pd(ptr, data);
   }
 
+  DEAL_II_ALWAYS_INLINE
+  void
+  store(float *ptr) const
+  {
+    _mm256_storeu_ps(ptr, _mm512_cvtpd_ps(data));
+  }
+
   /**
    * @copydoc VectorizedArray<Number>::streaming_store()
    * @note Memory must be aligned by 64 bytes.
@@ -2322,6 +2336,13 @@ public:
     data = _mm256_loadu_pd(ptr);
   }
 
+  DEAL_II_ALWAYS_INLINE
+  void
+  load(const float *ptr)
+  {
+    data = _mm256_cvtps_pd(_mm_loadu_ps(ptr));
+  }
+
   /**
    * Write the content of the calling class into memory in form of @p
    * size() to the given address. The memory need not be aligned by
@@ -2335,6 +2356,13 @@ public:
     _mm256_storeu_pd(ptr, data);
   }
 
+  DEAL_II_ALWAYS_INLINE
+  void
+  store(float *ptr) const
+  {
+    _mm_storeu_ps(ptr, _mm256_cvtpd_ps(data));
+  }
+
   /**
    * @copydoc VectorizedArray<Number>::streaming_store()
    * @note Memory must be aligned by 32 bytes.
@@ -3400,6 +3428,15 @@ public:
     data = _mm_loadu_pd(ptr);
   }
 
+  DEAL_II_ALWAYS_INLINE
+  void
+  load(const float *ptr)
+  {
+    DEAL_II_OPENMP_SIMD_PRAGMA
+    for (unsigned int i = 0; i < 2; ++i)
+      data[i] = ptr[i];
+  }
+
   /**
    * Write the content of the calling class into memory in form of @p
    * size() to the given address. The memory need not be aligned by
@@ -3413,6 +3450,15 @@ public:
     _mm_storeu_pd(ptr, data);
   }
 
+  DEAL_II_ALWAYS_INLINE
+  void
+  store(float *ptr) const
+  {
+    DEAL_II_OPENMP_SIMD_PRAGMA
+    for (unsigned int i = 0; i < 2; ++i)
+      ptr[i] = data[i];
+  }
+
   /**
    * @copydoc VectorizedArray<Number>::streaming_store()
    * @note Memory must be aligned by 16 bytes.

diff --git a/tests/base/vectorization_04.cc b/tests/base/vectorization_04.cc
index c7db3a7bd87a5b4a22bfa504f2af3144d26b3ef4..6fdc13b8c3357effa524a7893f54e03416b323e3 100644 (file)
--- a/tests/base/vectorization_04.cc
+++ b/tests/base/vectorization_04.cc
@@ -21,15 +21,15 @@
 
 #include "../tests.h"
 
-template <typename Number>
+template <typename Number, typename VectorizedArrayType>
 void
 test()
 {
-  const unsigned int  n_vectors = VectorizedArray<Number>::size();
+  const unsigned int  n_vectors = VectorizedArrayType::size();
   std::vector<Number> values(n_vectors * 5);
   for (unsigned int i = 0; i < values.size(); ++i)
     values[i] = i;
-  AlignedVector<VectorizedArray<Number>> copied(4);
+  AlignedVector<VectorizedArrayType> copied(4);
 
   // test load operation for all possible values of alignment
   for (unsigned int shift = 0; shift < n_vectors; ++shift)
@@ -49,7 +49,7 @@ test()
     {
       for (unsigned int i = 0; i < 4; ++i)
         {
-          VectorizedArray<Number> tmp;
+          VectorizedArrayType tmp;
           tmp.load(&values[i * n_vectors]);
           tmp.store(&stored[i * n_vectors + shift]);
         }
@@ -66,20 +66,59 @@ main()
 {
   initlog();
 
-  deallog.push("double");
-  test<double>();
+  deallog.push("double <-> VectorizedArray<double, 1>");
+  test<double, VectorizedArray<double, 1>>();
   deallog.pop();
-  deallog.push("float");
-  test<float>();
+  deallog.push("float <-> VectorizedArray<float, 1>");
+  test<float, VectorizedArray<float, 1>>();
   deallog.pop();
+  deallog.push("float <-> VectorizedArray<double, 1>");
+  test<float, VectorizedArray<double, 1>>();
+  deallog.pop();
+
+#if DEAL_II_VECTORIZATION_WIDTH_IN_BITS >= 128
+  deallog.push("double <-> VectorizedArray<double, 2>");
+  test<double, VectorizedArray<double, 2>>();
+  deallog.pop();
+  deallog.push("float <-> VectorizedArray<float, 4>");
+  test<float, VectorizedArray<float, 4>>();
+  deallog.pop();
+  deallog.push("float <-> VectorizedArray<double, 2>");
+  test<float, VectorizedArray<double, 2>>();
+  deallog.pop();
+#endif
+
+#if DEAL_II_VECTORIZATION_WIDTH_IN_BITS >= 256
+  deallog.push("double <-> VectorizedArray<double, 4>");
+  test<double, VectorizedArray<double, 4>>();
+  deallog.pop();
+  deallog.push("float <-> VectorizedArray<float, 8>");
+  test<float, VectorizedArray<float, 8>>();
+  deallog.pop();
+  deallog.push("float <-> VectorizedArray<double, 4>");
+  test<float, VectorizedArray<double, 4>>();
+  deallog.pop();
+#endif
+
+#if DEAL_II_VECTORIZATION_WIDTH_IN_BITS >= 512
+  deallog.push("double <-> VectorizedArray<double, 8>");
+  test<double, VectorizedArray<double, 8>>();
+  deallog.pop();
+  deallog.push("float <-> VectorizedArray<float, 16>");
+  test<float, VectorizedArray<float, 16>>();
+  deallog.pop();
+  deallog.push("float <-> VectorizedArray<double, 8>");
+  test<float, VectorizedArray<double, 8>>();
+  deallog.pop();
+#endif
 
   // test long double and unsigned int: in these cases, the default path of
   // VectorizedArray is taken no matter what was done for double or float
   deallog.push("long double");
-  test<long double>();
+  test<long double, VectorizedArray<long double>>();
   deallog.pop();
 
   deallog.push("unsigned int");
-  test<unsigned int>();
+  test<unsigned int, VectorizedArray<unsigned int>>();
   deallog.pop();
 }

diff --git a/tests/base/vectorization_04.output b/tests/base/vectorization_04.output
index 057015edea684a7a238e31ba5fbb692187725a7e..9c7840305bdddd6d803d5127974b844780af1b00 100644 (file)
--- a/tests/base/vectorization_04.output
+++ b/tests/base/vectorization_04.output
@@ -1,8 +1,10 @@
 
-DEAL:double::load OK
-DEAL:double::store OK
-DEAL:float::load OK
-DEAL:float::store OK
+DEAL:double <-> VectorizedArray<double, 1>::load OK
+DEAL:double <-> VectorizedArray<double, 1>::store OK
+DEAL:float <-> VectorizedArray<float, 1>::load OK
+DEAL:float <-> VectorizedArray<float, 1>::store OK
+DEAL:float <-> VectorizedArray<double, 1>::load OK
+DEAL:float <-> VectorizedArray<double, 1>::store OK
 DEAL:long double::load OK
 DEAL:long double::store OK
 DEAL:unsigned int::load OK

diff --git a/tests/base/vectorization_04.output.avx b/tests/base/vectorization_04.output.avx
new file mode 100644 (file)
index 0000000..d3b3722
--- /dev/null
+++ b/tests/base/vectorization_04.output.avx
@@ -0,0 +1,23 @@
+
+DEAL:double <-> VectorizedArray<double, 1>::load OK
+DEAL:double <-> VectorizedArray<double, 1>::store OK
+DEAL:float <-> VectorizedArray<float, 1>::load OK
+DEAL:float <-> VectorizedArray<float, 1>::store OK
+DEAL:float <-> VectorizedArray<double, 1>::load OK
+DEAL:float <-> VectorizedArray<double, 1>::store OK
+DEAL:double <-> VectorizedArray<double, 2>::load OK
+DEAL:double <-> VectorizedArray<double, 2>::store OK
+DEAL:float <-> VectorizedArray<float, 4>::load OK
+DEAL:float <-> VectorizedArray<float, 4>::store OK
+DEAL:float <-> VectorizedArray<double, 2>::load OK
+DEAL:float <-> VectorizedArray<double, 2>::store OK
+DEAL:double <-> VectorizedArray<double, 4>::load OK
+DEAL:double <-> VectorizedArray<double, 4>::store OK
+DEAL:float <-> VectorizedArray<float, 8>::load OK
+DEAL:float <-> VectorizedArray<float, 8>::store OK
+DEAL:float <-> VectorizedArray<double, 4>::load OK
+DEAL:float <-> VectorizedArray<double, 4>::store OK
+DEAL:long double::load OK
+DEAL:long double::store OK
+DEAL:unsigned int::load OK
+DEAL:unsigned int::store OK

diff --git a/tests/base/vectorization_04.output.avx512 b/tests/base/vectorization_04.output.avx512
new file mode 100644 (file)
index 0000000..ad7debc
--- /dev/null
+++ b/tests/base/vectorization_04.output.avx512
@@ -0,0 +1,29 @@
+
+DEAL:double <-> VectorizedArray<double, 1>::load OK
+DEAL:double <-> VectorizedArray<double, 1>::store OK
+DEAL:float <-> VectorizedArray<float, 1>::load OK
+DEAL:float <-> VectorizedArray<float, 1>::store OK
+DEAL:float <-> VectorizedArray<double, 1>::load OK
+DEAL:float <-> VectorizedArray<double, 1>::store OK
+DEAL:double <-> VectorizedArray<double, 2>::load OK
+DEAL:double <-> VectorizedArray<double, 2>::store OK
+DEAL:float <-> VectorizedArray<float, 4>::load OK
+DEAL:float <-> VectorizedArray<float, 4>::store OK
+DEAL:float <-> VectorizedArray<double, 2>::load OK
+DEAL:float <-> VectorizedArray<double, 2>::store OK
+DEAL:double <-> VectorizedArray<double, 4>::load OK
+DEAL:double <-> VectorizedArray<double, 4>::store OK
+DEAL:float <-> VectorizedArray<float, 8>::load OK
+DEAL:float <-> VectorizedArray<float, 8>::store OK
+DEAL:float <-> VectorizedArray<double, 4>::load OK
+DEAL:float <-> VectorizedArray<double, 4>::store OK
+DEAL:double <-> VectorizedArray<double, 8>::load OK
+DEAL:double <-> VectorizedArray<double, 8>::store OK
+DEAL:float <-> VectorizedArray<float, 16>::load OK
+DEAL:float <-> VectorizedArray<float, 16>::store OK
+DEAL:float <-> VectorizedArray<double, 8>::load OK
+DEAL:float <-> VectorizedArray<double, 8>::store OK
+DEAL:long double::load OK
+DEAL:long double::store OK
+DEAL:unsigned int::load OK
+DEAL:unsigned int::store OK

diff --git a/tests/base/vectorization_04.output.sse b/tests/base/vectorization_04.output.sse
new file mode 100644 (file)
index 0000000..6f5a83c
--- /dev/null
+++ b/tests/base/vectorization_04.output.sse
@@ -0,0 +1,17 @@
+
+DEAL:double <-> VectorizedArray<double, 1>::load OK
+DEAL:double <-> VectorizedArray<double, 1>::store OK
+DEAL:float <-> VectorizedArray<float, 1>::load OK
+DEAL:float <-> VectorizedArray<float, 1>::store OK
+DEAL:float <-> VectorizedArray<double, 1>::load OK
+DEAL:float <-> VectorizedArray<double, 1>::store OK
+DEAL:double <-> VectorizedArray<double, 2>::load OK
+DEAL:double <-> VectorizedArray<double, 2>::store OK
+DEAL:float <-> VectorizedArray<float, 4>::load OK
+DEAL:float <-> VectorizedArray<float, 4>::store OK
+DEAL:float <-> VectorizedArray<double, 2>::load OK
+DEAL:float <-> VectorizedArray<double, 2>::store OK
+DEAL:long double::load OK
+DEAL:long double::store OK
+DEAL:unsigned int::load OK
+DEAL:unsigned int::store OK