From ee70f82efc45b5067083e14d38fad89570ac464c Mon Sep 17 00:00:00 2001 From: peterrum Date: Sat, 6 Jul 2019 11:34:00 +0200 Subject: [PATCH] Add VectorizationArrayType to MatrixFree as template argument --- doc/news/changes/minor/20190706PeterMunch | 5 + doc/news/changes/minor/20190710PeterMunch | 4 + include/deal.II/base/vectorization.h | 45 + include/deal.II/matrix_free/dof_info.h | 2 +- include/deal.II/matrix_free/fe_evaluation.h | 3157 ++++++++++------- .../matrix_free/mapping_data_on_the_fly.h | 68 +- include/deal.II/matrix_free/mapping_info.h | 123 +- .../matrix_free/mapping_info.templates.h | 383 +- include/deal.II/matrix_free/matrix_free.h | 846 +++-- .../matrix_free/matrix_free.templates.h | 204 +- include/deal.II/matrix_free/operators.h | 141 +- .../matrix_free/shape_info.templates.h | 4 +- include/deal.II/numerics/vector_tools.h | 42 +- .../deal.II/numerics/vector_tools.templates.h | 12 +- source/matrix_free/mapping_info.cc | 28 +- source/matrix_free/mapping_info.inst.in | 206 +- source/matrix_free/matrix_free.inst.in | 535 ++- .../vector_tools_project_qpmf.inst.in | 12 +- tests/matrix_free/matrix_vector_faces_26.cc | 64 + .../matrix_free/matrix_vector_faces_26.output | 17 + .../matrix_vector_faces_26.output.avx | 49 + .../matrix_vector_faces_26.output.avx512 | 65 + .../matrix_vector_faces_26.output.sse2 | 33 + .../matrix_free/matrix_vector_faces_common.h | 429 ++- 24 files changed, 4133 insertions(+), 2341 deletions(-) create mode 100644 doc/news/changes/minor/20190706PeterMunch create mode 100644 doc/news/changes/minor/20190710PeterMunch create mode 100644 tests/matrix_free/matrix_vector_faces_26.cc create mode 100644 tests/matrix_free/matrix_vector_faces_26.output create mode 100644 tests/matrix_free/matrix_vector_faces_26.output.avx create mode 100644 tests/matrix_free/matrix_vector_faces_26.output.avx512 create mode 100644 tests/matrix_free/matrix_vector_faces_26.output.sse2 diff --git a/doc/news/changes/minor/20190706PeterMunch b/doc/news/changes/minor/20190706PeterMunch new file mode 100644 index 0000000000..329590e9e9 --- /dev/null +++ b/doc/news/changes/minor/20190706PeterMunch @@ -0,0 +1,5 @@ +New: An additional optional template parameter VectorizedArrayType has been +added to the MatrixFree framework. It lets the user switch between no +vectorization (only auto vectorization) and vectorization over elements. +
+(Peter Munch, 2019/07/06) diff --git a/doc/news/changes/minor/20190710PeterMunch b/doc/news/changes/minor/20190710PeterMunch new file mode 100644 index 0000000000..f275b6d66c --- /dev/null +++ b/doc/news/changes/minor/20190710PeterMunch @@ -0,0 +1,4 @@ +New: It is possible to access value type Number of VectorizedArray +directly via VectorizedArray::value_type. +
+(Peter Munch, 2019/07/10) diff --git a/include/deal.II/base/vectorization.h b/include/deal.II/base/vectorization.h index 7910a4e5a9..1e994fbd4a 100644 --- a/include/deal.II/base/vectorization.h +++ b/include/deal.II/base/vectorization.h @@ -206,6 +206,11 @@ template class VectorizedArray { public: + /** + * This gives the type of the array elements. + */ + using value_type = Number; + /** * This gives the number of elements collected in this class. In the general * case, there is only one element. Specializations use SIMD intrinsics and @@ -630,6 +635,11 @@ template <> class VectorizedArray { public: + /** + * This gives the type of the array elements. + */ + using value_type = double; + /** * This gives the number of vectors collected in this class. */ @@ -1046,6 +1056,11 @@ template <> class VectorizedArray { public: + /** + * This gives the type of the array elements. + */ + using value_type = float; + /** * This gives the number of vectors collected in this class. */ @@ -1513,6 +1528,11 @@ template <> class VectorizedArray { public: + /** + * This gives the type of the array elements. + */ + using value_type = double; + /** * This gives the number of vectors collected in this class. */ @@ -1898,6 +1918,11 @@ template <> class VectorizedArray { public: + /** + * This gives the type of the array elements. + */ + using value_type = float; + /** * This gives the number of vectors collected in this class. */ @@ -2304,6 +2329,11 @@ template <> class VectorizedArray { public: + /** + * This gives the type of the array elements. + */ + using value_type = double; + /** * This gives the number of vectors collected in this class. */ @@ -2648,6 +2678,11 @@ template <> class VectorizedArray { public: + /** + * This gives the type of the array elements. + */ + using value_type = float; + /** * This gives the number of vectors collected in this class. */ @@ -3013,6 +3048,11 @@ template <> class VectorizedArray { public: + /** + * This gives the type of the array elements. + */ + using value_type = double; + /** * This gives the number of vectors collected in this class. */ @@ -3228,6 +3268,11 @@ template <> class VectorizedArray { public: + /** + * This gives the type of the array elements. + */ + using value_type = float; + /** * This gives the number of vectors collected in this class. */ diff --git a/include/deal.II/matrix_free/dof_info.h b/include/deal.II/matrix_free/dof_info.h index 66f8b47ad4..63a2f10453 100644 --- a/include/deal.II/matrix_free/dof_info.h +++ b/include/deal.II/matrix_free/dof_info.h @@ -374,7 +374,7 @@ namespace internal * cells batched together. As opposed to the other classes which are * templated on the number type, this class as a pure index container is * not templated, so we need to keep the information otherwise contained - * in VectorizedArray::n_array_elements. + * in VectorizedArrayType::n_array_elements. */ unsigned int vectorization_length; diff --git a/include/deal.II/matrix_free/fe_evaluation.h b/include/deal.II/matrix_free/fe_evaluation.h index 0fcb53267a..9d91c0d55f 100644 --- a/include/deal.II/matrix_free/fe_evaluation.h +++ b/include/deal.II/matrix_free/fe_evaluation.h @@ -49,9 +49,10 @@ namespace internal template + int n_q_points_1d = fe_degree + 1, + int n_components_ = 1, + typename Number = double, + typename VectorizedArrayType = VectorizedArray> class FEEvaluation; @@ -76,18 +77,33 @@ class FEEvaluation; * * @tparam Number Number format, usually @p double or @p float * + * @tparam VectorizedArrayType Type of array to be woked on in a vectorized + * fashion, defaults to VectorizedArray + * + * @note Currently only VectorizedArray is supported as + * VectorizedArrayType. + * + * * @ingroup matrixfree * * @author Katharina Kormann and Martin Kronbichler, 2010-2018 */ -template +template > class FEEvaluationBase { + static_assert( + std::is_same::value, + "Type of Number and of VectorizedArrayType do not match."); + public: using number_type = Number; - using value_type = Tensor<1, n_components_, VectorizedArray>; + using value_type = Tensor<1, n_components_, VectorizedArrayType>; using gradient_type = - Tensor<1, n_components_, Tensor<1, dim, VectorizedArray>>; + Tensor<1, n_components_, Tensor<1, dim, VectorizedArrayType>>; static constexpr unsigned int dimension = dim; static constexpr unsigned int n_components = n_components_; @@ -130,7 +146,7 @@ public: /** * Return a reference to the ShapeInfo object currently in use. */ - const internal::MatrixFreeFunctions::ShapeInfo> & + const internal::MatrixFreeFunctions::ShapeInfo & get_shape_info() const; //@} @@ -241,10 +257,10 @@ public: template void distribute_local_to_global( - VectorType & dst, - const unsigned int first_index = 0, - const std::bitset::n_array_elements> &mask = - std::bitset::n_array_elements>().flip()) const; + VectorType & dst, + const unsigned int first_index = 0, + const std::bitset &mask = + std::bitset().flip()) const; /** * Takes the values stored internally on dof values of the current cell and @@ -281,10 +297,10 @@ public: template void set_dof_values( - VectorType & dst, - const unsigned int first_index = 0, - const std::bitset::n_array_elements> &mask = - std::bitset::n_array_elements>().flip()) const; + VectorType & dst, + const unsigned int first_index = 0, + const std::bitset &mask = + std::bitset().flip()) const; //@} @@ -426,7 +442,7 @@ public: * with specializations for the scalar case (n_components == 1) and for the * vector-valued case (n_components == dim). */ - Tensor<1, n_components_, Tensor<2, dim, VectorizedArray>> + Tensor<1, n_components_, Tensor<2, dim, VectorizedArrayType>> get_hessian(const unsigned int q_point) const; /** @@ -461,19 +477,19 @@ public: /** @copydoc FEEvaluationAccess::get_divergence() * @note Only available for n_components_==dim. */ - VectorizedArray + VectorizedArrayType get_divergence(const unsigned int q_point) const; /** @copydoc FEEvaluationAccess::get_symmetric_gradient() * @note Only available for n_components_==dim. */ - SymmetricTensor<2, dim, VectorizedArray> + SymmetricTensor<2, dim, VectorizedArrayType> get_symmetric_gradient(const unsigned int q_point) const; /** @copydoc FEEvaluationAccess::get_curl() * @note Only available for n_components_==dim. */ - Tensor<1, (dim == 2 ? 1 : dim), VectorizedArray> + Tensor<1, (dim == 2 ? 1 : dim), VectorizedArrayType> get_curl(const unsigned int q_point) const; /** @copydoc FEEvaluationAccess::submit_divergence() @@ -485,8 +501,8 @@ public: * be added into the diagonal of the contribution for submit_gradient(). */ void - submit_divergence(const VectorizedArray div_in, - const unsigned int q_point); + submit_divergence(const VectorizedArrayType div_in, + const unsigned int q_point); /** @copydoc FEEvaluationAccess::submit_symmetric_gradient() * @note Only available for n_components_==dim. @@ -499,8 +515,8 @@ public: */ void submit_symmetric_gradient( - const SymmetricTensor<2, dim, VectorizedArray> grad_in, - const unsigned int q_point); + const SymmetricTensor<2, dim, VectorizedArrayType> grad_in, + const unsigned int q_point); /** @copydoc FEEvaluationAccess::submit_curl() * @note Only available for n_components_==dim. @@ -512,9 +528,8 @@ public: * submit_gradient(). */ void - submit_curl( - const Tensor<1, dim == 2 ? 1 : dim, VectorizedArray> curl_in, - const unsigned int q_point); + submit_curl(const Tensor<1, dim == 2 ? 1 : dim, VectorizedArrayType> curl_in, + const unsigned int q_point); #endif @@ -541,7 +556,7 @@ public: * Return the determinant of the Jacobian from the unit to the real cell * times the quadrature weight. */ - VectorizedArray + VectorizedArrayType JxW(const unsigned int q_index) const; /** @@ -550,7 +565,7 @@ public: * @deprecated Use JxW() instead. */ DEAL_II_DEPRECATED void - fill_JxW_values(AlignedVector> &JxW_values) const; + fill_JxW_values(AlignedVector &JxW_values) const; /** * Return the inverse and transposed version of Jacobian of the mapping @@ -558,7 +573,7 @@ public: * transformation). This is exactly the matrix used internally to transform * the unit cell gradients to gradients on the real cell. */ - Tensor<2, dim, VectorizedArray> + Tensor<2, dim, VectorizedArrayType> inverse_jacobian(const unsigned int q_index) const; /** @@ -573,7 +588,7 @@ public: * * @note Only implemented in case `is_face == true`. */ - Tensor<1, dim, VectorizedArray> + Tensor<1, dim, VectorizedArrayType> get_normal_vector(const unsigned int q_point) const; /** @@ -582,8 +597,8 @@ public: * MatrixFree::n_macro_ghost_cells() for both cells (plain read) and faces * (indirect addressing). */ - VectorizedArray - read_cell_data(const AlignedVector> &array) const; + VectorizedArrayType + read_cell_data(const AlignedVector &array) const; //@} @@ -599,7 +614,7 @@ public: * used in this class. In general, it is safer to use the get_dof_value() * function instead. */ - const VectorizedArray * + const VectorizedArrayType * begin_dof_values() const; /** @@ -610,7 +625,7 @@ public: * structures used in this class. In general, it is safer to use the * get_dof_value() function instead. */ - VectorizedArray * + VectorizedArrayType * begin_dof_values(); /** @@ -623,7 +638,7 @@ public: * applied manually. In general, it is safer to use the get_value() function * instead, which does all the transformation internally. */ - const VectorizedArray * + const VectorizedArrayType * begin_values() const; /** @@ -636,7 +651,7 @@ public: * applied manually. In general, it is safer to use the get_value() function * instead, which does all the transformation internally. */ - VectorizedArray * + VectorizedArrayType * begin_values(); /** @@ -650,7 +665,7 @@ public: * manually. In general, it is safer to use the get_gradient() function * instead, which does all the transformation internally. */ - const VectorizedArray * + const VectorizedArrayType * begin_gradients() const; /** @@ -664,7 +679,7 @@ public: * manually. In general, it is safer to use the get_gradient() function * instead, which does all the transformation internally. */ - VectorizedArray * + VectorizedArrayType * begin_gradients(); /** @@ -679,7 +694,7 @@ public: * general, it is safer to use the get_laplacian() or get_hessian() * functions instead, which does all the transformation internally. */ - const VectorizedArray * + const VectorizedArrayType * begin_hessians() const; /** @@ -694,7 +709,7 @@ public: * general, it is safer to use the get_laplacian() or get_hessian() * functions instead, which does all the transformation internally. */ - VectorizedArray * + VectorizedArrayType * begin_hessians(); /** @@ -711,7 +726,7 @@ public: * do not assume it to be stable over those calls. The maximum size you can * write into is 3*dofs_per_cell+2*n_q_points. */ - ArrayView> + ArrayView get_scratch_data() const; //@} @@ -725,13 +740,14 @@ protected: * first_selected_component and @p quad_no allow to select the appropriate * components. */ - FEEvaluationBase(const MatrixFree &matrix_free, - const unsigned int dof_no, - const unsigned int first_selected_component, - const unsigned int quad_no, - const unsigned int fe_degree, - const unsigned int n_q_points, - const bool is_interior_face); + FEEvaluationBase( + const MatrixFree &matrix_free, + const unsigned int dof_no, + const unsigned int first_selected_component, + const unsigned int quad_no, + const unsigned int fe_degree, + const unsigned int n_q_points, + const bool is_interior_face); /** * Constructor that comes with reduced functionality and works similar as @@ -768,13 +784,16 @@ protected: * because otherwise the intended sharing may create race conditions. */ template - FEEvaluationBase( - const Mapping & mapping, - const FiniteElement &fe, - const Quadrature<1> & quadrature, - const UpdateFlags update_flags, - const unsigned int first_selected_component, - const FEEvaluationBase *other); + FEEvaluationBase(const Mapping & mapping, + const FiniteElement &fe, + const Quadrature<1> & quadrature, + const UpdateFlags update_flags, + const unsigned int first_selected_component, + const FEEvaluationBase *other); /** * Copy constructor. If FEEvaluationBase was constructed from a mapping, fe, @@ -802,9 +821,9 @@ protected: template void read_write_operation( - const VectorOperation & operation, - VectorType * vectors[], - const std::bitset::n_array_elements> &mask, + const VectorOperation & operation, + VectorType * vectors[], + const std::bitset &mask, const bool apply_constraints = true) const; /** @@ -817,9 +836,9 @@ protected: template void read_write_operation_contiguous( - const VectorOperation & operation, - VectorType * vectors[], - const std::bitset::n_array_elements> &mask) const; + const VectorOperation & operation, + VectorType * vectors[], + const std::bitset &mask) const; /** * A unified function to read from and write into vectors based on the given @@ -836,14 +855,14 @@ protected: /** * This is the general array for all data fields. */ - AlignedVector> *scratch_data_array; + AlignedVector *scratch_data_array; /** * This is the user-visible part of scratch_data_array, only showing the * last part of scratch_data_array. The first part is consumed by * values_dofs, values_quad, etc. */ - VectorizedArray *scratch_data; + VectorizedArrayType *scratch_data; /** * This field stores the values for local degrees of freedom (e.g. after @@ -857,7 +876,7 @@ protected: * memory on the stack, this approach allows for very large polynomial * degrees. */ - VectorizedArray *values_dofs[n_components]; + VectorizedArrayType *values_dofs[n_components]; /** * This field stores the values of the finite element function on quadrature @@ -870,7 +889,7 @@ protected: * memory on the stack, this approach allows for very large polynomial * degrees. */ - VectorizedArray *values_quad[n_components]; + VectorizedArrayType *values_quad[n_components]; /** * This field stores the gradients of the finite element function on @@ -885,7 +904,7 @@ protected: * memory on the stack, this approach allows for very large polynomial * degrees. */ - VectorizedArray *gradients_quad[n_components][dim]; + VectorizedArrayType *gradients_quad[n_components][dim]; /** * This field stores the Hessians of the finite element function on @@ -898,7 +917,7 @@ protected: * memory on the stack, this approach allows for very large polynomial * degrees. */ - VectorizedArray *hessians_quad[n_components][(dim * (dim + 1)) / 2]; + VectorizedArrayType *hessians_quad[n_components][(dim * (dim + 1)) / 2]; /** * Stores the number of the quadrature formula of the present cell. @@ -931,7 +950,7 @@ protected: /** * Stores a pointer to the underlying data. */ - const MatrixFree *matrix_info; + const MatrixFree *matrix_info; /** * Stores a pointer to the underlying DoF indices and constraint description @@ -946,8 +965,11 @@ protected: * Also contained in matrix_info, but it simplifies code if we store a * reference to it. */ - const internal::MatrixFreeFunctions:: - MappingInfoStorage<(is_face ? dim - 1 : dim), dim, Number> *mapping_data; + const internal::MatrixFreeFunctions::MappingInfoStorage< + (is_face ? dim - 1 : dim), + dim, + Number, + VectorizedArrayType> *mapping_data; /** * Stores a pointer to the unit cell shape data, i.e., values, gradients and @@ -955,13 +977,13 @@ protected: * product. Also contained in matrix_info, but it simplifies code if we * store a reference to it. */ - const internal::MatrixFreeFunctions::ShapeInfo> *data; + const internal::MatrixFreeFunctions::ShapeInfo *data; /** * A pointer to the Jacobian information of the present cell. Only set to a * useful value if on a non-Cartesian cell. */ - const Tensor<2, dim, VectorizedArray> *jacobian; + const Tensor<2, dim, VectorizedArrayType> *jacobian; /** * A pointer to the Jacobian determinant of the present cell. If on a @@ -969,17 +991,17 @@ protected: * Jacobian determinant, otherwise the Jacobian determinant times the * quadrature weight. */ - const VectorizedArray *J_value; + const VectorizedArrayType *J_value; /** * A pointer to the normal vectors at faces. */ - const Tensor<1, dim, VectorizedArray> *normal_vectors; + const Tensor<1, dim, VectorizedArrayType> *normal_vectors; /** * A pointer to the normal vectors times the jacobian at faces. */ - const Tensor<1, dim, VectorizedArray> *normal_x_jacobian; + const Tensor<1, dim, VectorizedArrayType> *normal_x_jacobian; /** * A pointer to the quadrature weights of the underlying quadrature formula. @@ -1079,8 +1101,8 @@ protected: * Geometry data that can be generated FEValues on the fly with the * respective constructor. */ - std::shared_ptr< - internal::MatrixFreeFunctions::MappingDataOnTheFly> + std::shared_ptr> mapped_geometry; /** @@ -1106,9 +1128,9 @@ private: /** * Make other FEEvaluationBase as well as FEEvaluation objects friends. */ - template + template friend class FEEvaluationBase; - template + template friend class FEEvaluation; }; @@ -1123,18 +1145,30 @@ private: * * @author Katharina Kormann and Martin Kronbichler, 2010, 2011 */ -template -class FEEvaluationAccess - : public FEEvaluationBase +template > +class FEEvaluationAccess : public FEEvaluationBase { + static_assert( + std::is_same::value, + "Type of Number and of VectorizedArrayType do not match."); + public: using number_type = Number; - using value_type = Tensor<1, n_components_, VectorizedArray>; + using value_type = Tensor<1, n_components_, VectorizedArrayType>; using gradient_type = - Tensor<1, n_components_, Tensor<1, dim, VectorizedArray>>; + Tensor<1, n_components_, Tensor<1, dim, VectorizedArrayType>>; static constexpr unsigned int dimension = dim; static constexpr unsigned int n_components = n_components_; - using BaseClass = FEEvaluationBase; + using BaseClass = + FEEvaluationBase; protected: /** @@ -1144,26 +1178,30 @@ protected: * construction of @p matrix_free, @p first_selected_component and @p * quad_no allow to select the appropriate components. */ - FEEvaluationAccess(const MatrixFree &matrix_free, - const unsigned int dof_no, - const unsigned int first_selected_component, - const unsigned int quad_no, - const unsigned int fe_degree, - const unsigned int n_q_points, - const bool is_interior_face = true); + FEEvaluationAccess( + const MatrixFree &matrix_free, + const unsigned int dof_no, + const unsigned int first_selected_component, + const unsigned int quad_no, + const unsigned int fe_degree, + const unsigned int n_q_points, + const bool is_interior_face = true); /** * Constructor with reduced functionality for similar usage of FEEvaluation * as FEValues, including matrix assembly. */ template - FEEvaluationAccess( - const Mapping & mapping, - const FiniteElement &fe, - const Quadrature<1> & quadrature, - const UpdateFlags update_flags, - const unsigned int first_selected_component, - const FEEvaluationBase *other); + FEEvaluationAccess(const Mapping & mapping, + const FiniteElement &fe, + const Quadrature<1> & quadrature, + const UpdateFlags update_flags, + const unsigned int first_selected_component, + const FEEvaluationBase *other); /** * Copy constructor @@ -1189,16 +1227,21 @@ protected: * * @author Katharina Kormann and Martin Kronbichler, 2010, 2011 */ -template -class FEEvaluationAccess - : public FEEvaluationBase +template +class FEEvaluationAccess + : public FEEvaluationBase { + static_assert( + std::is_same::value, + "Type of Number and of VectorizedArrayType do not match."); + public: - using number_type = Number; - using value_type = VectorizedArray; - using gradient_type = Tensor<1, dim, VectorizedArray>; + using number_type = Number; + using value_type = VectorizedArrayType; + using gradient_type = Tensor<1, dim, VectorizedArrayType>; static constexpr unsigned int dimension = dim; - using BaseClass = FEEvaluationBase; + using BaseClass = + FEEvaluationBase; /** @copydoc FEEvaluationBase::get_dof_value() */ @@ -1223,8 +1266,8 @@ public: /** @copydoc FEEvaluationBase::submit_value() */ void - submit_value(const Tensor<1, 1, VectorizedArray> val_in, - const unsigned int q_point); + submit_value(const Tensor<1, 1, VectorizedArrayType> val_in, + const unsigned int q_point); /** @copydoc FEEvaluationBase::get_gradient() */ @@ -1249,7 +1292,7 @@ public: /** @copydoc FEEvaluationBase::get_hessian() */ - Tensor<2, dim, VectorizedArray> + Tensor<2, dim, VectorizedArrayType> get_hessian(unsigned int q_point) const; /** @copydoc FEEvaluationBase::get_hessian_diagonal() @@ -1275,26 +1318,30 @@ protected: * construction of @p matrix_free, @p first_selected_component and @p * quad_no allow to select the appropriate components. */ - FEEvaluationAccess(const MatrixFree &matrix_free, - const unsigned int dof_no, - const unsigned int first_selected_component, - const unsigned int quad_no, - const unsigned int fe_degree, - const unsigned int n_q_points, - const bool is_interior_face = true); + FEEvaluationAccess( + const MatrixFree &matrix_free, + const unsigned int dof_no, + const unsigned int first_selected_component, + const unsigned int quad_no, + const unsigned int fe_degree, + const unsigned int n_q_points, + const bool is_interior_face = true); /** * Constructor with reduced functionality for similar usage of FEEvaluation * as FEValues, including matrix assembly. */ template - FEEvaluationAccess( - const Mapping & mapping, - const FiniteElement &fe, - const Quadrature<1> & quadrature, - const UpdateFlags update_flags, - const unsigned int first_selected_component, - const FEEvaluationBase *other); + FEEvaluationAccess(const Mapping & mapping, + const FiniteElement &fe, + const Quadrature<1> & quadrature, + const UpdateFlags update_flags, + const unsigned int first_selected_component, + const FEEvaluationBase *other); /** * Copy constructor @@ -1321,17 +1368,22 @@ protected: * * @author Katharina Kormann and Martin Kronbichler, 2010, 2011 */ -template -class FEEvaluationAccess - : public FEEvaluationBase +template +class FEEvaluationAccess + : public FEEvaluationBase { + static_assert( + std::is_same::value, + "Type of Number and of VectorizedArrayType do not match."); + public: - using number_type = Number; - using value_type = Tensor<1, dim, VectorizedArray>; - using gradient_type = Tensor<2, dim, VectorizedArray>; - static constexpr unsigned int dimension = dim; + using number_type = Number; + using value_type = Tensor<1, dim, VectorizedArrayType>; + using gradient_type = Tensor<2, dim, VectorizedArrayType>; + static constexpr unsigned int dimension = dim; static constexpr unsigned int n_components = dim; - using BaseClass = FEEvaluationBase; + using BaseClass = + FEEvaluationBase; /** @copydoc FEEvaluationBase::get_gradient() */ @@ -1342,7 +1394,7 @@ public: * Return the divergence of a vector-valued finite element at quadrature * point number @p q_point after a call to @p evaluate(...,true,...). */ - VectorizedArray + VectorizedArrayType get_divergence(const unsigned int q_point) const; /** @@ -1351,19 +1403,19 @@ public: * evaluate(...,true,...). It corresponds to 0.5 * (grad+gradT). */ - SymmetricTensor<2, dim, VectorizedArray> + SymmetricTensor<2, dim, VectorizedArrayType> get_symmetric_gradient(const unsigned int q_point) const; /** * Return the curl of the vector field, $\nabla \times v$ after a call to @p * evaluate(...,true,...). */ - Tensor<1, (dim == 2 ? 1 : dim), VectorizedArray> + Tensor<1, (dim == 2 ? 1 : dim), VectorizedArrayType> get_curl(const unsigned int q_point) const; /** @copydoc FEEvaluationBase::get_hessian() */ - Tensor<3, dim, VectorizedArray> + Tensor<3, dim, VectorizedArrayType> get_hessian(const unsigned int q_point) const; /** @copydoc FEEvaluationBase::get_hessian_diagonal() @@ -1386,8 +1438,8 @@ public: */ void submit_gradient( - const Tensor<1, dim, Tensor<1, dim, VectorizedArray>> grad_in, - const unsigned int q_point); + const Tensor<1, dim, Tensor<1, dim, VectorizedArrayType>> grad_in, + const unsigned int q_point); /** * Write a contribution that is tested by the divergence to the field @@ -1398,8 +1450,8 @@ public: * over. */ void - submit_divergence(const VectorizedArray div_in, - const unsigned int q_point); + submit_divergence(const VectorizedArrayType div_in, + const unsigned int q_point); /** * Write a contribution that is tested by the symmetric gradient to the field @@ -1411,17 +1463,16 @@ public: */ void submit_symmetric_gradient( - const SymmetricTensor<2, dim, VectorizedArray> grad_in, - const unsigned int q_point); + const SymmetricTensor<2, dim, VectorizedArrayType> grad_in, + const unsigned int q_point); /** * Write the components of a curl containing the values on quadrature point * @p q_point. Access to the same data field as through @p get_gradient. */ void - submit_curl( - const Tensor<1, dim == 2 ? 1 : dim, VectorizedArray> curl_in, - const unsigned int q_point); + submit_curl(const Tensor<1, dim == 2 ? 1 : dim, VectorizedArrayType> curl_in, + const unsigned int q_point); protected: /** @@ -1431,26 +1482,30 @@ protected: * construction of @p matrix_free, @p first_selected_component and @p * quad_no allow to select the appropriate components. */ - FEEvaluationAccess(const MatrixFree &matrix_free, - const unsigned int dof_no, - const unsigned int first_selected_component, - const unsigned int quad_no, - const unsigned int dofs_per_cell, - const unsigned int n_q_points, - const bool is_interior_face = true); + FEEvaluationAccess( + const MatrixFree &matrix_free, + const unsigned int dof_no, + const unsigned int first_selected_component, + const unsigned int quad_no, + const unsigned int dofs_per_cell, + const unsigned int n_q_points, + const bool is_interior_face = true); /** * Constructor with reduced functionality for similar usage of FEEvaluation * as FEValues, including matrix assembly. */ template - FEEvaluationAccess( - const Mapping & mapping, - const FiniteElement &fe, - const Quadrature<1> & quadrature, - const UpdateFlags update_flags, - const unsigned int first_selected_component, - const FEEvaluationBase *other); + FEEvaluationAccess(const Mapping & mapping, + const FiniteElement &fe, + const Quadrature<1> & quadrature, + const UpdateFlags update_flags, + const unsigned int first_selected_component, + const FEEvaluationBase *other); /** * Copy constructor @@ -1476,16 +1531,21 @@ protected: * @author Katharina Kormann and Martin Kronbichler, 2010, 2011, Shiva * Rudraraju, 2014 */ -template -class FEEvaluationAccess<1, 1, Number, is_face> - : public FEEvaluationBase<1, 1, Number, is_face> +template +class FEEvaluationAccess<1, 1, Number, is_face, VectorizedArrayType> + : public FEEvaluationBase<1, 1, Number, is_face, VectorizedArrayType> { + static_assert( + std::is_same::value, + "Type of Number and of VectorizedArrayType do not match."); + public: - using number_type = Number; - using value_type = VectorizedArray; - using gradient_type = Tensor<1, 1, VectorizedArray>; + using number_type = Number; + using value_type = VectorizedArrayType; + using gradient_type = Tensor<1, 1, VectorizedArrayType>; static constexpr unsigned int dimension = 1; - using BaseClass = FEEvaluationBase<1, 1, Number, is_face>; + using BaseClass = + FEEvaluationBase<1, 1, Number, is_face, VectorizedArrayType>; /** @copydoc FEEvaluationBase<1,1,Number,is_face>::get_dof_value() */ @@ -1546,7 +1606,7 @@ public: /** @copydoc FEEvaluationBase<1,1,Number,is_face>::get_hessian() */ - Tensor<2, 1, VectorizedArray> + Tensor<2, 1, VectorizedArrayType> get_hessian(unsigned int q_point) const; /** @copydoc FEEvaluationBase<1,1,Number,is_face>::get_hessian_diagonal() @@ -1572,26 +1632,30 @@ protected: * construction of @p matrix_free, @p first_selected_component and @p * quad_no allow to select the appropriate components. */ - FEEvaluationAccess(const MatrixFree<1, Number> &matrix_free, - const unsigned int dof_no, - const unsigned int first_selected_component, - const unsigned int quad_no, - const unsigned int fe_degree, - const unsigned int n_q_points, - const bool is_interior_face = true); + FEEvaluationAccess( + const MatrixFree<1, Number, VectorizedArrayType> &matrix_free, + const unsigned int dof_no, + const unsigned int first_selected_component, + const unsigned int quad_no, + const unsigned int fe_degree, + const unsigned int n_q_points, + const bool is_interior_face = true); /** * Constructor with reduced functionality for similar usage of FEEvaluation * as FEValues, including matrix assembly. */ template - FEEvaluationAccess( - const Mapping<1> & mapping, - const FiniteElement<1> &fe, - const Quadrature<1> & quadrature, - const UpdateFlags update_flags, - const unsigned int first_selected_component, - const FEEvaluationBase<1, n_components_other, Number, is_face> *other); + FEEvaluationAccess(const Mapping<1> & mapping, + const FiniteElement<1> &fe, + const Quadrature<1> & quadrature, + const UpdateFlags update_flags, + const unsigned int first_selected_component, + const FEEvaluationBase<1, + n_components_other, + Number, + is_face, + VectorizedArrayType> *other); /** * Copy constructor @@ -2160,15 +2224,24 @@ template -class FEEvaluation - : public FEEvaluationAccess + typename Number, + typename VectorizedArrayType> +class FEEvaluation : public FEEvaluationAccess { + static_assert( + std::is_same::value, + "Type of Number and of VectorizedArrayType do not match."); + public: /** * An alias to the base class. */ - using BaseClass = FEEvaluationAccess; + using BaseClass = + FEEvaluationAccess; /** * An underlying number type specified as template argument. @@ -2176,15 +2249,15 @@ public: using number_type = Number; /** - * The type of function values, e.g. `VectorizedArray` for - * `n_components=1` or `Tensor<1,dim,VectorizedArray >` for + * The type of function values, e.g. `VectorizedArrayType` for + * `n_components=1` or `Tensor<1,dim,VectorizedArrayType >` for * `n_components=dim`. */ using value_type = typename BaseClass::value_type; /** - * The type of gradients, e.g. `Tensor<1,dim,VectorizedArray>` for - * `n_components=1` or `Tensor<2,dim,VectorizedArray >` for + * The type of gradients, e.g. `Tensor<1,dim,VectorizedArrayType>` for + * `n_components=1` or `Tensor<2,dim,VectorizedArrayType >` for * `n_components=dim`. */ using gradient_type = typename BaseClass::gradient_type; @@ -2264,10 +2337,10 @@ public: * between @p first_selected_component and * first_selected_component+n_components_. */ - FEEvaluation(const MatrixFree &matrix_free, - const unsigned int dof_no = 0, - const unsigned int quad_no = 0, - const unsigned int first_selected_component = 0); + FEEvaluation(const MatrixFree &matrix_free, + const unsigned int dof_no = 0, + const unsigned int quad_no = 0, + const unsigned int first_selected_component = 0); /** * Constructor that comes with reduced functionality and works similar as @@ -2322,8 +2395,12 @@ public: * otherwise the intended sharing may create race conditions. */ template - FEEvaluation(const FiniteElement & fe, - const FEEvaluationBase &other, + FEEvaluation(const FiniteElement & fe, + const FEEvaluationBase &other, const unsigned int first_selected_component = 0); /** @@ -2411,10 +2488,10 @@ public: * useful information (unless these values have been set manually). */ void - evaluate(const VectorizedArray *values_array, - const bool evaluate_values, - const bool evaluate_gradients, - const bool evaluate_hessians = false); + evaluate(const VectorizedArrayType *values_array, + const bool evaluate_values, + const bool evaluate_gradients, + const bool evaluate_hessians = false); /** * Read from the input vector and evaluates the function values, the @@ -2461,9 +2538,9 @@ public: * internal data structures behind begin_dof_values(). */ void - integrate(const bool integrate_values, - const bool integrate_gradients, - VectorizedArray *values_array); + integrate(const bool integrate_values, + const bool integrate_gradients, + VectorizedArrayType *values_array); /** * This function takes the values and/or gradients that are stored on @@ -2488,7 +2565,7 @@ public: * Return the q-th quadrature point in real coordinates stored in * MappingInfo. */ - Point> + Point quadrature_point(const unsigned int q_point) const; /** @@ -2557,21 +2634,36 @@ private: * * @tparam Number Number format, usually @p double or @p float * + * @tparam VectorizedArrayType Type of array to be woked on in a vectorized + * fashion, defaults to VectorizedArray + * + * @note Currently only VectorizedArray is supported as + * VectorizedArrayType. + * * @author Katharina Kormann and Martin Kronbichler, 2018 */ template -class FEFaceEvaluation - : public FEEvaluationAccess + int n_q_points_1d = fe_degree + 1, + int n_components_ = 1, + typename Number = double, + typename VectorizedArrayType = VectorizedArray> +class FEFaceEvaluation : public FEEvaluationAccess { + static_assert( + std::is_same::value, + "Type of Number and of VectorizedArrayType do not match."); + public: /** * An alias to the base class. */ - using BaseClass = FEEvaluationAccess; + using BaseClass = + FEEvaluationAccess; /** * A underlying number type specified as template argument. @@ -2579,15 +2671,15 @@ public: using number_type = Number; /** - * The type of function values, e.g. `VectorizedArray` for - * `n_components=1` or `Tensor<1,dim,VectorizedArray >` for + * The type of function values, e.g. `VectorizedArrayType` for + * `n_components=1` or `Tensor<1,dim,VectorizedArrayType >` for * `n_components=dim`. */ using value_type = typename BaseClass::value_type; /** - * The type of gradients, e.g. `Tensor<1,dim,VectorizedArray>` for - * `n_components=1` or `Tensor<2,dim,VectorizedArray >` for + * The type of gradients, e.g. `Tensor<1,dim,VectorizedArrayType>` for + * `n_components=1` or `Tensor<2,dim,VectorizedArrayType >` for * `n_components=dim`. */ using gradient_type = typename BaseClass::gradient_type; @@ -2678,11 +2770,12 @@ public: * not directly relate to the component of the respective element due to the * possibility for a multiplicity in the element. */ - FEFaceEvaluation(const MatrixFree &matrix_free, - const bool is_interior_face = true, - const unsigned int dof_no = 0, - const unsigned int quad_no = 0, - const unsigned int first_selected_component = 0); + FEFaceEvaluation( + const MatrixFree &matrix_free, + const bool is_interior_face = true, + const unsigned int dof_no = 0, + const unsigned int quad_no = 0, + const unsigned int first_selected_component = 0); /** * Initializes the operation pointer to the current face. This method is the @@ -2733,9 +2826,9 @@ public: * been set manually). */ void - evaluate(const VectorizedArray *values_array, - const bool evaluate_values, - const bool evaluate_gradients); + evaluate(const VectorizedArrayType *values_array, + const bool evaluate_values, + const bool evaluate_gradients); /** * Reads from the input vector and evaluates the function values, the @@ -2775,9 +2868,9 @@ public: * call stores the result of the testing in the given array `values_array`. */ void - integrate(const bool integrate_values, - const bool integrate_gradients, - VectorizedArray *values_array); + integrate(const bool integrate_values, + const bool integrate_gradients, + VectorizedArrayType *values_array); /** * This function takes the values and/or gradients that are stored on @@ -2800,7 +2893,7 @@ public: * Returns the q-th quadrature point on the face in real coordinates stored * in MappingInfo. */ - Point> + Point quadrature_point(const unsigned int q_point) const; /** @@ -2874,15 +2967,23 @@ namespace internal /*----------------------- FEEvaluationBase ----------------------------------*/ -template -inline FEEvaluationBase::FEEvaluationBase( - const MatrixFree &data_in, - const unsigned int dof_no, - const unsigned int first_selected_component, - const unsigned int quad_no_in, - const unsigned int fe_degree, - const unsigned int n_q_points, - const bool is_interior_face) +template +inline FEEvaluationBase:: + FEEvaluationBase(const MatrixFree &data_in, + const unsigned int dof_no, + const unsigned int first_selected_component, + const unsigned int quad_no_in, + const unsigned int fe_degree, + const unsigned int n_q_points, + const bool is_interior_face) : scratch_data_array(data_in.acquire_scratch_data()) , quad_no(quad_no_in) , n_fe_components(data_in.get_dof_info(dof_no).start_components.back()) @@ -2915,10 +3016,11 @@ inline FEEvaluationBase::FEEvaluationBase( .n_q_points)) , matrix_info(&data_in) , dof_info(&data_in.get_dof_info(dof_no)) - , mapping_data(internal::MatrixFreeFunctions:: - MappingInfoCellsOrFaces::get( - data_in.get_mapping_info(), - quad_no)) + , mapping_data( + internal::MatrixFreeFunctions:: + MappingInfoCellsOrFaces::get( + data_in.get_mapping_info(), + quad_no)) , data(&data_in.get_shape_info( dof_no, quad_no_in, @@ -2951,7 +3053,7 @@ inline FEEvaluationBase::FEEvaluationBase( set_data_pointers(); Assert(matrix_info->mapping_initialized() == true, ExcNotInitialized()); AssertDimension(matrix_info->get_size_info().vectorization_length, - VectorizedArray::n_array_elements); + VectorizedArrayType::n_array_elements); AssertDimension((is_face ? data->n_q_points_face : data->n_q_points), n_quadrature_points); AssertDimension(n_quadrature_points, @@ -2986,16 +3088,28 @@ inline FEEvaluationBase::FEEvaluationBase( -template +template template -inline FEEvaluationBase::FEEvaluationBase( - const Mapping & mapping, - const FiniteElement &fe, - const Quadrature<1> & quadrature, - const UpdateFlags update_flags, - const unsigned int first_selected_component, - const FEEvaluationBase *other) - : scratch_data_array(new AlignedVector>()) +inline FEEvaluationBase:: + FEEvaluationBase(const Mapping & mapping, + const FiniteElement &fe, + const Quadrature<1> & quadrature, + const UpdateFlags update_flags, + const unsigned int first_selected_component, + const FEEvaluationBase *other) + : scratch_data_array(new AlignedVector()) , quad_no(numbers::invalid_unsigned_int) , n_fe_components(n_components_) , active_fe_index(numbers::invalid_unsigned_int) @@ -3007,7 +3121,7 @@ inline FEEvaluationBase::FEEvaluationBase( , mapping_data(nullptr) , // select the correct base element from the given FE component - data(new internal::MatrixFreeFunctions::ShapeInfo>( + data(new internal::MatrixFreeFunctions::ShapeInfo( quadrature, fe, fe.component_to_base_index(first_selected_component).first)) @@ -3039,9 +3153,10 @@ inline FEEvaluationBase::FEEvaluationBase( other->mapped_geometry->get_quadrature() == quadrature) mapped_geometry = other->mapped_geometry; else - mapped_geometry = std::make_shared< - internal::MatrixFreeFunctions::MappingDataOnTheFly>( - mapping, quadrature, update_flags); + mapped_geometry = + std::make_shared>( + mapping, quadrature, update_flags); cell = 0; mapping_data = &mapped_geometry->get_data_storage(); @@ -3061,11 +3176,23 @@ inline FEEvaluationBase::FEEvaluationBase( -template -inline FEEvaluationBase::FEEvaluationBase( - const FEEvaluationBase &other) +template +inline FEEvaluationBase:: + FEEvaluationBase(const FEEvaluationBase &other) : scratch_data_array(other.matrix_info == nullptr ? - new AlignedVector>() : + new AlignedVector() : other.matrix_info->acquire_scratch_data()) , quad_no(other.quad_no) , n_fe_components(other.n_fe_components) @@ -3075,11 +3202,10 @@ inline FEEvaluationBase::FEEvaluationBase( , matrix_info(other.matrix_info) , dof_info(other.dof_info) , mapping_data(other.mapping_data) - , data( - other.matrix_info == nullptr ? - new internal::MatrixFreeFunctions::ShapeInfo>( - *other.data) : - other.data) + , data(other.matrix_info == nullptr ? + new internal::MatrixFreeFunctions::ShapeInfo( + *other.data) : + other.data) , jacobian(nullptr) , J_value(nullptr) , normal_vectors(nullptr) @@ -3106,10 +3232,11 @@ inline FEEvaluationBase::FEEvaluationBase( if (other.mapped_geometry.get() != nullptr) { mapped_geometry.reset( - new internal::MatrixFreeFunctions::MappingDataOnTheFly( - other.mapped_geometry->get_fe_values().get_mapping(), - other.mapped_geometry->get_quadrature(), - other.mapped_geometry->get_fe_values().get_update_flags())); + new internal::MatrixFreeFunctions:: + MappingDataOnTheFly( + other.mapped_geometry->get_fe_values().get_mapping(), + other.mapped_geometry->get_quadrature(), + other.mapped_geometry->get_fe_values().get_update_flags())); mapping_data = &mapped_geometry->get_data_storage(); cell = 0; @@ -3120,10 +3247,22 @@ inline FEEvaluationBase::FEEvaluationBase( -template -inline FEEvaluationBase & -FEEvaluationBase:: -operator=(const FEEvaluationBase &other) +template +inline FEEvaluationBase & +FEEvaluationBase:: +operator=(const FEEvaluationBase &other) { AssertDimension(quad_no, other.quad_no); AssertDimension(n_fe_components, other.n_fe_components); @@ -3147,10 +3286,9 @@ operator=(const FEEvaluationBase &other) mapping_data = other.mapping_data; if (other.matrix_info == nullptr) { - data = - new internal::MatrixFreeFunctions::ShapeInfo>( - *other.data); - scratch_data_array = new AlignedVector>(); + data = new internal::MatrixFreeFunctions::ShapeInfo( + *other.data); + scratch_data_array = new AlignedVector(); } else { @@ -3172,10 +3310,11 @@ operator=(const FEEvaluationBase &other) if (other.mapped_geometry.get() != nullptr) { mapped_geometry.reset( - new internal::MatrixFreeFunctions::MappingDataOnTheFly( - other.mapped_geometry->get_fe_values().get_mapping(), - other.mapped_geometry->get_quadrature(), - other.mapped_geometry->get_fe_values().get_update_flags())); + new internal::MatrixFreeFunctions:: + MappingDataOnTheFly( + other.mapped_geometry->get_fe_values().get_mapping(), + other.mapped_geometry->get_quadrature(), + other.mapped_geometry->get_fe_values().get_update_flags())); cell = 0; mapping_data = &mapped_geometry->get_data_storage(); jacobian = mapped_geometry->get_data_storage().jacobians[0].begin(); @@ -3187,9 +3326,16 @@ operator=(const FEEvaluationBase &other) -template -inline FEEvaluationBase:: - ~FEEvaluationBase() +template +inline FEEvaluationBase::~FEEvaluationBase() { if (matrix_info != nullptr) { @@ -3211,9 +3357,14 @@ inline FEEvaluationBase:: -template +template inline void -FEEvaluationBase::set_data_pointers() +FEEvaluationBase:: + set_data_pointers() { Assert(scratch_data_array != nullptr, ExcInternalError()); @@ -3260,19 +3411,27 @@ FEEvaluationBase::set_data_pointers() -template +template inline unsigned int -FEEvaluationBase::get_cell_data_number() - const +FEEvaluationBase:: + get_cell_data_number() const { return get_mapping_data_index_offset(); } -template +template inline unsigned int -FEEvaluationBase:: +FEEvaluationBase:: get_mapping_data_index_offset() const { if (matrix_info == nullptr) @@ -3286,9 +3445,14 @@ FEEvaluationBase:: -template +template inline internal::MatrixFreeFunctions::GeometryType -FEEvaluationBase::get_cell_type() const +FEEvaluationBase:: + get_cell_type() const { Assert(cell != numbers::invalid_unsigned_int, ExcNotInitialized()); return cell_type; @@ -3296,9 +3460,14 @@ FEEvaluationBase::get_cell_type() const -template -inline const internal::MatrixFreeFunctions::ShapeInfo> & -FEEvaluationBase::get_shape_info() const +template +inline const internal::MatrixFreeFunctions::ShapeInfo & +FEEvaluationBase:: + get_shape_info() const { Assert(data != nullptr, ExcInternalError()); return *data; @@ -3306,16 +3475,20 @@ FEEvaluationBase::get_shape_info() const -template +template inline void -FEEvaluationBase::fill_JxW_values( - AlignedVector> &JxW_values) const +FEEvaluationBase:: + fill_JxW_values(AlignedVector &JxW_values) const { AssertDimension(JxW_values.size(), n_quadrature_points); Assert(J_value != nullptr, ExcNotInitialized()); if (this->cell_type <= internal::MatrixFreeFunctions::affine) { - VectorizedArray J = J_value[0]; + VectorizedArrayType J = J_value[0]; for (unsigned int q = 0; q < this->n_quadrature_points; ++q) JxW_values[q] = J * this->quadrature_weights[q]; } @@ -3326,10 +3499,14 @@ FEEvaluationBase::fill_JxW_values( -template -inline DEAL_II_ALWAYS_INLINE Tensor<1, dim, VectorizedArray> - FEEvaluationBase::get_normal_vector( - const unsigned int q_index) const +template +inline DEAL_II_ALWAYS_INLINE Tensor<1, dim, VectorizedArrayType> + FEEvaluationBase:: + get_normal_vector(const unsigned int q_index) const { AssertIndexRange(q_index, n_quadrature_points); Assert(normal_vectors != nullptr, ExcMessage("Did not call reinit()!")); @@ -3341,9 +3518,13 @@ inline DEAL_II_ALWAYS_INLINE Tensor<1, dim, VectorizedArray> -template -inline DEAL_II_ALWAYS_INLINE VectorizedArray - FEEvaluationBase::JxW( +template +inline DEAL_II_ALWAYS_INLINE VectorizedArrayType + FEEvaluationBase::JxW( const unsigned int q_index) const { AssertIndexRange(q_index, n_quadrature_points); @@ -3359,10 +3540,14 @@ inline DEAL_II_ALWAYS_INLINE VectorizedArray -template -inline Tensor<2, dim, VectorizedArray> -FEEvaluationBase::inverse_jacobian( - const unsigned int q_index) const +template +inline Tensor<2, dim, VectorizedArrayType> +FEEvaluationBase:: + inverse_jacobian(const unsigned int q_index) const { AssertIndexRange(q_index, n_quadrature_points); Assert(this->jacobian != nullptr, ExcNotImplemented()); @@ -3374,26 +3559,29 @@ FEEvaluationBase::inverse_jacobian( -template -inline VectorizedArray -FEEvaluationBase::read_cell_data( - const AlignedVector> &array) const +template +inline VectorizedArrayType +FEEvaluationBase:: + read_cell_data(const AlignedVector &array) const { Assert(matrix_info != nullptr, ExcNotImplemented()); AssertDimension(array.size(), matrix_info->get_task_info().cell_partition_data.back()); if (is_face) { - VectorizedArray out = make_vectorized_array(Number(1.)); - const unsigned int * cells = + VectorizedArrayType out = make_vectorized_array(Number(1.)); + const unsigned int *cells = is_interior_face ? &this->matrix_info->get_face_info(cell).cells_interior[0] : &this->matrix_info->get_face_info(cell).cells_exterior[0]; - for (unsigned int i = 0; i < VectorizedArray::n_array_elements; - ++i) + for (unsigned int i = 0; i < VectorizedArrayType::n_array_elements; ++i) if (cells[i] != numbers::invalid_unsigned_int) - out[i] = array[cells[i] / VectorizedArray::n_array_elements] - [cells[i] % VectorizedArray::n_array_elements]; + out[i] = array[cells[i] / VectorizedArrayType::n_array_elements] + [cells[i] % VectorizedArrayType::n_array_elements]; return out; } else @@ -3587,7 +3775,7 @@ namespace internal // of the vector. // 1. A class to read data from vector - template + template struct VectorReader { template @@ -3603,15 +3791,15 @@ namespace internal template void - process_dofs_vectorized(const unsigned int dofs_per_cell, - const unsigned int dof_index, - VectorType & vec, - VectorizedArray *dof_values, + process_dofs_vectorized(const unsigned int dofs_per_cell, + const unsigned int dof_index, + VectorType & vec, + VectorizedArrayType *dof_values, std::integral_constant) const { const Number *vec_ptr = vec.begin() + dof_index; for (unsigned int i = 0; i < dofs_per_cell; - ++i, vec_ptr += VectorizedArray::n_array_elements) + ++i, vec_ptr += VectorizedArrayType::n_array_elements) dof_values[i].load(vec_ptr); } @@ -3619,27 +3807,26 @@ namespace internal template void - process_dofs_vectorized(const unsigned int dofs_per_cell, - const unsigned int dof_index, - const VectorType & vec, - VectorizedArray *dof_values, + process_dofs_vectorized(const unsigned int dofs_per_cell, + const unsigned int dof_index, + const VectorType & vec, + VectorizedArrayType *dof_values, std::integral_constant) const { for (unsigned int i = 0; i < dofs_per_cell; ++i) - for (unsigned int v = 0; v < VectorizedArray::n_array_elements; - ++v) + for (unsigned int v = 0; v < VectorizedArrayType::n_array_elements; ++v) dof_values[i][v] = vector_access( - vec, dof_index + v + i * VectorizedArray::n_array_elements); + vec, dof_index + v + i * VectorizedArrayType::n_array_elements); } template void - process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, - const unsigned int * dof_indices, - VectorType & vec, - VectorizedArray *dof_values, + process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, + const unsigned int * dof_indices, + VectorType & vec, + VectorizedArrayType *dof_values, std::integral_constant) const { dealii::vectorized_load_and_transpose(dofs_per_cell, @@ -3652,15 +3839,14 @@ namespace internal template void - process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, - const unsigned int * dof_indices, - const VectorType & vec, - VectorizedArray *dof_values, + process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, + const unsigned int * dof_indices, + const VectorType & vec, + VectorizedArrayType *dof_values, std::integral_constant) const { for (unsigned int d = 0; d < dofs_per_cell; ++d) - for (unsigned int v = 0; v < VectorizedArray::n_array_elements; - ++v) + for (unsigned int v = 0; v < VectorizedArrayType::n_array_elements; ++v) dof_values[d][v] = vector_access(vec, dof_indices[v] + d); } @@ -3670,10 +3856,10 @@ namespace internal // gather template void - process_dof_gather(const unsigned int * indices, - VectorType & vec, - const unsigned int constant_offset, - VectorizedArray &res, + process_dof_gather(const unsigned int * indices, + VectorType & vec, + const unsigned int constant_offset, + VectorizedArrayType &res, std::integral_constant) const { res.gather(vec.begin() + constant_offset, indices); @@ -3685,14 +3871,13 @@ namespace internal // manually load the data template void - process_dof_gather(const unsigned int * indices, - const VectorType & vec, - const unsigned int constant_offset, - VectorizedArray &res, + process_dof_gather(const unsigned int * indices, + const VectorType & vec, + const unsigned int constant_offset, + VectorizedArrayType &res, std::integral_constant) const { - for (unsigned int v = 0; v < VectorizedArray::n_array_elements; - ++v) + for (unsigned int v = 0; v < VectorizedArrayType::n_array_elements; ++v) res[v] = vector_access(vec, indices[v] + constant_offset); } @@ -3738,9 +3923,9 @@ namespace internal void - process_empty(VectorizedArray &res) const + process_empty(VectorizedArrayType &res) const { - res = VectorizedArray(); + res = VectorizedArrayType(); } }; @@ -3748,7 +3933,7 @@ namespace internal // 2. A class to add values to the vector during // FEEvaluation::distribute_local_to_global() call - template + template struct VectorDistributorLocalToGlobal { template @@ -3762,17 +3947,17 @@ namespace internal template void - process_dofs_vectorized(const unsigned int dofs_per_cell, - const unsigned int dof_index, - VectorType & vec, - VectorizedArray *dof_values, + process_dofs_vectorized(const unsigned int dofs_per_cell, + const unsigned int dof_index, + VectorType & vec, + VectorizedArrayType *dof_values, std::integral_constant) const { Number *vec_ptr = vec.begin() + dof_index; for (unsigned int i = 0; i < dofs_per_cell; - ++i, vec_ptr += VectorizedArray::n_array_elements) + ++i, vec_ptr += VectorizedArrayType::n_array_elements) { - VectorizedArray tmp; + VectorizedArrayType tmp; tmp.load(vec_ptr); tmp += dof_values[i]; tmp.store(vec_ptr); @@ -3783,18 +3968,17 @@ namespace internal template void - process_dofs_vectorized(const unsigned int dofs_per_cell, - const unsigned int dof_index, - VectorType & vec, - VectorizedArray *dof_values, + process_dofs_vectorized(const unsigned int dofs_per_cell, + const unsigned int dof_index, + VectorType & vec, + VectorizedArrayType *dof_values, std::integral_constant) const { for (unsigned int i = 0; i < dofs_per_cell; ++i) - for (unsigned int v = 0; v < VectorizedArray::n_array_elements; - ++v) + for (unsigned int v = 0; v < VectorizedArrayType::n_array_elements; ++v) vector_access_add(vec, dof_index + v + - i * VectorizedArray::n_array_elements, + i * VectorizedArrayType::n_array_elements, dof_values[i][v]); } @@ -3802,10 +3986,10 @@ namespace internal template void - process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, - const unsigned int * dof_indices, - VectorType & vec, - VectorizedArray *dof_values, + process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, + const unsigned int * dof_indices, + VectorType & vec, + VectorizedArrayType *dof_values, std::integral_constant) const { vectorized_transpose_and_store( @@ -3816,15 +4000,14 @@ namespace internal template void - process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, - const unsigned int * dof_indices, - VectorType & vec, - VectorizedArray *dof_values, + process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, + const unsigned int * dof_indices, + VectorType & vec, + VectorizedArrayType *dof_values, std::integral_constant) const { for (unsigned int d = 0; d < dofs_per_cell; ++d) - for (unsigned int v = 0; v < VectorizedArray::n_array_elements; - ++v) + for (unsigned int v = 0; v < VectorizedArrayType::n_array_elements; ++v) vector_access_add(vec, dof_indices[v] + d, dof_values[d][v]); } @@ -3834,19 +4017,18 @@ namespace internal // scatter template void - process_dof_gather(const unsigned int * indices, - VectorType & vec, - const unsigned int constant_offset, - VectorizedArray &res, + process_dof_gather(const unsigned int * indices, + VectorType & vec, + const unsigned int constant_offset, + VectorizedArrayType &res, std::integral_constant) const { # if DEAL_II_COMPILER_VECTORIZATION_LEVEL < 3 - for (unsigned int v = 0; v < VectorizedArray::n_array_elements; - ++v) + for (unsigned int v = 0; v < VectorizedArrayType::n_array_elements; ++v) vector_access(vec, indices[v] + constant_offset) += res[v]; # else // only use gather in case there is also scatter. - VectorizedArray tmp; + VectorizedArrayType tmp; tmp.gather(vec.begin() + constant_offset, indices); tmp += res; tmp.scatter(indices, vec.begin() + constant_offset); @@ -3859,14 +4041,13 @@ namespace internal // manually append all data template void - process_dof_gather(const unsigned int * indices, - VectorType & vec, - const unsigned int constant_offset, - VectorizedArray &res, + process_dof_gather(const unsigned int * indices, + VectorType & vec, + const unsigned int constant_offset, + VectorizedArrayType &res, std::integral_constant) const { - for (unsigned int v = 0; v < VectorizedArray::n_array_elements; - ++v) + for (unsigned int v = 0; v < VectorizedArrayType::n_array_elements; ++v) vector_access_add(vec, indices[v] + constant_offset, res[v]); } @@ -3910,14 +4091,14 @@ namespace internal void - process_empty(VectorizedArray &) const + process_empty(VectorizedArrayType &) const {} }; // 3. A class to set elements of the vector - template + template struct VectorSetter { template @@ -3931,15 +4112,15 @@ namespace internal template void - process_dofs_vectorized(const unsigned int dofs_per_cell, - const unsigned int dof_index, - VectorType & vec, - VectorizedArray *dof_values, + process_dofs_vectorized(const unsigned int dofs_per_cell, + const unsigned int dof_index, + VectorType & vec, + VectorizedArrayType *dof_values, std::integral_constant) const { Number *vec_ptr = vec.begin() + dof_index; for (unsigned int i = 0; i < dofs_per_cell; - ++i, vec_ptr += VectorizedArray::n_array_elements) + ++i, vec_ptr += VectorizedArrayType::n_array_elements) dof_values[i].store(vec_ptr); } @@ -3947,18 +4128,17 @@ namespace internal template void - process_dofs_vectorized(const unsigned int dofs_per_cell, - const unsigned int dof_index, - VectorType & vec, - VectorizedArray *dof_values, + process_dofs_vectorized(const unsigned int dofs_per_cell, + const unsigned int dof_index, + VectorType & vec, + VectorizedArrayType *dof_values, std::integral_constant) const { for (unsigned int i = 0; i < dofs_per_cell; ++i) - for (unsigned int v = 0; v < VectorizedArray::n_array_elements; - ++v) + for (unsigned int v = 0; v < VectorizedArrayType::n_array_elements; ++v) vector_access(vec, dof_index + v + - i * VectorizedArray::n_array_elements) = + i * VectorizedArrayType::n_array_elements) = dof_values[i][v]; } @@ -3966,10 +4146,10 @@ namespace internal template void - process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, - const unsigned int * dof_indices, - VectorType & vec, - VectorizedArray *dof_values, + process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, + const unsigned int * dof_indices, + VectorType & vec, + VectorizedArrayType *dof_values, std::integral_constant) const { vectorized_transpose_and_store( @@ -3980,15 +4160,14 @@ namespace internal template void - process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, - const unsigned int * dof_indices, - VectorType & vec, - VectorizedArray *dof_values, + process_dofs_vectorized_transpose(const unsigned int dofs_per_cell, + const unsigned int * dof_indices, + VectorType & vec, + VectorizedArrayType *dof_values, std::integral_constant) const { for (unsigned int i = 0; i < dofs_per_cell; ++i) - for (unsigned int v = 0; v < VectorizedArray::n_array_elements; - ++v) + for (unsigned int v = 0; v < VectorizedArrayType::n_array_elements; ++v) vector_access(vec, dof_indices[v] + i) = dof_values[i][v]; } @@ -3996,10 +4175,10 @@ namespace internal template void - process_dof_gather(const unsigned int * indices, - VectorType & vec, - const unsigned int constant_offset, - VectorizedArray &res, + process_dof_gather(const unsigned int * indices, + VectorType & vec, + const unsigned int constant_offset, + VectorizedArrayType &res, std::integral_constant) const { res.scatter(indices, vec.begin() + constant_offset); @@ -4009,14 +4188,13 @@ namespace internal template void - process_dof_gather(const unsigned int * indices, - VectorType & vec, - const unsigned int constant_offset, - VectorizedArray &res, + process_dof_gather(const unsigned int * indices, + VectorType & vec, + const unsigned int constant_offset, + VectorizedArrayType &res, std::integral_constant) const { - for (unsigned int v = 0; v < VectorizedArray::n_array_elements; - ++v) + for (unsigned int v = 0; v < VectorizedArrayType::n_array_elements; ++v) vector_access(vec, indices[v] + constant_offset) = res[v]; } @@ -4056,7 +4234,7 @@ namespace internal void - process_empty(VectorizedArray &) const + process_empty(VectorizedArrayType &) const {} }; @@ -4138,14 +4316,19 @@ namespace internal -template +template template inline void -FEEvaluationBase::read_write_operation( - const VectorOperation & operation, - VectorType * src[], - const std::bitset::n_array_elements> &mask, - const bool apply_constraints) const +FEEvaluationBase:: + read_write_operation( + const VectorOperation & operation, + VectorType * src[], + const std::bitset &mask, + const bool apply_constraints) const { // Case 1: No MatrixFree object given, simple case because we do not need to // process constraints and need not care about vectorization -> go to @@ -4183,7 +4366,7 @@ FEEvaluationBase::read_write_operation( // Case 3: standard operation with one index per degree of freedom -> go on // here constexpr unsigned int n_vectorization = - VectorizedArray::n_array_elements; + VectorizedArrayType::n_array_elements; Assert(mask.count() == n_vectorization, ExcNotImplemented("Masking currently not implemented for " "non-contiguous DoF storage")); @@ -4224,9 +4407,9 @@ FEEvaluationBase::read_write_operation( return; } - const unsigned int * dof_indices[n_vectorization]; - VectorizedArray **values_dofs = - const_cast **>(&this->values_dofs[0]); + const unsigned int * dof_indices[n_vectorization]; + VectorizedArrayType **values_dofs = + const_cast(&this->values_dofs[0]); unsigned int cells_copied[n_vectorization]; const unsigned int *cells; @@ -4238,8 +4421,7 @@ FEEvaluationBase::read_write_operation( if (dof_access_index == internal::MatrixFreeFunctions::DoFInfo::dof_access_cell) for (unsigned int v = 0; v < n_vectorization_actual; ++v) - cells_copied[v] = - cell * VectorizedArray::n_array_elements + v; + cells_copied[v] = cell * VectorizedArrayType::n_array_elements + v; cells = dof_access_index == internal::MatrixFreeFunctions::DoFInfo::dof_access_cell ? &cells_copied[0] : @@ -4552,10 +4734,14 @@ FEEvaluationBase::read_write_operation( -template +template template inline void -FEEvaluationBase:: +FEEvaluationBase:: read_write_operation_global(const VectorOperation &operation, VectorType * src[]) const { @@ -4579,14 +4765,18 @@ FEEvaluationBase:: -template +template template inline void -FEEvaluationBase:: +FEEvaluationBase:: read_write_operation_contiguous( - const VectorOperation & operation, - VectorType * src[], - const std::bitset::n_array_elements> &mask) const + const VectorOperation & operation, + VectorType * src[], + const std::bitset &mask) const { // This functions processes the functions read_dof_values, // distribute_local_to_global, and set_dof_values with the same code for @@ -4611,13 +4801,13 @@ FEEvaluationBase:: if (dof_info->index_storage_variants[ind][cell] == internal::MatrixFreeFunctions::DoFInfo::IndexStorageVariants:: interleaved_contiguous && - n_lanes == VectorizedArray::n_array_elements) + n_lanes == VectorizedArrayType::n_array_elements) { const unsigned int dof_index = - dof_indices_cont[cell * VectorizedArray::n_array_elements] + + dof_indices_cont[cell * VectorizedArrayType::n_array_elements] + dof_info->component_dof_indices_offset[active_fe_index] [first_selected_component] * - VectorizedArray::n_array_elements; + VectorizedArrayType::n_array_elements; if (n_components == 1 || n_fe_components == 1) for (unsigned int comp = 0; comp < n_components; ++comp) operation.process_dofs_vectorized(data->dofs_per_component_on_cell, @@ -4640,24 +4830,24 @@ FEEvaluationBase:: const unsigned int vectorization_populated = dof_info->n_vectorization_lanes_filled[ind][this->cell]; - unsigned int dof_indices[VectorizedArray::n_array_elements]; + unsigned int dof_indices[VectorizedArrayType::n_array_elements]; for (unsigned int v = 0; v < vectorization_populated; ++v) dof_indices[v] = - dof_indices_cont[cell * VectorizedArray::n_array_elements + v] + + dof_indices_cont[cell * VectorizedArrayType::n_array_elements + v] + dof_info->component_dof_indices_offset[active_fe_index] [first_selected_component] * dof_info->dof_indices_interleave_strides - [ind][cell * VectorizedArray::n_array_elements + v]; + [ind][cell * VectorizedArrayType::n_array_elements + v]; for (unsigned int v = vectorization_populated; - v < VectorizedArray::n_array_elements; + v < VectorizedArrayType::n_array_elements; ++v) dof_indices[v] = numbers::invalid_unsigned_int; // In the case with contiguous cell indices, we know that there are no // constraints and that the indices within each element are contiguous - if (vectorization_populated == VectorizedArray::n_array_elements && - n_lanes == VectorizedArray::n_array_elements) + if (vectorization_populated == VectorizedArrayType::n_array_elements && + n_lanes == VectorizedArrayType::n_array_elements) { if (dof_info->index_storage_variants[ind][cell] == internal::MatrixFreeFunctions::DoFInfo::IndexStorageVariants:: @@ -4690,7 +4880,7 @@ FEEvaluationBase:: operation.process_dof_gather( dof_indices, *src[comp], - i * VectorizedArray::n_array_elements, + i * VectorizedArrayType::n_array_elements, values_dofs[comp][i], vector_selector); } @@ -4703,7 +4893,7 @@ FEEvaluationBase:: dof_indices, *src[0], (comp * data->dofs_per_component_on_cell + i) * - VectorizedArray::n_array_elements, + VectorizedArrayType::n_array_elements, values_dofs[comp][i], vector_selector); } @@ -4716,7 +4906,7 @@ FEEvaluationBase:: ExcNotImplemented()); const unsigned int *offsets = &dof_info->dof_indices_interleave_strides - [ind][VectorizedArray::n_array_elements * cell]; + [ind][VectorizedArrayType::n_array_elements * cell]; if (n_components == 1 || n_fe_components == 1) for (unsigned int i = 0; i < data->dofs_per_component_on_cell; ++i) { @@ -4728,7 +4918,7 @@ FEEvaluationBase:: vector_selector); DEAL_II_OPENMP_SIMD_PRAGMA for (unsigned int v = 0; - v < VectorizedArray::n_array_elements; + v < VectorizedArrayType::n_array_elements; ++v) dof_indices[v] += offsets[v]; } @@ -4744,7 +4934,7 @@ FEEvaluationBase:: vector_selector); DEAL_II_OPENMP_SIMD_PRAGMA for (unsigned int v = 0; - v < VectorizedArray::n_array_elements; + v < VectorizedArrayType::n_array_elements; ++v) dof_indices[v] += offsets[v]; } @@ -4788,10 +4978,10 @@ FEEvaluationBase:: { const unsigned int *offsets = &dof_info->dof_indices_interleave_strides - [ind][VectorizedArray::n_array_elements * cell]; + [ind][VectorizedArrayType::n_array_elements * cell]; for (unsigned int v = 0; v < vectorization_populated; ++v) AssertIndexRange(offsets[v], - VectorizedArray::n_array_elements + 1); + VectorizedArrayType::n_array_elements + 1); if (n_components == 1 || n_fe_components == 1) for (unsigned int v = 0; v < vectorization_populated; ++v) { @@ -4823,12 +5013,15 @@ FEEvaluationBase:: -template +template template inline void -FEEvaluationBase::read_dof_values( - const VectorType & src, - const unsigned int first_index) +FEEvaluationBase:: + read_dof_values(const VectorType &src, const unsigned int first_index) { // select between block vectors and non-block vectors. Note that the number // of components is checked in the internal data @@ -4841,11 +5034,11 @@ FEEvaluationBase::read_dof_values( IsBlockVector::value>:: get_vector_component(const_cast(src), d + first_index); - internal::VectorReader reader; + internal::VectorReader reader; read_write_operation( reader, src_data, - std::bitset::n_array_elements>().flip(), + std::bitset().flip(), true); # ifdef DEBUG @@ -4855,12 +5048,15 @@ FEEvaluationBase::read_dof_values( -template +template template inline void -FEEvaluationBase::read_dof_values_plain( - const VectorType & src, - const unsigned int first_index) +FEEvaluationBase:: + read_dof_values_plain(const VectorType &src, const unsigned int first_index) { // select between block vectors and non-block vectors. Note that the number // of components is checked in the internal data @@ -4873,11 +5069,11 @@ FEEvaluationBase::read_dof_values_plain( IsBlockVector::value>:: get_vector_component(const_cast(src), d + first_index); - internal::VectorReader reader; + internal::VectorReader reader; read_write_operation( reader, src_data, - std::bitset::n_array_elements>().flip(), + std::bitset().flip(), false); # ifdef DEBUG @@ -4887,14 +5083,18 @@ FEEvaluationBase::read_dof_values_plain( -template +template template inline void -FEEvaluationBase:: +FEEvaluationBase:: distribute_local_to_global( - VectorType & dst, - const unsigned int first_index, - const std::bitset::n_array_elements> &mask) const + VectorType & dst, + const unsigned int first_index, + const std::bitset &mask) const { Assert(dof_values_initialized == true, internal::ExcAccessToUninitializedField()); @@ -4910,19 +5110,25 @@ FEEvaluationBase:: IsBlockVector::value>::get_vector_component(dst, d + first_index); - internal::VectorDistributorLocalToGlobal distributor; + internal::VectorDistributorLocalToGlobal + distributor; read_write_operation(distributor, dst_data, mask); } -template +template template inline void -FEEvaluationBase::set_dof_values( - VectorType & dst, - const unsigned int first_index, - const std::bitset::n_array_elements> &mask) const +FEEvaluationBase:: + set_dof_values( + VectorType & dst, + const unsigned int first_index, + const std::bitset &mask) const { Assert(dof_values_initialized == true, internal::ExcAccessToUninitializedField()); @@ -4938,7 +5144,7 @@ FEEvaluationBase::set_dof_values( IsBlockVector::value>::get_vector_component(dst, d + first_index); - internal::VectorSetter setter; + internal::VectorSetter setter; read_write_operation(setter, dst_data, mask); } @@ -4946,9 +5152,13 @@ FEEvaluationBase::set_dof_values( /*------------------------------ access to data fields ----------------------*/ -template +template inline const std::vector & -FEEvaluationBase:: +FEEvaluationBase:: get_internal_dof_numbering() const { return data->lexicographic_numbering; @@ -4956,29 +5166,44 @@ FEEvaluationBase:: -template -inline ArrayView> -FEEvaluationBase::get_scratch_data() const +template +inline ArrayView +FEEvaluationBase:: + get_scratch_data() const { - return ArrayView>( - const_cast *>(scratch_data), + return ArrayView( + const_cast(scratch_data), scratch_data_array->end() - scratch_data); } -template -inline const VectorizedArray * -FEEvaluationBase::begin_dof_values() const +template +inline const VectorizedArrayType * +FEEvaluationBase:: + begin_dof_values() const { return &values_dofs[0][0]; } -template -inline VectorizedArray * -FEEvaluationBase::begin_dof_values() +template +inline VectorizedArrayType * +FEEvaluationBase:: + begin_dof_values() { # ifdef DEBUG dof_values_initialized = true; @@ -4988,9 +5213,14 @@ FEEvaluationBase::begin_dof_values() -template -inline const VectorizedArray * -FEEvaluationBase::begin_values() const +template +inline const VectorizedArrayType * +FEEvaluationBase:: + begin_values() const { Assert(values_quad_initialized || values_quad_submitted, ExcNotInitialized()); return &values_quad[0][0]; @@ -4998,9 +5228,14 @@ FEEvaluationBase::begin_values() const -template -inline VectorizedArray * -FEEvaluationBase::begin_values() +template +inline VectorizedArrayType * +FEEvaluationBase:: + begin_values() { # ifdef DEBUG values_quad_initialized = true; @@ -5011,9 +5246,14 @@ FEEvaluationBase::begin_values() -template -inline const VectorizedArray * -FEEvaluationBase::begin_gradients() const +template +inline const VectorizedArrayType * +FEEvaluationBase:: + begin_gradients() const { Assert(gradients_quad_initialized || gradients_quad_submitted, ExcNotInitialized()); @@ -5022,9 +5262,14 @@ FEEvaluationBase::begin_gradients() const -template -inline VectorizedArray * -FEEvaluationBase::begin_gradients() +template +inline VectorizedArrayType * +FEEvaluationBase:: + begin_gradients() { # ifdef DEBUG gradients_quad_submitted = true; @@ -5035,9 +5280,14 @@ FEEvaluationBase::begin_gradients() -template -inline const VectorizedArray * -FEEvaluationBase::begin_hessians() const +template +inline const VectorizedArrayType * +FEEvaluationBase:: + begin_hessians() const { Assert(hessians_quad_initialized, ExcNotInitialized()); return &hessians_quad[0][0][0]; @@ -5045,9 +5295,14 @@ FEEvaluationBase::begin_hessians() const -template -inline VectorizedArray * -FEEvaluationBase::begin_hessians() +template +inline VectorizedArrayType * +FEEvaluationBase:: + begin_hessians() { # ifdef DEBUG hessians_quad_initialized = true; @@ -5057,13 +5312,17 @@ FEEvaluationBase::begin_hessians() -template -inline DEAL_II_ALWAYS_INLINE Tensor<1, n_components_, VectorizedArray> - FEEvaluationBase::get_dof_value( - const unsigned int dof) const +template +inline DEAL_II_ALWAYS_INLINE Tensor<1, n_components_, VectorizedArrayType> + FEEvaluationBase:: + get_dof_value(const unsigned int dof) const { AssertIndexRange(dof, this->data->dofs_per_component_on_cell); - Tensor<1, n_components_, VectorizedArray> return_value; + Tensor<1, n_components_, VectorizedArrayType> return_value; for (unsigned int comp = 0; comp < n_components; comp++) return_value[comp] = this->values_dofs[comp][dof]; return return_value; @@ -5071,15 +5330,19 @@ inline DEAL_II_ALWAYS_INLINE Tensor<1, n_components_, VectorizedArray> -template -inline DEAL_II_ALWAYS_INLINE Tensor<1, n_components_, VectorizedArray> - FEEvaluationBase::get_value( - const unsigned int q_point) const +template +inline DEAL_II_ALWAYS_INLINE Tensor<1, n_components_, VectorizedArrayType> + FEEvaluationBase:: + get_value(const unsigned int q_point) const { Assert(this->values_quad_initialized == true, internal::ExcAccessToUninitializedField()); AssertIndexRange(q_point, this->n_quadrature_points); - Tensor<1, n_components_, VectorizedArray> return_value; + Tensor<1, n_components_, VectorizedArrayType> return_value; for (unsigned int comp = 0; comp < n_components; comp++) return_value[comp] = this->values_quad[comp][q_point]; return return_value; @@ -5087,11 +5350,15 @@ inline DEAL_II_ALWAYS_INLINE Tensor<1, n_components_, VectorizedArray> -template +template inline DEAL_II_ALWAYS_INLINE - Tensor<1, n_components_, Tensor<1, dim, VectorizedArray>> - FEEvaluationBase::get_gradient( - const unsigned int q_point) const + Tensor<1, n_components_, Tensor<1, dim, VectorizedArrayType>> + FEEvaluationBase:: + get_gradient(const unsigned int q_point) const { Assert(this->gradients_quad_initialized == true, internal::ExcAccessToUninitializedField()); @@ -5099,7 +5366,7 @@ inline DEAL_II_ALWAYS_INLINE Assert(jacobian != nullptr, ExcNotInitialized()); - Tensor<1, n_components_, Tensor<1, dim, VectorizedArray>> grad_out; + Tensor<1, n_components_, Tensor<1, dim, VectorizedArrayType>> grad_out; // Cartesian cell if (!is_face && this->cell_type == internal::MatrixFreeFunctions::cartesian) @@ -5112,7 +5379,7 @@ inline DEAL_II_ALWAYS_INLINE // cell with general/affine Jacobian else { - const Tensor<2, dim, VectorizedArray> &jac = + const Tensor<2, dim, VectorizedArrayType> &jac = jacobian[this->cell_type > internal::MatrixFreeFunctions::affine ? q_point : 0]; @@ -5131,10 +5398,14 @@ inline DEAL_II_ALWAYS_INLINE -template -inline DEAL_II_ALWAYS_INLINE Tensor<1, n_components_, VectorizedArray> - FEEvaluationBase::get_normal_derivative( - const unsigned int q_point) const +template +inline DEAL_II_ALWAYS_INLINE Tensor<1, n_components_, VectorizedArrayType> + FEEvaluationBase:: + get_normal_derivative(const unsigned int q_point) const { AssertIndexRange(q_point, this->n_quadrature_points); Assert(this->gradients_quad_initialized == true, @@ -5142,7 +5413,7 @@ inline DEAL_II_ALWAYS_INLINE Tensor<1, n_components_, VectorizedArray> Assert(normal_x_jacobian != nullptr, ExcNotInitialized()); - Tensor<1, n_components, VectorizedArray> grad_out; + Tensor<1, n_components, VectorizedArrayType> grad_out; if (this->cell_type == internal::MatrixFreeFunctions::cartesian) for (unsigned int comp = 0; comp < n_components; comp++) grad_out[comp] = this->gradients_quad[comp][dim - 1][q_point] * @@ -5169,22 +5440,22 @@ namespace internal { // compute tmp = hess_unit(u) * J^T. do this manually because we do not // store the lower diagonal because of symmetry - template + template inline void - hessian_unit_times_jac(const Tensor<2, 1, VectorizedArray> &jac, - const VectorizedArray *const hessians_quad[1], - const unsigned int q_point, - VectorizedArray (&tmp)[1][1]) + hessian_unit_times_jac(const Tensor<2, 1, VectorizedArrayType> &jac, + const VectorizedArrayType *const hessians_quad[1], + const unsigned int q_point, + VectorizedArrayType (&tmp)[1][1]) { tmp[0][0] = jac[0][0] * hessians_quad[0][q_point]; } - template + template inline void - hessian_unit_times_jac(const Tensor<2, 2, VectorizedArray> &jac, - const VectorizedArray *const hessians_quad[3], - const unsigned int q_point, - VectorizedArray (&tmp)[2][2]) + hessian_unit_times_jac(const Tensor<2, 2, VectorizedArrayType> &jac, + const VectorizedArrayType *const hessians_quad[3], + const unsigned int q_point, + VectorizedArrayType (&tmp)[2][2]) { for (unsigned int d = 0; d < 2; ++d) { @@ -5195,12 +5466,12 @@ namespace internal } } - template + template inline void - hessian_unit_times_jac(const Tensor<2, 3, VectorizedArray> &jac, - const VectorizedArray *const hessians_quad[6], - const unsigned int q_point, - VectorizedArray (&tmp)[3][3]) + hessian_unit_times_jac(const Tensor<2, 3, VectorizedArrayType> &jac, + const VectorizedArrayType *const hessians_quad[6], + const unsigned int q_point, + VectorizedArrayType (&tmp)[3][3]) { for (unsigned int d = 0; d < 3; ++d) { @@ -5219,10 +5490,14 @@ namespace internal -template -inline Tensor<1, n_components_, Tensor<2, dim, VectorizedArray>> -FEEvaluationBase::get_hessian( - const unsigned int q_point) const +template +inline Tensor<1, n_components_, Tensor<2, dim, VectorizedArrayType>> +FEEvaluationBase:: + get_hessian(const unsigned int q_point) const { Assert(!is_face, ExcNotImplemented()); Assert(this->hessians_quad_initialized == true, @@ -5230,12 +5505,12 @@ FEEvaluationBase::get_hessian( AssertIndexRange(q_point, this->n_quadrature_points); Assert(jacobian != nullptr, ExcNotImplemented()); - const Tensor<2, dim, VectorizedArray> &jac = + const Tensor<2, dim, VectorizedArrayType> &jac = jacobian[this->cell_type <= internal::MatrixFreeFunctions::affine ? 0 : q_point]; - Tensor<2, dim, VectorizedArray> hessian_out[n_components]; + Tensor<2, dim, VectorizedArrayType> hessian_out[n_components]; // Cartesian cell if (this->cell_type == internal::MatrixFreeFunctions::cartesian) @@ -5279,7 +5554,7 @@ FEEvaluationBase::get_hessian( { // compute laplacian before the gradient because it needs to access // unscaled gradient data - VectorizedArray tmp[dim][dim]; + VectorizedArrayType tmp[dim][dim]; internal::hessian_unit_times_jac(jac, this->hessians_quad[comp], q_point, @@ -5306,17 +5581,16 @@ FEEvaluationBase::get_hessian( // cell with general Jacobian else { - const Tensor<1, - dim *(dim + 1) / 2, - Tensor<1, dim, VectorizedArray>> &jac_grad = - mapping_data->jacobian_gradients - [1 - this->is_interior_face] - [this->mapping_data->data_index_offsets[this->cell] + q_point]; + const Tensor<1, dim *(dim + 1) / 2, Tensor<1, dim, VectorizedArrayType>> + &jac_grad = + mapping_data->jacobian_gradients + [1 - this->is_interior_face] + [this->mapping_data->data_index_offsets[this->cell] + q_point]; for (unsigned int comp = 0; comp < n_components; comp++) { // compute laplacian before the gradient because it needs to access // unscaled gradient data - VectorizedArray tmp[dim][dim]; + VectorizedArrayType tmp[dim][dim]; internal::hessian_unit_times_jac(jac, this->hessians_quad[comp], q_point, @@ -5350,16 +5624,20 @@ FEEvaluationBase::get_hessian( hessian_out[comp][e][d] = hessian_out[comp][d][e]; } } - return Tensor<1, n_components_, Tensor<2, dim, VectorizedArray>>( + return Tensor<1, n_components_, Tensor<2, dim, VectorizedArrayType>>( hessian_out); } -template -inline Tensor<1, n_components_, Tensor<1, dim, VectorizedArray>> -FEEvaluationBase::get_hessian_diagonal( - const unsigned int q_point) const +template +inline Tensor<1, n_components_, Tensor<1, dim, VectorizedArrayType>> +FEEvaluationBase:: + get_hessian_diagonal(const unsigned int q_point) const { Assert(!is_face, ExcNotImplemented()); Assert(this->hessians_quad_initialized == true, @@ -5367,12 +5645,12 @@ FEEvaluationBase::get_hessian_diagonal( AssertIndexRange(q_point, this->n_quadrature_points); Assert(jacobian != nullptr, ExcNotImplemented()); - const Tensor<2, dim, VectorizedArray> &jac = + const Tensor<2, dim, VectorizedArrayType> &jac = jacobian[this->cell_type <= internal::MatrixFreeFunctions::affine ? 0 : q_point]; - Tensor<1, n_components_, Tensor<1, dim, VectorizedArray>> hessian_out; + Tensor<1, n_components_, Tensor<1, dim, VectorizedArrayType>> hessian_out; // Cartesian cell if (this->cell_type == internal::MatrixFreeFunctions::cartesian) @@ -5389,7 +5667,7 @@ FEEvaluationBase::get_hessian_diagonal( { // compute laplacian before the gradient because it needs to access // unscaled gradient data - VectorizedArray tmp[dim][dim]; + VectorizedArrayType tmp[dim][dim]; internal::hessian_unit_times_jac(jac, this->hessians_quad[comp], q_point, @@ -5408,16 +5686,15 @@ FEEvaluationBase::get_hessian_diagonal( // cell with general Jacobian else { - const Tensor<1, - dim *(dim + 1) / 2, - Tensor<1, dim, VectorizedArray>> &jac_grad = - mapping_data->jacobian_gradients - [0][this->mapping_data->data_index_offsets[this->cell] + q_point]; + const Tensor<1, dim *(dim + 1) / 2, Tensor<1, dim, VectorizedArrayType>> + &jac_grad = + mapping_data->jacobian_gradients + [0][this->mapping_data->data_index_offsets[this->cell] + q_point]; for (unsigned int comp = 0; comp < n_components; comp++) { // compute laplacian before the gradient because it needs to access // unscaled gradient data - VectorizedArray tmp[dim][dim]; + VectorizedArrayType tmp[dim][dim]; internal::hessian_unit_times_jac(jac, this->hessians_quad[comp], q_point, @@ -5443,18 +5720,22 @@ FEEvaluationBase::get_hessian_diagonal( -template -inline Tensor<1, n_components_, VectorizedArray> -FEEvaluationBase::get_laplacian( - const unsigned int q_point) const +template +inline Tensor<1, n_components_, VectorizedArrayType> +FEEvaluationBase:: + get_laplacian(const unsigned int q_point) const { Assert(is_face == false, ExcNotImplemented()); Assert(this->hessians_quad_initialized == true, internal::ExcAccessToUninitializedField()); AssertIndexRange(q_point, this->n_quadrature_points); - Tensor<1, n_components_, VectorizedArray> laplacian_out; - const Tensor<1, n_components_, Tensor<1, dim, VectorizedArray>> + Tensor<1, n_components_, VectorizedArrayType> laplacian_out; + const Tensor<1, n_components_, Tensor<1, dim, VectorizedArrayType>> hess_diag = get_hessian_diagonal(q_point); for (unsigned int comp = 0; comp < n_components; ++comp) { @@ -5467,11 +5748,15 @@ FEEvaluationBase::get_laplacian( -template +template inline DEAL_II_ALWAYS_INLINE void -FEEvaluationBase::submit_dof_value( - const Tensor<1, n_components_, VectorizedArray> val_in, - const unsigned int dof) +FEEvaluationBase:: + submit_dof_value(const Tensor<1, n_components_, VectorizedArrayType> val_in, + const unsigned int dof) { # ifdef DEBUG this->dof_values_initialized = true; @@ -5483,11 +5768,15 @@ FEEvaluationBase::submit_dof_value( -template +template inline DEAL_II_ALWAYS_INLINE void -FEEvaluationBase::submit_value( - const Tensor<1, n_components_, VectorizedArray> val_in, - const unsigned int q_point) +FEEvaluationBase:: + submit_value(const Tensor<1, n_components_, VectorizedArrayType> val_in, + const unsigned int q_point) { # ifdef DEBUG Assert(this->cell != numbers::invalid_unsigned_int, ExcNotInitialized()); @@ -5498,14 +5787,13 @@ FEEvaluationBase::submit_value( if (this->cell_type <= internal::MatrixFreeFunctions::affine) { - const VectorizedArray JxW = - J_value[0] * quadrature_weights[q_point]; + const VectorizedArrayType JxW = J_value[0] * quadrature_weights[q_point]; for (unsigned int comp = 0; comp < n_components; ++comp) this->values_quad[comp][q_point] = val_in[comp] * JxW; } else { - const VectorizedArray JxW = J_value[q_point]; + const VectorizedArrayType JxW = J_value[q_point]; for (unsigned int comp = 0; comp < n_components; ++comp) this->values_quad[comp][q_point] = val_in[comp] * JxW; } @@ -5513,12 +5801,16 @@ FEEvaluationBase::submit_value( -template +template inline DEAL_II_ALWAYS_INLINE void -FEEvaluationBase::submit_gradient( - const Tensor<1, n_components_, Tensor<1, dim, VectorizedArray>> - grad_in, - const unsigned int q_point) +FEEvaluationBase:: + submit_gradient( + const Tensor<1, n_components_, Tensor<1, dim, VectorizedArrayType>> grad_in, + const unsigned int q_point) { # ifdef DEBUG Assert(this->cell != numbers::invalid_unsigned_int, ExcNotInitialized()); @@ -5530,8 +5822,7 @@ FEEvaluationBase::submit_gradient( if (!is_face && this->cell_type == internal::MatrixFreeFunctions::cartesian) { - const VectorizedArray JxW = - J_value[0] * quadrature_weights[q_point]; + const VectorizedArrayType JxW = J_value[0] * quadrature_weights[q_point]; for (unsigned int comp = 0; comp < n_components; comp++) for (unsigned int d = 0; d < dim; ++d) this->gradients_quad[comp][d][q_point] = @@ -5539,18 +5830,18 @@ FEEvaluationBase::submit_gradient( } else { - const Tensor<2, dim, VectorizedArray> &jac = + const Tensor<2, dim, VectorizedArrayType> &jac = this->cell_type > internal::MatrixFreeFunctions::affine ? jacobian[q_point] : jacobian[0]; - const VectorizedArray JxW = + const VectorizedArrayType JxW = this->cell_type > internal::MatrixFreeFunctions::affine ? J_value[q_point] : J_value[0] * quadrature_weights[q_point]; for (unsigned int comp = 0; comp < n_components; ++comp) for (unsigned int d = 0; d < dim; ++d) { - VectorizedArray new_val = jac[0][d] * grad_in[comp][0]; + VectorizedArrayType new_val = jac[0][d] * grad_in[comp][0]; for (unsigned int e = 1; e < dim; ++e) new_val += (jac[e][d] * grad_in[comp][e]); this->gradients_quad[comp][d][q_point] = new_val * JxW; @@ -5560,11 +5851,16 @@ FEEvaluationBase::submit_gradient( -template +template inline DEAL_II_ALWAYS_INLINE void -FEEvaluationBase::submit_normal_derivative( - const Tensor<1, n_components_, VectorizedArray> grad_in, - const unsigned int q_point) +FEEvaluationBase:: + submit_normal_derivative( + const Tensor<1, n_components_, VectorizedArrayType> grad_in, + const unsigned int q_point) { # ifdef DEBUG AssertIndexRange(q_point, this->n_quadrature_points); @@ -5576,7 +5872,7 @@ FEEvaluationBase::submit_normal_derivative( for (unsigned int comp = 0; comp < n_components; comp++) { for (unsigned int d = 0; d < dim - 1; ++d) - this->gradients_quad[comp][d][q_point] = VectorizedArray(); + this->gradients_quad[comp][d][q_point] = VectorizedArrayType(); this->gradients_quad[comp][dim - 1][q_point] = grad_in[comp] * (this->normal_x_jacobian[0][dim - 1] * this->J_value[0] * @@ -5588,7 +5884,7 @@ FEEvaluationBase::submit_normal_derivative( this->cell_type <= internal::MatrixFreeFunctions::affine ? 0 : q_point; for (unsigned int comp = 0; comp < n_components; comp++) { - VectorizedArray factor = grad_in[comp] * this->J_value[index]; + VectorizedArrayType factor = grad_in[comp] * this->J_value[index]; if (this->cell_type <= internal::MatrixFreeFunctions::affine) factor = factor * this->quadrature_weights[q_point]; for (unsigned int d = 0; d < dim; ++d) @@ -5600,16 +5896,21 @@ FEEvaluationBase::submit_normal_derivative( -template -inline Tensor<1, n_components_, VectorizedArray> -FEEvaluationBase::integrate_value() const +template +inline Tensor<1, n_components_, VectorizedArrayType> +FEEvaluationBase:: + integrate_value() const { # ifdef DEBUG Assert(this->cell != numbers::invalid_unsigned_int, ExcNotInitialized()); Assert(this->values_quad_submitted == true, internal::ExcAccessToUninitializedField()); # endif - Tensor<1, n_components_, VectorizedArray> return_value; + Tensor<1, n_components_, VectorizedArrayType> return_value; for (unsigned int comp = 0; comp < n_components; ++comp) return_value[comp] = this->values_quad[comp][0]; const unsigned int n_q_points = this->n_quadrature_points; @@ -5624,16 +5925,25 @@ FEEvaluationBase::integrate_value() const /*----------------------- FEEvaluationAccess --------------------------------*/ -template -inline FEEvaluationAccess:: - FEEvaluationAccess(const MatrixFree &data_in, - const unsigned int dof_no, - const unsigned int first_selected_component, - const unsigned int quad_no_in, - const unsigned int fe_degree, - const unsigned int n_q_points, - const bool is_interior_face) - : FEEvaluationBase( +template +inline FEEvaluationAccess:: + FEEvaluationAccess( + const MatrixFree &data_in, + const unsigned int dof_no, + const unsigned int first_selected_component, + const unsigned int quad_no_in, + const unsigned int fe_degree, + const unsigned int n_q_points, + const bool is_interior_face) + : FEEvaluationBase( data_in, dof_no, first_selected_component, @@ -5645,17 +5955,28 @@ inline FEEvaluationAccess:: -template +template template -inline FEEvaluationAccess:: - FEEvaluationAccess( - const Mapping & mapping, - const FiniteElement &fe, - const Quadrature<1> & quadrature, - const UpdateFlags update_flags, - const unsigned int first_selected_component, - const FEEvaluationBase *other) - : FEEvaluationBase( +inline FEEvaluationAccess:: + FEEvaluationAccess(const Mapping & mapping, + const FiniteElement &fe, + const Quadrature<1> & quadrature, + const UpdateFlags update_flags, + const unsigned int first_selected_component, + const FEEvaluationBase *other) + : FEEvaluationBase( mapping, fe, quadrature, @@ -5666,21 +5987,49 @@ inline FEEvaluationAccess:: -template -inline FEEvaluationAccess:: - FEEvaluationAccess( - const FEEvaluationAccess &other) - : FEEvaluationBase(other) +template +inline FEEvaluationAccess:: + FEEvaluationAccess(const FEEvaluationAccess &other) + : FEEvaluationBase( + other) {} -template -inline FEEvaluationAccess & -FEEvaluationAccess:: -operator=(const FEEvaluationAccess &other) +template +inline FEEvaluationAccess & +FEEvaluationAccess:: +operator=(const FEEvaluationAccess &other) { - this->FEEvaluationBase::operator=(other); + this->FEEvaluationBase::operator=(other); return *this; } @@ -5689,67 +6038,77 @@ operator=(const FEEvaluationAccess &other) /*-------------------- FEEvaluationAccess scalar ----------------------------*/ -template -inline FEEvaluationAccess::FEEvaluationAccess( - const MatrixFree &data_in, - const unsigned int dof_no, - const unsigned int first_selected_component, - const unsigned int quad_no_in, - const unsigned int fe_degree, - const unsigned int n_q_points, - const bool is_interior_face) - : FEEvaluationBase(data_in, - dof_no, - first_selected_component, - quad_no_in, - fe_degree, - n_q_points, - is_interior_face) +template +inline FEEvaluationAccess:: + FEEvaluationAccess( + const MatrixFree &data_in, + const unsigned int dof_no, + const unsigned int first_selected_component, + const unsigned int quad_no_in, + const unsigned int fe_degree, + const unsigned int n_q_points, + const bool is_interior_face) + : FEEvaluationBase( + data_in, + dof_no, + first_selected_component, + quad_no_in, + fe_degree, + n_q_points, + is_interior_face) {} -template +template template -inline FEEvaluationAccess::FEEvaluationAccess( - const Mapping & mapping, - const FiniteElement &fe, - const Quadrature<1> & quadrature, - const UpdateFlags update_flags, - const unsigned int first_selected_component, - const FEEvaluationBase *other) - : FEEvaluationBase(mapping, - fe, - quadrature, - update_flags, - first_selected_component, - other) +inline FEEvaluationAccess:: + FEEvaluationAccess(const Mapping & mapping, + const FiniteElement &fe, + const Quadrature<1> & quadrature, + const UpdateFlags update_flags, + const unsigned int first_selected_component, + const FEEvaluationBase *other) + : FEEvaluationBase( + mapping, + fe, + quadrature, + update_flags, + first_selected_component, + other) {} -template -inline FEEvaluationAccess::FEEvaluationAccess( - const FEEvaluationAccess &other) - : FEEvaluationBase(other) +template +inline FEEvaluationAccess:: + FEEvaluationAccess( + const FEEvaluationAccess + &other) + : FEEvaluationBase(other) {} -template -inline FEEvaluationAccess & -FEEvaluationAccess:: -operator=(const FEEvaluationAccess &other) +template +inline FEEvaluationAccess & +FEEvaluationAccess::operator=( + const FEEvaluationAccess &other) { - this->FEEvaluationBase::operator=(other); + this->FEEvaluationBase:: + operator=(other); return *this; } -template -inline DEAL_II_ALWAYS_INLINE VectorizedArray - FEEvaluationAccess::get_dof_value( +template +inline DEAL_II_ALWAYS_INLINE VectorizedArrayType + FEEvaluationAccess::get_dof_value( const unsigned int dof) const { AssertIndexRange(dof, this->data->dofs_per_component_on_cell); @@ -5758,9 +6117,9 @@ inline DEAL_II_ALWAYS_INLINE VectorizedArray -template -inline DEAL_II_ALWAYS_INLINE VectorizedArray - FEEvaluationAccess::get_value( +template +inline DEAL_II_ALWAYS_INLINE VectorizedArrayType + FEEvaluationAccess::get_value( const unsigned int q_point) const { Assert(this->values_quad_initialized == true, @@ -5771,19 +6130,19 @@ inline DEAL_II_ALWAYS_INLINE VectorizedArray -template -inline DEAL_II_ALWAYS_INLINE VectorizedArray - FEEvaluationAccess::get_normal_derivative( - const unsigned int q_point) const +template +inline DEAL_II_ALWAYS_INLINE VectorizedArrayType + FEEvaluationAccess:: + get_normal_derivative(const unsigned int q_point) const { return BaseClass::get_normal_derivative(q_point)[0]; } -template -inline DEAL_II_ALWAYS_INLINE Tensor<1, dim, VectorizedArray> - FEEvaluationAccess::get_gradient( +template +inline DEAL_II_ALWAYS_INLINE Tensor<1, dim, VectorizedArrayType> + FEEvaluationAccess::get_gradient( const unsigned int q_point) const { // could use the base class gradient, but that involves too many expensive @@ -5795,7 +6154,7 @@ inline DEAL_II_ALWAYS_INLINE Tensor<1, dim, VectorizedArray> Assert(this->jacobian != nullptr, ExcNotInitialized()); - Tensor<1, dim, VectorizedArray> grad_out; + Tensor<1, dim, VectorizedArrayType> grad_out; if (!is_face && this->cell_type == internal::MatrixFreeFunctions::cartesian) { @@ -5806,7 +6165,7 @@ inline DEAL_II_ALWAYS_INLINE Tensor<1, dim, VectorizedArray> // cell with general/affine Jacobian else { - const Tensor<2, dim, VectorizedArray> &jac = + const Tensor<2, dim, VectorizedArrayType> &jac = this->jacobian[this->cell_type > internal::MatrixFreeFunctions::affine ? q_point : 0]; @@ -5822,9 +6181,9 @@ inline DEAL_II_ALWAYS_INLINE Tensor<1, dim, VectorizedArray> -template -inline Tensor<2, dim, VectorizedArray> -FEEvaluationAccess::get_hessian( +template +inline Tensor<2, dim, VectorizedArrayType> +FEEvaluationAccess::get_hessian( const unsigned int q_point) const { return BaseClass::get_hessian(q_point)[0]; @@ -5832,19 +6191,19 @@ FEEvaluationAccess::get_hessian( -template -inline Tensor<1, dim, VectorizedArray> -FEEvaluationAccess::get_hessian_diagonal( - const unsigned int q_point) const +template +inline Tensor<1, dim, VectorizedArrayType> +FEEvaluationAccess:: + get_hessian_diagonal(const unsigned int q_point) const { return BaseClass::get_hessian_diagonal(q_point)[0]; } -template -inline VectorizedArray -FEEvaluationAccess::get_laplacian( +template +inline VectorizedArrayType +FEEvaluationAccess::get_laplacian( const unsigned int q_point) const { return BaseClass::get_laplacian(q_point)[0]; @@ -5852,11 +6211,10 @@ FEEvaluationAccess::get_laplacian( -template +template inline void DEAL_II_ALWAYS_INLINE - FEEvaluationAccess::submit_dof_value( - const VectorizedArray val_in, - const unsigned int dof) + FEEvaluationAccess:: + submit_dof_value(const VectorizedArrayType val_in, const unsigned int dof) { # ifdef DEBUG this->dof_values_initialized = true; @@ -5867,11 +6225,11 @@ inline void DEAL_II_ALWAYS_INLINE -template +template inline void DEAL_II_ALWAYS_INLINE - FEEvaluationAccess::submit_value( - const VectorizedArray val_in, - const unsigned int q_index) + FEEvaluationAccess::submit_value( + const VectorizedArrayType val_in, + const unsigned int q_index) { # ifdef DEBUG Assert(this->cell != numbers::invalid_unsigned_int, ExcNotInitialized()); @@ -5881,7 +6239,7 @@ inline void DEAL_II_ALWAYS_INLINE # endif if (this->cell_type <= internal::MatrixFreeFunctions::affine) { - const VectorizedArray JxW = + const VectorizedArrayType JxW = this->J_value[0] * this->quadrature_weights[q_index]; this->values_quad[0][q_index] = val_in * JxW; } @@ -5893,35 +6251,35 @@ inline void DEAL_II_ALWAYS_INLINE -template +template inline DEAL_II_ALWAYS_INLINE void -FEEvaluationAccess::submit_value( - const Tensor<1, 1, VectorizedArray> val_in, - const unsigned int q_point) +FEEvaluationAccess::submit_value( + const Tensor<1, 1, VectorizedArrayType> val_in, + const unsigned int q_point) { submit_value(val_in[0], q_point); } -template +template inline DEAL_II_ALWAYS_INLINE void -FEEvaluationAccess::submit_normal_derivative( - const VectorizedArray grad_in, - const unsigned int q_point) +FEEvaluationAccess:: + submit_normal_derivative(const VectorizedArrayType grad_in, + const unsigned int q_point) { - Tensor<1, 1, VectorizedArray> grad; + Tensor<1, 1, VectorizedArrayType> grad; grad[0] = grad_in; BaseClass::submit_normal_derivative(grad, q_point); } -template +template inline DEAL_II_ALWAYS_INLINE void -FEEvaluationAccess::submit_gradient( - const Tensor<1, dim, VectorizedArray> grad_in, - const unsigned int q_index) +FEEvaluationAccess:: + submit_gradient(const Tensor<1, dim, VectorizedArrayType> grad_in, + const unsigned int q_index) { # ifdef DEBUG Assert(this->cell != numbers::invalid_unsigned_int, ExcNotInitialized()); @@ -5933,7 +6291,7 @@ FEEvaluationAccess::submit_gradient( if (!is_face && this->cell_type == internal::MatrixFreeFunctions::cartesian) { - const VectorizedArray JxW = + const VectorizedArrayType JxW = this->J_value[0] * this->quadrature_weights[q_index]; for (unsigned int d = 0; d < dim; ++d) this->gradients_quad[0][d][q_index] = @@ -5942,17 +6300,17 @@ FEEvaluationAccess::submit_gradient( // general/affine cell type else { - const Tensor<2, dim, VectorizedArray> &jac = + const Tensor<2, dim, VectorizedArrayType> &jac = this->cell_type > internal::MatrixFreeFunctions::affine ? this->jacobian[q_index] : this->jacobian[0]; - const VectorizedArray JxW = + const VectorizedArrayType JxW = this->cell_type > internal::MatrixFreeFunctions::affine ? this->J_value[q_index] : this->J_value[0] * this->quadrature_weights[q_index]; for (unsigned int d = 0; d < dim; ++d) { - VectorizedArray new_val = jac[0][d] * grad_in[0]; + VectorizedArrayType new_val = jac[0][d] * grad_in[0]; for (unsigned int e = 1; e < dim; ++e) new_val += jac[e][d] * grad_in[e]; this->gradients_quad[0][d][q_index] = new_val * JxW; @@ -5962,9 +6320,10 @@ FEEvaluationAccess::submit_gradient( -template -inline VectorizedArray -FEEvaluationAccess::integrate_value() const +template +inline VectorizedArrayType +FEEvaluationAccess:: + integrate_value() const { return BaseClass::integrate_value()[0]; } @@ -5974,85 +6333,96 @@ FEEvaluationAccess::integrate_value() const /*----------------- FEEvaluationAccess vector-valued ------------------------*/ -template -inline FEEvaluationAccess::FEEvaluationAccess( - const MatrixFree &data_in, - const unsigned int dof_no, - const unsigned int first_selected_component, - const unsigned int quad_no_in, - const unsigned int fe_degree, - const unsigned int n_q_points, - const bool is_interior_face) - : FEEvaluationBase(data_in, - dof_no, - first_selected_component, - quad_no_in, - fe_degree, - n_q_points, - is_interior_face) +template +inline FEEvaluationAccess:: + FEEvaluationAccess( + const MatrixFree &data_in, + const unsigned int dof_no, + const unsigned int first_selected_component, + const unsigned int quad_no_in, + const unsigned int fe_degree, + const unsigned int n_q_points, + const bool is_interior_face) + : FEEvaluationBase( + data_in, + dof_no, + first_selected_component, + quad_no_in, + fe_degree, + n_q_points, + is_interior_face) {} -template +template template -inline FEEvaluationAccess::FEEvaluationAccess( - const Mapping & mapping, - const FiniteElement &fe, - const Quadrature<1> & quadrature, - const UpdateFlags update_flags, - const unsigned int first_selected_component, - const FEEvaluationBase *other) - : FEEvaluationBase(mapping, - fe, - quadrature, - update_flags, - first_selected_component, - other) +inline FEEvaluationAccess:: + FEEvaluationAccess(const Mapping & mapping, + const FiniteElement &fe, + const Quadrature<1> & quadrature, + const UpdateFlags update_flags, + const unsigned int first_selected_component, + const FEEvaluationBase *other) + : FEEvaluationBase( + mapping, + fe, + quadrature, + update_flags, + first_selected_component, + other) {} -template -inline FEEvaluationAccess::FEEvaluationAccess( - const FEEvaluationAccess &other) - : FEEvaluationBase(other) +template +inline FEEvaluationAccess:: + FEEvaluationAccess( + const FEEvaluationAccess + &other) + : FEEvaluationBase(other) {} -template -inline FEEvaluationAccess & -FEEvaluationAccess:: -operator=(const FEEvaluationAccess &other) +template +inline FEEvaluationAccess & +FEEvaluationAccess::operator=( + const FEEvaluationAccess + &other) { - this->FEEvaluationBase::operator=(other); + this->FEEvaluationBase:: + operator=(other); return *this; } -template -inline DEAL_II_ALWAYS_INLINE Tensor<2, dim, VectorizedArray> - FEEvaluationAccess::get_gradient( - const unsigned int q_point) const +template +inline DEAL_II_ALWAYS_INLINE Tensor<2, dim, VectorizedArrayType> + FEEvaluationAccess:: + get_gradient(const unsigned int q_point) const { return BaseClass::get_gradient(q_point); } -template -inline DEAL_II_ALWAYS_INLINE VectorizedArray - FEEvaluationAccess::get_divergence( - const unsigned int q_point) const +template +inline DEAL_II_ALWAYS_INLINE VectorizedArrayType + FEEvaluationAccess:: + get_divergence(const unsigned int q_point) const { Assert(this->gradients_quad_initialized == true, internal::ExcAccessToUninitializedField()); AssertIndexRange(q_point, this->n_quadrature_points); Assert(this->jacobian != nullptr, ExcNotInitialized()); - VectorizedArray divergence; + VectorizedArrayType divergence; // Cartesian cell if (!is_face && this->cell_type == internal::MatrixFreeFunctions::cartesian) @@ -6066,7 +6436,7 @@ inline DEAL_II_ALWAYS_INLINE VectorizedArray // cell with general/constant Jacobian else { - const Tensor<2, dim, VectorizedArray> &jac = + const Tensor<2, dim, VectorizedArrayType> &jac = this->cell_type == internal::MatrixFreeFunctions::general ? this->jacobian[q_point] : this->jacobian[0]; @@ -6082,15 +6452,15 @@ inline DEAL_II_ALWAYS_INLINE VectorizedArray -template -inline DEAL_II_ALWAYS_INLINE SymmetricTensor<2, dim, VectorizedArray> - FEEvaluationAccess::get_symmetric_gradient( - const unsigned int q_point) const +template +inline DEAL_II_ALWAYS_INLINE SymmetricTensor<2, dim, VectorizedArrayType> + FEEvaluationAccess:: + get_symmetric_gradient(const unsigned int q_point) const { // copy from generic function into dim-specialization function - const Tensor<2, dim, VectorizedArray> grad = get_gradient(q_point); - VectorizedArray symmetrized[(dim * dim + dim) / 2]; - VectorizedArray half = make_vectorized_array(0.5); + const Tensor<2, dim, VectorizedArrayType> grad = get_gradient(q_point); + VectorizedArrayType symmetrized[(dim * dim + dim) / 2]; + VectorizedArrayType half = make_vectorized_array(0.5); for (unsigned int d = 0; d < dim; ++d) symmetrized[d] = grad[d][d]; switch (dim) @@ -6112,20 +6482,20 @@ inline DEAL_II_ALWAYS_INLINE SymmetricTensor<2, dim, VectorizedArray> default: Assert(false, ExcNotImplemented()); } - return SymmetricTensor<2, dim, VectorizedArray>(symmetrized); + return SymmetricTensor<2, dim, VectorizedArrayType>(symmetrized); } -template +template inline DEAL_II_ALWAYS_INLINE - Tensor<1, (dim == 2 ? 1 : dim), VectorizedArray> - FEEvaluationAccess::get_curl( + Tensor<1, (dim == 2 ? 1 : dim), VectorizedArrayType> + FEEvaluationAccess::get_curl( const unsigned int q_point) const { // copy from generic function into dim-specialization function - const Tensor<2, dim, VectorizedArray> grad = get_gradient(q_point); - Tensor<1, (dim == 2 ? 1 : dim), VectorizedArray> curl; + const Tensor<2, dim, VectorizedArrayType> grad = get_gradient(q_point); + Tensor<1, (dim == 2 ? 1 : dim), VectorizedArrayType> curl; switch (dim) { case 1: @@ -6149,19 +6519,19 @@ inline DEAL_II_ALWAYS_INLINE -template -inline DEAL_II_ALWAYS_INLINE Tensor<2, dim, VectorizedArray> - FEEvaluationAccess::get_hessian_diagonal( - const unsigned int q_point) const +template +inline DEAL_II_ALWAYS_INLINE Tensor<2, dim, VectorizedArrayType> + FEEvaluationAccess:: + get_hessian_diagonal(const unsigned int q_point) const { return BaseClass::get_hessian_diagonal(q_point); } -template -inline DEAL_II_ALWAYS_INLINE Tensor<3, dim, VectorizedArray> - FEEvaluationAccess::get_hessian( +template +inline DEAL_II_ALWAYS_INLINE Tensor<3, dim, VectorizedArrayType> + FEEvaluationAccess::get_hessian( const unsigned int q_point) const { Assert(this->hessians_quad_initialized == true, @@ -6172,33 +6542,34 @@ inline DEAL_II_ALWAYS_INLINE Tensor<3, dim, VectorizedArray> -template +template inline DEAL_II_ALWAYS_INLINE void -FEEvaluationAccess::submit_gradient( - const Tensor<2, dim, VectorizedArray> grad_in, - const unsigned int q_point) +FEEvaluationAccess:: + submit_gradient(const Tensor<2, dim, VectorizedArrayType> grad_in, + const unsigned int q_point) { BaseClass::submit_gradient(grad_in, q_point); } -template +template inline DEAL_II_ALWAYS_INLINE void -FEEvaluationAccess::submit_gradient( - const Tensor<1, dim, Tensor<1, dim, VectorizedArray>> grad_in, - const unsigned int q_point) +FEEvaluationAccess:: + submit_gradient( + const Tensor<1, dim, Tensor<1, dim, VectorizedArrayType>> grad_in, + const unsigned int q_point) { BaseClass::submit_gradient(grad_in, q_point); } -template +template inline DEAL_II_ALWAYS_INLINE void -FEEvaluationAccess::submit_divergence( - const VectorizedArray div_in, - const unsigned int q_point) +FEEvaluationAccess:: + submit_divergence(const VectorizedArrayType div_in, + const unsigned int q_point) { # ifdef DEBUG Assert(this->cell != numbers::invalid_unsigned_int, ExcNotInitialized()); @@ -6210,25 +6581,25 @@ FEEvaluationAccess::submit_divergence( if (!is_face && this->cell_type == internal::MatrixFreeFunctions::cartesian) { - const VectorizedArray fac = + const VectorizedArrayType fac = this->J_value[0] * this->quadrature_weights[q_point] * div_in; for (unsigned int d = 0; d < dim; ++d) { this->gradients_quad[d][d][q_point] = (fac * this->jacobian[0][d][d]); for (unsigned int e = d + 1; e < dim; ++e) { - this->gradients_quad[d][e][q_point] = VectorizedArray(); - this->gradients_quad[e][d][q_point] = VectorizedArray(); + this->gradients_quad[d][e][q_point] = VectorizedArrayType(); + this->gradients_quad[e][d][q_point] = VectorizedArrayType(); } } } else { - const Tensor<2, dim, VectorizedArray> &jac = + const Tensor<2, dim, VectorizedArrayType> &jac = this->cell_type == internal::MatrixFreeFunctions::general ? this->jacobian[q_point] : this->jacobian[0]; - const VectorizedArray fac = + const VectorizedArrayType fac = (this->cell_type == internal::MatrixFreeFunctions::general ? this->J_value[q_point] : this->J_value[0] * this->quadrature_weights[q_point]) * @@ -6243,11 +6614,12 @@ FEEvaluationAccess::submit_divergence( -template +template inline DEAL_II_ALWAYS_INLINE void -FEEvaluationAccess::submit_symmetric_gradient( - const SymmetricTensor<2, dim, VectorizedArray> sym_grad, - const unsigned int q_point) +FEEvaluationAccess:: + submit_symmetric_gradient( + const SymmetricTensor<2, dim, VectorizedArrayType> sym_grad, + const unsigned int q_point) { // could have used base class operator, but that involves some overhead // which is inefficient. it is nice to have the symmetric tensor because @@ -6262,7 +6634,7 @@ FEEvaluationAccess::submit_symmetric_gradient( if (!is_face && this->cell_type == internal::MatrixFreeFunctions::cartesian) { - const VectorizedArray JxW = + const VectorizedArrayType JxW = this->J_value[0] * this->quadrature_weights[q_point]; for (unsigned int d = 0; d < dim; ++d) this->gradients_quad[d][d][q_point] = @@ -6270,7 +6642,7 @@ FEEvaluationAccess::submit_symmetric_gradient( for (unsigned int e = 0, counter = dim; e < dim; ++e) for (unsigned int d = e + 1; d < dim; ++d, ++counter) { - const VectorizedArray value = + const VectorizedArrayType value = sym_grad.access_raw_entry(counter) * JxW; this->gradients_quad[e][d][q_point] = (value * this->jacobian[0][d][d]); @@ -6281,21 +6653,21 @@ FEEvaluationAccess::submit_symmetric_gradient( // general/affine cell type else { - const VectorizedArray JxW = + const VectorizedArrayType JxW = this->cell_type == internal::MatrixFreeFunctions::general ? this->J_value[q_point] : this->J_value[0] * this->quadrature_weights[q_point]; - const Tensor<2, dim, VectorizedArray> &jac = + const Tensor<2, dim, VectorizedArrayType> &jac = this->cell_type == internal::MatrixFreeFunctions::general ? this->jacobian[q_point] : this->jacobian[0]; - VectorizedArray weighted[dim][dim]; + VectorizedArrayType weighted[dim][dim]; for (unsigned int i = 0; i < dim; ++i) weighted[i][i] = sym_grad.access_raw_entry(i) * JxW; for (unsigned int i = 0, counter = dim; i < dim; ++i) for (unsigned int j = i + 1; j < dim; ++j, ++counter) { - const VectorizedArray value = + const VectorizedArrayType value = sym_grad.access_raw_entry(counter) * JxW; weighted[i][j] = value; weighted[j][i] = value; @@ -6303,7 +6675,7 @@ FEEvaluationAccess::submit_symmetric_gradient( for (unsigned int comp = 0; comp < dim; ++comp) for (unsigned int d = 0; d < dim; ++d) { - VectorizedArray new_val = jac[0][d] * weighted[comp][0]; + VectorizedArrayType new_val = jac[0][d] * weighted[comp][0]; for (unsigned int e = 1; e < dim; ++e) new_val += jac[e][d] * weighted[comp][e]; this->gradients_quad[comp][d][q_point] = new_val; @@ -6313,13 +6685,13 @@ FEEvaluationAccess::submit_symmetric_gradient( -template +template inline DEAL_II_ALWAYS_INLINE void -FEEvaluationAccess::submit_curl( - const Tensor<1, dim == 2 ? 1 : dim, VectorizedArray> curl, - const unsigned int q_point) +FEEvaluationAccess::submit_curl( + const Tensor<1, dim == 2 ? 1 : dim, VectorizedArrayType> curl, + const unsigned int q_point) { - Tensor<2, dim, VectorizedArray> grad; + Tensor<2, dim, VectorizedArrayType> grad; switch (dim) { case 1: @@ -6349,67 +6721,75 @@ FEEvaluationAccess::submit_curl( /*-------------------- FEEvaluationAccess scalar for 1d ---------------------*/ -template -inline FEEvaluationAccess<1, 1, Number, is_face>::FEEvaluationAccess( - const MatrixFree<1, Number> &data_in, - const unsigned int dof_no, - const unsigned int first_selected_component, - const unsigned int quad_no_in, - const unsigned int fe_degree, - const unsigned int n_q_points, - const bool is_interior_face) - : FEEvaluationBase<1, 1, Number, is_face>(data_in, - dof_no, - first_selected_component, - quad_no_in, - fe_degree, - n_q_points, - is_interior_face) +template +inline FEEvaluationAccess<1, 1, Number, is_face, VectorizedArrayType>:: + FEEvaluationAccess(const MatrixFree<1, Number, VectorizedArrayType> &data_in, + const unsigned int dof_no, + const unsigned int first_selected_component, + const unsigned int quad_no_in, + const unsigned int fe_degree, + const unsigned int n_q_points, + const bool is_interior_face) + : FEEvaluationBase<1, 1, Number, is_face, VectorizedArrayType>( + data_in, + dof_no, + first_selected_component, + quad_no_in, + fe_degree, + n_q_points, + is_interior_face) {} -template +template template -inline FEEvaluationAccess<1, 1, Number, is_face>::FEEvaluationAccess( - const Mapping<1> & mapping, - const FiniteElement<1> &fe, - const Quadrature<1> & quadrature, - const UpdateFlags update_flags, - const unsigned int first_selected_component, - const FEEvaluationBase<1, n_components_other, Number, is_face> *other) - : FEEvaluationBase<1, 1, Number, is_face>(mapping, - fe, - quadrature, - update_flags, - first_selected_component, - other) +inline FEEvaluationAccess<1, 1, Number, is_face, VectorizedArrayType>:: + FEEvaluationAccess(const Mapping<1> & mapping, + const FiniteElement<1> &fe, + const Quadrature<1> & quadrature, + const UpdateFlags update_flags, + const unsigned int first_selected_component, + const FEEvaluationBase<1, + n_components_other, + Number, + is_face, + VectorizedArrayType> *other) + : FEEvaluationBase<1, 1, Number, is_face, VectorizedArrayType>( + mapping, + fe, + quadrature, + update_flags, + first_selected_component, + other) {} -template -inline FEEvaluationAccess<1, 1, Number, is_face>::FEEvaluationAccess( - const FEEvaluationAccess<1, 1, Number, is_face> &other) - : FEEvaluationBase<1, 1, Number, is_face>(other) +template +inline FEEvaluationAccess<1, 1, Number, is_face, VectorizedArrayType>:: + FEEvaluationAccess( + const FEEvaluationAccess<1, 1, Number, is_face, VectorizedArrayType> &other) + : FEEvaluationBase<1, 1, Number, is_face, VectorizedArrayType>(other) {} -template -inline FEEvaluationAccess<1, 1, Number, is_face> & -FEEvaluationAccess<1, 1, Number, is_face>:: -operator=(const FEEvaluationAccess<1, 1, Number, is_face> &other) +template +inline FEEvaluationAccess<1, 1, Number, is_face, VectorizedArrayType> & +FEEvaluationAccess<1, 1, Number, is_face, VectorizedArrayType>::operator=( + const FEEvaluationAccess<1, 1, Number, is_face, VectorizedArrayType> &other) { - this->FEEvaluationBase<1, 1, Number, is_face>::operator=(other); + this->FEEvaluationBase<1, 1, Number, is_face, VectorizedArrayType>::operator=( + other); return *this; } -template -inline DEAL_II_ALWAYS_INLINE VectorizedArray - FEEvaluationAccess<1, 1, Number, is_face>::get_dof_value( +template +inline DEAL_II_ALWAYS_INLINE VectorizedArrayType + FEEvaluationAccess<1, 1, Number, is_face, VectorizedArrayType>::get_dof_value( const unsigned int dof) const { AssertIndexRange(dof, this->data->dofs_per_component_on_cell); @@ -6418,9 +6798,9 @@ inline DEAL_II_ALWAYS_INLINE VectorizedArray -template -inline DEAL_II_ALWAYS_INLINE VectorizedArray - FEEvaluationAccess<1, 1, Number, is_face>::get_value( +template +inline DEAL_II_ALWAYS_INLINE VectorizedArrayType + FEEvaluationAccess<1, 1, Number, is_face, VectorizedArrayType>::get_value( const unsigned int q_point) const { Assert(this->values_quad_initialized == true, @@ -6431,9 +6811,9 @@ inline DEAL_II_ALWAYS_INLINE VectorizedArray -template -inline DEAL_II_ALWAYS_INLINE Tensor<1, 1, VectorizedArray> - FEEvaluationAccess<1, 1, Number, is_face>::get_gradient( +template +inline DEAL_II_ALWAYS_INLINE Tensor<1, 1, VectorizedArrayType> + FEEvaluationAccess<1, 1, Number, is_face, VectorizedArrayType>::get_gradient( const unsigned int q_point) const { // could use the base class gradient, but that involves too many inefficient @@ -6443,12 +6823,12 @@ inline DEAL_II_ALWAYS_INLINE Tensor<1, 1, VectorizedArray> internal::ExcAccessToUninitializedField()); AssertIndexRange(q_point, this->n_quadrature_points); - const Tensor<2, 1, VectorizedArray> &jac = + const Tensor<2, 1, VectorizedArrayType> &jac = this->cell_type == internal::MatrixFreeFunctions::general ? this->jacobian[q_point] : this->jacobian[0]; - Tensor<1, 1, VectorizedArray> grad_out; + Tensor<1, 1, VectorizedArrayType> grad_out; grad_out[0] = jac[0][0] * this->gradients_quad[0][0][q_point]; return grad_out; @@ -6456,19 +6836,19 @@ inline DEAL_II_ALWAYS_INLINE Tensor<1, 1, VectorizedArray> -template -inline DEAL_II_ALWAYS_INLINE VectorizedArray - FEEvaluationAccess<1, 1, Number, is_face>::get_normal_derivative( - const unsigned int q_point) const +template +inline DEAL_II_ALWAYS_INLINE VectorizedArrayType + FEEvaluationAccess<1, 1, Number, is_face, VectorizedArrayType>:: + get_normal_derivative(const unsigned int q_point) const { return BaseClass::get_normal_derivative(q_point)[0]; } -template -inline DEAL_II_ALWAYS_INLINE Tensor<2, 1, VectorizedArray> - FEEvaluationAccess<1, 1, Number, is_face>::get_hessian( +template +inline DEAL_II_ALWAYS_INLINE Tensor<2, 1, VectorizedArrayType> + FEEvaluationAccess<1, 1, Number, is_face, VectorizedArrayType>::get_hessian( const unsigned int q_point) const { return BaseClass::get_hessian(q_point)[0]; @@ -6476,19 +6856,19 @@ inline DEAL_II_ALWAYS_INLINE Tensor<2, 1, VectorizedArray> -template -inline DEAL_II_ALWAYS_INLINE Tensor<1, 1, VectorizedArray> - FEEvaluationAccess<1, 1, Number, is_face>::get_hessian_diagonal( - const unsigned int q_point) const +template +inline DEAL_II_ALWAYS_INLINE Tensor<1, 1, VectorizedArrayType> + FEEvaluationAccess<1, 1, Number, is_face, VectorizedArrayType>:: + get_hessian_diagonal(const unsigned int q_point) const { return BaseClass::get_hessian_diagonal(q_point)[0]; } -template -inline DEAL_II_ALWAYS_INLINE VectorizedArray - FEEvaluationAccess<1, 1, Number, is_face>::get_laplacian( +template +inline DEAL_II_ALWAYS_INLINE VectorizedArrayType + FEEvaluationAccess<1, 1, Number, is_face, VectorizedArrayType>::get_laplacian( const unsigned int q_point) const { return BaseClass::get_laplacian(q_point)[0]; @@ -6496,11 +6876,10 @@ inline DEAL_II_ALWAYS_INLINE VectorizedArray -template +template inline DEAL_II_ALWAYS_INLINE void DEAL_II_ALWAYS_INLINE - FEEvaluationAccess<1, 1, Number, is_face>::submit_dof_value( - const VectorizedArray val_in, - const unsigned int dof) + FEEvaluationAccess<1, 1, Number, is_face, VectorizedArrayType>:: + submit_dof_value(const VectorizedArrayType val_in, const unsigned int dof) { # ifdef DEBUG this->dof_values_initialized = true; @@ -6511,11 +6890,11 @@ inline DEAL_II_ALWAYS_INLINE void DEAL_II_ALWAYS_INLINE -template +template inline DEAL_II_ALWAYS_INLINE void -FEEvaluationAccess<1, 1, Number, is_face>::submit_value( - const VectorizedArray val_in, - const unsigned int q_point) +FEEvaluationAccess<1, 1, Number, is_face, VectorizedArrayType>::submit_value( + const VectorizedArrayType val_in, + const unsigned int q_point) { # ifdef DEBUG Assert(this->cell != numbers::invalid_unsigned_int, ExcNotInitialized()); @@ -6524,12 +6903,12 @@ FEEvaluationAccess<1, 1, Number, is_face>::submit_value( # endif if (this->cell_type == internal::MatrixFreeFunctions::general) { - const VectorizedArray JxW = this->J_value[q_point]; - this->values_quad[0][q_point] = val_in * JxW; + const VectorizedArrayType JxW = this->J_value[q_point]; + this->values_quad[0][q_point] = val_in * JxW; } else // if (this->cell_type == internal::MatrixFreeFunctions::general) { - const VectorizedArray JxW = + const VectorizedArrayType JxW = this->J_value[0] * this->quadrature_weights[q_point]; this->values_quad[0][q_point] = val_in * JxW; } @@ -6537,33 +6916,33 @@ FEEvaluationAccess<1, 1, Number, is_face>::submit_value( -template +template inline DEAL_II_ALWAYS_INLINE void -FEEvaluationAccess<1, 1, Number, is_face>::submit_value( - const Tensor<1, 1, VectorizedArray> val_in, - const unsigned int q_point) +FEEvaluationAccess<1, 1, Number, is_face, VectorizedArrayType>::submit_value( + const Tensor<1, 1, VectorizedArrayType> val_in, + const unsigned int q_point) { submit_value(val_in[0], q_point); } -template +template inline DEAL_II_ALWAYS_INLINE void -FEEvaluationAccess<1, 1, Number, is_face>::submit_gradient( - const Tensor<1, 1, VectorizedArray> grad_in, - const unsigned int q_point) +FEEvaluationAccess<1, 1, Number, is_face, VectorizedArrayType>::submit_gradient( + const Tensor<1, 1, VectorizedArrayType> grad_in, + const unsigned int q_point) { submit_gradient(grad_in[0], q_point); } -template +template inline DEAL_II_ALWAYS_INLINE void -FEEvaluationAccess<1, 1, Number, is_face>::submit_gradient( - const VectorizedArray grad_in, - const unsigned int q_point) +FEEvaluationAccess<1, 1, Number, is_face, VectorizedArrayType>::submit_gradient( + const VectorizedArrayType grad_in, + const unsigned int q_point) { # ifdef DEBUG Assert(this->cell != numbers::invalid_unsigned_int, ExcNotInitialized()); @@ -6571,11 +6950,11 @@ FEEvaluationAccess<1, 1, Number, is_face>::submit_gradient( this->gradients_quad_submitted = true; # endif - const Tensor<2, 1, VectorizedArray> &jac = + const Tensor<2, 1, VectorizedArrayType> &jac = this->cell_type == internal::MatrixFreeFunctions::general ? this->jacobian[q_point] : this->jacobian[0]; - const VectorizedArray JxW = + const VectorizedArrayType JxW = this->cell_type == internal::MatrixFreeFunctions::general ? this->J_value[q_point] : this->J_value[0] * this->quadrature_weights[q_point]; @@ -6585,33 +6964,34 @@ FEEvaluationAccess<1, 1, Number, is_face>::submit_gradient( -template +template inline DEAL_II_ALWAYS_INLINE void -FEEvaluationAccess<1, 1, Number, is_face>::submit_normal_derivative( - const VectorizedArray grad_in, - const unsigned int q_point) +FEEvaluationAccess<1, 1, Number, is_face, VectorizedArrayType>:: + submit_normal_derivative(const VectorizedArrayType grad_in, + const unsigned int q_point) { - Tensor<1, 1, VectorizedArray> grad; + Tensor<1, 1, VectorizedArrayType> grad; grad[0] = grad_in; BaseClass::submit_normal_derivative(grad, q_point); } -template +template inline DEAL_II_ALWAYS_INLINE void -FEEvaluationAccess<1, 1, Number, is_face>::submit_normal_derivative( - const Tensor<1, 1, VectorizedArray> grad_in, - const unsigned int q_point) +FEEvaluationAccess<1, 1, Number, is_face, VectorizedArrayType>:: + submit_normal_derivative(const Tensor<1, 1, VectorizedArrayType> grad_in, + const unsigned int q_point) { BaseClass::submit_normal_derivative(grad_in, q_point); } -template -inline VectorizedArray -FEEvaluationAccess<1, 1, Number, is_face>::integrate_value() const +template +inline VectorizedArrayType +FEEvaluationAccess<1, 1, Number, is_face, VectorizedArrayType>:: + integrate_value() const { return BaseClass::integrate_value()[0]; } @@ -6625,12 +7005,18 @@ template -inline FEEvaluation:: - FEEvaluation(const MatrixFree &data_in, - const unsigned int fe_no, - const unsigned int quad_no, - const unsigned int first_selected_component) + typename Number, + typename VectorizedArrayType> +inline FEEvaluation:: + FEEvaluation(const MatrixFree &data_in, + const unsigned int fe_no, + const unsigned int quad_no, + const unsigned int first_selected_component) : BaseClass(data_in, fe_no, first_selected_component, @@ -6650,8 +7036,14 @@ template -inline FEEvaluation:: + typename Number, + typename VectorizedArrayType> +inline FEEvaluation:: FEEvaluation(const Mapping & mapping, const FiniteElement &fe, const Quadrature<1> & quadrature, @@ -6662,7 +7054,9 @@ inline FEEvaluation:: quadrature, update_flags, first_selected_component, - static_cast *>(nullptr)) + static_cast< + FEEvaluationBase *>( + nullptr)) , dofs_per_component(this->data->dofs_per_component_on_cell) , dofs_per_cell(this->data->dofs_per_component_on_cell * n_components_) , n_q_points(this->data->n_q_points) @@ -6676,8 +7070,14 @@ template -inline FEEvaluation:: + typename Number, + typename VectorizedArrayType> +inline FEEvaluation:: FEEvaluation(const FiniteElement &fe, const Quadrature<1> & quadrature, const UpdateFlags update_flags, @@ -6687,7 +7087,9 @@ inline FEEvaluation:: quadrature, update_flags, first_selected_component, - static_cast *>(nullptr)) + static_cast< + FEEvaluationBase *>( + nullptr)) , dofs_per_component(this->data->dofs_per_component_on_cell) , dofs_per_cell(this->data->dofs_per_component_on_cell * n_components_) , n_q_points(this->data->n_q_points) @@ -6701,11 +7103,21 @@ template + typename Number, + typename VectorizedArrayType> template -inline FEEvaluation:: - FEEvaluation(const FiniteElement & fe, - const FEEvaluationBase &other, +inline FEEvaluation:: + FEEvaluation(const FiniteElement & fe, + const FEEvaluationBase &other, const unsigned int first_selected_component) : BaseClass(other.mapped_geometry->get_fe_values().get_mapping(), fe, @@ -6726,9 +7138,15 @@ template -inline FEEvaluation:: - FEEvaluation(const FEEvaluation &other) + typename Number, + typename VectorizedArrayType> +inline FEEvaluation::FEEvaluation(const FEEvaluation + &other) : BaseClass(other) , dofs_per_component(this->data->dofs_per_component_on_cell) , dofs_per_cell(this->data->dofs_per_component_on_cell * n_components_) @@ -6743,10 +7161,20 @@ template -inline FEEvaluation & -FEEvaluation:: -operator=(const FEEvaluation &other) + typename Number, + typename VectorizedArrayType> +inline FEEvaluation & +FEEvaluation::operator=(const FEEvaluation &other) { BaseClass::operator=(other); check_template_arguments(numbers::invalid_unsigned_int, 0); @@ -6759,9 +7187,15 @@ template + typename Number, + typename VectorizedArrayType> inline void -FEEvaluation:: +FEEvaluation:: check_template_arguments(const unsigned int dof_no, const unsigned int first_selected_component) { @@ -6915,10 +7349,15 @@ template + typename Number, + typename VectorizedArrayType> inline void -FEEvaluation::reinit( - const unsigned int cell_index) +FEEvaluation::reinit(const unsigned int cell_index) { Assert(this->mapped_geometry == nullptr, ExcMessage("FEEvaluation was initialized without a matrix-free object." @@ -6951,11 +7390,18 @@ template + typename Number, + typename VectorizedArrayType> template inline void -FEEvaluation::reinit( - const TriaIterator> &cell) +FEEvaluation:: + reinit( + const TriaIterator> &cell) { Assert(this->matrix_info == nullptr, ExcMessage("Cannot use initialization from cell iterator if " @@ -6978,10 +7424,16 @@ template + typename Number, + typename VectorizedArrayType> inline void -FEEvaluation::reinit( - const typename Triangulation::cell_iterator &cell) +FEEvaluation:: + reinit(const typename Triangulation::cell_iterator &cell) { Assert(this->matrix_info == 0, ExcMessage("Cannot use initialization from cell iterator if " @@ -6998,10 +7450,15 @@ template -inline Point> -FEEvaluation:: - quadrature_point(const unsigned int q) const + typename Number, + typename VectorizedArrayType> +inline Point +FEEvaluation::quadrature_point(const unsigned int q) const { if (this->matrix_info == nullptr) { @@ -7023,12 +7480,12 @@ FEEvaluation:: // Cartesian mesh: not all quadrature points are stored, only the // diagonal. Hence, need to find the tensor product index and retrieve the // value from that - const Point> *quadrature_points = + const Point *quadrature_points = &this->mapping_data->quadrature_points [this->mapping_data->quadrature_point_offsets[this->cell]]; if (this->cell_type == internal::MatrixFreeFunctions::cartesian) { - Point> point; + Point point; switch (dim) { case 1: @@ -7060,12 +7517,17 @@ template + typename Number, + typename VectorizedArrayType> inline void -FEEvaluation::evaluate( - const bool evaluate_values, - const bool evaluate_gradients, - const bool evaluate_hessians) +FEEvaluation::evaluate(const bool evaluate_values, + const bool evaluate_gradients, + const bool evaluate_hessians) { Assert(this->dof_values_initialized == true, internal::ExcAccessToUninitializedField()); @@ -7081,29 +7543,35 @@ template + typename Number, + typename VectorizedArrayType> inline void -FEEvaluation::evaluate( - const VectorizedArray *values_array, - const bool evaluate_values, - const bool evaluate_gradients, - const bool evaluate_hessians) +FEEvaluation::evaluate(const VectorizedArrayType + * values_array, + const bool evaluate_values, + const bool evaluate_gradients, + const bool evaluate_hessians) { SelectEvaluator< dim, fe_degree, n_q_points_1d, n_components, - VectorizedArray>::evaluate(*this->data, - const_cast *>( - values_array), - this->values_quad[0], - this->gradients_quad[0][0], - this->hessians_quad[0][0], - this->scratch_data, - evaluate_values, - evaluate_gradients, - evaluate_hessians); + VectorizedArrayType>::evaluate(*this->data, + const_cast( + values_array), + this->values_quad[0], + this->gradients_quad[0][0], + this->hessians_quad[0][0], + this->scratch_data, + evaluate_values, + evaluate_gradients, + evaluate_hessians); # ifdef DEBUG if (evaluate_values == true) @@ -7121,14 +7589,20 @@ template + typename Number, + typename VectorizedArrayType> template inline void -FEEvaluation:: - gather_evaluate(const VectorType &input_vector, - const bool evaluate_values, - const bool evaluate_gradients, - const bool evaluate_hessians) +FEEvaluation< + dim, + fe_degree, + n_q_points_1d, + n_components_, + Number, + VectorizedArrayType>::gather_evaluate(const VectorType &input_vector, + const bool evaluate_values, + const bool evaluate_gradients, + const bool evaluate_hessians) { // If the index storage is interleaved and contiguous and the vector storage // has the correct alignment, we can directly pass the pointer into the @@ -7143,20 +7617,20 @@ FEEvaluation:: input_vector.begin() + this->dof_info->dof_indices_contiguous [internal::MatrixFreeFunctions::DoFInfo::dof_access_cell] - [this->cell * VectorizedArray::n_array_elements]) % - sizeof(VectorizedArray) == + [this->cell * VectorizedArrayType::n_array_elements]) % + sizeof(VectorizedArrayType) == 0) { - const VectorizedArray *vec_values = - reinterpret_cast *>( + const VectorizedArrayType *vec_values = + reinterpret_cast( input_vector.begin() + this->dof_info->dof_indices_contiguous [internal::MatrixFreeFunctions::DoFInfo::dof_access_cell] - [this->cell * VectorizedArray::n_array_elements] + + [this->cell * VectorizedArrayType::n_array_elements] + this->dof_info ->component_dof_indices_offset[this->active_fe_index] [this->first_selected_component] * - VectorizedArray::n_array_elements); + VectorizedArrayType::n_array_elements); evaluate(vec_values, evaluate_values, @@ -7179,11 +7653,16 @@ template + typename Number, + typename VectorizedArrayType> inline void -FEEvaluation::integrate( - const bool integrate_values, - const bool integrate_gradients) +FEEvaluation::integrate(const bool integrate_values, + const bool integrate_gradients) { integrate(integrate_values, integrate_gradients, this->values_dofs[0]); @@ -7198,12 +7677,17 @@ template + typename Number, + typename VectorizedArrayType> inline void -FEEvaluation::integrate( - const bool integrate_values, - const bool integrate_gradients, - VectorizedArray *values_array) +FEEvaluation::integrate(const bool integrate_values, + const bool integrate_gradients, + VectorizedArrayType *values_array) { if (integrate_values == true) Assert(this->values_quad_submitted == true, @@ -7219,15 +7703,14 @@ FEEvaluation::integrate( fe_degree, n_q_points_1d, n_components, - VectorizedArray>::integrate(*this->data, - values_array, - this->values_quad[0], - this - ->gradients_quad[0][0], - this->scratch_data, - integrate_values, - integrate_gradients, - false); + VectorizedArrayType>::integrate(*this->data, + values_array, + this->values_quad[0], + this->gradients_quad[0][0], + this->scratch_data, + integrate_values, + integrate_gradients, + false); # ifdef DEBUG this->dof_values_initialized = true; @@ -7240,13 +7723,19 @@ template + typename Number, + typename VectorizedArrayType> template inline void -FEEvaluation:: - integrate_scatter(const bool integrate_values, - const bool integrate_gradients, - VectorType &destination) +FEEvaluation< + dim, + fe_degree, + n_q_points_1d, + n_components_, + Number, + VectorizedArrayType>::integrate_scatter(const bool integrate_values, + const bool integrate_gradients, + VectorType &destination) { // If the index storage is interleaved and contiguous and the vector storage // has the correct alignment, we can directly pass the pointer into the @@ -7262,33 +7751,32 @@ FEEvaluation:: destination.begin() + this->dof_info->dof_indices_contiguous [internal::MatrixFreeFunctions::DoFInfo::dof_access_cell] - [this->cell * VectorizedArray::n_array_elements]) % - sizeof(VectorizedArray) == + [this->cell * VectorizedArrayType::n_array_elements]) % + sizeof(VectorizedArrayType) == 0) { - VectorizedArray *vec_values = - reinterpret_cast *>( - destination.begin() + - this->dof_info->dof_indices_contiguous - [internal::MatrixFreeFunctions::DoFInfo::dof_access_cell] - [this->cell * VectorizedArray::n_array_elements] + - this->dof_info - ->component_dof_indices_offset[this->active_fe_index] - [this->first_selected_component] * - VectorizedArray::n_array_elements); - SelectEvaluator< - dim, - fe_degree, - n_q_points_1d, - n_components, - VectorizedArray>::integrate(*this->data, - vec_values, - this->values_quad[0], - this->gradients_quad[0][0], - this->scratch_data, - integrate_values, - integrate_gradients, - true); + VectorizedArrayType *vec_values = reinterpret_cast( + destination.begin() + + this->dof_info->dof_indices_contiguous + [internal::MatrixFreeFunctions::DoFInfo::dof_access_cell] + [this->cell * VectorizedArrayType::n_array_elements] + + this->dof_info + ->component_dof_indices_offset[this->active_fe_index] + [this->first_selected_component] * + VectorizedArrayType::n_array_elements); + SelectEvaluator::integrate(*this->data, + vec_values, + this->values_quad[0], + this + ->gradients_quad[0][0], + this->scratch_data, + integrate_values, + integrate_gradients, + true); } else { @@ -7309,13 +7797,20 @@ template -inline FEFaceEvaluation:: - FEFaceEvaluation(const MatrixFree &matrix_free, - const bool is_interior_face, - const unsigned int dof_no, - const unsigned int quad_no, - const unsigned int first_selected_component) + typename Number, + typename VectorizedArrayType> +inline FEFaceEvaluation:: + FEFaceEvaluation( + const MatrixFree &matrix_free, + const bool is_interior_face, + const unsigned int dof_no, + const unsigned int quad_no, + const unsigned int first_selected_component) : BaseClass(matrix_free, dof_no, first_selected_component, @@ -7334,10 +7829,15 @@ template + typename Number, + typename VectorizedArrayType> inline void -FEFaceEvaluation::reinit( - const unsigned int face_index) +FEFaceEvaluation::reinit(const unsigned int face_index) { Assert(this->mapped_geometry == nullptr, ExcMessage("FEEvaluation was initialized without a matrix-free object." @@ -7351,7 +7851,7 @@ FEFaceEvaluation::reinit( internal::MatrixFreeFunctions::DoFInfo::dof_access_face_interior : internal::MatrixFreeFunctions::DoFInfo::dof_access_face_exterior; Assert(this->mapping_data != nullptr, ExcNotInitialized()); - const unsigned int n_vectors = VectorizedArray::n_array_elements; + const unsigned int n_vectors = VectorizedArrayType::n_array_elements; const internal::MatrixFreeFunctions::FaceToCellTopology &faces = this->matrix_info->get_face_info(face_index); if (face_index >= @@ -7407,11 +7907,16 @@ template + typename Number, + typename VectorizedArrayType> inline void -FEFaceEvaluation::reinit( - const unsigned int cell_index, - const unsigned int face_number) +FEFaceEvaluation::reinit(const unsigned int cell_index, + const unsigned int face_number) { Assert( this->quad_no < @@ -7476,11 +7981,16 @@ template + typename Number, + typename VectorizedArrayType> inline void -FEFaceEvaluation::evaluate( - const bool evaluate_values, - const bool evaluate_gradients) +FEFaceEvaluation::evaluate(const bool evaluate_values, + const bool evaluate_gradients) { Assert(this->dof_values_initialized, ExcNotInitialized()); @@ -7493,12 +8003,18 @@ template + typename Number, + typename VectorizedArrayType> inline void -FEFaceEvaluation::evaluate( - const VectorizedArray *values_array, - const bool evaluate_values, - const bool evaluate_gradients) +FEFaceEvaluation::evaluate(const VectorizedArrayType + * values_array, + const bool evaluate_values, + const bool evaluate_gradients) { if (!(evaluate_values + evaluate_gradients)) return; @@ -7518,22 +8034,25 @@ FEFaceEvaluation::evaluate( // variable to not risk a stack overflow. constexpr unsigned int stack_array_size_threshold = 100; - VectorizedArray - temp_data[static_dofs_per_face < stack_array_size_threshold ? + VectorizedArrayType + temp_data[static_dofs_per_face < stack_array_size_threshold ? n_components * 2 * static_dofs_per_face : 1]; - VectorizedArray *temp1; + VectorizedArrayType *temp1; if (static_dofs_per_face < stack_array_size_threshold) temp1 = &temp_data[0]; else temp1 = this->scratch_data; - internal::FEFaceNormalEvaluationImpl>:: - template interpolate( - *this->data, values_array, temp1, evaluate_gradients, this->face_no); + internal::FEFaceNormalEvaluationImpl< + dim, + fe_degree, + n_components, + VectorizedArrayType>::template interpolate(*this->data, + values_array, + temp1, + evaluate_gradients, + this->face_no); const unsigned int n_q_points_1d_actual = fe_degree > -1 ? n_q_points_1d : 0; if (fe_degree > -1 && @@ -7546,16 +8065,15 @@ FEFaceEvaluation::evaluate( fe_degree, n_q_points_1d_actual, n_components, - VectorizedArray>::evaluate_in_face(*this->data, - temp1, - this->begin_values(), - this->begin_gradients(), - this->scratch_data + - 2 * n_components * - dofs_per_face, - evaluate_values, - evaluate_gradients, - this->subface_index); + VectorizedArrayType>::evaluate_in_face(*this->data, + temp1, + this->begin_values(), + this->begin_gradients(), + this->scratch_data + + 2 * n_components * dofs_per_face, + evaluate_values, + evaluate_gradients, + this->subface_index); else internal::FEFaceEvaluationImpl< false, @@ -7563,16 +8081,15 @@ FEFaceEvaluation::evaluate( fe_degree, n_q_points_1d_actual, n_components, - VectorizedArray>::evaluate_in_face(*this->data, - temp1, - this->begin_values(), - this->begin_gradients(), - this->scratch_data + - 2 * n_components * - dofs_per_face, - evaluate_values, - evaluate_gradients, - this->subface_index); + VectorizedArrayType>::evaluate_in_face(*this->data, + temp1, + this->begin_values(), + this->begin_gradients(), + this->scratch_data + + 2 * n_components * dofs_per_face, + evaluate_values, + evaluate_gradients, + this->subface_index); if (this->face_orientation) adjust_for_face_orientation(false, evaluate_values, evaluate_gradients); @@ -7591,10 +8108,16 @@ template + typename Number, + typename VectorizedArrayType> inline void -FEFaceEvaluation:: - integrate(const bool integrate_values, const bool integrate_gradients) +FEFaceEvaluation::integrate(const bool integrate_values, + const bool integrate_gradients) { integrate(integrate_values, integrate_gradients, this->values_dofs[0]); @@ -7609,12 +8132,18 @@ template + typename Number, + typename VectorizedArrayType> inline void -FEFaceEvaluation:: - integrate(const bool integrate_values, - const bool integrate_gradients, - VectorizedArray *values_array) +FEFaceEvaluation::integrate(const bool integrate_values, + const bool integrate_gradients, + VectorizedArrayType + *values_array) { if (!(integrate_values + integrate_gradients)) return; @@ -7631,11 +8160,11 @@ FEFaceEvaluation:: constexpr unsigned int stack_array_size_threshold = 100; - VectorizedArray - temp_data[static_dofs_per_face < stack_array_size_threshold ? + VectorizedArrayType + temp_data[static_dofs_per_face < stack_array_size_threshold ? n_components * 2 * static_dofs_per_face : 1]; - VectorizedArray *temp1; + VectorizedArrayType *temp1; if (static_dofs_per_face < stack_array_size_threshold) temp1 = &temp_data[0]; else @@ -7652,16 +8181,16 @@ FEFaceEvaluation:: fe_degree, n_q_points_1d_actual, n_components, - VectorizedArray>::integrate_in_face(*this->data, - temp1, - this->begin_values(), - this->begin_gradients(), - this->scratch_data + - 2 * n_components * - dofs_per_face, - integrate_values, - integrate_gradients, - this->subface_index); + VectorizedArrayType>::integrate_in_face(*this->data, + temp1, + this->begin_values(), + this->begin_gradients(), + this->scratch_data + + 2 * n_components * + dofs_per_face, + integrate_values, + integrate_gradients, + this->subface_index); else internal::FEFaceEvaluationImpl< false, @@ -7669,21 +8198,21 @@ FEFaceEvaluation:: fe_degree, n_q_points_1d_actual, n_components, - VectorizedArray>::integrate_in_face(*this->data, - temp1, - this->begin_values(), - this->begin_gradients(), - this->scratch_data + - 2 * n_components * - dofs_per_face, - integrate_values, - integrate_gradients, - this->subface_index); + VectorizedArrayType>::integrate_in_face(*this->data, + temp1, + this->begin_values(), + this->begin_gradients(), + this->scratch_data + + 2 * n_components * + dofs_per_face, + integrate_values, + integrate_gradients, + this->subface_index); internal::FEFaceNormalEvaluationImpl>:: + VectorizedArrayType>:: template interpolate( *this->data, temp1, values_array, integrate_gradients, this->face_no); } @@ -7694,13 +8223,19 @@ template + typename Number, + typename VectorizedArrayType> template inline void -FEFaceEvaluation:: - gather_evaluate(const VectorType &input_vector, - const bool evaluate_values, - const bool evaluate_gradients) +FEFaceEvaluation< + dim, + fe_degree, + n_q_points_1d, + n_components_, + Number, + VectorizedArrayType>::gather_evaluate(const VectorType &input_vector, + const bool evaluate_values, + const bool evaluate_gradients) { const unsigned int side = this->face_no % 2; @@ -7713,17 +8248,17 @@ FEFaceEvaluation:: constexpr unsigned int stack_array_size_threshold = 100; - VectorizedArray + VectorizedArrayType temp_data[static_dofs_per_face < stack_array_size_threshold ? n_components_ * 2 * dofs_per_face : 1]; - VectorizedArray *__restrict temp1; + VectorizedArrayType *__restrict temp1; if (static_dofs_per_face < stack_array_size_threshold) temp1 = &temp_data[0]; else temp1 = this->scratch_data; - internal::VectorReader reader; + internal::VectorReader reader; std::integral_constant::value> vector_selector; @@ -7742,23 +8277,23 @@ FEFaceEvaluation:: AssertDimension( this->dof_info ->n_vectorization_lanes_filled[this->dof_access_index][this->cell], - VectorizedArray::n_array_elements); + VectorizedArrayType::n_array_elements); const unsigned int dof_index = this->dof_info ->dof_indices_contiguous[this->dof_access_index] [this->cell * - VectorizedArray::n_array_elements] + + VectorizedArrayType::n_array_elements] + this->dof_info ->component_dof_indices_offset[this->active_fe_index] [this->first_selected_component] * - VectorizedArray::n_array_elements; + VectorizedArrayType::n_array_elements; if (fe_degree > 1 && evaluate_gradients == true) { // we know that the gradient weights for the Hermite case on the // right (side==1) are the negative from the value at the left // (side==0), so we only read out one of them. - const VectorizedArray grad_weight = + const VectorizedArrayType grad_weight = this->data->shape_data_on_face[0][fe_degree + 1 + side]; AssertDimension(this->data->face_to_cell_index_hermite.size(1), 2 * dofs_per_face); @@ -7775,14 +8310,14 @@ FEFaceEvaluation:: reader.process_dofs_vectorized( 1, dof_index + (ind1 + comp * static_dofs_per_component) * - VectorizedArray::n_array_elements, + VectorizedArrayType::n_array_elements, input_vector, temp1 + i + 2 * comp * dofs_per_face, vector_selector); reader.process_dofs_vectorized( 1, dof_index + (ind2 + comp * static_dofs_per_component) * - VectorizedArray::n_array_elements, + VectorizedArrayType::n_array_elements, input_vector, temp1 + dofs_per_face + i + 2 * comp * dofs_per_face, vector_selector); @@ -7806,7 +8341,7 @@ FEFaceEvaluation:: reader.process_dofs_vectorized( 1, dof_index + (ind + comp * static_dofs_per_component) * - VectorizedArray::n_array_elements, + VectorizedArrayType::n_array_elements, input_vector, temp1 + i + 2 * comp * dofs_per_face, vector_selector); @@ -7828,18 +8363,18 @@ FEFaceEvaluation:: AssertDimension( this->dof_info ->n_vectorization_lanes_filled[this->dof_access_index][this->cell], - VectorizedArray::n_array_elements); + VectorizedArrayType::n_array_elements); const unsigned int *indices = &this->dof_info ->dof_indices_contiguous[this->dof_access_index] [this->cell * - VectorizedArray::n_array_elements]; + VectorizedArrayType::n_array_elements]; if (fe_degree > 1 && evaluate_gradients == true) { // we know that the gradient weights for the Hermite case on the // right (side==1) are the negative from the value at the left // (side==0), so we only read out one of them. - const VectorizedArray grad_weight = + const VectorizedArrayType grad_weight = this->data->shape_data_on_face[0][fe_degree + 1 + side]; AssertDimension(this->data->face_to_cell_index_hermite.size(1), 2 * dofs_per_face); @@ -7849,10 +8384,9 @@ FEFaceEvaluation:: for (unsigned int i = 0; i < dofs_per_face; ++i) { const unsigned int ind1 = - index_array[2 * i] * VectorizedArray::n_array_elements; + index_array[2 * i] * VectorizedArrayType::n_array_elements; const unsigned int ind2 = - index_array[2 * i + 1] * - VectorizedArray::n_array_elements; + index_array[2 * i + 1] * VectorizedArrayType::n_array_elements; for (unsigned int comp = 0; comp < n_components_; ++comp) { reader.process_dof_gather( @@ -7860,24 +8394,24 @@ FEFaceEvaluation:: input_vector, ind1 + comp * static_dofs_per_component * - VectorizedArray::n_array_elements + + VectorizedArrayType::n_array_elements + this->dof_info->component_dof_indices_offset [this->active_fe_index] [this->first_selected_component] * - VectorizedArray::n_array_elements, + VectorizedArrayType::n_array_elements, temp1[i + 2 * comp * dofs_per_face], vector_selector); - VectorizedArray grad; + VectorizedArrayType grad; reader.process_dof_gather( indices, input_vector, ind2 + comp * static_dofs_per_component * - VectorizedArray::n_array_elements + + VectorizedArrayType::n_array_elements + this->dof_info->component_dof_indices_offset [this->active_fe_index] [this->first_selected_component] * - VectorizedArray::n_array_elements, + VectorizedArrayType::n_array_elements, grad, vector_selector); temp1[i + dofs_per_face + 2 * comp * dofs_per_face] = @@ -7894,18 +8428,18 @@ FEFaceEvaluation:: for (unsigned int i = 0; i < dofs_per_face; ++i) { const unsigned int ind = - index_array[i] * VectorizedArray::n_array_elements; + index_array[i] * VectorizedArrayType::n_array_elements; for (unsigned int comp = 0; comp < n_components_; ++comp) reader.process_dof_gather( indices, input_vector, ind + comp * static_dofs_per_component * - VectorizedArray::n_array_elements + + VectorizedArrayType::n_array_elements + this->dof_info->component_dof_indices_offset [this->active_fe_index] [this->first_selected_component] * - VectorizedArray::n_array_elements, + VectorizedArrayType::n_array_elements, temp1[i + 2 * comp * dofs_per_face], vector_selector); } @@ -7926,14 +8460,13 @@ FEFaceEvaluation:: const unsigned int *strides = &this->dof_info->dof_indices_interleave_strides [this->dof_access_index] - [this->cell * VectorizedArray::n_array_elements]; - unsigned int indices[VectorizedArray::n_array_elements]; - for (unsigned int v = 0; v < VectorizedArray::n_array_elements; - ++v) + [this->cell * VectorizedArrayType::n_array_elements]; + unsigned int indices[VectorizedArrayType::n_array_elements]; + for (unsigned int v = 0; v < VectorizedArrayType::n_array_elements; ++v) indices[v] = this->dof_info->dof_indices_contiguous [this->dof_access_index] - [this->cell * VectorizedArray::n_array_elements + v] + + [this->cell * VectorizedArrayType::n_array_elements + v] + this->dof_info ->component_dof_indices_offset[this->active_fe_index] [this->first_selected_component] * @@ -7947,29 +8480,29 @@ FEFaceEvaluation:: // we know that the gradient weights for the Hermite case on the // right (side==1) are the negative from the value at the left // (side==0), so we only read out one of them. - const VectorizedArray grad_weight = + const VectorizedArrayType grad_weight = this->data->shape_data_on_face[0][fe_degree + 1 + side]; AssertDimension(this->data->face_to_cell_index_hermite.size(1), 2 * dofs_per_face); const unsigned int *index_array = &this->data->face_to_cell_index_hermite(this->face_no, 0); - if (nvec == VectorizedArray::n_array_elements) + if (nvec == VectorizedArrayType::n_array_elements) for (unsigned int comp = 0; comp < n_components_; ++comp) for (unsigned int i = 0; i < dofs_per_face; ++i) { - unsigned int ind1[VectorizedArray::n_array_elements]; + unsigned int ind1[VectorizedArrayType::n_array_elements]; DEAL_II_OPENMP_SIMD_PRAGMA for (unsigned int v = 0; - v < VectorizedArray::n_array_elements; + v < VectorizedArrayType::n_array_elements; ++v) ind1[v] = indices[v] + (comp * static_dofs_per_component + index_array[2 * i]) * strides[v]; - unsigned int ind2[VectorizedArray::n_array_elements]; + unsigned int ind2[VectorizedArrayType::n_array_elements]; DEAL_II_OPENMP_SIMD_PRAGMA for (unsigned int v = 0; - v < VectorizedArray::n_array_elements; + v < VectorizedArrayType::n_array_elements; ++v) ind2[v] = indices[v] + (comp * static_dofs_per_component + index_array[2 * i + 1]) * @@ -7979,7 +8512,7 @@ FEFaceEvaluation:: 0, temp1[i + 2 * comp * dofs_per_face], vector_selector); - VectorizedArray grad; + VectorizedArrayType grad; reader.process_dof_gather( ind2, input_vector, 0, grad, vector_selector); temp1[i + dofs_per_face + 2 * comp * dofs_per_face] = @@ -7989,7 +8522,7 @@ FEFaceEvaluation:: { for (unsigned int i = 0; i < n_components_ * 2 * dofs_per_face; ++i) - temp1[i] = VectorizedArray(); + temp1[i] = VectorizedArrayType(); for (unsigned int v = 0; v < nvec; ++v) for (unsigned int comp = 0; comp < n_components_; ++comp) for (unsigned int i = 0; i < dofs_per_face; ++i) @@ -8022,14 +8555,14 @@ FEFaceEvaluation:: dofs_per_face); const unsigned int *index_array = &this->data->face_to_cell_index_nodal(this->face_no, 0); - if (nvec == VectorizedArray::n_array_elements) + if (nvec == VectorizedArrayType::n_array_elements) for (unsigned int comp = 0; comp < n_components_; ++comp) for (unsigned int i = 0; i < dofs_per_face; ++i) { - unsigned int ind[VectorizedArray::n_array_elements]; + unsigned int ind[VectorizedArrayType::n_array_elements]; DEAL_II_OPENMP_SIMD_PRAGMA for (unsigned int v = 0; - v < VectorizedArray::n_array_elements; + v < VectorizedArrayType::n_array_elements; ++v) ind[v] = indices[v] + (comp * static_dofs_per_component + index_array[i]) * @@ -8043,7 +8576,7 @@ FEFaceEvaluation:: else { for (unsigned int i = 0; i < n_components_ * dofs_per_face; ++i) - temp1[i] = VectorizedArray(); + temp1[i] = VectorizedArrayType(); for (unsigned int v = 0; v < nvec; ++v) for (unsigned int comp = 0; comp < n_components_; ++comp) for (unsigned int i = 0; i < dofs_per_face; ++i) @@ -8073,13 +8606,13 @@ FEFaceEvaluation:: contiguous && this->dof_info->n_vectorization_lanes_filled[this->dof_access_index] [this->cell] == - VectorizedArray::n_array_elements) + VectorizedArrayType::n_array_elements) { const unsigned int *indices = &this->dof_info ->dof_indices_contiguous[this->dof_access_index] [this->cell * - VectorizedArray::n_array_elements]; + VectorizedArrayType::n_array_elements]; if (evaluate_gradients == true && this->data->element_type == internal::MatrixFreeFunctions::tensor_symmetric_hermite) @@ -8087,7 +8620,7 @@ FEFaceEvaluation:: // we know that the gradient weights for the Hermite case on the // right (side==1) are the negative from the value at the left // (side==0), so we only read out one of them. - const VectorizedArray grad_weight = + const VectorizedArrayType grad_weight = this->data->shape_data_on_face[0][fe_degree + 1 + side]; AssertDimension(this->data->face_to_cell_index_hermite.size(1), 2 * dofs_per_face); @@ -8108,7 +8641,7 @@ FEFaceEvaluation:: [this->active_fe_index][this->first_selected_component], temp1[i + 2 * comp * dofs_per_face], vector_selector); - VectorizedArray grad; + VectorizedArrayType grad; reader.process_dof_gather( indices, input_vector, @@ -8151,7 +8684,7 @@ FEFaceEvaluation:: internal::FEFaceNormalEvaluationImpl>:: + VectorizedArrayType>:: template interpolate(*this->data, this->values_dofs[0], temp1, @@ -8169,16 +8702,16 @@ FEFaceEvaluation:: fe_degree, n_q_points_1d, n_components_, - VectorizedArray>::evaluate_in_face(*this->data, - temp1, - this->values_quad[0], - this->gradients_quad[0][0], - this->scratch_data + - 2 * n_components_ * - dofs_per_face, - evaluate_values, - evaluate_gradients, - this->subface_index); + VectorizedArrayType>::evaluate_in_face(*this->data, + temp1, + this->values_quad[0], + this->gradients_quad[0][0], + this->scratch_data + + 2 * n_components_ * + dofs_per_face, + evaluate_values, + evaluate_gradients, + this->subface_index); else internal::FEFaceEvaluationImpl< false, @@ -8186,16 +8719,16 @@ FEFaceEvaluation:: fe_degree, n_q_points_1d, n_components_, - VectorizedArray>::evaluate_in_face(*this->data, - temp1, - this->values_quad[0], - this->gradients_quad[0][0], - this->scratch_data + - 2 * n_components_ * - dofs_per_face, - evaluate_values, - evaluate_gradients, - this->subface_index); + VectorizedArrayType>::evaluate_in_face(*this->data, + temp1, + this->values_quad[0], + this->gradients_quad[0][0], + this->scratch_data + + 2 * n_components_ * + dofs_per_face, + evaluate_values, + evaluate_gradients, + this->subface_index); if (this->face_orientation) adjust_for_face_orientation(false, evaluate_values, evaluate_gradients); @@ -8214,13 +8747,19 @@ template + typename Number, + typename VectorizedArrayType> template inline void -FEFaceEvaluation:: - integrate_scatter(const bool integrate_values, - const bool integrate_gradients, - VectorType &destination) +FEFaceEvaluation< + dim, + fe_degree, + n_q_points_1d, + n_components_, + Number, + VectorizedArrayType>::integrate_scatter(const bool integrate_values, + const bool integrate_gradients, + VectorType &destination) { const unsigned int side = this->face_no % 2; const unsigned int dofs_per_face = @@ -8229,10 +8768,10 @@ FEFaceEvaluation:: constexpr unsigned int stack_array_size_threshold = 100; - VectorizedArray temp_data[dofs_per_face < stack_array_size_threshold ? - n_components_ * 2 * dofs_per_face : - 1]; - VectorizedArray *__restrict temp1; + VectorizedArrayType temp_data[dofs_per_face < stack_array_size_threshold ? + n_components_ * 2 * dofs_per_face : + 1]; + VectorizedArrayType *__restrict temp1; if (dofs_per_face < stack_array_size_threshold) temp1 = &temp_data[0]; else @@ -8250,16 +8789,16 @@ FEFaceEvaluation:: fe_degree, n_q_points_1d, n_components_, - VectorizedArray>::integrate_in_face(*this->data, - temp1, - this->values_quad[0], - this->gradients_quad[0][0], - this->scratch_data + - 2 * n_components_ * - dofs_per_face, - integrate_values, - integrate_gradients, - this->subface_index); + VectorizedArrayType>::integrate_in_face(*this->data, + temp1, + this->values_quad[0], + this->gradients_quad[0][0], + this->scratch_data + + 2 * n_components_ * + dofs_per_face, + integrate_values, + integrate_gradients, + this->subface_index); else internal::FEFaceEvaluationImpl< false, @@ -8267,22 +8806,22 @@ FEFaceEvaluation:: fe_degree, n_q_points_1d, n_components_, - VectorizedArray>::integrate_in_face(*this->data, - temp1, - this->values_quad[0], - this->gradients_quad[0][0], - this->scratch_data + - 2 * n_components_ * - dofs_per_face, - integrate_values, - integrate_gradients, - this->subface_index); + VectorizedArrayType>::integrate_in_face(*this->data, + temp1, + this->values_quad[0], + this->gradients_quad[0][0], + this->scratch_data + + 2 * n_components_ * + dofs_per_face, + integrate_values, + integrate_gradients, + this->subface_index); # ifdef DEBUG this->dof_values_initialized = true; # endif - internal::VectorDistributorLocalToGlobal writer; + internal::VectorDistributorLocalToGlobal writer; std::integral_constant::value> vector_selector; @@ -8301,23 +8840,23 @@ FEFaceEvaluation:: AssertDimension( this->dof_info ->n_vectorization_lanes_filled[this->dof_access_index][this->cell], - VectorizedArray::n_array_elements); + VectorizedArrayType::n_array_elements); const unsigned int dof_index = this->dof_info ->dof_indices_contiguous[this->dof_access_index] [this->cell * - VectorizedArray::n_array_elements] + + VectorizedArrayType::n_array_elements] + this->dof_info ->component_dof_indices_offset[this->active_fe_index] [this->first_selected_component] * - VectorizedArray::n_array_elements; + VectorizedArrayType::n_array_elements; if (fe_degree > 1 && integrate_gradients == true) { // we know that the gradient weights for the Hermite case on the // right (side==1) are the negative from the value at the left // (side==0), so we only read out one of them. - const VectorizedArray grad_weight = + const VectorizedArrayType grad_weight = this->data->shape_data_on_face[0][fe_degree + 2 - side]; AssertDimension(this->data->face_to_cell_index_hermite.size(1), 2 * dofs_per_face); @@ -8331,24 +8870,24 @@ FEFaceEvaluation:: AssertIndexRange(ind2, dofs_per_cell); for (unsigned int comp = 0; comp < n_components_; ++comp) { - VectorizedArray val = + VectorizedArrayType val = temp1[i + 2 * comp * dofs_per_face] - grad_weight * temp1[i + dofs_per_face + 2 * comp * dofs_per_face]; - VectorizedArray grad = + VectorizedArrayType grad = grad_weight * temp1[i + dofs_per_face + 2 * comp * dofs_per_face]; writer.process_dofs_vectorized( 1, dof_index + (ind1 + comp * static_dofs_per_component) * - VectorizedArray::n_array_elements, + VectorizedArrayType::n_array_elements, destination, &val, vector_selector); writer.process_dofs_vectorized( 1, dof_index + (ind2 + comp * static_dofs_per_component) * - VectorizedArray::n_array_elements, + VectorizedArrayType::n_array_elements, destination, &grad, vector_selector); @@ -8368,7 +8907,7 @@ FEFaceEvaluation:: writer.process_dofs_vectorized( 1, dof_index + (ind + comp * static_dofs_per_component) * - VectorizedArray::n_array_elements, + VectorizedArrayType::n_array_elements, destination, temp1 + i + 2 * comp * dofs_per_face, vector_selector); @@ -8390,18 +8929,18 @@ FEFaceEvaluation:: AssertDimension( this->dof_info ->n_vectorization_lanes_filled[this->dof_access_index][this->cell], - VectorizedArray::n_array_elements); + VectorizedArrayType::n_array_elements); const unsigned int *indices = &this->dof_info ->dof_indices_contiguous[this->dof_access_index] [this->cell * - VectorizedArray::n_array_elements]; + VectorizedArrayType::n_array_elements]; if (fe_degree > 1 && integrate_gradients == true) { // we know that the gradient weights for the Hermite case on the // right (side==1) are the negative from the value at the left // (side==0), so we only read out one of them. - const VectorizedArray grad_weight = + const VectorizedArrayType grad_weight = this->data->shape_data_on_face[0][fe_degree + 2 - side]; AssertDimension(this->data->face_to_cell_index_hermite.size(1), 2 * dofs_per_face); @@ -8411,17 +8950,16 @@ FEFaceEvaluation:: for (unsigned int i = 0; i < dofs_per_face; ++i) { const unsigned int ind1 = - index_array[2 * i] * VectorizedArray::n_array_elements; + index_array[2 * i] * VectorizedArrayType::n_array_elements; const unsigned int ind2 = - index_array[2 * i + 1] * - VectorizedArray::n_array_elements; + index_array[2 * i + 1] * VectorizedArrayType::n_array_elements; for (unsigned int comp = 0; comp < n_components_; ++comp) { - VectorizedArray val = + VectorizedArrayType val = temp1[i + 2 * comp * dofs_per_face] - grad_weight * temp1[i + dofs_per_face + 2 * comp * dofs_per_face]; - VectorizedArray grad = + VectorizedArrayType grad = grad_weight * temp1[i + dofs_per_face + 2 * comp * dofs_per_face]; writer.process_dof_gather( @@ -8429,11 +8967,11 @@ FEFaceEvaluation:: destination, ind1 + comp * static_dofs_per_component * - VectorizedArray::n_array_elements + + VectorizedArrayType::n_array_elements + this->dof_info->component_dof_indices_offset [this->active_fe_index] [this->first_selected_component] * - VectorizedArray::n_array_elements, + VectorizedArrayType::n_array_elements, val, vector_selector); writer.process_dof_gather( @@ -8441,11 +8979,11 @@ FEFaceEvaluation:: destination, ind2 + comp * static_dofs_per_component * - VectorizedArray::n_array_elements + + VectorizedArrayType::n_array_elements + this->dof_info->component_dof_indices_offset [this->active_fe_index] [this->first_selected_component] * - VectorizedArray::n_array_elements, + VectorizedArrayType::n_array_elements, grad, vector_selector); } @@ -8460,18 +8998,18 @@ FEFaceEvaluation:: for (unsigned int i = 0; i < dofs_per_face; ++i) { const unsigned int ind = - index_array[i] * VectorizedArray::n_array_elements; + index_array[i] * VectorizedArrayType::n_array_elements; for (unsigned int comp = 0; comp < n_components_; ++comp) writer.process_dof_gather( indices, destination, ind + comp * static_dofs_per_component * - VectorizedArray::n_array_elements + + VectorizedArrayType::n_array_elements + this->dof_info->component_dof_indices_offset [this->active_fe_index] [this->first_selected_component] * - VectorizedArray::n_array_elements, + VectorizedArrayType::n_array_elements, temp1[i + 2 * comp * dofs_per_face], vector_selector); } @@ -8492,14 +9030,13 @@ FEFaceEvaluation:: const unsigned int *strides = &this->dof_info->dof_indices_interleave_strides [this->dof_access_index] - [this->cell * VectorizedArray::n_array_elements]; - unsigned int indices[VectorizedArray::n_array_elements]; - for (unsigned int v = 0; v < VectorizedArray::n_array_elements; - ++v) + [this->cell * VectorizedArrayType::n_array_elements]; + unsigned int indices[VectorizedArrayType::n_array_elements]; + for (unsigned int v = 0; v < VectorizedArrayType::n_array_elements; ++v) indices[v] = this->dof_info->dof_indices_contiguous [this->dof_access_index] - [this->cell * VectorizedArray::n_array_elements + v] + + [this->cell * VectorizedArrayType::n_array_elements + v] + this->dof_info ->component_dof_indices_offset[this->active_fe_index] [this->first_selected_component] * @@ -8513,38 +9050,38 @@ FEFaceEvaluation:: // we know that the gradient weights for the Hermite case on the // right (side==1) are the negative from the value at the left // (side==0), so we only read out one of them. - const VectorizedArray grad_weight = + const VectorizedArrayType grad_weight = this->data->shape_data_on_face[0][fe_degree + 2 - side]; AssertDimension(this->data->face_to_cell_index_hermite.size(1), 2 * dofs_per_face); const unsigned int *index_array = &this->data->face_to_cell_index_hermite(this->face_no, 0); - if (nvec == VectorizedArray::n_array_elements) + if (nvec == VectorizedArrayType::n_array_elements) for (unsigned int comp = 0; comp < n_components_; ++comp) for (unsigned int i = 0; i < dofs_per_face; ++i) { - unsigned int ind1[VectorizedArray::n_array_elements]; + unsigned int ind1[VectorizedArrayType::n_array_elements]; DEAL_II_OPENMP_SIMD_PRAGMA for (unsigned int v = 0; - v < VectorizedArray::n_array_elements; + v < VectorizedArrayType::n_array_elements; ++v) ind1[v] = indices[v] + (comp * static_dofs_per_component + index_array[2 * i]) * strides[v]; - unsigned int ind2[VectorizedArray::n_array_elements]; + unsigned int ind2[VectorizedArrayType::n_array_elements]; DEAL_II_OPENMP_SIMD_PRAGMA for (unsigned int v = 0; - v < VectorizedArray::n_array_elements; + v < VectorizedArrayType::n_array_elements; ++v) ind2[v] = indices[v] + (comp * static_dofs_per_component + index_array[2 * i + 1]) * strides[v]; - VectorizedArray val = + VectorizedArrayType val = temp1[i + 2 * comp * dofs_per_face] - grad_weight * temp1[i + dofs_per_face + 2 * comp * dofs_per_face]; - VectorizedArray grad = + VectorizedArrayType grad = grad_weight * temp1[i + dofs_per_face + 2 * comp * dofs_per_face]; writer.process_dof_gather( @@ -8584,14 +9121,14 @@ FEFaceEvaluation:: dofs_per_face); const unsigned int *index_array = &this->data->face_to_cell_index_nodal(this->face_no, 0); - if (nvec == VectorizedArray::n_array_elements) + if (nvec == VectorizedArrayType::n_array_elements) for (unsigned int comp = 0; comp < n_components_; ++comp) for (unsigned int i = 0; i < dofs_per_face; ++i) { - unsigned int ind[VectorizedArray::n_array_elements]; + unsigned int ind[VectorizedArrayType::n_array_elements]; DEAL_II_OPENMP_SIMD_PRAGMA for (unsigned int v = 0; - v < VectorizedArray::n_array_elements; + v < VectorizedArrayType::n_array_elements; ++v) ind[v] = indices[v] + (comp * static_dofs_per_component + index_array[i]) * @@ -8633,13 +9170,13 @@ FEFaceEvaluation:: contiguous && this->dof_info->n_vectorization_lanes_filled[this->dof_access_index] [this->cell] == - VectorizedArray::n_array_elements) + VectorizedArrayType::n_array_elements) { const unsigned int *indices = &this->dof_info ->dof_indices_contiguous[this->dof_access_index] [this->cell * - VectorizedArray::n_array_elements]; + VectorizedArrayType::n_array_elements]; if (integrate_gradients == true && this->data->element_type == @@ -8648,7 +9185,7 @@ FEFaceEvaluation:: // we know that the gradient weights for the Hermite case on the // right (side==1) are the negative from the value at the left // (side==0), so we only read out one of them. - const VectorizedArray grad_weight = + const VectorizedArrayType grad_weight = this->data->shape_data_on_face[0][fe_degree + 2 - side]; AssertDimension(this->data->face_to_cell_index_hermite.size(1), 2 * dofs_per_face); @@ -8660,11 +9197,11 @@ FEFaceEvaluation:: const unsigned int ind2 = index_array[2 * i + 1]; for (unsigned int comp = 0; comp < n_components_; ++comp) { - VectorizedArray val = + VectorizedArrayType val = temp1[i + 2 * comp * dofs_per_face] - grad_weight * temp1[i + dofs_per_face + 2 * comp * dofs_per_face]; - VectorizedArray grad = + VectorizedArrayType grad = grad_weight * temp1[i + dofs_per_face + 2 * comp * dofs_per_face]; writer.process_dof_gather( @@ -8714,7 +9251,7 @@ FEFaceEvaluation:: internal::FEFaceNormalEvaluationImpl>:: + VectorizedArrayType>:: template interpolate(*this->data, temp1, this->values_dofs[0], @@ -8730,15 +9267,21 @@ template + typename Number, + typename VectorizedArrayType> inline void -FEFaceEvaluation:: - adjust_for_face_orientation(const bool integrate, - const bool values, - const bool gradients) +FEFaceEvaluation< + dim, + fe_degree, + n_q_points_1d, + n_components, + Number, + VectorizedArrayType>::adjust_for_face_orientation(const bool integrate, + const bool values, + const bool gradients) { - VectorizedArray *tmp_values = this->scratch_data; - const unsigned int * orientations = + VectorizedArrayType *tmp_values = this->scratch_data; + const unsigned int * orientations = &this->mapping_data->descriptor[this->active_fe_index] .face_orientations[this->face_orientation][0]; for (unsigned int c = 0; c < n_components; ++c) @@ -8775,10 +9318,16 @@ template -inline Point> -FEFaceEvaluation:: - quadrature_point(const unsigned int q) const + typename Number, + typename VectorizedArrayType> +inline Point +FEFaceEvaluation::quadrature_point(const unsigned int q) + const { AssertIndexRange(q, n_q_points); if (this->dof_access_index < 2) diff --git a/include/deal.II/matrix_free/mapping_data_on_the_fly.h b/include/deal.II/matrix_free/mapping_data_on_the_fly.h index f42bc67008..ce9327d682 100644 --- a/include/deal.II/matrix_free/mapping_data_on_the_fly.h +++ b/include/deal.II/matrix_free/mapping_data_on_the_fly.h @@ -58,9 +58,13 @@ namespace internal * * @author Martin Kronbichler, 2014 */ - template + template class MappingDataOnTheFly { + static_assert( + std::is_same::value, + "Type of Number and of VectorizedArrayType do not match."); + public: /** * Constructor, similar to FEValues. Since this class only evaluates the @@ -114,7 +118,7 @@ namespace internal * MappingInfo. This ensures compatibility with the precomputed data * fields in the MappingInfo class. */ - const MappingInfoStorage & + const MappingInfoStorage & get_data_storage() const; /** @@ -150,21 +154,24 @@ namespace internal * The storage part created for a single cell and held in analogy to * MappingInfo. */ - MappingInfoStorage mapping_info_storage; + MappingInfoStorage + mapping_info_storage; }; /*-------------------------- Inline functions ---------------------------*/ - template - inline MappingDataOnTheFly::MappingDataOnTheFly( - const Mapping & mapping, - const Quadrature<1> &quadrature, - const UpdateFlags update_flags) - : fe_values(mapping, - fe_dummy, - Quadrature(quadrature), - MappingInfo::compute_update_flags(update_flags)) + template + inline MappingDataOnTheFly:: + MappingDataOnTheFly(const Mapping & mapping, + const Quadrature<1> &quadrature, + const UpdateFlags update_flags) + : fe_values( + mapping, + fe_dummy, + Quadrature(quadrature), + MappingInfo::compute_update_flags( + update_flags)) , quadrature_1d(quadrature) { mapping_info_storage.descriptor.resize(1); @@ -191,10 +198,10 @@ namespace internal - template - inline MappingDataOnTheFly::MappingDataOnTheFly( - const Quadrature<1> &quadrature, - const UpdateFlags update_flags) + template + inline MappingDataOnTheFly:: + MappingDataOnTheFly(const Quadrature<1> &quadrature, + const UpdateFlags update_flags) : MappingDataOnTheFly(::dealii::StaticMappingQ1::mapping, quadrature, update_flags) @@ -202,9 +209,9 @@ namespace internal - template + template inline void - MappingDataOnTheFly::reinit( + MappingDataOnTheFly::reinit( typename dealii::Triangulation::cell_iterator cell) { if (present_cell == cell) @@ -241,9 +248,10 @@ namespace internal - template + template inline bool - MappingDataOnTheFly::is_initialized() const + MappingDataOnTheFly::is_initialized() + const { return present_cell != typename dealii::Triangulation::cell_iterator(); @@ -251,36 +259,38 @@ namespace internal - template + template inline typename dealii::Triangulation::cell_iterator - MappingDataOnTheFly::get_cell() const + MappingDataOnTheFly::get_cell() const { return fe_values.get_cell(); } - template + template inline const dealii::FEValues & - MappingDataOnTheFly::get_fe_values() const + MappingDataOnTheFly::get_fe_values() const { return fe_values; } - template - inline const MappingInfoStorage & - MappingDataOnTheFly::get_data_storage() const + template + inline const MappingInfoStorage & + MappingDataOnTheFly::get_data_storage() + const { return mapping_info_storage; } - template + template inline const Quadrature<1> & - MappingDataOnTheFly::get_quadrature() const + MappingDataOnTheFly::get_quadrature() + const { return quadrature_1d; } diff --git a/include/deal.II/matrix_free/mapping_info.h b/include/deal.II/matrix_free/mapping_info.h index bc5abce313..3adf978006 100644 --- a/include/deal.II/matrix_free/mapping_info.h +++ b/include/deal.II/matrix_free/mapping_info.h @@ -108,9 +108,16 @@ namespace internal * * @author Katharina Kormann, Martin Kronbichler, 2018 */ - template + template struct MappingInfoStorage { + static_assert( + std::is_same::value, + "Type of Number and of VectorizedArrayType do not match."); + struct QuadratureDescriptor { /** @@ -187,15 +194,14 @@ namespace internal * * Indexed by @p data_index_offsets. */ - AlignedVector> JxW_values; + AlignedVector JxW_values; /** * Stores the normal vectors. * * Indexed by @p data_index_offsets. */ - AlignedVector>> - normal_vectors; + AlignedVector> normal_vectors; /** * The storage of covariant transformation on quadrature points, i.e., @@ -208,7 +214,7 @@ namespace internal * but the default case (cell integrals or boundary integrals) only * fills the zeroth component and ignores the first one. */ - AlignedVector>> jacobians[2]; + AlignedVector> jacobians[2]; /** * The storage of the gradients of the inverse Jacobian @@ -227,7 +233,7 @@ namespace internal */ AlignedVector>>> + Tensor<1, spacedim, VectorizedArrayType>>> jacobian_gradients[2]; /** @@ -237,7 +243,7 @@ namespace internal * * Indexed by @p data_index_offsets. */ - AlignedVector>> + AlignedVector> normals_times_jacobians[2]; /** @@ -255,7 +261,7 @@ namespace internal * * Indexed by @p quadrature_point_offsets. */ - AlignedVector>> quadrature_points; + AlignedVector> quadrature_points; /** * Returns the quadrature index for a given number of quadrature @@ -292,7 +298,7 @@ namespace internal * * @author Katharina Kormann and Martin Kronbichler, 2010, 2011, 2017 */ - template + template struct MappingInfo { /** @@ -305,9 +311,9 @@ namespace internal */ void initialize( - const dealii::Triangulation & tria, - const std::vector> & cells, - const FaceInfo::n_array_elements> &faces, + const dealii::Triangulation & tria, + const std::vector> &cells, + const FaceInfo & faces, const std::vector & active_fe_index, const Mapping & mapping, const std::vector> &quad, @@ -363,18 +369,21 @@ namespace internal /** * The data cache for the cells. */ - std::vector> cell_data; + std::vector> + cell_data; /** * The data cache for the faces. */ - std::vector> face_data; + std::vector> + face_data; /** * The data cache for the face-associated-with-cell topology, following * the @p cell_type variable for the cell types. */ - std::vector> face_data_by_cells; + std::vector> + face_data_by_cells; /** * Computes the information in the given cells, called within @@ -398,9 +407,9 @@ namespace internal const dealii::Triangulation & tria, const std::vector> &cells, const std::vector< - FaceToCellTopology::n_array_elements>> &faces, - const Mapping & mapping, - const std::vector> &quad, + FaceToCellTopology> &faces, + const Mapping & mapping, + const std::vector> & quad, const UpdateFlags update_flags_boundary_faces, const UpdateFlags update_flags_inner_faces); @@ -434,27 +443,28 @@ namespace internal * * @author Katharina Kormann, Martin Kronbichler, 2018 */ - template + template struct MappingInfoCellsOrFaces; - template - struct MappingInfoCellsOrFaces + template + struct MappingInfoCellsOrFaces { - static const MappingInfoStorage * - get(const MappingInfo &mapping_info, - const unsigned int quad_no) + static const MappingInfoStorage * + get(const MappingInfo &mapping_info, + const unsigned int quad_no) { AssertIndexRange(quad_no, mapping_info.cell_data.size()); return &mapping_info.cell_data[quad_no]; } }; - template - struct MappingInfoCellsOrFaces + template + struct MappingInfoCellsOrFaces { - static const MappingInfoStorage * - get(const MappingInfo &mapping_info, - const unsigned int quad_no) + static const MappingInfoStorage + * + get(const MappingInfo &mapping_info, + const unsigned int quad_no) { AssertIndexRange(quad_no, mapping_info.face_data.size()); return &mapping_info.face_data[quad_no]; @@ -474,7 +484,8 @@ namespace internal * satisfied). This is not a problem in the use cases for this class, but be * careful when using it in other contexts. */ - template + template > struct FPArrayComparator { FPArrayComparator(const Number scaling); @@ -485,35 +496,33 @@ namespace internal bool operator()( - const Tensor<1, VectorizedArray::n_array_elements, Number> &t1, - const Tensor<1, VectorizedArray::n_array_elements, Number> &t2) + const Tensor<1, VectorizedArrayType::n_array_elements, Number> &t1, + const Tensor<1, VectorizedArrayType::n_array_elements, Number> &t2) const; template bool operator()( - const Tensor< - 1, - dim, - Tensor<1, VectorizedArray::n_array_elements, Number>> &t1, - const Tensor< - 1, - dim, - Tensor<1, VectorizedArray::n_array_elements, Number>> &t2) - const; + const Tensor<1, + dim, + Tensor<1, VectorizedArrayType::n_array_elements, Number>> + &t1, + const Tensor<1, + dim, + Tensor<1, VectorizedArrayType::n_array_elements, Number>> + &t2) const; template bool operator()( - const Tensor< - 2, - dim, - Tensor<1, VectorizedArray::n_array_elements, Number>> &t1, - const Tensor< - 2, - dim, - Tensor<1, VectorizedArray::n_array_elements, Number>> &t2) - const; + const Tensor<2, + dim, + Tensor<1, VectorizedArrayType::n_array_elements, Number>> + &t1, + const Tensor<2, + dim, + Tensor<1, VectorizedArrayType::n_array_elements, Number>> + &t2) const; Number tolerance; }; @@ -522,10 +531,13 @@ namespace internal /* ------------------- inline functions ----------------------------- */ - template + template inline unsigned int - MappingInfoStorage::quad_index_from_n_q_points( - const unsigned int n_q_points) const + MappingInfoStorage:: + quad_index_from_n_q_points(const unsigned int n_q_points) const { for (unsigned int i = 0; i < descriptor.size(); ++i) if (n_q_points == descriptor[i].n_q_points) @@ -535,9 +547,10 @@ namespace internal - template + template inline GeometryType - MappingInfo::get_cell_type(const unsigned int cell_no) const + MappingInfo::get_cell_type( + const unsigned int cell_no) const { AssertIndexRange(cell_no, cell_type.size()); return cell_type[cell_no]; diff --git a/include/deal.II/matrix_free/mapping_info.templates.h b/include/deal.II/matrix_free/mapping_info.templates.h index 1734b36f3e..6e8d4fb7ec 100644 --- a/include/deal.II/matrix_free/mapping_info.templates.h +++ b/include/deal.II/matrix_free/mapping_info.templates.h @@ -37,19 +37,26 @@ namespace internal { /* ------------------------ MappingInfoStorage implementation ---------- */ - template - MappingInfoStorage::QuadratureDescriptor :: - QuadratureDescriptor() + template + MappingInfoStorage:: + QuadratureDescriptor ::QuadratureDescriptor() : n_q_points(numbers::invalid_unsigned_int) {} - template + template void - MappingInfoStorage::QuadratureDescriptor :: - initialize(const Quadrature<1> &quadrature_1d, - const UpdateFlags update_flags_inner_faces) + MappingInfoStorage:: + QuadratureDescriptor ::initialize( + const Quadrature<1> &quadrature_1d, + const UpdateFlags update_flags_inner_faces) { Assert(structdim + 1 <= spacedim || update_flags_inner_faces == update_default, @@ -98,10 +105,13 @@ namespace internal - template + template std::size_t - MappingInfoStorage::QuadratureDescriptor :: - memory_consumption() const + MappingInfoStorage:: + QuadratureDescriptor ::memory_consumption() const { std::size_t memory = sizeof(this) + quadrature.memory_consumption() + quadrature_weights.memory_consumption() + @@ -113,9 +123,13 @@ namespace internal - template + template std::size_t - MappingInfoStorage::memory_consumption() const + MappingInfoStorage:: + memory_consumption() const { return MemoryConsumption::memory_consumption(descriptor) + MemoryConsumption::memory_consumption(data_index_offsets) + @@ -133,12 +147,14 @@ namespace internal - template + template template void - MappingInfoStorage::print_memory_consumption( - StreamType & out, - const SizeInfo &task_info) const + MappingInfoStorage:: + print_memory_consumption(StreamType &out, const SizeInfo &task_info) const { // print_memory_statistics involves global communication, so we can // disable the check here only if no processor has any such data @@ -192,9 +208,9 @@ namespace internal /* ------------------------ MappingInfo implementation ----------------- */ - template + template void - MappingInfo::clear() + MappingInfo::clear() { cell_data.clear(); face_data.clear(); @@ -205,9 +221,9 @@ namespace internal - template + template UpdateFlags - MappingInfo::compute_update_flags( + MappingInfo::compute_update_flags( const UpdateFlags update_flags, const std::vector> &quad) { @@ -252,16 +268,16 @@ namespace internal - template + template void - MappingInfo::initialize( - const dealii::Triangulation & tria, - const std::vector> & cells, - const FaceInfo::n_array_elements> &face_info, - const std::vector & active_fe_index, - const Mapping & mapping, - const std::vector> &quad, - const UpdateFlags update_flags_cells, + MappingInfo::initialize( + const dealii::Triangulation & tria, + const std::vector> &cells, + const FaceInfo & face_info, + const std::vector & active_fe_index, + const Mapping & mapping, + const std::vector> & quad, + const UpdateFlags update_flags_cells, const UpdateFlags update_flags_boundary_faces, const UpdateFlags update_flags_inner_faces, const UpdateFlags update_flags_faces_by_cells) @@ -303,53 +319,55 @@ namespace internal - template + template struct CompressedCellData { CompressedCellData(const double expected_size) - : data(FPArrayComparator(expected_size)) + : data(FPArrayComparator(expected_size)) {} std::map< Tensor<2, dim, - Tensor<1, VectorizedArray::n_array_elements, Number>>, + Tensor<1, VectorizedArrayType::n_array_elements, Number>>, unsigned int, - FPArrayComparator> + FPArrayComparator> data; }; /** * Internal temporary data used for the initialization. */ - template + template struct LocalData { LocalData(const double jac_size); void resize(const unsigned int size); - AlignedVector>> quadrature_points; - AlignedVector>> general_jac; - AlignedVector> JxW_values; - AlignedVector>> general_jac_grad; - AlignedVector>> normal_vectors; - Tensor<2, dim, VectorizedArray> const_jac; - const double jac_size; + AlignedVector> quadrature_points; + AlignedVector> general_jac; + AlignedVector JxW_values; + AlignedVector> general_jac_grad; + AlignedVector> normal_vectors; + Tensor<2, dim, VectorizedArrayType> const_jac; + const double jac_size; }; - template - LocalData::LocalData(const double jac_size_in) + template + LocalData::LocalData( + const double jac_size_in) : jac_size(jac_size_in) {} - template + template void - LocalData::resize(const unsigned int size) + LocalData::resize( + const unsigned int size) { if (JxW_values.size() != size) { @@ -364,28 +382,27 @@ namespace internal /** * Helper function called internally during the initialize function. */ - template + template void evaluate_on_cell( const dealii::Triangulation & tria, const std::pair *cells, const unsigned int my_q, GeometryType & cell_t_prev, - GeometryType (&cell_t)[VectorizedArray::n_array_elements], - dealii::FEValues &fe_val, - LocalData & cell_data) + GeometryType (&cell_t)[VectorizedArrayType::n_array_elements], + dealii::FEValues & fe_val, + LocalData &cell_data) { const unsigned int n_q_points = fe_val.n_quadrature_points; const UpdateFlags update_flags = fe_val.get_update_flags(); - cell_data.const_jac = Tensor<2, dim, VectorizedArray>(); + cell_data.const_jac = Tensor<2, dim, VectorizedArrayType>(); // this should be the same value as used in HashValue::scaling (but we // not have that field here) const double zero_tolerance_double = cell_data.jac_size * std::numeric_limits::epsilon() * 1024.; - for (unsigned int j = 0; j < VectorizedArray::n_array_elements; - ++j) + for (unsigned int j = 0; j < VectorizedArrayType::n_array_elements; ++j) { typename dealii::Triangulation::cell_iterator cell_it( &tria, cells[j].first, cells[j].second); @@ -550,12 +567,12 @@ namespace internal } // end loop over entries of vectorization (n_array_elements cells) // set information for next cell - cell_t_prev = cell_t[VectorizedArray::n_array_elements - 1]; + cell_t_prev = cell_t[VectorizedArrayType::n_array_elements - 1]; } - template + template void initialize_cell_range( const std::pair cell_range, @@ -565,14 +582,15 @@ namespace internal const Mapping & mapping, const std::vector> &quad, const UpdateFlags update_flags, - MappingInfo & mapping_info, - std::pair>, - CompressedCellData> & data) + MappingInfo &mapping_info, + std::pair>, + CompressedCellData> &data) { FE_Nothing dummy_fe; - Tensor<3, dim, VectorizedArray> jac_grad, grad_jac_inv; - Tensor<1, dim, VectorizedArray> tmp; + Tensor<3, dim, VectorizedArrayType> jac_grad, grad_jac_inv; + Tensor<1, dim, VectorizedArrayType> tmp; // when we make comparisons about the size of Jacobians we need to // know the approximate size of typical entries in Jacobians. We need @@ -587,12 +605,12 @@ namespace internal // we fill them up. Only after all vectorization_width cells have been // processed, we can insert the data into the data structures of this // class - LocalData cell_data(jacobian_size); + LocalData cell_data(jacobian_size); // encodes the cell types of the current cell. Since several cells // must be considered together, this variable holds the individual // info of the last chunk of cells - GeometryType cell_t[VectorizedArray::n_array_elements]; + GeometryType cell_t[VectorizedArrayType::n_array_elements]; GeometryType cell_t_prev = general; // fe_values object that is used to compute the mapping data. for @@ -670,9 +688,9 @@ namespace internal active_fe_index[cell] != active_fe_index[cell - 1]) cell_t_prev = general; - evaluate_on_cell( + evaluate_on_cell( tria, - &cells[cell * VectorizedArray::n_array_elements], + &cells[cell * VectorizedArrayType::n_array_elements], my_q, cell_t_prev, cell_t, @@ -690,7 +708,7 @@ namespace internal // (general cell)) GeometryType most_general_type = cartesian; for (unsigned int j = 0; - j < VectorizedArray::n_array_elements; + j < VectorizedArrayType::n_array_elements; ++j) if (cell_t[j] > most_general_type) most_general_type = cell_t[j]; @@ -718,7 +736,7 @@ namespace internal Tensor<2, dim, Tensor<1, - VectorizedArray::n_array_elements, + VectorizedArrayType::n_array_elements, Number>>, unsigned int> new_entry; @@ -729,7 +747,7 @@ namespace internal for (unsigned int d = 0; d < dim; ++d) for (unsigned int e = 0; e < dim; ++e) for (unsigned int v = 0; - v < VectorizedArray::n_array_elements; + v < VectorizedArrayType::n_array_elements; ++v) new_entry.first[d][e][v] = cell_data.const_jac[d][e][v]; @@ -750,12 +768,12 @@ namespace internal { for (unsigned int q = 0; q < n_q_points; ++q) { - Tensor<2, dim, VectorizedArray> &jac = + Tensor<2, dim, VectorizedArrayType> &jac = cell_data.general_jac[q]; - Tensor<3, dim, VectorizedArray> &jacobian_grad = + Tensor<3, dim, VectorizedArrayType> &jacobian_grad = cell_data.general_jac_grad[q]; for (unsigned int j = 0; - j < VectorizedArray::n_array_elements; + j < VectorizedArrayType::n_array_elements; ++j) if (cell_t[j] < general) { @@ -770,7 +788,7 @@ namespace internal data.first[my_q].JxW_values.push_back( determinant(jac) * fe_val.get_quadrature().weight(q)); - Tensor<2, dim, VectorizedArray> inv_jac = + Tensor<2, dim, VectorizedArrayType> inv_jac = transpose(invert(jac)); data.first[my_q].jacobians[0].push_back(inv_jac); @@ -810,7 +828,7 @@ namespace internal { for (unsigned int f = 0; f < dim; ++f) { - tmp[f] = VectorizedArray(); + tmp[f] = VectorizedArrayType(); for (unsigned int g = 0; g < dim; ++g) tmp[f] -= jac_grad[d][f][g] * inv_jac[g][e]; @@ -831,7 +849,7 @@ namespace internal // the diagonal part of Jacobian gradient comes first Tensor<1, dim *(dim + 1) / 2, - Tensor<1, dim, VectorizedArray>> + Tensor<1, dim, VectorizedArrayType>> final_grad; for (unsigned int d = 0; d < dim; ++d) for (unsigned int e = 0; e < dim; ++e) @@ -871,7 +889,7 @@ namespace internal q < n_q_points_1d[my_q][fe_index]; ++q) { - Point> quad_point; + Point quad_point; for (unsigned int d = 0; d < dim; ++d) quad_point[d] = cell_data.quadrature_points @@ -916,14 +934,19 @@ namespace internal - template + template void - copy_data(const unsigned int first_cell, - const std::array & data_shift, - const std::vector & indices_compressed, - const std::vector & cell_type, - MappingInfoStorage &data_cells_local, - MappingInfoStorage &data_cells) + copy_data(const unsigned int first_cell, + const std::array &data_shift, + const std::vector & indices_compressed, + const std::vector & cell_type, + MappingInfoStorage + &data_cells_local, + MappingInfoStorage + &data_cells) { // Copy the index offsets and shift by the appropriate value for (unsigned int lcell = 0; @@ -944,9 +967,9 @@ namespace internal // Copy quadrature points if (data_cells.quadrature_point_offsets.empty() == false) { - Point> *out_point = + Point *out_point = &data_cells.quadrature_points[data_shift[1]]; - for (const Point> *point = + for (const Point *point = data_cells_local.quadrature_points.begin(); point != data_cells_local.quadrature_points.end(); ++point, ++out_point) @@ -989,9 +1012,9 @@ namespace internal - template + template void - MappingInfo::initialize_cells( + MappingInfo::initialize_cells( const dealii::Triangulation & tria, const std::vector> &cells, const std::vector & active_fe_index, @@ -1002,7 +1025,7 @@ namespace internal const unsigned int n_quads = quad.size(); const unsigned int n_cells = cells.size(); const unsigned int vectorization_width = - VectorizedArray::n_array_elements; + VectorizedArrayType::n_array_elements; Assert(n_cells % vectorization_width == 0, ExcInternalError()); const unsigned int n_macro_cells = n_cells / vectorization_width; cell_data.resize(n_quads); @@ -1032,8 +1055,10 @@ namespace internal (n_macro_cells + MultithreadInfo::n_threads() - 1) / MultithreadInfo::n_threads()); - std::vector>, - ExtractCellHelper::CompressedCellData>> + std::vector>, + ExtractCellHelper:: + CompressedCellData>> data_cells_local; // Reserve enough space to avoid re-allocation (which would break the // references to the data fields passed to the tasks!) @@ -1045,11 +1070,15 @@ namespace internal while (cell_range.first < n_macro_cells) { data_cells_local.push_back(std::make_pair( - std::vector>(n_quads), - ExtractCellHelper::CompressedCellData( - ExtractCellHelper::get_jacobian_size(tria)))); + std::vector< + MappingInfoStorage>( + n_quads), + ExtractCellHelper:: + CompressedCellData( + ExtractCellHelper::get_jacobian_size(tria)))); tasks += Threads::new_task( - &ExtractCellHelper::initialize_cell_range, + &ExtractCellHelper:: + initialize_cell_range, cell_range, tria, cells, @@ -1114,14 +1143,15 @@ namespace internal // Start tasks that copy the local data Threads::TaskGroup<> tasks; for (unsigned int i = 0; i < data_cells_local.size(); ++i) - tasks += - Threads::new_task(&ExtractCellHelper::copy_data, - work_per_chunk * i, - shift[i], - indices_compressed[i], - cell_type, - data_cells_local[i].first[my_q], - cell_data[my_q]); + tasks += Threads::new_task( + &ExtractCellHelper:: + copy_data, + work_per_chunk * i, + shift[i], + indices_compressed[i], + cell_type, + data_cells_local[i].first[my_q], + cell_data[my_q]); // finally, insert the constant cell data at the beginning (the // other tasks can already start copying the non-constant data) @@ -1129,11 +1159,11 @@ namespace internal { for (const auto &it : data_cells_local[0].second.data) { - Tensor<2, dim, VectorizedArray> jac; + Tensor<2, dim, VectorizedArrayType> jac; for (unsigned int d = 0; d < dim; ++d) for (unsigned int e = 0; e < dim; ++e) for (unsigned int v = 0; - v < VectorizedArray::n_array_elements; + v < VectorizedArrayType::n_array_elements; ++v) jac[d][e][v] = it.first[d][e][v]; AssertIndexRange(it.second, n_constant_jacobians); @@ -1167,7 +1197,7 @@ namespace internal // range namespace ExtractFaceHelper { - template + template struct CompressedFaceData { // Constructor. As a scaling factor for the FPArrayComparator, we @@ -1175,7 +1205,8 @@ namespace internal // CompressedCellData) and add another factor of 512 to account for // some roundoff effects. CompressedFaceData(const Number jacobian_size) - : data(FPArrayComparator(512. / jacobian_size)) + : data(FPArrayComparator(512. / + jacobian_size)) , jacobian_size(jacobian_size) {} @@ -1188,9 +1219,9 @@ namespace internal std::map< Tensor<1, 2 * dim * dim + dim + 1, - Tensor<1, VectorizedArray::n_array_elements, Number>>, + Tensor<1, VectorizedArrayType::n_array_elements, Number>>, unsigned int, - FPArrayComparator> + FPArrayComparator> data; // Store the scaling factor @@ -1230,20 +1261,22 @@ namespace internal - template + template void initialize_face_range( const std::pair face_range, const dealii::Triangulation & tria, const std::vector> &cells, const std::vector< - FaceToCellTopology::n_array_elements>> &faces, - const Mapping & mapping, - const UpdateFlags update_flags_boundary, - const UpdateFlags update_flags_inner, - MappingInfo & mapping_info, - std::pair>, - CompressedFaceData> &data) + FaceToCellTopology> &faces, + const Mapping & mapping, + const UpdateFlags update_flags_boundary, + const UpdateFlags update_flags_inner, + MappingInfo &mapping_info, + std::pair< + std::vector< + MappingInfoStorage>, + CompressedFaceData> &data) { FE_Nothing dummy_fe; @@ -1268,8 +1301,8 @@ namespace internal fe_subface_values_container[my_q].resize( mapping_info.face_data[my_q].descriptor.size()); - ExtractCellHelper::LocalData face_data( - ExtractCellHelper::get_jacobian_size(tria)); + ExtractCellHelper::LocalData + face_data(ExtractCellHelper::get_jacobian_size(tria)); const unsigned int end_face = std::min(std::size_t(face_range.second), faces.size()); @@ -1306,7 +1339,7 @@ namespace internal bool JxW_is_similar = true; bool cell_is_cartesian = true; for (unsigned int v = 0; - v < VectorizedArray::n_array_elements; + v < VectorizedArrayType::n_array_elements; ++v) { Tensor<2, dim> jacobian_0; @@ -1529,7 +1562,7 @@ namespace internal } using VEC_ARRAY = - Tensor<1, VectorizedArray::n_array_elements, Number>; + Tensor<1, VectorizedArrayType::n_array_elements, Number>; unsigned int insert_position = data.first[my_q].JxW_values.size(); // Fill in JxW values, apply compression @@ -1543,7 +1576,7 @@ namespace internal new_entry; new_entry.second = data.second.data.size(); for (unsigned int v = 0; - v < VectorizedArray::n_array_elements; + v < VectorizedArrayType::n_array_elements; ++v) new_entry.first[2 * dim * dim + dim][v] = face_data.JxW_values[0][v] / quadrature.weight(0) / @@ -1553,7 +1586,7 @@ namespace internal for (unsigned int d = 0; d < dim; ++d) for (unsigned int e = 0; e < dim; ++e) for (unsigned int v = 0; - v < VectorizedArray::n_array_elements; + v < VectorizedArrayType::n_array_elements; ++v) new_entry.first[d * dim + e][v] = face_data.general_jac[0][d][e][v]; @@ -1561,7 +1594,7 @@ namespace internal for (unsigned int d = 0; d < dim; ++d) for (unsigned int e = 0; e < dim; ++e) for (unsigned int v = 0; - v < VectorizedArray::n_array_elements; + v < VectorizedArrayType::n_array_elements; ++v) new_entry.first[dim * dim + d * dim + e][v] = face_data.general_jac[n_q_points][d][e][v]; @@ -1574,7 +1607,7 @@ namespace internal // size to get the right scaling for (unsigned int d = 0; d < dim; ++d) for (unsigned int v = 0; - v < VectorizedArray::n_array_elements; + v < VectorizedArrayType::n_array_elements; ++v) new_entry.first[2 * dim * dim + d][v] = face_data.normal_vectors[0][d][v] / @@ -1606,15 +1639,16 @@ namespace internal } } - template + template void compute_normal_times_jacobian( const unsigned int first_face, const unsigned int last_face, const std::vector &face_type, const std::vector< - FaceToCellTopology::n_array_elements>> &faces, - MappingInfoStorage &data_faces) + FaceToCellTopology> &faces, + MappingInfoStorage + &data_faces) { for (unsigned int face = first_face; face < last_face; ++face) { @@ -1642,15 +1676,15 @@ namespace internal - template + template void - MappingInfo::initialize_faces( + MappingInfo::initialize_faces( const dealii::Triangulation & tria, const std::vector> &cells, const std::vector< - FaceToCellTopology::n_array_elements>> &faces, - const Mapping & mapping, - const std::vector> & quad, + FaceToCellTopology> &faces, + const Mapping & mapping, + const std::vector> & quad, const UpdateFlags update_flags_boundary_faces, const UpdateFlags update_flags_inner_faces) { @@ -1692,9 +1726,11 @@ namespace internal (faces.size() + MultithreadInfo::n_threads() - 1) / MultithreadInfo::n_threads()); - std::vector< - std::pair>, - ExtractFaceHelper::CompressedFaceData>> + std::vector>, + ExtractFaceHelper:: + CompressedFaceData>> data_faces_local; // Reserve enough space to avoid re-allocation (which would destroy the // references passed to the tasks!) @@ -1706,12 +1742,15 @@ namespace internal while (face_range.first < faces.size()) { data_faces_local.push_back(std::make_pair( - std::vector>( + std::vector< + MappingInfoStorage>( quad.size()), - ExtractFaceHelper::CompressedFaceData( - ExtractCellHelper::get_jacobian_size(tria)))); + ExtractFaceHelper:: + CompressedFaceData( + ExtractCellHelper::get_jacobian_size(tria)))); tasks += Threads::new_task( - &ExtractFaceHelper::initialize_face_range, + &ExtractFaceHelper:: + initialize_face_range, face_range, tria, cells, @@ -1790,7 +1829,8 @@ namespace internal Threads::TaskGroup<> tasks; for (unsigned int i = 0; i < data_faces_local.size(); ++i) tasks += Threads::new_task( - &ExtractCellHelper::copy_data, + &ExtractCellHelper:: + copy_data, work_per_chunk * i, shift[i], indices_compressed[i], @@ -1807,7 +1847,7 @@ namespace internal { // JxW values; invert previously applied scaling for (unsigned int v = 0; - v < VectorizedArray::n_array_elements; + v < VectorizedArrayType::n_array_elements; ++v) face_data[my_q].JxW_values[it.second][v] = it.first[2 * dim * dim + dim][v] * @@ -1818,7 +1858,7 @@ namespace internal for (unsigned int d = 0; d < dim; ++d) for (unsigned int e = 0; e < dim; ++e) for (unsigned int v = 0; - v < VectorizedArray::n_array_elements; + v < VectorizedArrayType::n_array_elements; ++v) face_data[my_q].jacobians[i][it.second][d][e][v] = it.first[i * dim * dim + d * dim + e][v]; @@ -1826,7 +1866,7 @@ namespace internal // normal vectors; invert previously applied scaling for (unsigned int d = 0; d < dim; ++d) for (unsigned int v = 0; - v < VectorizedArray::n_array_elements; + v < VectorizedArrayType::n_array_elements; ++v) face_data[my_q].normal_vectors[it.second][d][v] = it.first[2 * dim * dim + d][v] * jac_size; @@ -1851,7 +1891,8 @@ namespace internal // finally compute the normal times the jacobian for (unsigned int i = 0; i < data_faces_local.size(); ++i) tasks += Threads::new_task( - &ExtractFaceHelper::compute_normal_times_jacobian, + &ExtractFaceHelper:: + compute_normal_times_jacobian, work_per_chunk * i, std::min(work_per_chunk * (i + 1), faces.size()), face_type, @@ -1863,9 +1904,9 @@ namespace internal - template + template void - MappingInfo::initialize_faces_by_cells( + MappingInfo::initialize_faces_by_cells( const dealii::Triangulation & tria, const std::vector> &cells, const Mapping & mapping, @@ -1878,7 +1919,7 @@ namespace internal face_data_by_cells.resize(quad.size()); const unsigned int n_quads = quad.size(); const unsigned int vectorization_width = - VectorizedArray::n_array_elements; + VectorizedArrayType::n_array_elements; UpdateFlags update_flags = (update_flags_faces_by_cells & update_quadrature_points ? update_quadrature_points : @@ -2078,9 +2119,9 @@ namespace internal - template + template std::size_t - MappingInfo::memory_consumption() const + MappingInfo::memory_consumption() const { std::size_t memory = MemoryConsumption::memory_consumption(cell_data); memory += MemoryConsumption::memory_consumption(face_data); @@ -2092,10 +2133,10 @@ namespace internal - template + template template void - MappingInfo::print_memory_consumption( + MappingInfo::print_memory_consumption( StreamType & out, const SizeInfo &task_info) const { @@ -2119,17 +2160,19 @@ namespace internal /* ------------------------------------------------------------------ */ - template - FPArrayComparator::FPArrayComparator(const Number scaling) + template + FPArrayComparator::FPArrayComparator( + const Number scaling) : tolerance(scaling * std::numeric_limits::epsilon() * 1024.) {} - template + template bool - FPArrayComparator::operator()(const std::vector &v1, - const std::vector &v2) const + FPArrayComparator:: + operator()(const std::vector &v1, + const std::vector &v2) const { const unsigned int s1 = v1.size(), s2 = v2.size(); if (s1 < s2) @@ -2147,15 +2190,13 @@ namespace internal - template + template bool - FPArrayComparator::operator()( - const Tensor<1, VectorizedArray::n_array_elements, Number> &t1, - const Tensor<1, VectorizedArray::n_array_elements, Number> &t2) - const + FPArrayComparator::operator()( + const Tensor<1, VectorizedArrayType::n_array_elements, Number> &t1, + const Tensor<1, VectorizedArrayType::n_array_elements, Number> &t2) const { - for (unsigned int k = 0; k < VectorizedArray::n_array_elements; - ++k) + for (unsigned int k = 0; k < VectorizedArrayType::n_array_elements; ++k) if (t1[k] < t2[k] - tolerance) return true; else if (t1[k] > t2[k] + tolerance) @@ -2165,22 +2206,21 @@ namespace internal - template + template template bool - FPArrayComparator::operator()( + FPArrayComparator::operator()( const Tensor<1, dim, - Tensor<1, VectorizedArray::n_array_elements, Number>> + Tensor<1, VectorizedArrayType::n_array_elements, Number>> &t1, const Tensor<1, dim, - Tensor<1, VectorizedArray::n_array_elements, Number>> + Tensor<1, VectorizedArrayType::n_array_elements, Number>> &t2) const { for (unsigned int d = 0; d < dim; ++d) - for (unsigned int k = 0; k < VectorizedArray::n_array_elements; - ++k) + for (unsigned int k = 0; k < VectorizedArrayType::n_array_elements; ++k) if (t1[d][k] < t2[d][k] - tolerance) return true; else if (t1[d][k] > t2[d][k] + tolerance) @@ -2190,23 +2230,22 @@ namespace internal - template + template template bool - FPArrayComparator::operator()( + FPArrayComparator::operator()( const Tensor<2, dim, - Tensor<1, VectorizedArray::n_array_elements, Number>> + Tensor<1, VectorizedArrayType::n_array_elements, Number>> &t1, const Tensor<2, dim, - Tensor<1, VectorizedArray::n_array_elements, Number>> + Tensor<1, VectorizedArrayType::n_array_elements, Number>> &t2) const { for (unsigned int d = 0; d < dim; ++d) for (unsigned int e = 0; e < dim; ++e) - for (unsigned int k = 0; - k < VectorizedArray::n_array_elements; + for (unsigned int k = 0; k < VectorizedArrayType::n_array_elements; ++k) if (t1[d][e][k] < t2[d][e][k] - tolerance) return true; diff --git a/include/deal.II/matrix_free/matrix_free.h b/include/deal.II/matrix_free/matrix_free.h index fb456f1203..9749a4c319 100644 --- a/include/deal.II/matrix_free/matrix_free.h +++ b/include/deal.II/matrix_free/matrix_free.h @@ -109,15 +109,22 @@ DEAL_II_NAMESPACE_OPEN * @author Katharina Kormann, Martin Kronbichler, 2010, 2011 */ -template +template > class MatrixFree : public Subscriptor { + static_assert( + std::is_same::value, + "Type of Number and of VectorizedArrayType do not match."); + public: /** * An alias for the underlying number type specified by the template * argument. */ - using value_type = Number; + using value_type = Number; + using vectorized_value_type = VectorizedArrayType; /** * The dimension set by the template argument `dim`. @@ -469,7 +476,7 @@ public: /** * Copy constructor, calls copy_from */ - MatrixFree(const MatrixFree &other); + MatrixFree(const MatrixFree &other); /** * Destructor. @@ -611,7 +618,8 @@ public: * should not be needed very often. */ void - copy_from(const MatrixFree &matrix_free_base); + copy_from( + const MatrixFree &matrix_free_base); /** * Clear all data fields and brings the class into a condition similar to @@ -717,15 +725,14 @@ public: */ template void - cell_loop( - const std::function &, - OutVector &, - const InVector &, - const std::pair &)> - & cell_operation, - OutVector & dst, - const InVector &src, - const bool zero_dst_vector = false) const; + cell_loop(const std::function &, + OutVector &, + const InVector &, + const std::pair &)> &cell_operation, + OutVector & dst, + const InVector & src, + const bool zero_dst_vector = false) const; /** * This is the second variant to run the loop over all cells, now providing @@ -872,23 +879,23 @@ public: */ template void - loop(const std::function &, - OutVector &, - const InVector &, - const std::pair &)> - &cell_operation, - const std::function &, - OutVector &, - const InVector &, - const std::pair &)> - &face_operation, - const std::function &, - OutVector &, - const InVector &, - const std::pair &)> - & boundary_operation, - OutVector & dst, - const InVector & src, + loop(const std::function< + void(const MatrixFree &, + OutVector &, + const InVector &, + const std::pair &)> &cell_operation, + const std::function< + void(const MatrixFree &, + OutVector &, + const InVector &, + const std::pair &)> &face_operation, + const std::function &, + OutVector &, + const InVector &, + const std::pair &)> &boundary_operation, + OutVector & dst, + const InVector & src, const bool zero_dst_vector = false, const DataAccessOnFaces dst_vector_face_access = DataAccessOnFaces::unspecified, @@ -1276,7 +1283,7 @@ public: * Return the boundary ids for the faces within a cell, using the cells' * sorting by lanes in the VectorizedArray. */ - std::array::n_array_elements> + std::array get_faces_by_cells_boundary_id(const unsigned int macro_cell, const unsigned int face_number) const; @@ -1485,8 +1492,9 @@ public: /* * Return geometry-dependent information on the cells. */ - const internal::MatrixFreeFunctions::MappingInfo & - get_mapping_info() const; + const internal::MatrixFreeFunctions:: + MappingInfo & + get_mapping_info() const; /** * Return information on indexation degrees of freedom. @@ -1518,7 +1526,7 @@ public: /** * Return the unit cell information for given hp index. */ - const internal::MatrixFreeFunctions::ShapeInfo> & + const internal::MatrixFreeFunctions::ShapeInfo & get_shape_info(const unsigned int dof_handler_index_component = 0, const unsigned int quad_index = 0, const unsigned int fe_base_element = 0, @@ -1529,7 +1537,7 @@ public: * Return the connectivity information of a face. */ const internal::MatrixFreeFunctions::FaceToCellTopology< - VectorizedArray::n_array_elements> & + VectorizedArrayType::n_array_elements> & get_face_info(const unsigned int face_batch_number) const; /** @@ -1545,15 +1553,14 @@ public: * local contributions of WorkStream, see * @ref workstream_paper "the WorkStream paper". */ - AlignedVector> * + AlignedVector * acquire_scratch_data() const; /** * Makes the object of the scratchpad available again. */ void - release_scratch_data( - const AlignedVector> *memory) const; + release_scratch_data(const AlignedVector *memory) const; /** * Obtains a scratch data object for internal use. Make sure to release it @@ -1699,12 +1706,13 @@ private: * Holds information on transformation of cells from reference cell to real * cell that is needed for evaluating integrals. */ - internal::MatrixFreeFunctions::MappingInfo mapping_info; + internal::MatrixFreeFunctions::MappingInfo + mapping_info; /** * Contains shape value information on the unit cell. */ - Table<4, internal::MatrixFreeFunctions::ShapeInfo>> + Table<4, internal::MatrixFreeFunctions::ShapeInfo> shape_info; /** @@ -1735,8 +1743,7 @@ private: * Vector holding face information. Only initialized if * build_face_info=true. */ - internal::MatrixFreeFunctions::FaceInfo< - VectorizedArray::n_array_elements> + internal::MatrixFreeFunctions::FaceInfo face_info; /** @@ -1757,7 +1764,7 @@ private: * objects. */ mutable Threads::ThreadLocalStorage< - std::list>>>> + std::list>>> scratch_pad; /** @@ -1776,11 +1783,12 @@ private: -template +template template inline void -MatrixFree::initialize_dof_vector(VectorType & vec, - const unsigned int comp) const +MatrixFree::initialize_dof_vector( + VectorType & vec, + const unsigned int comp) const { AssertIndexRange(comp, n_components()); vec.reinit(dof_info[comp].vector_partitioner->size()); @@ -1788,10 +1796,10 @@ MatrixFree::initialize_dof_vector(VectorType & vec, -template +template template inline void -MatrixFree::initialize_dof_vector( +MatrixFree::initialize_dof_vector( LinearAlgebra::distributed::Vector &vec, const unsigned int comp) const { @@ -1801,9 +1809,10 @@ MatrixFree::initialize_dof_vector( -template +template inline const std::shared_ptr & -MatrixFree::get_vector_partitioner(const unsigned int comp) const +MatrixFree::get_vector_partitioner( + const unsigned int comp) const { AssertIndexRange(comp, n_components()); return dof_info[comp].vector_partitioner; @@ -1811,9 +1820,10 @@ MatrixFree::get_vector_partitioner(const unsigned int comp) const -template +template inline const std::vector & -MatrixFree::get_constrained_dofs(const unsigned int comp) const +MatrixFree::get_constrained_dofs( + const unsigned int comp) const { AssertIndexRange(comp, n_components()); return dof_info[comp].constrained_dofs; @@ -1821,9 +1831,9 @@ MatrixFree::get_constrained_dofs(const unsigned int comp) const -template +template inline unsigned int -MatrixFree::n_components() const +MatrixFree::n_components() const { AssertDimension(dof_handlers.n_dof_handlers, dof_info.size()); return dof_handlers.n_dof_handlers; @@ -1831,9 +1841,10 @@ MatrixFree::n_components() const -template +template inline unsigned int -MatrixFree::n_base_elements(const unsigned int dof_no) const +MatrixFree::n_base_elements( + const unsigned int dof_no) const { AssertDimension(dof_handlers.n_dof_handlers, dof_info.size()); AssertIndexRange(dof_no, dof_handlers.n_dof_handlers); @@ -1842,54 +1853,54 @@ MatrixFree::n_base_elements(const unsigned int dof_no) const -template +template inline const internal::MatrixFreeFunctions::TaskInfo & -MatrixFree::get_task_info() const +MatrixFree::get_task_info() const { return task_info; } -template +template inline const internal::MatrixFreeFunctions::TaskInfo & -MatrixFree::get_size_info() const +MatrixFree::get_size_info() const { return task_info; } -template +template inline unsigned int -MatrixFree::n_macro_cells() const +MatrixFree::n_macro_cells() const { return *(task_info.cell_partition_data.end() - 2); } -template +template inline unsigned int -MatrixFree::n_physical_cells() const +MatrixFree::n_physical_cells() const { return task_info.n_active_cells; } -template +template inline unsigned int -MatrixFree::n_cell_batches() const +MatrixFree::n_cell_batches() const { return *(task_info.cell_partition_data.end() - 2); } -template +template inline unsigned int -MatrixFree::n_ghost_cell_batches() const +MatrixFree::n_ghost_cell_batches() const { return *(task_info.cell_partition_data.end() - 1) - *(task_info.cell_partition_data.end() - 2); @@ -1897,9 +1908,9 @@ MatrixFree::n_ghost_cell_batches() const -template +template inline unsigned int -MatrixFree::n_inner_face_batches() const +MatrixFree::n_inner_face_batches() const { if (task_info.face_partition_data.size() == 0) return 0; @@ -1908,9 +1919,9 @@ MatrixFree::n_inner_face_batches() const -template +template inline unsigned int -MatrixFree::n_boundary_face_batches() const +MatrixFree::n_boundary_face_batches() const { if (task_info.face_partition_data.size() == 0) return 0; @@ -1920,9 +1931,9 @@ MatrixFree::n_boundary_face_batches() const -template +template inline unsigned int -MatrixFree::n_ghost_inner_face_batches() const +MatrixFree::n_ghost_inner_face_batches() const { if (task_info.face_partition_data.size() == 0) return 0; @@ -1931,9 +1942,10 @@ MatrixFree::n_ghost_inner_face_batches() const -template +template inline types::boundary_id -MatrixFree::get_boundary_id(const unsigned int macro_face) const +MatrixFree::get_boundary_id( + const unsigned int macro_face) const { Assert(macro_face >= task_info.boundary_partition_data[0] && macro_face < task_info.boundary_partition_data.back(), @@ -1945,9 +1957,9 @@ MatrixFree::get_boundary_id(const unsigned int macro_face) const -template -inline std::array::n_array_elements> -MatrixFree::get_faces_by_cells_boundary_id( +template +inline std::array +MatrixFree::get_faces_by_cells_boundary_id( const unsigned int macro_cell, const unsigned int face_number) const { @@ -1955,8 +1967,7 @@ MatrixFree::get_faces_by_cells_boundary_id( AssertIndexRange(face_number, GeometryInfo::faces_per_cell); Assert(face_info.cell_and_face_boundary_id.size(0) >= n_macro_cells(), ExcNotInitialized()); - std::array::n_array_elements> - result; + std::array result; result.fill(numbers::invalid_boundary_id); for (unsigned int v = 0; v < n_active_entries_per_cell_batch(macro_cell); ++v) result[v] = face_info.cell_and_face_boundary_id(macro_cell, face_number, v); @@ -1965,18 +1976,20 @@ MatrixFree::get_faces_by_cells_boundary_id( -template -inline const internal::MatrixFreeFunctions::MappingInfo & -MatrixFree::get_mapping_info() const +template +inline const internal::MatrixFreeFunctions:: + MappingInfo & + MatrixFree::get_mapping_info() const { return mapping_info; } -template +template inline const internal::MatrixFreeFunctions::DoFInfo & -MatrixFree::get_dof_info(const unsigned int dof_index) const +MatrixFree::get_dof_info( + const unsigned int dof_index) const { AssertIndexRange(dof_index, n_components()); return dof_info[dof_index]; @@ -1984,18 +1997,19 @@ MatrixFree::get_dof_info(const unsigned int dof_index) const -template +template inline unsigned int -MatrixFree::n_constraint_pool_entries() const +MatrixFree::n_constraint_pool_entries() const { return constraint_pool_row_index.size() - 1; } -template +template inline const Number * -MatrixFree::constraint_pool_begin(const unsigned int row) const +MatrixFree::constraint_pool_begin( + const unsigned int row) const { AssertIndexRange(row, constraint_pool_row_index.size() - 1); return constraint_pool_data.empty() ? @@ -2005,9 +2019,10 @@ MatrixFree::constraint_pool_begin(const unsigned int row) const -template +template inline const Number * -MatrixFree::constraint_pool_end(const unsigned int row) const +MatrixFree::constraint_pool_end( + const unsigned int row) const { AssertIndexRange(row, constraint_pool_row_index.size() - 1); return constraint_pool_data.empty() ? @@ -2017,9 +2032,9 @@ MatrixFree::constraint_pool_end(const unsigned int row) const -template +template inline std::pair -MatrixFree::create_cell_subrange_hp( +MatrixFree::create_cell_subrange_hp( const std::pair &range, const unsigned int degree, const unsigned int dof_handler_component) const @@ -2048,25 +2063,26 @@ MatrixFree::create_cell_subrange_hp( -template +template inline bool -MatrixFree::at_irregular_cell(const unsigned int macro_cell) const +MatrixFree::at_irregular_cell( + const unsigned int macro_cell) const { AssertIndexRange(macro_cell, task_info.cell_partition_data.back()); - return VectorizedArray::n_array_elements > 1 && + return VectorizedArrayType::n_array_elements > 1 && cell_level_index[(macro_cell + 1) * - VectorizedArray::n_array_elements - + VectorizedArrayType::n_array_elements - 1] == cell_level_index[(macro_cell + 1) * - VectorizedArray::n_array_elements - + VectorizedArrayType::n_array_elements - 2]; } -template +template inline unsigned int -MatrixFree::n_components_filled( +MatrixFree::n_components_filled( const unsigned int cell_batch_number) const { return n_active_entries_per_cell_batch(cell_batch_number); @@ -2074,47 +2090,47 @@ MatrixFree::n_components_filled( -template +template inline unsigned int -MatrixFree::n_active_entries_per_cell_batch( +MatrixFree::n_active_entries_per_cell_batch( const unsigned int cell_batch_number) const { AssertIndexRange(cell_batch_number, task_info.cell_partition_data.back()); - unsigned int n_components = VectorizedArray::n_array_elements; - while (n_components > 1 && - cell_level_index[cell_batch_number * - VectorizedArray::n_array_elements + - n_components - 1] == - cell_level_index[cell_batch_number * - VectorizedArray::n_array_elements + - n_components - 2]) + unsigned int n_components = VectorizedArrayType::n_array_elements; + while ( + n_components > 1 && + cell_level_index[cell_batch_number * VectorizedArrayType::n_array_elements + + n_components - 1] == + cell_level_index[cell_batch_number * + VectorizedArrayType::n_array_elements + + n_components - 2]) --n_components; - AssertIndexRange(n_components - 1, VectorizedArray::n_array_elements); + AssertIndexRange(n_components - 1, VectorizedArrayType::n_array_elements); return n_components; } -template +template inline unsigned int -MatrixFree::n_active_entries_per_face_batch( +MatrixFree::n_active_entries_per_face_batch( const unsigned int face_batch_number) const { AssertIndexRange(face_batch_number, face_info.faces.size()); - unsigned int n_components = VectorizedArray::n_array_elements; + unsigned int n_components = VectorizedArrayType::n_array_elements; while (n_components > 1 && face_info.faces[face_batch_number].cells_interior[n_components - 1] == numbers::invalid_unsigned_int) --n_components; - AssertIndexRange(n_components - 1, VectorizedArray::n_array_elements); + AssertIndexRange(n_components - 1, VectorizedArrayType::n_array_elements); return n_components; } -template +template inline unsigned int -MatrixFree::get_dofs_per_cell( +MatrixFree::get_dofs_per_cell( const unsigned int dof_handler_index, const unsigned int active_fe_index) const { @@ -2123,9 +2139,9 @@ MatrixFree::get_dofs_per_cell( -template +template inline unsigned int -MatrixFree::get_n_q_points( +MatrixFree::get_n_q_points( const unsigned int quad_index, const unsigned int active_fe_index) const { @@ -2137,9 +2153,9 @@ MatrixFree::get_n_q_points( -template +template inline unsigned int -MatrixFree::get_dofs_per_face( +MatrixFree::get_dofs_per_face( const unsigned int dof_handler_index, const unsigned int active_fe_index) const { @@ -2148,9 +2164,9 @@ MatrixFree::get_dofs_per_face( -template +template inline unsigned int -MatrixFree::get_n_q_points_face( +MatrixFree::get_n_q_points_face( const unsigned int quad_index, const unsigned int active_fe_index) const { @@ -2162,9 +2178,9 @@ MatrixFree::get_n_q_points_face( -template +template inline const IndexSet & -MatrixFree::get_locally_owned_set( +MatrixFree::get_locally_owned_set( const unsigned int dof_handler_index) const { return dof_info[dof_handler_index].vector_partitioner->locally_owned_range(); @@ -2172,9 +2188,9 @@ MatrixFree::get_locally_owned_set( -template +template inline const IndexSet & -MatrixFree::get_ghost_set( +MatrixFree::get_ghost_set( const unsigned int dof_handler_index) const { return dof_info[dof_handler_index].vector_partitioner->ghost_indices(); @@ -2182,9 +2198,9 @@ MatrixFree::get_ghost_set( -template -inline const internal::MatrixFreeFunctions::ShapeInfo> & -MatrixFree::get_shape_info( +template +inline const internal::MatrixFreeFunctions::ShapeInfo & +MatrixFree::get_shape_info( const unsigned int dof_handler_index, const unsigned int index_quad, const unsigned int index_fe, @@ -2203,10 +2219,11 @@ MatrixFree::get_shape_info( -template +template inline const internal::MatrixFreeFunctions::FaceToCellTopology< - VectorizedArray::n_array_elements> & -MatrixFree::get_face_info(const unsigned int macro_face) const + VectorizedArrayType::n_array_elements> & +MatrixFree::get_face_info( + const unsigned int macro_face) const { AssertIndexRange(macro_face, face_info.faces.size()); return face_info.faces[macro_face]; @@ -2214,9 +2231,9 @@ MatrixFree::get_face_info(const unsigned int macro_face) const -template +template inline const Quadrature & -MatrixFree::get_quadrature( +MatrixFree::get_quadrature( const unsigned int quad_index, const unsigned int active_fe_index) const { @@ -2228,9 +2245,9 @@ MatrixFree::get_quadrature( -template +template inline const Quadrature & -MatrixFree::get_face_quadrature( +MatrixFree::get_face_quadrature( const unsigned int quad_index, const unsigned int active_fe_index) const { @@ -2242,9 +2259,10 @@ MatrixFree::get_face_quadrature( -template +template inline unsigned int -MatrixFree::get_cell_category(const unsigned int macro_cell) const +MatrixFree::get_cell_category( + const unsigned int macro_cell) const { AssertIndexRange(0, dof_info.size()); AssertIndexRange(macro_cell, dof_info[0].cell_active_fe_index.size()); @@ -2256,16 +2274,17 @@ MatrixFree::get_cell_category(const unsigned int macro_cell) const -template +template inline std::pair -MatrixFree::get_face_category(const unsigned int macro_face) const +MatrixFree::get_face_category( + const unsigned int macro_face) const { AssertIndexRange(macro_face, face_info.faces.size()); if (dof_info[0].cell_active_fe_index.empty()) return std::make_pair(0U, 0U); std::pair result; - for (unsigned int v = 0; v < VectorizedArray::n_array_elements && + for (unsigned int v = 0; v < VectorizedArrayType::n_array_elements && face_info.faces[macro_face].cells_interior[v] != numbers::invalid_unsigned_int; ++v) @@ -2275,7 +2294,7 @@ MatrixFree::get_face_category(const unsigned int macro_face) const .cell_active_fe_index[face_info.faces[macro_face].cells_interior[v]]); if (face_info.faces[macro_face].cells_exterior[0] != numbers::invalid_unsigned_int) - for (unsigned int v = 0; v < VectorizedArray::n_array_elements && + for (unsigned int v = 0; v < VectorizedArrayType::n_array_elements && face_info.faces[macro_face].cells_exterior[v] != numbers::invalid_unsigned_int; ++v) @@ -2290,30 +2309,30 @@ MatrixFree::get_face_category(const unsigned int macro_face) const -template +template inline bool -MatrixFree::indices_initialized() const +MatrixFree::indices_initialized() const { return indices_are_initialized; } -template +template inline bool -MatrixFree::mapping_initialized() const +MatrixFree::mapping_initialized() const { return mapping_is_initialized; } -template -AlignedVector> * -MatrixFree::acquire_scratch_data() const +template +AlignedVector * +MatrixFree::acquire_scratch_data() const { using list_type = - std::list>>>; + std::list>>; list_type &data = scratch_pad.get(); for (typename list_type::iterator it = data.begin(); it != data.end(); ++it) if (it->first == false) @@ -2321,20 +2340,19 @@ MatrixFree::acquire_scratch_data() const it->first = true; return &it->second; } - data.push_front( - std::make_pair(true, AlignedVector>())); + data.emplace_front(true, AlignedVector()); return &data.front().second; } -template +template void -MatrixFree::release_scratch_data( - const AlignedVector> *scratch) const +MatrixFree::release_scratch_data( + const AlignedVector *scratch) const { using list_type = - std::list>>>; + std::list>>; list_type &data = scratch_pad.get(); for (typename list_type::iterator it = data.begin(); it != data.end(); ++it) if (&it->second == scratch) @@ -2348,9 +2366,10 @@ MatrixFree::release_scratch_data( -template +template AlignedVector * -MatrixFree::acquire_scratch_data_non_threadsafe() const +MatrixFree:: + acquire_scratch_data_non_threadsafe() const { for (typename std::list>>::iterator it = scratch_pad_non_threadsafe.begin(); @@ -2368,10 +2387,11 @@ MatrixFree::acquire_scratch_data_non_threadsafe() const -template +template void -MatrixFree::release_scratch_data_non_threadsafe( - const AlignedVector *scratch) const +MatrixFree:: + release_scratch_data_non_threadsafe( + const AlignedVector *scratch) const { for (typename std::list>>::iterator it = scratch_pad_non_threadsafe.begin(); @@ -2430,14 +2450,15 @@ namespace internal -template +template template void -MatrixFree::reinit( - const DoFHandlerType & dof_handler, - const AffineConstraints & constraints_in, - const QuadratureType & quad, - const typename MatrixFree::AdditionalData additional_data) +MatrixFree::reinit( + const DoFHandlerType & dof_handler, + const AffineConstraints &constraints_in, + const QuadratureType & quad, + const typename MatrixFree::AdditionalData + additional_data) { std::vector dof_handlers; std::vector *> constraints; @@ -2464,15 +2485,16 @@ MatrixFree::reinit( -template +template template void -MatrixFree::reinit( - const Mapping & mapping, - const DoFHandlerType & dof_handler, - const AffineConstraints & constraints_in, - const QuadratureType & quad, - const typename MatrixFree::AdditionalData additional_data) +MatrixFree::reinit( + const Mapping & mapping, + const DoFHandlerType & dof_handler, + const AffineConstraints &constraints_in, + const QuadratureType & quad, + const typename MatrixFree::AdditionalData + additional_data) { std::vector dof_handlers; std::vector *> constraints; @@ -2497,14 +2519,15 @@ MatrixFree::reinit( -template +template template void -MatrixFree::reinit( +MatrixFree::reinit( const std::vector & dof_handler, const std::vector *> &constraint, const std::vector & quad, - const typename MatrixFree::AdditionalData additional_data) + const typename MatrixFree::AdditionalData + additional_data) { std::vector locally_owned_set = internal::MatrixFreeImplementation::extract_locally_owned_index_sets( @@ -2522,14 +2545,15 @@ MatrixFree::reinit( -template +template template void -MatrixFree::reinit( +MatrixFree::reinit( const std::vector & dof_handler, const std::vector *> &constraint, const QuadratureType & quad, - const typename MatrixFree::AdditionalData additional_data) + const typename MatrixFree::AdditionalData + additional_data) { std::vector locally_owned_set = internal::MatrixFreeImplementation::extract_locally_owned_index_sets( @@ -2546,15 +2570,16 @@ MatrixFree::reinit( -template +template template void -MatrixFree::reinit( +MatrixFree::reinit( const Mapping & mapping, const std::vector & dof_handler, const std::vector *> &constraint, const QuadratureType & quad, - const typename MatrixFree::AdditionalData additional_data) + const typename MatrixFree::AdditionalData + additional_data) { std::vector locally_owned_set = internal::MatrixFreeImplementation::extract_locally_owned_index_sets( @@ -2571,15 +2596,16 @@ MatrixFree::reinit( -template +template template void -MatrixFree::reinit( +MatrixFree::reinit( const Mapping & mapping, const std::vector & dof_handler, const std::vector *> &constraint, const std::vector & quad, - const typename MatrixFree::AdditionalData additional_data) + const typename MatrixFree::AdditionalData + additional_data) { std::vector locally_owned_set = internal::MatrixFreeImplementation::extract_locally_owned_index_sets( @@ -2597,16 +2623,17 @@ MatrixFree::reinit( -template +template template void -MatrixFree::reinit( +MatrixFree::reinit( const Mapping & mapping, const std::vector & dof_handler, const std::vector *> &constraint, const std::vector & locally_owned_set, const std::vector & quad, - const typename MatrixFree::AdditionalData additional_data) + const typename MatrixFree::AdditionalData + additional_data) { // find out whether we use a hp Quadrature or a standard quadrature std::vector> quad_hp; @@ -2637,7 +2664,7 @@ namespace internal /** * Internal class for exchanging data between vectors. */ - template + template struct VectorDataExchange { // An arbitrary shift for communication to reduce the risk for accidental @@ -2652,14 +2679,15 @@ namespace internal * number of components. */ VectorDataExchange( - const dealii::MatrixFree &matrix_free, - const typename dealii::MatrixFree::DataAccessOnFaces - vector_face_access, - const unsigned int n_components) + const dealii::MatrixFree &matrix_free, + const typename dealii::MatrixFree:: + DataAccessOnFaces vector_face_access, + const unsigned int n_components) : matrix_free(matrix_free) , vector_face_access( matrix_free.get_task_info().face_partition_data.empty() ? - dealii::MatrixFree::DataAccessOnFaces::unspecified : + dealii::MatrixFree:: + DataAccessOnFaces::unspecified : vector_face_access) , ghosts_were_set(false) # ifdef DEAL_II_WITH_MPI @@ -2669,7 +2697,8 @@ namespace internal { (void)n_components; if (this->vector_face_access != - dealii::MatrixFree::DataAccessOnFaces::unspecified) + dealii::MatrixFree:: + DataAccessOnFaces::unspecified) for (unsigned int c = 0; c < matrix_free.n_components(); ++c) AssertDimension( matrix_free.get_dof_info(c).vector_partitioner_face_variants.size(), @@ -2732,11 +2761,13 @@ namespace internal .vector_partitioner_face_variants.size(), 3); if (vector_face_access == - dealii::MatrixFree::DataAccessOnFaces::none) + dealii::MatrixFree:: + DataAccessOnFaces::none) return *matrix_free.get_dof_info(mf_component) .vector_partitioner_face_variants[0]; else if (vector_face_access == - dealii::MatrixFree::DataAccessOnFaces::values) + dealii::MatrixFree:: + DataAccessOnFaces::values) return *matrix_free.get_dof_info(mf_component) .vector_partitioner_face_variants[1]; else @@ -2828,7 +2859,8 @@ namespace internal if (ghosts_set) ghosts_were_set = true; if (vector_face_access == - dealii::MatrixFree::DataAccessOnFaces::unspecified || + dealii::MatrixFree:: + DataAccessOnFaces::unspecified || vec.size() == 0) vec.update_ghost_values_start(component_in_block_vector + channel_shift); @@ -2925,7 +2957,8 @@ namespace internal "Type mismatch between VectorType and VectorDataExchange"); (void)component_in_block_vector; if (vector_face_access == - dealii::MatrixFree::DataAccessOnFaces::unspecified || + dealii::MatrixFree:: + DataAccessOnFaces::unspecified || vec.size() == 0) vec.update_ghost_values_finish(); else @@ -3041,7 +3074,8 @@ namespace internal (void)component_in_block_vector; Assert(vec.has_ghost_elements() == false, ExcNotImplemented()); if (vector_face_access == - dealii::MatrixFree::DataAccessOnFaces::unspecified || + dealii::MatrixFree:: + DataAccessOnFaces::unspecified || vec.size() == 0) vec.compress_start(component_in_block_vector + channel_shift); else @@ -3135,7 +3169,8 @@ namespace internal "Type mismatch between VectorType and VectorDataExchange"); (void)component_in_block_vector; if (vector_face_access == - dealii::MatrixFree::DataAccessOnFaces::unspecified || + dealii::MatrixFree:: + DataAccessOnFaces::unspecified || vec.size() == 0) vec.compress_finish(dealii::VectorOperation::add); else @@ -3227,7 +3262,8 @@ namespace internal return; if (vector_face_access == - dealii::MatrixFree::DataAccessOnFaces::unspecified || + dealii::MatrixFree:: + DataAccessOnFaces::unspecified || vec.size() == 0) vec.zero_out_ghosts(); else @@ -3354,10 +3390,10 @@ namespace internal - const dealii::MatrixFree &matrix_free; - const typename dealii::MatrixFree::DataAccessOnFaces - vector_face_access; - bool ghosts_were_set; + const dealii::MatrixFree &matrix_free; + const typename dealii::MatrixFree:: + DataAccessOnFaces vector_face_access; + bool ghosts_were_set; # ifdef DEAL_II_WITH_MPI std::vector *> tmp_data; std::vector> requests; @@ -3461,12 +3497,14 @@ namespace internal template ::value, VectorStruct>::type * = nullptr> void - update_ghost_values_start(const VectorStruct & vec, - VectorDataExchange &exchanger, - const unsigned int channel = 0) + update_ghost_values_start( + const VectorStruct & vec, + VectorDataExchange &exchanger, + const unsigned int channel = 0) { if (get_communication_block_size(vec) < vec.n_blocks()) { @@ -3488,12 +3526,14 @@ namespace internal template ::value, VectorStruct>::type * = nullptr> void - update_ghost_values_start(const VectorStruct & vec, - VectorDataExchange &exchanger, - const unsigned int channel = 0) + update_ghost_values_start( + const VectorStruct & vec, + VectorDataExchange &exchanger, + const unsigned int channel = 0) { exchanger.update_ghost_values_start(channel, vec); } @@ -3501,10 +3541,14 @@ namespace internal // update_ghost_values_start() for vector of vectors - template + template inline void - update_ghost_values_start(const std::vector &vec, - VectorDataExchange &exchanger) + update_ghost_values_start( + const std::vector & vec, + VectorDataExchange &exchanger) { unsigned int component_index = 0; for (unsigned int comp = 0; comp < vec.size(); comp++) @@ -3517,10 +3561,14 @@ namespace internal // update_ghost_values_start() for vector of pointers to vectors - template + template inline void - update_ghost_values_start(const std::vector &vec, - VectorDataExchange & exchanger) + update_ghost_values_start( + const std::vector & vec, + VectorDataExchange &exchanger) { unsigned int component_index = 0; for (unsigned int comp = 0; comp < vec.size(); comp++) @@ -3540,12 +3588,14 @@ namespace internal template ::value, VectorStruct>::type * = nullptr> void - update_ghost_values_finish(const VectorStruct & vec, - VectorDataExchange &exchanger, - const unsigned int channel = 0) + update_ghost_values_finish( + const VectorStruct & vec, + VectorDataExchange &exchanger, + const unsigned int channel = 0) { if (get_communication_block_size(vec) < vec.n_blocks()) { @@ -3563,12 +3613,14 @@ namespace internal template ::value, VectorStruct>::type * = nullptr> void - update_ghost_values_finish(const VectorStruct & vec, - VectorDataExchange &exchanger, - const unsigned int channel = 0) + update_ghost_values_finish( + const VectorStruct & vec, + VectorDataExchange &exchanger, + const unsigned int channel = 0) { exchanger.update_ghost_values_finish(channel, vec); } @@ -3576,10 +3628,14 @@ namespace internal // for vector of vectors - template + template inline void - update_ghost_values_finish(const std::vector &vec, - VectorDataExchange &exchanger) + update_ghost_values_finish( + const std::vector & vec, + VectorDataExchange &exchanger) { unsigned int component_index = 0; for (unsigned int comp = 0; comp < vec.size(); comp++) @@ -3592,10 +3648,14 @@ namespace internal // for vector of pointers to vectors - template + template inline void - update_ghost_values_finish(const std::vector &vec, - VectorDataExchange & exchanger) + update_ghost_values_finish( + const std::vector & vec, + VectorDataExchange &exchanger) { unsigned int component_index = 0; for (unsigned int comp = 0; comp < vec.size(); comp++) @@ -3615,12 +3675,14 @@ namespace internal template ::value, VectorStruct>::type * = nullptr> inline void - compress_start(VectorStruct & vec, - VectorDataExchange &exchanger, - const unsigned int channel = 0) + compress_start( + VectorStruct & vec, + VectorDataExchange &exchanger, + const unsigned int channel = 0) { if (get_communication_block_size(vec) < vec.n_blocks()) vec.compress(dealii::VectorOperation::add); @@ -3635,12 +3697,14 @@ namespace internal template ::value, VectorStruct>::type * = nullptr> inline void - compress_start(VectorStruct & vec, - VectorDataExchange &exchanger, - const unsigned int channel = 0) + compress_start( + VectorStruct & vec, + VectorDataExchange &exchanger, + const unsigned int channel = 0) { exchanger.compress_start(channel, vec); } @@ -3648,10 +3712,14 @@ namespace internal // for std::vector of vectors - template + template inline void - compress_start(std::vector & vec, - VectorDataExchange &exchanger) + compress_start( + std::vector & vec, + VectorDataExchange &exchanger) { unsigned int component_index = 0; for (unsigned int comp = 0; comp < vec.size(); comp++) @@ -3664,10 +3732,14 @@ namespace internal // for std::vector of pointer to vectors - template + template inline void - compress_start(std::vector & vec, - VectorDataExchange &exchanger) + compress_start( + std::vector & vec, + VectorDataExchange &exchanger) { unsigned int component_index = 0; for (unsigned int comp = 0; comp < vec.size(); comp++) @@ -3687,12 +3759,14 @@ namespace internal template ::value, VectorStruct>::type * = nullptr> inline void - compress_finish(VectorStruct & vec, - VectorDataExchange &exchanger, - const unsigned int channel = 0) + compress_finish( + VectorStruct & vec, + VectorDataExchange &exchanger, + const unsigned int channel = 0) { if (get_communication_block_size(vec) < vec.n_blocks()) { @@ -3710,12 +3784,14 @@ namespace internal template ::value, VectorStruct>::type * = nullptr> inline void - compress_finish(VectorStruct & vec, - VectorDataExchange &exchanger, - const unsigned int channel = 0) + compress_finish( + VectorStruct & vec, + VectorDataExchange &exchanger, + const unsigned int channel = 0) { exchanger.compress_finish(channel, vec); } @@ -3723,10 +3799,14 @@ namespace internal // for std::vector of vectors - template + template inline void - compress_finish(std::vector & vec, - VectorDataExchange &exchanger) + compress_finish( + std::vector & vec, + VectorDataExchange &exchanger) { unsigned int component_index = 0; for (unsigned int comp = 0; comp < vec.size(); comp++) @@ -3739,10 +3819,14 @@ namespace internal // for std::vector of pointer to vectors - template + template inline void - compress_finish(std::vector & vec, - VectorDataExchange &exchanger) + compress_finish( + std::vector & vec, + VectorDataExchange &exchanger) { unsigned int component_index = 0; for (unsigned int comp = 0; comp < vec.size(); comp++) @@ -3766,11 +3850,13 @@ namespace internal template ::value, VectorStruct>::type * = nullptr> inline void - reset_ghost_values(const VectorStruct & vec, - VectorDataExchange &exchanger) + reset_ghost_values( + const VectorStruct & vec, + VectorDataExchange &exchanger) { // return immediately if there is nothing to do. if (exchanger.ghosts_were_set == true) @@ -3786,11 +3872,13 @@ namespace internal template ::value, VectorStruct>::type * = nullptr> inline void - reset_ghost_values(const VectorStruct & vec, - VectorDataExchange &exchanger) + reset_ghost_values( + const VectorStruct & vec, + VectorDataExchange &exchanger) { exchanger.reset_ghost_values(vec); } @@ -3798,10 +3886,14 @@ namespace internal // for std::vector of vectors - template + template inline void - reset_ghost_values(const std::vector &vec, - VectorDataExchange &exchanger) + reset_ghost_values( + const std::vector & vec, + VectorDataExchange &exchanger) { // return immediately if there is nothing to do. if (exchanger.ghosts_were_set == true) @@ -3814,10 +3906,14 @@ namespace internal // for std::vector of pointer to vectors - template + template inline void - reset_ghost_values(const std::vector &vec, - VectorDataExchange & exchanger) + reset_ghost_values( + const std::vector & vec, + VectorDataExchange &exchanger) { // return immediately if there is nothing to do. if (exchanger.ghosts_were_set == true) @@ -3837,12 +3933,14 @@ namespace internal template ::value, VectorStruct>::type * = nullptr> inline void - zero_vector_region(const unsigned int range_index, - VectorStruct & vec, - VectorDataExchange &exchanger) + zero_vector_region( + const unsigned int range_index, + VectorStruct & vec, + VectorDataExchange &exchanger) { for (unsigned int i = 0; i < vec.n_blocks(); ++i) exchanger.zero_vector_region(range_index, vec.block(i)); @@ -3854,12 +3952,14 @@ namespace internal template ::value, VectorStruct>::type * = nullptr> inline void - zero_vector_region(const unsigned int range_index, - VectorStruct & vec, - VectorDataExchange &exchanger) + zero_vector_region( + const unsigned int range_index, + VectorStruct & vec, + VectorDataExchange &exchanger) { exchanger.zero_vector_region(range_index, vec); } @@ -3867,11 +3967,15 @@ namespace internal // for std::vector of vectors - template + template inline void - zero_vector_region(const unsigned int range_index, - std::vector & vec, - VectorDataExchange &exchanger) + zero_vector_region( + const unsigned int range_index, + std::vector & vec, + VectorDataExchange &exchanger) { for (unsigned int comp = 0; comp < vec.size(); comp++) zero_vector_region(range_index, vec[comp], exchanger); @@ -3880,11 +3984,15 @@ namespace internal // for std::vector of pointers to vectors - template + template inline void - zero_vector_region(const unsigned int range_index, - std::vector & vec, - VectorDataExchange &exchanger) + zero_vector_region( + const unsigned int range_index, + std::vector & vec, + VectorDataExchange &exchanger) { for (unsigned int comp = 0; comp < vec.size(); comp++) zero_vector_region(range_index, *vec[comp], exchanger); @@ -4060,9 +4168,13 @@ namespace internal const InVector &src; OutVector & dst; - VectorDataExchange + VectorDataExchange src_data_exchanger; - VectorDataExchange + VectorDataExchange dst_data_exchanger; const bool src_and_dst_are_same; const bool zero_dst_vector_setting; @@ -4131,11 +4243,11 @@ namespace internal -template +template template inline void -MatrixFree::cell_loop( - const std::function &, +MatrixFree::cell_loop( + const std::function &, OutVector &, const InVector &, const std::pair &)> @@ -4145,39 +4257,44 @@ MatrixFree::cell_loop( const bool zero_dst_vector) const { using Wrapper = - internal::MFClassWrapper, InVector, OutVector>; + internal::MFClassWrapper, + InVector, + OutVector>; Wrapper wrap(cell_operation, nullptr, nullptr); - internal:: - MFWorker, InVector, OutVector, Wrapper, true> - worker(*this, - src, - dst, - zero_dst_vector, - wrap, - &Wrapper::cell_integrator, - &Wrapper::face_integrator, - &Wrapper::boundary_integrator); + internal::MFWorker, + InVector, + OutVector, + Wrapper, + true> + worker(*this, + src, + dst, + zero_dst_vector, + wrap, + &Wrapper::cell_integrator, + &Wrapper::face_integrator, + &Wrapper::boundary_integrator); task_info.loop(worker); } -template +template template inline void -MatrixFree::loop( - const std::function &, +MatrixFree::loop( + const std::function &, OutVector &, const InVector &, const std::pair &)> &cell_operation, - const std::function &, + const std::function &, OutVector &, const InVector &, const std::pair &)> &face_operation, - const std::function &, + const std::function &, OutVector &, const InVector &, const std::pair &)> @@ -4189,41 +4306,50 @@ MatrixFree::loop( const DataAccessOnFaces src_vector_face_access) const { using Wrapper = - internal::MFClassWrapper, InVector, OutVector>; + internal::MFClassWrapper, + InVector, + OutVector>; Wrapper wrap(cell_operation, face_operation, boundary_operation); - internal:: - MFWorker, InVector, OutVector, Wrapper, true> - worker(*this, - src, - dst, - zero_dst_vector, - wrap, - &Wrapper::cell_integrator, - &Wrapper::face_integrator, - &Wrapper::boundary_integrator, - src_vector_face_access, - dst_vector_face_access); + internal::MFWorker, + InVector, + OutVector, + Wrapper, + true> + worker(*this, + src, + dst, + zero_dst_vector, + wrap, + &Wrapper::cell_integrator, + &Wrapper::face_integrator, + &Wrapper::boundary_integrator, + src_vector_face_access, + dst_vector_face_access); task_info.loop(worker); } -template +template template inline void -MatrixFree::cell_loop( - void (CLASS::*function_pointer)(const MatrixFree &, - OutVector &, - const InVector &, - const std::pair &) - const, +MatrixFree::cell_loop( + void (CLASS::*function_pointer)( + const MatrixFree &, + OutVector &, + const InVector &, + const std::pair &) const, const CLASS * owning_class, OutVector & dst, const InVector &src, const bool zero_dst_vector) const { - internal::MFWorker, InVector, OutVector, CLASS, true> + internal::MFWorker, + InVector, + OutVector, + CLASS, + true> worker(*this, src, dst, @@ -4237,22 +4363,22 @@ MatrixFree::cell_loop( -template +template template inline void -MatrixFree::loop( - void (CLASS::*cell_operation)(const MatrixFree &, - OutVector &, - const InVector &, - const std::pair &) - const, - void (CLASS::*face_operation)(const MatrixFree &, - OutVector &, - const InVector &, - const std::pair &) - const, +MatrixFree::loop( + void (CLASS::*cell_operation)( + const MatrixFree &, + OutVector &, + const InVector &, + const std::pair &) const, + void (CLASS::*face_operation)( + const MatrixFree &, + OutVector &, + const InVector &, + const std::pair &) const, void (CLASS::*boundary_operation)( - const MatrixFree &, + const MatrixFree &, OutVector &, const InVector &, const std::pair &) const, @@ -4263,7 +4389,11 @@ MatrixFree::loop( const DataAccessOnFaces dst_vector_face_access, const DataAccessOnFaces src_vector_face_access) const { - internal::MFWorker, InVector, OutVector, CLASS, true> + internal::MFWorker, + InVector, + OutVector, + CLASS, + true> worker(*this, src, dst, @@ -4279,12 +4409,12 @@ MatrixFree::loop( -template +template template inline void -MatrixFree::cell_loop( +MatrixFree::cell_loop( void (CLASS::*function_pointer)( - const MatrixFree &, + const MatrixFree &, OutVector &, const InVector &, const std::pair &), @@ -4293,7 +4423,11 @@ MatrixFree::cell_loop( const InVector &src, const bool zero_dst_vector) const { - internal::MFWorker, InVector, OutVector, CLASS, false> + internal::MFWorker, + InVector, + OutVector, + CLASS, + false> worker(*this, src, dst, @@ -4307,20 +4441,22 @@ MatrixFree::cell_loop( -template +template template inline void -MatrixFree::loop( - void (CLASS::*cell_operation)(const MatrixFree &, - OutVector &, - const InVector &, - const std::pair &), - void (CLASS::*face_operation)(const MatrixFree &, - OutVector &, - const InVector &, - const std::pair &), +MatrixFree::loop( + void (CLASS::*cell_operation)( + const MatrixFree &, + OutVector &, + const InVector &, + const std::pair &), + void (CLASS::*face_operation)( + const MatrixFree &, + OutVector &, + const InVector &, + const std::pair &), void (CLASS::*boundary_operation)( - const MatrixFree &, + const MatrixFree &, OutVector &, const InVector &, const std::pair &), @@ -4331,7 +4467,11 @@ MatrixFree::loop( const DataAccessOnFaces dst_vector_face_access, const DataAccessOnFaces src_vector_face_access) const { - internal::MFWorker, InVector, OutVector, CLASS, false> + internal::MFWorker, + InVector, + OutVector, + CLASS, + false> worker(*this, src, dst, diff --git a/include/deal.II/matrix_free/matrix_free.templates.h b/include/deal.II/matrix_free/matrix_free.templates.h index 764a26d11c..fa41620d39 100644 --- a/include/deal.II/matrix_free/matrix_free.templates.h +++ b/include/deal.II/matrix_free/matrix_free.templates.h @@ -53,8 +53,8 @@ DEAL_II_NAMESPACE_OPEN // --------------------- MatrixFree ----------------------------------- -template -MatrixFree::MatrixFree() +template +MatrixFree::MatrixFree() : Subscriptor() , indices_are_initialized(false) , mapping_is_initialized(false) @@ -62,8 +62,9 @@ MatrixFree::MatrixFree() -template -MatrixFree::MatrixFree(const MatrixFree &other) +template +MatrixFree::MatrixFree( + const MatrixFree &other) : Subscriptor() { copy_from(other); @@ -71,9 +72,9 @@ MatrixFree::MatrixFree(const MatrixFree &other) -template +template std::pair -MatrixFree::create_cell_subrange_hp_by_index( +MatrixFree::create_cell_subrange_hp_by_index( const std::pair &range, const unsigned int fe_index, const unsigned int vector_component) const @@ -115,9 +116,9 @@ MatrixFree::create_cell_subrange_hp_by_index( -template +template void -MatrixFree::renumber_dofs( +MatrixFree::renumber_dofs( std::vector &renumbering, const unsigned int vector_component) { @@ -127,9 +128,10 @@ MatrixFree::renumber_dofs( -template +template const DoFHandler & -MatrixFree::get_dof_handler(const unsigned int dof_index) const +MatrixFree::get_dof_handler( + const unsigned int dof_index) const { AssertIndexRange(dof_index, n_components()); if (dof_handlers.active_dof_handler == DoFHandlers::usual) @@ -149,14 +151,15 @@ MatrixFree::get_dof_handler(const unsigned int dof_index) const -template +template typename DoFHandler::cell_iterator -MatrixFree::get_cell_iterator(const unsigned int macro_cell_number, - const unsigned int vector_number, - const unsigned int dof_index) const +MatrixFree::get_cell_iterator( + const unsigned int macro_cell_number, + const unsigned int vector_number, + const unsigned int dof_index) const { const unsigned int vectorization_length = - VectorizedArray::n_array_elements; + VectorizedArrayType::n_array_elements; AssertIndexRange(dof_index, dof_handlers.n_dof_handlers); AssertIndexRange(macro_cell_number, task_info.cell_partition_data.back()); AssertIndexRange(vector_number, n_components_filled(macro_cell_number)); @@ -185,15 +188,15 @@ MatrixFree::get_cell_iterator(const unsigned int macro_cell_number, -template +template typename hp::DoFHandler::active_cell_iterator -MatrixFree::get_hp_cell_iterator( +MatrixFree::get_hp_cell_iterator( const unsigned int macro_cell_number, const unsigned int vector_number, const unsigned int dof_index) const { constexpr unsigned int vectorization_length = - VectorizedArray::n_array_elements; + VectorizedArrayType::n_array_elements; AssertIndexRange(dof_index, dof_handlers.n_dof_handlers); AssertIndexRange(macro_cell_number, task_info.cell_partition_data.back()); AssertIndexRange(vector_number, n_components_filled(macro_cell_number)); @@ -211,9 +214,10 @@ MatrixFree::get_hp_cell_iterator( -template +template void -MatrixFree::copy_from(const MatrixFree &v) +MatrixFree::copy_from( + const MatrixFree &v) { clear(); dof_handlers = v.dof_handlers; @@ -230,16 +234,17 @@ MatrixFree::copy_from(const MatrixFree &v) -template +template template void -MatrixFree::internal_reinit( - const Mapping & mapping, - const std::vector *> & dof_handler, - const std::vector *> & constraint, - const std::vector & locally_owned_set, - const std::vector> & quad, - const typename MatrixFree::AdditionalData &additional_data) +MatrixFree::internal_reinit( + const Mapping & mapping, + const std::vector *> & dof_handler, + const std::vector *> &constraint, + const std::vector & locally_owned_set, + const std::vector> & quad, + const typename MatrixFree::AdditionalData + &additional_data) { // Reads out the FE information and stores the shape function values, // gradients and Hessians for quadrature points. @@ -327,11 +332,10 @@ MatrixFree::internal_reinit( initialize_dof_handlers(dof_handler, additional_data); std::vector dummy; std::vector dummy2; - task_info.collect_boundary_cells( - cell_level_index.size(), - cell_level_index.size(), - VectorizedArray::n_array_elements, - dummy); + task_info.collect_boundary_cells(cell_level_index.size(), + cell_level_index.size(), + VectorizedArrayType::n_array_elements, + dummy); task_info.create_blocks_serial( dummy, dummy, 1, dummy, false, dummy, dummy2); for (unsigned int i = 0; i < dof_info.size(); ++i) @@ -357,7 +361,7 @@ MatrixFree::internal_reinit( // divisible by the vectorization length. But it must be for // mapping_info... while (cell_level_index.size() % - VectorizedArray::n_array_elements != + VectorizedArrayType::n_array_elements != 0) cell_level_index.push_back(cell_level_index.back()); } @@ -390,16 +394,17 @@ MatrixFree::internal_reinit( -template +template template void -MatrixFree::internal_reinit( - const Mapping & mapping, - const std::vector *> & dof_handler, - const std::vector *> & constraint, - const std::vector & locally_owned_set, - const std::vector> & quad, - const typename MatrixFree::AdditionalData &additional_data) +MatrixFree::internal_reinit( + const Mapping & mapping, + const std::vector *> & dof_handler, + const std::vector *> &constraint, + const std::vector & locally_owned_set, + const std::vector> & quad, + const typename MatrixFree::AdditionalData + &additional_data) { // Reads out the FE information and stores the shape function values, // gradients and Hessians for quadrature points. @@ -496,11 +501,10 @@ MatrixFree::internal_reinit( initialize_dof_handlers(dof_handler, additional_data); std::vector dummy; std::vector dummy2; - task_info.collect_boundary_cells( - cell_level_index.size(), - cell_level_index.size(), - VectorizedArray::n_array_elements, - dummy); + task_info.collect_boundary_cells(cell_level_index.size(), + cell_level_index.size(), + VectorizedArrayType::n_array_elements, + dummy); task_info.create_blocks_serial( dummy, dummy, 1, dummy, false, dummy, dummy2); for (unsigned int i = 0; i < dof_info.size(); ++i) @@ -528,7 +532,7 @@ MatrixFree::internal_reinit( // divisible by the vectorization length. But it must be for // mapping_info... while (cell_level_index.size() % - VectorizedArray::n_array_elements != + VectorizedArrayType::n_array_elements != 0) cell_level_index.push_back(cell_level_index.back()); } @@ -556,10 +560,11 @@ MatrixFree::internal_reinit( } -template +template template bool -MatrixFree::is_supported(const FiniteElement &fe) +MatrixFree::is_supported( + const FiniteElement &fe) { if (dim != spacedim) return false; @@ -618,9 +623,9 @@ namespace internal -template +template void -MatrixFree::initialize_dof_handlers( +MatrixFree::initialize_dof_handlers( const std::vector *> &dof_handler, const AdditionalData & additional_data) { @@ -634,8 +639,7 @@ MatrixFree::initialize_dof_handlers( dof_info.resize(dof_handlers.n_dof_handlers); for (unsigned int no = 0; no < dof_handlers.n_dof_handlers; ++no) - dof_info[no].vectorization_length = - VectorizedArray::n_array_elements; + dof_info[no].vectorization_length = VectorizedArrayType::n_array_elements; // Go through cells on zeroth level and then successively step down into // children. This gives a z-ordering of the cells, which is beneficial when @@ -688,9 +692,9 @@ MatrixFree::initialize_dof_handlers( -template +template void -MatrixFree::initialize_dof_handlers( +MatrixFree::initialize_dof_handlers( const std::vector *> &dof_handler, const AdditionalData & additional_data) { @@ -706,8 +710,7 @@ MatrixFree::initialize_dof_handlers( dof_info.resize(dof_handlers.n_dof_handlers); for (unsigned int no = 0; no < dof_handlers.n_dof_handlers; ++no) - dof_info[no].vectorization_length = - VectorizedArray::n_array_elements; + dof_info[no].vectorization_length = VectorizedArrayType::n_array_elements; // go through cells on zeroth level and then successively step down into // children. This gives a z-ordering of the cells, which is beneficial when @@ -746,10 +749,10 @@ MatrixFree::initialize_dof_handlers( -template +template template void -MatrixFree::initialize_indices( +MatrixFree::initialize_indices( const std::vector *> &constraint, const std::vector & locally_owned_set, const AdditionalData & additional_data) @@ -995,7 +998,7 @@ MatrixFree::initialize_indices( } const unsigned int vectorization_length = - VectorizedArray::n_array_elements; + VectorizedArrayType::n_array_elements; task_info.collect_boundary_cells(cell_level_index_end_local, n_active_cells, vectorization_length, @@ -1374,7 +1377,7 @@ MatrixFree::initialize_indices( cell_level_index.resize( cell_level_index.size() + - VectorizedArray::n_array_elements * + VectorizedArrayType::n_array_elements * (task_info.refinement_edge_face_partition_data[1] - task_info.refinement_edge_face_partition_data[0])); @@ -1386,50 +1389,48 @@ MatrixFree::initialize_indices( face_info.cell_and_face_to_plain_faces.reinit( TableIndices<3>(task_info.cell_partition_data.back(), GeometryInfo::faces_per_cell, - VectorizedArray::n_array_elements), + VectorizedArrayType::n_array_elements), true); face_info.cell_and_face_to_plain_faces.fill( numbers::invalid_unsigned_int); face_info.cell_and_face_boundary_id.reinit( TableIndices<3>(task_info.cell_partition_data.back(), GeometryInfo::faces_per_cell, - VectorizedArray::n_array_elements), + VectorizedArrayType::n_array_elements), true); face_info.cell_and_face_boundary_id.fill(numbers::invalid_boundary_id); for (unsigned int f = 0; f < task_info.boundary_partition_data.back(); ++f) - for (unsigned int v = 0; - v < VectorizedArray::n_array_elements && - face_info.faces[f].cells_interior[v] != - numbers::invalid_unsigned_int; + for (unsigned int v = 0; v < VectorizedArrayType::n_array_elements && + face_info.faces[f].cells_interior[v] != + numbers::invalid_unsigned_int; ++v) { TableIndices<3> index(face_info.faces[f].cells_interior[v] / - VectorizedArray::n_array_elements, + VectorizedArrayType::n_array_elements, face_info.faces[f].interior_face_no, face_info.faces[f].cells_interior[v] % - VectorizedArray::n_array_elements); + VectorizedArrayType::n_array_elements); // Assert(cell_and_face_to_plain_faces(index) == // numbers::invalid_unsigned_int, // ExcInternalError("Should only visit each face once")); face_info.cell_and_face_to_plain_faces(index) = - f * VectorizedArray::n_array_elements + v; + f * VectorizedArrayType::n_array_elements + v; if (face_info.faces[f].cells_exterior[v] != numbers::invalid_unsigned_int) { - TableIndices<3> index( - face_info.faces[f].cells_exterior[v] / - VectorizedArray::n_array_elements, - face_info.faces[f].exterior_face_no, - face_info.faces[f].cells_exterior[v] % - VectorizedArray::n_array_elements); + TableIndices<3> index(face_info.faces[f].cells_exterior[v] / + VectorizedArrayType::n_array_elements, + face_info.faces[f].exterior_face_no, + face_info.faces[f].cells_exterior[v] % + VectorizedArrayType::n_array_elements); // Assert(cell_and_face_to_plain_faces(index) == // numbers::invalid_unsigned_int, // ExcInternalError("Should only visit each face once")); face_info.cell_and_face_to_plain_faces(index) = - f * VectorizedArray::n_array_elements + v; + f * VectorizedArrayType::n_array_elements + v; } else face_info.cell_and_face_boundary_id(index) = @@ -1449,8 +1450,8 @@ MatrixFree::initialize_indices( std::vector ghost_indices; { for (unsigned int cell = 0; - cell < VectorizedArray::n_array_elements * - n_macro_cells_before; + cell < + VectorizedArrayType::n_array_elements * n_macro_cells_before; ++cell) if (cell > 0 && cell_level_index[cell] != cell_level_index[cell - 1]) @@ -1520,15 +1521,14 @@ MatrixFree::initialize_indices( bool has_noncontiguous_cell = false; for (unsigned int f = 0; f < n_inner_face_batches(); ++f) for (unsigned int v = 0; - v < VectorizedArray::n_array_elements && + v < VectorizedArrayType::n_array_elements && face_info.faces[f].cells_interior[v] != numbers::invalid_unsigned_int; ++v) { - AssertIndexRange( - face_info.faces[f].cells_interior[v], - n_macro_cells_before * - VectorizedArray::n_array_elements); + AssertIndexRange(face_info.faces[f].cells_interior[v], + n_macro_cells_before * + VectorizedArrayType::n_array_elements); if (dof_info[no].index_storage_variants [internal::MatrixFreeFunctions::DoFInfo:: dof_access_face_exterior][f] >= @@ -1553,7 +1553,7 @@ MatrixFree::initialize_indices( ++c) { const internal::MatrixFreeFunctions::ShapeInfo< - VectorizedArray> &shape = + VectorizedArrayType> &shape = shape_info( dof_info[no].global_base_element_offset + e, 0, @@ -1638,15 +1638,14 @@ MatrixFree::initialize_indices( { for (unsigned int f = 0; f < n_inner_face_batches(); ++f) for (unsigned int v = 0; - v < VectorizedArray::n_array_elements && + v < VectorizedArrayType::n_array_elements && face_info.faces[f].cells_interior[v] != numbers::invalid_unsigned_int; ++v) { - AssertIndexRange( - face_info.faces[f].cells_interior[v], - n_macro_cells_before * - VectorizedArray::n_array_elements); + AssertIndexRange(face_info.faces[f].cells_interior[v], + n_macro_cells_before * + VectorizedArrayType::n_array_elements); if (dof_info[no].index_storage_variants [internal::MatrixFreeFunctions::DoFInfo:: dof_access_face_exterior][f] >= @@ -1671,7 +1670,7 @@ MatrixFree::initialize_indices( ++c) { const internal::MatrixFreeFunctions::ShapeInfo< - VectorizedArray> &shape = + VectorizedArrayType> &shape = shape_info( dof_info[no].global_base_element_offset + e, 0, @@ -1738,9 +1737,9 @@ MatrixFree::initialize_indices( -template +template void -MatrixFree::clear() +MatrixFree::clear() { dof_info.clear(); mapping_info.clear(); @@ -1858,9 +1857,9 @@ namespace internal -template +template void -MatrixFree::make_connectivity_graph_faces( +MatrixFree::make_connectivity_graph_faces( DynamicSparsityPattern &connectivity) { (void)connectivity; @@ -1916,9 +1915,9 @@ MatrixFree::make_connectivity_graph_faces( -template +template std::size_t -MatrixFree::memory_consumption() const +MatrixFree::memory_consumption() const { std::size_t memory = MemoryConsumption::memory_consumption(dof_info); memory += MemoryConsumption::memory_consumption(cell_level_index); @@ -1934,10 +1933,11 @@ MatrixFree::memory_consumption() const -template +template template void -MatrixFree::print_memory_consumption(StreamType &out) const +MatrixFree::print_memory_consumption( + StreamType &out) const { out << " Memory matrix-free data total: --> "; task_info.print_memory_statistics(out, memory_consumption()); @@ -1972,9 +1972,9 @@ MatrixFree::print_memory_consumption(StreamType &out) const -template +template void -MatrixFree::print(std::ostream &out) const +MatrixFree::print(std::ostream &out) const { // print indices local to global for (unsigned int no = 0; no < dof_info.size(); ++no) diff --git a/include/deal.II/matrix_free/operators.h b/include/deal.II/matrix_free/operators.h index d320f9f30a..b1fb390ef3 100644 --- a/include/deal.II/matrix_free/operators.h +++ b/include/deal.II/matrix_free/operators.h @@ -610,17 +610,26 @@ namespace MatrixFreeOperators */ template + int n_components = 1, + typename Number = double, + typename VectorizedArrayType = VectorizedArray> class CellwiseInverseMassMatrix { + static_assert( + std::is_same::value, + "Type of Number and of VectorizedArrayType do not match."); + public: /** * Constructor. Initializes the shape information from the ShapeInfo field * in the class FEEval. */ CellwiseInverseMassMatrix( - const FEEvaluationBase &fe_eval); + const FEEvaluationBase &fe_eval); /** * Applies the inverse mass matrix operation on an input array. It is @@ -631,10 +640,10 @@ namespace MatrixFreeOperators * coefficient is allowed. */ void - apply(const AlignedVector> &inverse_coefficient, - const unsigned int n_actual_components, - const VectorizedArray * in_array, - VectorizedArray * out_array) const; + apply(const AlignedVector &inverse_coefficient, + const unsigned int n_actual_components, + const VectorizedArrayType * in_array, + VectorizedArrayType * out_array) const; /** * This operation performs a projection from the data given in quadrature @@ -671,8 +680,8 @@ namespace MatrixFreeOperators */ void transform_from_q_points_to_basis(const unsigned int n_actual_components, - const VectorizedArray *in_array, - VectorizedArray *out_array) const; + const VectorizedArrayType *in_array, + VectorizedArrayType *out_array) const; /** * Fills the given array with the inverse of the JxW values, i.e., a mass @@ -681,18 +690,22 @@ namespace MatrixFreeOperators */ void fill_inverse_JxW_values( - AlignedVector> &inverse_jxw) const; + AlignedVector &inverse_jxw) const; private: /** * A reference to the FEEvaluation object for getting the JxW_values. */ - const FEEvaluationBase &fe_eval; + const FEEvaluationBase &fe_eval; /** * A structure to hold inverse shape functions */ - AlignedVector> inverse_shape; + AlignedVector inverse_shape; }; @@ -912,10 +925,22 @@ namespace MatrixFreeOperators // ------------------------------------ inline functions --------------------- - template - inline CellwiseInverseMassMatrix:: + template + inline CellwiseInverseMassMatrix:: CellwiseInverseMassMatrix( - const FEEvaluationBase &fe_eval) + const FEEvaluationBase &fe_eval) : fe_eval(fe_eval) { FullMatrix shapes_1d(fe_degree + 1, fe_degree + 1); @@ -940,11 +965,19 @@ namespace MatrixFreeOperators - template + template inline void - CellwiseInverseMassMatrix:: + CellwiseInverseMassMatrix:: fill_inverse_JxW_values( - AlignedVector> &inverse_jxw) const + AlignedVector &inverse_jxw) const { constexpr unsigned int dofs_per_component_on_cell = Utilities::pow(fe_degree + 1, dim); @@ -965,13 +998,21 @@ namespace MatrixFreeOperators - template + template inline void - CellwiseInverseMassMatrix::apply( - const AlignedVector> &inverse_coefficients, - const unsigned int n_actual_components, - const VectorizedArray * in_array, - VectorizedArray * out_array) const + CellwiseInverseMassMatrix:: + apply(const AlignedVector &inverse_coefficients, + const unsigned int n_actual_components, + const VectorizedArrayType * in_array, + VectorizedArrayType * out_array) const { constexpr unsigned int dofs_per_component = Utilities::pow(fe_degree + 1, dim); @@ -989,18 +1030,17 @@ namespace MatrixFreeOperators dim, fe_degree + 1, fe_degree + 1, - VectorizedArray> + VectorizedArrayType> evaluator(inverse_shape, inverse_shape, inverse_shape); const unsigned int shift_coefficient = inverse_coefficients.size() > dofs_per_component ? dofs_per_component : 0; - const VectorizedArray *inv_coefficient = - inverse_coefficients.data(); - VectorizedArray temp_data_field[dofs_per_component]; + const VectorizedArrayType *inv_coefficient = inverse_coefficients.data(); + VectorizedArrayType temp_data_field[dofs_per_component]; for (unsigned int d = 0; d < n_actual_components; ++d) { - const VectorizedArray *in = in_array + d * dofs_per_component; - VectorizedArray * out = out_array + d * dofs_per_component; + const VectorizedArrayType *in = in_array + d * dofs_per_component; + VectorizedArrayType * out = out_array + d * dofs_per_component; // Need to select 'apply' method with hessian slot because values // assume symmetries that do not exist in the inverse shapes evaluator.template hessians<0, false, false>(in, temp_data_field); @@ -1036,27 +1076,35 @@ namespace MatrixFreeOperators - template + template inline void - CellwiseInverseMassMatrix:: + CellwiseInverseMassMatrix:: transform_from_q_points_to_basis(const unsigned int n_actual_components, - const VectorizedArray *in_array, - VectorizedArray *out_array) const + const VectorizedArrayType *in_array, + VectorizedArrayType * out_array) const { constexpr unsigned int dofs_per_cell = Utilities::pow(fe_degree + 1, dim); internal::EvaluatorTensorProduct> - evaluator(AlignedVector>(), - AlignedVector>(), + VectorizedArrayType> + evaluator(AlignedVector(), + AlignedVector(), inverse_shape); for (unsigned int d = 0; d < n_actual_components; ++d) { - const VectorizedArray *in = in_array + d * dofs_per_cell; - VectorizedArray * out = out_array + d * dofs_per_cell; + const VectorizedArrayType *in = in_array + d * dofs_per_cell; + VectorizedArrayType * out = out_array + d * dofs_per_cell; if (dim == 3) { @@ -1937,12 +1985,11 @@ namespace MatrixFreeOperators namespace Implementation { - template + template bool - non_negative(const VectorizedArray &n) + non_negative(const VectorizedArrayType &n) { - for (unsigned int v = 0; v < VectorizedArray::n_array_elements; - ++v) + for (unsigned int v = 0; v < VectorizedArrayType::n_array_elements; ++v) if (n[v] < 0.) return false; @@ -2031,16 +2078,20 @@ namespace MatrixFreeOperators const std::pair &cell_range) const { using Number = typename Base::value_type; + using vectorized_value_type = + typename MatrixFree::value_type>:: + vectorized_value_type; + FEEvaluation phi( data, this->selected_rows[0]); for (unsigned int cell = cell_range.first; cell < cell_range.second; ++cell) { phi.reinit(cell); - VectorizedArray local_diagonal_vector[phi.static_dofs_per_cell]; + vectorized_value_type local_diagonal_vector[phi.static_dofs_per_cell]; for (unsigned int i = 0; i < phi.dofs_per_component; ++i) { for (unsigned int j = 0; j < phi.dofs_per_component; ++j) - phi.begin_dof_values()[j] = VectorizedArray(); + phi.begin_dof_values()[j] = vectorized_value_type(); phi.begin_dof_values()[i] = 1.; do_operation_on_cell(phi, cell); local_diagonal_vector[i] = phi.begin_dof_values()[i]; diff --git a/include/deal.II/matrix_free/shape_info.templates.h b/include/deal.II/matrix_free/shape_info.templates.h index 90ddcfe3f4..8459115ac5 100644 --- a/include/deal.II/matrix_free/shape_info.templates.h +++ b/include/deal.II/matrix_free/shape_info.templates.h @@ -47,9 +47,9 @@ namespace internal return a; } - template + template Number - get_first_array_element(const VectorizedArray a) + get_first_array_element(const VectorizedArray a) { return a[0]; } diff --git a/include/deal.II/numerics/vector_tools.h b/include/deal.II/numerics/vector_tools.h index bff31509d9..1b93154ea0 100644 --- a/include/deal.II/numerics/vector_tools.h +++ b/include/deal.II/numerics/vector_tools.h @@ -44,7 +44,7 @@ template class Quadrature; template class QGauss; -template +template class MatrixFree; template @@ -1007,15 +1007,18 @@ namespace VectorTools */ template void - project(std::shared_ptr< - const MatrixFree> data, - const AffineConstraints &constraints, - const unsigned int n_q_points_1d, - const std::function( - const unsigned int, - const unsigned int)> &func, - VectorType & vec_result, - const unsigned int fe_component = 0); + project( + std::shared_ptr< + const MatrixFree>> data, + const AffineConstraints &constraints, + const unsigned int n_q_points_1d, + const std::function( + const unsigned int, + const unsigned int)> & func, + VectorType & vec_result, + const unsigned int fe_component = 0); /** * Same as above but for n_q_points_1d = @@ -1023,14 +1026,17 @@ namespace VectorTools */ template void - project(std::shared_ptr< - const MatrixFree> data, - const AffineConstraints &constraints, - const std::function( - const unsigned int, - const unsigned int)> & func, - VectorType & vec_result, - const unsigned int fe_component = 0); + project( + std::shared_ptr< + const MatrixFree>> data, + const AffineConstraints &constraints, + const std::function( + const unsigned int, + const unsigned int)> & func, + VectorType & vec_result, + const unsigned int fe_component = 0); /** * Compute Dirichlet boundary conditions. This function makes up a map of diff --git a/include/deal.II/numerics/vector_tools.templates.h b/include/deal.II/numerics/vector_tools.templates.h index 0e49213734..be663b841b 100644 --- a/include/deal.II/numerics/vector_tools.templates.h +++ b/include/deal.II/numerics/vector_tools.templates.h @@ -1723,8 +1723,10 @@ namespace VectorTools template void - project(std::shared_ptr< - const MatrixFree> matrix_free, + project(std::shared_ptr>> matrix_free, const AffineConstraints &constraints, const unsigned int n_q_points_1d, const std::function( @@ -1764,8 +1766,10 @@ namespace VectorTools template void - project(std::shared_ptr< - const MatrixFree> matrix_free, + project(std::shared_ptr>> matrix_free, const AffineConstraints &constraints, const std::function( const unsigned int, diff --git a/source/matrix_free/mapping_info.cc b/source/matrix_free/mapping_info.cc index fd8cff5596..4ee9d63a8f 100644 --- a/source/matrix_free/mapping_info.cc +++ b/source/matrix_free/mapping_info.cc @@ -25,7 +25,31 @@ DEAL_II_NAMESPACE_OPEN #include "mapping_info.inst" -template struct internal::MatrixFreeFunctions::FPArrayComparator; -template struct internal::MatrixFreeFunctions::FPArrayComparator; +template struct internal::MatrixFreeFunctions:: + FPArrayComparator>; +template struct internal::MatrixFreeFunctions:: + FPArrayComparator>; + +#if (DEAL_II_COMPILER_VECTORIZATION_LEVEL >= 1 && defined(__SSE2__)) || \ + (DEAL_II_COMPILER_VECTORIZATION_LEVEL >= 1 && defined(__ALTIVEC__)) +template struct internal::MatrixFreeFunctions:: + FPArrayComparator>; +template struct internal::MatrixFreeFunctions:: + FPArrayComparator>; +#endif + +#if DEAL_II_COMPILER_VECTORIZATION_LEVEL >= 2 && defined(__AVX__) +template struct internal::MatrixFreeFunctions:: + FPArrayComparator>; +template struct internal::MatrixFreeFunctions:: + FPArrayComparator>; +#endif + +#if DEAL_II_COMPILER_VECTORIZATION_LEVEL >= 3 && defined(__AVX512F__) +template struct internal::MatrixFreeFunctions:: + FPArrayComparator>; +template struct internal::MatrixFreeFunctions:: + FPArrayComparator>; +#endif DEAL_II_NAMESPACE_CLOSE diff --git a/source/matrix_free/mapping_info.inst.in b/source/matrix_free/mapping_info.inst.in index 48de91e411..2adf59c3f6 100644 --- a/source/matrix_free/mapping_info.inst.in +++ b/source/matrix_free/mapping_info.inst.in @@ -17,37 +17,199 @@ for (deal_II_dimension : DIMENSIONS) { template struct internal::MatrixFreeFunctions:: - MappingInfo; + MappingInfo>; template struct internal::MatrixFreeFunctions:: - MappingInfo; + MappingInfo>; - template struct internal::MatrixFreeFunctions:: - MappingInfoStorage; - template struct internal::MatrixFreeFunctions:: - MappingInfoStorage; + template struct internal::MatrixFreeFunctions::MappingInfoStorage< + deal_II_dimension, + deal_II_dimension, + double, + VectorizedArray>; + template struct internal::MatrixFreeFunctions::MappingInfoStorage< + deal_II_dimension, + deal_II_dimension, + float, + VectorizedArray>; #if deal_II_dimension > 1 + template struct internal::MatrixFreeFunctions::MappingInfoStorage< + deal_II_dimension - 1, + deal_II_dimension, + double, + VectorizedArray>; + template struct internal::MatrixFreeFunctions::MappingInfoStorage< + deal_II_dimension - 1, + deal_II_dimension, + float, + VectorizedArray>; +#endif + + template void internal::MatrixFreeFunctions:: + MappingInfo>:: + print_memory_consumption(std::ostream &, const TaskInfo &) + const; + template void internal::MatrixFreeFunctions:: + MappingInfo>:: + print_memory_consumption(ConditionalOStream &, + const TaskInfo &) const; + + template void internal::MatrixFreeFunctions:: + MappingInfo>:: + print_memory_consumption(std::ostream &, const TaskInfo &) + const; + template void internal::MatrixFreeFunctions:: + MappingInfo>:: + print_memory_consumption(ConditionalOStream &, + const TaskInfo &) const; + + + +#if (DEAL_II_COMPILER_VECTORIZATION_LEVEL >= 1 && defined(__SSE2__)) || \ + (DEAL_II_COMPILER_VECTORIZATION_LEVEL >= 1 && defined(__ALTIVEC__)) template struct internal::MatrixFreeFunctions:: - MappingInfoStorage; + MappingInfo>; template struct internal::MatrixFreeFunctions:: - MappingInfoStorage; -#endif + MappingInfo>; - template void internal::MatrixFreeFunctions::MappingInfo< + template struct internal::MatrixFreeFunctions::MappingInfoStorage< + deal_II_dimension, + deal_II_dimension, + double, + VectorizedArray>; + template struct internal::MatrixFreeFunctions::MappingInfoStorage< + deal_II_dimension, + deal_II_dimension, + float, + VectorizedArray>; +# if deal_II_dimension > 1 + template struct internal::MatrixFreeFunctions::MappingInfoStorage< + deal_II_dimension - 1, + deal_II_dimension, + double, + VectorizedArray>; + template struct internal::MatrixFreeFunctions::MappingInfoStorage< + deal_II_dimension - 1, deal_II_dimension, - double>::print_memory_consumption(std::ostream &, - const TaskInfo &) const; - template void - internal::MatrixFreeFunctions::MappingInfo:: - print_memory_consumption(ConditionalOStream &, - const TaskInfo &) const; + float, + VectorizedArray>; +# endif + + template void internal::MatrixFreeFunctions:: + MappingInfo>:: + print_memory_consumption(std::ostream &, const TaskInfo &) + const; + template void internal::MatrixFreeFunctions:: + MappingInfo>:: + print_memory_consumption(ConditionalOStream &, + const TaskInfo &) const; + + template void internal::MatrixFreeFunctions:: + MappingInfo>:: + print_memory_consumption(std::ostream &, const TaskInfo &) + const; + template void internal::MatrixFreeFunctions:: + MappingInfo>:: + print_memory_consumption(ConditionalOStream &, + const TaskInfo &) const; +#endif + + - template void internal::MatrixFreeFunctions::MappingInfo< +#if DEAL_II_COMPILER_VECTORIZATION_LEVEL >= 2 && defined(__AVX__) + template struct internal::MatrixFreeFunctions:: + MappingInfo>; + template struct internal::MatrixFreeFunctions:: + MappingInfo>; + + template struct internal::MatrixFreeFunctions::MappingInfoStorage< + deal_II_dimension, + deal_II_dimension, + double, + VectorizedArray>; + template struct internal::MatrixFreeFunctions::MappingInfoStorage< + deal_II_dimension, + deal_II_dimension, + float, + VectorizedArray>; +# if deal_II_dimension > 1 + template struct internal::MatrixFreeFunctions::MappingInfoStorage< + deal_II_dimension - 1, deal_II_dimension, - float>::print_memory_consumption(std::ostream &, + double, + VectorizedArray>; + template struct internal::MatrixFreeFunctions::MappingInfoStorage< + deal_II_dimension - 1, + deal_II_dimension, + float, + VectorizedArray>; +# endif + + template void internal::MatrixFreeFunctions:: + MappingInfo>:: + print_memory_consumption(std::ostream &, const TaskInfo &) + const; + template void internal::MatrixFreeFunctions:: + MappingInfo>:: + print_memory_consumption(ConditionalOStream &, + const TaskInfo &) const; + + template void internal::MatrixFreeFunctions:: + MappingInfo>:: + print_memory_consumption(std::ostream &, const TaskInfo &) + const; + template void internal::MatrixFreeFunctions:: + MappingInfo>:: + print_memory_consumption(ConditionalOStream &, const TaskInfo &) const; - template void internal::MatrixFreeFunctions::MappingInfo< +#endif + + + +#if DEAL_II_COMPILER_VECTORIZATION_LEVEL >= 3 && defined(__AVX512F__) + template struct internal::MatrixFreeFunctions:: + MappingInfo>; + template struct internal::MatrixFreeFunctions:: + MappingInfo>; + + template struct internal::MatrixFreeFunctions::MappingInfoStorage< + deal_II_dimension, + deal_II_dimension, + double, + VectorizedArray>; + template struct internal::MatrixFreeFunctions::MappingInfoStorage< + deal_II_dimension, deal_II_dimension, - float>::print_memory_consumption(ConditionalOStream &, - const TaskInfo &) - const; + float, + VectorizedArray>; +# if deal_II_dimension > 1 + template struct internal::MatrixFreeFunctions::MappingInfoStorage< + deal_II_dimension - 1, + deal_II_dimension, + double, + VectorizedArray>; + template struct internal::MatrixFreeFunctions::MappingInfoStorage< + deal_II_dimension - 1, + deal_II_dimension, + float, + VectorizedArray>; +# endif + + template void internal::MatrixFreeFunctions:: + MappingInfo>:: + print_memory_consumption(std::ostream &, const TaskInfo &) + const; + template void internal::MatrixFreeFunctions:: + MappingInfo>:: + print_memory_consumption(ConditionalOStream &, + const TaskInfo &) const; + + template void internal::MatrixFreeFunctions:: + MappingInfo>:: + print_memory_consumption(std::ostream &, const TaskInfo &) + const; + template void internal::MatrixFreeFunctions:: + MappingInfo>:: + print_memory_consumption(ConditionalOStream &, + const TaskInfo &) const; +#endif } diff --git a/source/matrix_free/matrix_free.inst.in b/source/matrix_free/matrix_free.inst.in index ed0485d287..73b00dcaef 100644 --- a/source/matrix_free/matrix_free.inst.in +++ b/source/matrix_free/matrix_free.inst.in @@ -16,104 +16,497 @@ for (deal_II_dimension : DIMENSIONS) { - template class MatrixFree; - template class MatrixFree; + template void + internal::MatrixFreeFunctions::ShapeInfo::reinit( + const Quadrature<1> &, + const FiniteElement &, + const unsigned int); template void - MatrixFree::print_memory_consumption(std::ostream &) - const; - template void MatrixFree:: + internal::MatrixFreeFunctions::ShapeInfo::reinit( + const Quadrature<1> &, + const FiniteElement &, + const unsigned int); + } + + + +for (deal_II_dimension : DIMENSIONS) + { + template class MatrixFree>; + template class MatrixFree>; + + template void + MatrixFree>:: + print_memory_consumption(std::ostream &) const; + + template void + MatrixFree>:: print_memory_consumption(ConditionalOStream &) const; template void - MatrixFree::print_memory_consumption(std::ostream &) - const; + MatrixFree>:: + print_memory_consumption(std::ostream &) const; - template void MatrixFree:: + template void + MatrixFree>:: print_memory_consumption(ConditionalOStream &) const; - template void MatrixFree::internal_reinit< - double>(const Mapping &, - const std::vector *> &, - const std::vector *> &, - const std::vector &, - const std::vector> &, - const AdditionalData &); - - template void MatrixFree::internal_reinit< - double>(const Mapping &, - const std::vector *> &, - const std::vector *> &, - const std::vector &, - const std::vector> &, - const AdditionalData &); - - template void MatrixFree::internal_reinit( - const Mapping &, - const std::vector *> &, - const std::vector *> &, - const std::vector &, - const std::vector> &, - const AdditionalData &); - - template void MatrixFree::internal_reinit( - const Mapping &, - const std::vector *> &, - const std::vector *> &, - const std::vector &, - const std::vector> &, - const AdditionalData &); - - template void MatrixFree::internal_reinit( - const Mapping &, - const std::vector *> &, - const std::vector *> &, - const std::vector &, - const std::vector> &, - const AdditionalData &); - - template void MatrixFree::internal_reinit( - const Mapping &, - const std::vector *> &, - const std::vector *> &, - const std::vector &, - const std::vector> &, - const AdditionalData &); + template void + MatrixFree>:: + internal_reinit( + const Mapping &, + const std::vector *> &, + const std::vector *> &, + const std::vector &, + const std::vector> &, + const AdditionalData &); template void - internal::MatrixFreeFunctions::ShapeInfo::reinit( - const Quadrature<1> &, - const FiniteElement &, - const unsigned int); + MatrixFree>:: + internal_reinit( + const Mapping &, + const std::vector *> &, + const std::vector *> &, + const std::vector &, + const std::vector> &, + const AdditionalData &); template void - internal::MatrixFreeFunctions::ShapeInfo::reinit( - const Quadrature<1> &, - const FiniteElement &, - const unsigned int); + MatrixFree>:: + internal_reinit( + const Mapping &, + const std::vector *> &, + const std::vector *> &, + const std::vector &, + const std::vector> &, + const AdditionalData &); + + template void + MatrixFree>:: + internal_reinit( + const Mapping &, + const std::vector *> &, + const std::vector *> &, + const std::vector &, + const std::vector> &, + const AdditionalData &); + + template void + MatrixFree>:: + internal_reinit( + const Mapping &, + const std::vector *> &, + const std::vector *> &, + const std::vector &, + const std::vector> &, + const AdditionalData &); + + template void + MatrixFree>:: + internal_reinit( + const Mapping &, + const std::vector *> &, + const std::vector *> &, + const std::vector &, + const std::vector> &, + const AdditionalData &); template void internal::MatrixFreeFunctions:: - ShapeInfo>::reinit( + ShapeInfo>::reinit( const Quadrature<1> &, const FiniteElement &, const unsigned int); template void internal::MatrixFreeFunctions:: - ShapeInfo>::reinit( + ShapeInfo>::reinit( const Quadrature<1> &, const FiniteElement &, const unsigned int); } + + for (deal_II_dimension : DIMENSIONS; deal_II_space_dimension : SPACE_DIMENSIONS) { #if deal_II_dimension <= deal_II_space_dimension - template bool MatrixFree::is_supported( - const FiniteElement &); + template bool + MatrixFree>:: + is_supported( + const FiniteElement &); + + template bool + MatrixFree>:: + is_supported( + const FiniteElement &); +#endif + } + + + +for (deal_II_dimension : DIMENSIONS) + { +#if DEAL_II_COMPILER_VECTORIZATION_LEVEL >= 2 && defined(__AVX__) + template class MatrixFree>; + template class MatrixFree>; + + template void + MatrixFree>:: + print_memory_consumption(std::ostream &) const; + + template void + MatrixFree>:: + print_memory_consumption(ConditionalOStream &) const; + + template void + MatrixFree>:: + print_memory_consumption(std::ostream &) const; + + template void + MatrixFree>:: + print_memory_consumption(ConditionalOStream &) const; + + template void + MatrixFree>:: + internal_reinit( + const Mapping &, + const std::vector *> &, + const std::vector *> &, + const std::vector &, + const std::vector> &, + const AdditionalData &); + + template void + MatrixFree>:: + internal_reinit( + const Mapping &, + const std::vector *> &, + const std::vector *> &, + const std::vector &, + const std::vector> &, + const AdditionalData &); + + template void + MatrixFree>:: + internal_reinit( + const Mapping &, + const std::vector *> &, + const std::vector *> &, + const std::vector &, + const std::vector> &, + const AdditionalData &); + + template void + MatrixFree>:: + internal_reinit( + const Mapping &, + const std::vector *> &, + const std::vector *> &, + const std::vector &, + const std::vector> &, + const AdditionalData &); + + template void + MatrixFree>:: + internal_reinit( + const Mapping &, + const std::vector *> &, + const std::vector *> &, + const std::vector &, + const std::vector> &, + const AdditionalData &); + + template void + MatrixFree>:: + internal_reinit( + const Mapping &, + const std::vector *> &, + const std::vector *> &, + const std::vector &, + const std::vector> &, + const AdditionalData &); + + template void internal::MatrixFreeFunctions:: + ShapeInfo>::reinit( + const Quadrature<1> &, + const FiniteElement &, + const unsigned int); + + template void internal::MatrixFreeFunctions:: + ShapeInfo>::reinit( + const Quadrature<1> &, + const FiniteElement &, + const unsigned int); +#endif + } + + + +for (deal_II_dimension : DIMENSIONS; deal_II_space_dimension : SPACE_DIMENSIONS) + { +#if DEAL_II_COMPILER_VECTORIZATION_LEVEL >= 2 && defined(__AVX__) +# if deal_II_dimension <= deal_II_space_dimension + template bool + MatrixFree>:: + is_supported( + const FiniteElement &); + + template bool + MatrixFree>:: + is_supported( + const FiniteElement &); +# endif +#endif + } + + + +for (deal_II_dimension : DIMENSIONS) + { +#if (DEAL_II_COMPILER_VECTORIZATION_LEVEL >= 1 && defined(__SSE2__)) || \ + (DEAL_II_COMPILER_VECTORIZATION_LEVEL >= 1 && defined(__ALTIVEC__)) + template class MatrixFree>; + template class MatrixFree>; + + template void + MatrixFree>:: + print_memory_consumption(std::ostream &) const; + + template void + MatrixFree>:: + print_memory_consumption(ConditionalOStream &) const; + + template void + MatrixFree>:: + print_memory_consumption(std::ostream &) const; + + template void + MatrixFree>:: + print_memory_consumption(ConditionalOStream &) const; + + template void + MatrixFree>:: + internal_reinit( + const Mapping &, + const std::vector *> &, + const std::vector *> &, + const std::vector &, + const std::vector> &, + const AdditionalData &); + + template void + MatrixFree>:: + internal_reinit( + const Mapping &, + const std::vector *> &, + const std::vector *> &, + const std::vector &, + const std::vector> &, + const AdditionalData &); + + template void + MatrixFree>:: + internal_reinit( + const Mapping &, + const std::vector *> &, + const std::vector *> &, + const std::vector &, + const std::vector> &, + const AdditionalData &); + + template void + MatrixFree>:: + internal_reinit( + const Mapping &, + const std::vector *> &, + const std::vector *> &, + const std::vector &, + const std::vector> &, + const AdditionalData &); + + template void + MatrixFree>:: + internal_reinit( + const Mapping &, + const std::vector *> &, + const std::vector *> &, + const std::vector &, + const std::vector> &, + const AdditionalData &); + + template void + MatrixFree>:: + internal_reinit( + const Mapping &, + const std::vector *> &, + const std::vector *> &, + const std::vector &, + const std::vector> &, + const AdditionalData &); + + template void internal::MatrixFreeFunctions:: + ShapeInfo>::reinit( + const Quadrature<1> &, + const FiniteElement &, + const unsigned int); + + template void internal::MatrixFreeFunctions:: + ShapeInfo>::reinit( + const Quadrature<1> &, + const FiniteElement &, + const unsigned int); +#endif + } + + + +for (deal_II_dimension : DIMENSIONS; deal_II_space_dimension : SPACE_DIMENSIONS) + { +#if (DEAL_II_COMPILER_VECTORIZATION_LEVEL >= 1 && defined(__SSE2__)) || \ + (DEAL_II_COMPILER_VECTORIZATION_LEVEL >= 1 && defined(__ALTIVEC__)) +# if deal_II_dimension <= deal_II_space_dimension + template bool + MatrixFree>:: + is_supported( + const FiniteElement &); + + template bool + MatrixFree>:: + is_supported( + const FiniteElement &); +# endif +#endif + } + + + +for (deal_II_dimension : DIMENSIONS) + { +#if DEAL_II_COMPILER_VECTORIZATION_LEVEL >= 3 && defined(__AVX512F__) + template class MatrixFree>; + template class MatrixFree>; + + template void + MatrixFree>:: + print_memory_consumption(std::ostream &) const; + + template void + MatrixFree>:: + print_memory_consumption(ConditionalOStream &) const; + + template void + MatrixFree>:: + print_memory_consumption(std::ostream &) const; + + template void + MatrixFree>:: + print_memory_consumption(ConditionalOStream &) const; + + template void + MatrixFree>:: + internal_reinit( + const Mapping &, + const std::vector *> &, + const std::vector *> &, + const std::vector &, + const std::vector> &, + const AdditionalData &); + + template void + MatrixFree>:: + internal_reinit( + const Mapping &, + const std::vector *> &, + const std::vector *> &, + const std::vector &, + const std::vector> &, + const AdditionalData &); + + template void + MatrixFree>:: + internal_reinit( + const Mapping &, + const std::vector *> &, + const std::vector *> &, + const std::vector &, + const std::vector> &, + const AdditionalData &); + + template void + MatrixFree>:: + internal_reinit( + const Mapping &, + const std::vector *> &, + const std::vector *> &, + const std::vector &, + const std::vector> &, + const AdditionalData &); + + template void + MatrixFree>:: + internal_reinit( + const Mapping &, + const std::vector *> &, + const std::vector *> &, + const std::vector &, + const std::vector> &, + const AdditionalData &); + + template void + MatrixFree>:: + internal_reinit( + const Mapping &, + const std::vector *> &, + const std::vector *> &, + const std::vector &, + const std::vector> &, + const AdditionalData &); + + template void internal::MatrixFreeFunctions:: + ShapeInfo>::reinit( + const Quadrature<1> &, + const FiniteElement &, + const unsigned int); + + template void internal::MatrixFreeFunctions:: + ShapeInfo>::reinit( + const Quadrature<1> &, + const FiniteElement &, + const unsigned int); +#endif + } + + + +for (deal_II_dimension : DIMENSIONS; deal_II_space_dimension : SPACE_DIMENSIONS) + { +#if DEAL_II_COMPILER_VECTORIZATION_LEVEL >= 3 && defined(__AVX512F__) +# if deal_II_dimension <= deal_II_space_dimension + template bool + MatrixFree>:: + is_supported( + const FiniteElement &); - template bool MatrixFree::is_supported( - const FiniteElement &); + template bool + MatrixFree>:: + is_supported( + const FiniteElement &); +# endif #endif } diff --git a/source/numerics/vector_tools_project_qpmf.inst.in b/source/numerics/vector_tools_project_qpmf.inst.in index b1d1bbd068..12526638f1 100644 --- a/source/numerics/vector_tools_project_qpmf.inst.in +++ b/source/numerics/vector_tools_project_qpmf.inst.in @@ -19,10 +19,11 @@ for (VEC : REAL_NONBLOCK_VECTORS; deal_II_dimension : DIMENSIONS) { namespace VectorTools \{ - template void project( - std::shared_ptr> + std::shared_ptr>> matrix_free, const AffineConstraints &constraints, const std::function( - std::shared_ptr> + std::shared_ptr>> matrix_free, const AffineConstraints &constraints, const unsigned int, @@ -41,5 +44,6 @@ for (VEC : REAL_NONBLOCK_VECTORS; deal_II_dimension : DIMENSIONS) VEC &, const unsigned int); + \} - } + } \ No newline at end of file diff --git a/tests/matrix_free/matrix_vector_faces_26.cc b/tests/matrix_free/matrix_vector_faces_26.cc new file mode 100644 index 0000000000..64a2ceaa9a --- /dev/null +++ b/tests/matrix_free/matrix_vector_faces_26.cc @@ -0,0 +1,64 @@ +// --------------------------------------------------------------------- +// +// Copyright (C) 2018 by the deal.II authors +// +// This file is part of the deal.II library. +// +// The deal.II library is free software; you can use it, redistribute +// it, and/or modify it under the terms of the GNU Lesser General +// Public License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// The full text of the license can be found in the file LICENSE.md at +// the top level directory of deal.II. +// +// --------------------------------------------------------------------- + + + +// tests matrix-free face evaluation, matrix-vector products as compared to +// the same implementation with MeshWorker. This example uses a hypercube mesh +// without any hanging nodes + +#include + +#include + +#include "../tests.h" + +std::ofstream logfile("output"); + +#include "matrix_vector_faces_common.h" + +template +void +test() +{ + Triangulation tria; + GridGenerator::hyper_cube(tria); + tria.refine_global(5 - dim); + + FE_DGQ fe(fe_degree); + DoFHandler dof(tria); + dof.distribute_dofs(fe); + AffineConstraints constraints; + constraints.close(); + + do_test>( + dof, constraints, true); + +#if (DEAL_II_COMPILER_VECTORIZATION_LEVEL >= 1 && defined(__SSE2__)) || \ + (DEAL_II_COMPILER_VECTORIZATION_LEVEL >= 1 && defined(__ALTIVEC__)) + do_test>( + dof, constraints, true); +#endif + +#if DEAL_II_COMPILER_VECTORIZATION_LEVEL >= 2 && defined(__AVX__) + do_test>( + dof, constraints, true); +#endif + +#if DEAL_II_COMPILER_VECTORIZATION_LEVEL >= 3 && defined(__AVX512F__) + do_test>( + dof, constraints, true); +#endif +} diff --git a/tests/matrix_free/matrix_vector_faces_26.output b/tests/matrix_free/matrix_vector_faces_26.output new file mode 100644 index 0000000000..07c965df3b --- /dev/null +++ b/tests/matrix_free/matrix_vector_faces_26.output @@ -0,0 +1,17 @@ + +DEAL:2d::Testing FE_DGQ<2>(1) +DEAL:2d::Norm of difference: 0 +DEAL:2d::Norm of difference parallel: 0 +DEAL:2d:: +DEAL:2d::Testing FE_DGQ<2>(2) +DEAL:2d::Norm of difference: 0 +DEAL:2d::Norm of difference parallel: 0 +DEAL:2d:: +DEAL:3d::Testing FE_DGQ<3>(1) +DEAL:3d::Norm of difference: 0 +DEAL:3d::Norm of difference parallel: 0 +DEAL:3d:: +DEAL:3d::Testing FE_DGQ<3>(2) +DEAL:3d::Norm of difference: 0 +DEAL:3d::Norm of difference parallel: 0 +DEAL:3d:: diff --git a/tests/matrix_free/matrix_vector_faces_26.output.avx b/tests/matrix_free/matrix_vector_faces_26.output.avx new file mode 100644 index 0000000000..ea2a47eae6 --- /dev/null +++ b/tests/matrix_free/matrix_vector_faces_26.output.avx @@ -0,0 +1,49 @@ + +DEAL:2d::Testing FE_DGQ<2>(1) +DEAL:2d::Norm of difference: 0 +DEAL:2d::Norm of difference parallel: 0 +DEAL:2d:: +DEAL:2d::Testing FE_DGQ<2>(1) +DEAL:2d::Norm of difference: 0 +DEAL:2d::Norm of difference parallel: 0 +DEAL:2d:: +DEAL:2d::Testing FE_DGQ<2>(1) +DEAL:2d::Norm of difference: 0 +DEAL:2d::Norm of difference parallel: 0 +DEAL:2d:: +DEAL:2d::Testing FE_DGQ<2>(2) +DEAL:2d::Norm of difference: 0 +DEAL:2d::Norm of difference parallel: 0 +DEAL:2d:: +DEAL:2d::Testing FE_DGQ<2>(2) +DEAL:2d::Norm of difference: 0 +DEAL:2d::Norm of difference parallel: 0 +DEAL:2d:: +DEAL:2d::Testing FE_DGQ<2>(2) +DEAL:2d::Norm of difference: 0 +DEAL:2d::Norm of difference parallel: 0 +DEAL:2d:: +DEAL:3d::Testing FE_DGQ<3>(1) +DEAL:3d::Norm of difference: 0 +DEAL:3d::Norm of difference parallel: 0 +DEAL:3d:: +DEAL:3d::Testing FE_DGQ<3>(1) +DEAL:3d::Norm of difference: 0 +DEAL:3d::Norm of difference parallel: 0 +DEAL:3d:: +DEAL:3d::Testing FE_DGQ<3>(1) +DEAL:3d::Norm of difference: 0 +DEAL:3d::Norm of difference parallel: 0 +DEAL:3d:: +DEAL:3d::Testing FE_DGQ<3>(2) +DEAL:3d::Norm of difference: 0 +DEAL:3d::Norm of difference parallel: 0 +DEAL:3d:: +DEAL:3d::Testing FE_DGQ<3>(2) +DEAL:3d::Norm of difference: 0 +DEAL:3d::Norm of difference parallel: 0 +DEAL:3d:: +DEAL:3d::Testing FE_DGQ<3>(2) +DEAL:3d::Norm of difference: 0 +DEAL:3d::Norm of difference parallel: 0 +DEAL:3d:: diff --git a/tests/matrix_free/matrix_vector_faces_26.output.avx512 b/tests/matrix_free/matrix_vector_faces_26.output.avx512 new file mode 100644 index 0000000000..785d86e8eb --- /dev/null +++ b/tests/matrix_free/matrix_vector_faces_26.output.avx512 @@ -0,0 +1,65 @@ + +DEAL:2d::Testing FE_DGQ<2>(1) +DEAL:2d::Norm of difference: 0 +DEAL:2d::Norm of difference parallel: 0 +DEAL:2d:: +DEAL:2d::Testing FE_DGQ<2>(1) +DEAL:2d::Norm of difference: 0 +DEAL:2d::Norm of difference parallel: 0 +DEAL:2d:: +DEAL:2d::Testing FE_DGQ<2>(1) +DEAL:2d::Norm of difference: 0 +DEAL:2d::Norm of difference parallel: 0 +DEAL:2d:: +DEAL:2d::Testing FE_DGQ<2>(1) +DEAL:2d::Norm of difference: 0 +DEAL:2d::Norm of difference parallel: 0 +DEAL:2d:: +DEAL:2d::Testing FE_DGQ<2>(2) +DEAL:2d::Norm of difference: 0 +DEAL:2d::Norm of difference parallel: 0 +DEAL:2d:: +DEAL:2d::Testing FE_DGQ<2>(2) +DEAL:2d::Norm of difference: 0 +DEAL:2d::Norm of difference parallel: 0 +DEAL:2d:: +DEAL:2d::Testing FE_DGQ<2>(2) +DEAL:2d::Norm of difference: 0 +DEAL:2d::Norm of difference parallel: 0 +DEAL:2d:: +DEAL:2d::Testing FE_DGQ<2>(2) +DEAL:2d::Norm of difference: 0 +DEAL:2d::Norm of difference parallel: 0 +DEAL:2d:: +DEAL:3d::Testing FE_DGQ<3>(1) +DEAL:3d::Norm of difference: 0 +DEAL:3d::Norm of difference parallel: 0 +DEAL:3d:: +DEAL:3d::Testing FE_DGQ<3>(1) +DEAL:3d::Norm of difference: 0 +DEAL:3d::Norm of difference parallel: 0 +DEAL:3d:: +DEAL:3d::Testing FE_DGQ<3>(1) +DEAL:3d::Norm of difference: 0 +DEAL:3d::Norm of difference parallel: 0 +DEAL:3d:: +DEAL:3d::Testing FE_DGQ<3>(1) +DEAL:3d::Norm of difference: 0 +DEAL:3d::Norm of difference parallel: 0 +DEAL:3d:: +DEAL:3d::Testing FE_DGQ<3>(2) +DEAL:3d::Norm of difference: 0 +DEAL:3d::Norm of difference parallel: 0 +DEAL:3d:: +DEAL:3d::Testing FE_DGQ<3>(2) +DEAL:3d::Norm of difference: 0 +DEAL:3d::Norm of difference parallel: 0 +DEAL:3d:: +DEAL:3d::Testing FE_DGQ<3>(2) +DEAL:3d::Norm of difference: 0 +DEAL:3d::Norm of difference parallel: 0 +DEAL:3d:: +DEAL:3d::Testing FE_DGQ<3>(2) +DEAL:3d::Norm of difference: 0 +DEAL:3d::Norm of difference parallel: 0 +DEAL:3d:: diff --git a/tests/matrix_free/matrix_vector_faces_26.output.sse2 b/tests/matrix_free/matrix_vector_faces_26.output.sse2 new file mode 100644 index 0000000000..5c89f98bad --- /dev/null +++ b/tests/matrix_free/matrix_vector_faces_26.output.sse2 @@ -0,0 +1,33 @@ + +DEAL:2d::Testing FE_DGQ<2>(1) +DEAL:2d::Norm of difference: 0 +DEAL:2d::Norm of difference parallel: 0 +DEAL:2d:: +DEAL:2d::Testing FE_DGQ<2>(1) +DEAL:2d::Norm of difference: 0 +DEAL:2d::Norm of difference parallel: 0 +DEAL:2d:: +DEAL:2d::Testing FE_DGQ<2>(2) +DEAL:2d::Norm of difference: 0 +DEAL:2d::Norm of difference parallel: 0 +DEAL:2d:: +DEAL:2d::Testing FE_DGQ<2>(2) +DEAL:2d::Norm of difference: 0 +DEAL:2d::Norm of difference parallel: 0 +DEAL:2d:: +DEAL:3d::Testing FE_DGQ<3>(1) +DEAL:3d::Norm of difference: 0 +DEAL:3d::Norm of difference parallel: 0 +DEAL:3d:: +DEAL:3d::Testing FE_DGQ<3>(1) +DEAL:3d::Norm of difference: 0 +DEAL:3d::Norm of difference parallel: 0 +DEAL:3d:: +DEAL:3d::Testing FE_DGQ<3>(2) +DEAL:3d::Norm of difference: 0 +DEAL:3d::Norm of difference parallel: 0 +DEAL:3d:: +DEAL:3d::Testing FE_DGQ<3>(2) +DEAL:3d::Norm of difference: 0 +DEAL:3d::Norm of difference parallel: 0 +DEAL:3d:: diff --git a/tests/matrix_free/matrix_vector_faces_common.h b/tests/matrix_free/matrix_vector_faces_common.h index 6054c09199..1d93b0a7c2 100644 --- a/tests/matrix_free/matrix_vector_faces_common.h +++ b/tests/matrix_free/matrix_vector_faces_common.h @@ -64,14 +64,15 @@ test(); template , - int n_components = 1> + int n_q_points_1d = fe_degree + 1, + typename number = double, + typename VectorType = Vector, + int n_components = 1, + typename VectorizedArrayType = VectorizedArray> class MatrixFreeTest { public: - MatrixFreeTest(const MatrixFree &data) + MatrixFreeTest(const MatrixFree &data) : data(data) {} @@ -89,12 +90,18 @@ public: private: void - local_apply(const MatrixFree & data, - VectorType & dst, - const VectorType & src, + local_apply(const MatrixFree &data, + VectorType & dst, + const VectorType & src, const std::pair &cell_range) const { - FEEvaluation phi(data); + FEEvaluation + phi(data); for (unsigned int cell = cell_range.first; cell < cell_range.second; ++cell) { @@ -110,20 +117,32 @@ private: void local_apply_face( - const MatrixFree & data, - VectorType & dst, - const VectorType & src, - const std::pair &face_range) const + const MatrixFree &data, + VectorType & dst, + const VectorType & src, + const std::pair & face_range) const { - FEFaceEvaluation + FEFaceEvaluation fe_eval(data, true); - FEFaceEvaluation - fe_eval_neighbor(data, false); - typedef typename FEFaceEvaluation::value_type value_type; + FEFaceEvaluation + fe_eval_neighbor(data, false); + typedef + typename FEFaceEvaluation::value_type value_type; const int actual_degree = data.get_dof_handler().get_fe().degree; for (unsigned int face = face_range.first; face < face_range.second; face++) @@ -135,7 +154,7 @@ private: fe_eval.evaluate(true, true); fe_eval_neighbor.read_dof_values(src); fe_eval_neighbor.evaluate(true, true); - VectorizedArray sigmaF = + VectorizedArrayType sigmaF = (std::abs((fe_eval.get_normal_vector(0) * fe_eval.inverse_jacobian(0))[dim - 1]) + std::abs((fe_eval.get_normal_vector(0) * @@ -146,13 +165,17 @@ private: { value_type average_value = (fe_eval.get_value(q) - fe_eval_neighbor.get_value(q)) * - make_vectorized_array(0.5); + make_vectorized_array(0.5); value_type average_valgrad = fe_eval.get_normal_derivative(q) + fe_eval_neighbor.get_normal_derivative(q); average_valgrad = average_value * sigmaF - - average_valgrad * make_vectorized_array(0.5); + average_valgrad * + make_vectorized_array( + 0.5); fe_eval.submit_normal_derivative(-average_value, q); fe_eval_neighbor.submit_normal_derivative(-average_value, q); fe_eval.submit_value(average_valgrad, q); @@ -168,25 +191,32 @@ private: void local_apply_boundary_face( - const MatrixFree & data, - VectorType & dst, - const VectorType & src, - const std::pair &face_range) const + const MatrixFree &data, + VectorType & dst, + const VectorType & src, + const std::pair & face_range) const { - FEFaceEvaluation - fe_eval(data, true); - typedef typename FEFaceEvaluation::value_type value_type; + FEFaceEvaluation + fe_eval(data, true); + typedef + typename FEFaceEvaluation::value_type value_type; const int actual_degree = data.get_dof_handler().get_fe().degree; for (unsigned int face = face_range.first; face < face_range.second; face++) { fe_eval.reinit(face); fe_eval.read_dof_values(src); fe_eval.evaluate(true, true); - VectorizedArray sigmaF = + VectorizedArrayType sigmaF = 2.0 * std::abs((fe_eval.get_normal_vector(0) * fe_eval.inverse_jacobian(0))[dim - 1]) * @@ -206,7 +236,7 @@ private: } } - const MatrixFree &data; + const MatrixFree &data; }; @@ -215,16 +245,17 @@ private: // vector-access/evaluate routines that typically give better performance template , - int n_components = 1> + int n_q_points_1d = fe_degree + 1, + typename number = double, + typename VectorType = Vector, + int n_components = 1, + typename VectorizedArrayType = VectorizedArray> class MatrixFreeVariant { public: - MatrixFreeVariant(const MatrixFree &data, - const bool zero_within_loop = true, - const unsigned int start_vector_component = 0) + MatrixFreeVariant(const MatrixFree &data, + const bool zero_within_loop = true, + const unsigned int start_vector_component = 0) : data(data) , zero_within_loop(zero_within_loop) , start_vector_component(start_vector_component) @@ -242,8 +273,10 @@ public: dst, src, zero_within_loop, - MatrixFree::DataAccessOnFaces::gradients, - MatrixFree::DataAccessOnFaces::gradients); + MatrixFree::DataAccessOnFaces:: + gradients, + MatrixFree::DataAccessOnFaces:: + gradients); } void @@ -256,19 +289,26 @@ public: dst, src, false, - MatrixFree::DataAccessOnFaces::gradients, - MatrixFree::DataAccessOnFaces::gradients); + MatrixFree::DataAccessOnFaces:: + gradients, + MatrixFree::DataAccessOnFaces:: + gradients); } private: void - local_apply(const MatrixFree & data, - VectorType & dst, - const VectorType & src, + local_apply(const MatrixFree &data, + VectorType & dst, + const VectorType & src, const std::pair &cell_range) const { - FEEvaluation phi( - data, 0, 0, start_vector_component); + FEEvaluation + phi(data, 0, 0, start_vector_component); for (unsigned int cell = cell_range.first; cell < cell_range.second; ++cell) { @@ -282,20 +322,32 @@ private: void local_apply_face( - const MatrixFree & data, - VectorType & dst, - const VectorType & src, - const std::pair &face_range) const + const MatrixFree &data, + VectorType & dst, + const VectorType & src, + const std::pair & face_range) const { - FEFaceEvaluation + FEFaceEvaluation fe_eval(data, true, 0, 0, start_vector_component); - FEFaceEvaluation - fe_eval_neighbor(data, false, 0, 0, start_vector_component); - typedef typename FEFaceEvaluation::value_type value_type; + FEFaceEvaluation + fe_eval_neighbor(data, false, 0, 0, start_vector_component); + typedef + typename FEFaceEvaluation::value_type value_type; const int actual_degree = data.get_dof_handler().get_fe().degree; for (unsigned int face = face_range.first; face < face_range.second; face++) @@ -306,7 +358,7 @@ private: fe_eval.gather_evaluate(src, true, true); fe_eval_neighbor.gather_evaluate(src, true, true); - VectorizedArray sigmaF = + VectorizedArrayType sigmaF = (std::abs((fe_eval.get_normal_vector(0) * fe_eval.inverse_jacobian(0))[dim - 1]) + std::abs((fe_eval.get_normal_vector(0) * @@ -317,13 +369,17 @@ private: { value_type average_value = (fe_eval.get_value(q) - fe_eval_neighbor.get_value(q)) * - make_vectorized_array(0.5); + make_vectorized_array(0.5); value_type average_valgrad = fe_eval.get_normal_derivative(q) + fe_eval_neighbor.get_normal_derivative(q); average_valgrad = average_value * sigmaF - - average_valgrad * make_vectorized_array(0.5); + average_valgrad * + make_vectorized_array( + 0.5); fe_eval.submit_normal_derivative(-average_value, q); fe_eval_neighbor.submit_normal_derivative(-average_value, q); fe_eval.submit_value(average_valgrad, q); @@ -336,24 +392,31 @@ private: void local_apply_boundary_face( - const MatrixFree & data, - VectorType & dst, - const VectorType & src, - const std::pair &face_range) const + const MatrixFree &data, + VectorType & dst, + const VectorType & src, + const std::pair & face_range) const { - FEFaceEvaluation - fe_eval(data, true, 0, 0, start_vector_component); - typedef typename FEFaceEvaluation::value_type value_type; + FEFaceEvaluation + fe_eval(data, true, 0, 0, start_vector_component); + typedef + typename FEFaceEvaluation::value_type value_type; const int actual_degree = data.get_dof_handler().get_fe().degree; for (unsigned int face = face_range.first; face < face_range.second; face++) { fe_eval.reinit(face); fe_eval.gather_evaluate(src, true, true); - VectorizedArray sigmaF = + VectorizedArrayType sigmaF = std::abs((fe_eval.get_normal_vector(0) * fe_eval.inverse_jacobian(0))[dim - 1]) * number(std::max(actual_degree, 1) * (actual_degree + 1.0)) * 2.0; @@ -371,9 +434,9 @@ private: } } - const MatrixFree &data; - const bool zero_within_loop; - const unsigned int start_vector_component; + const MatrixFree &data; + const bool zero_within_loop; + const unsigned int start_vector_component; }; @@ -408,16 +471,17 @@ multiply_by_advection(const Tensor<1, dim, Number> &advection, // An implementation of matrix-free advection operator template , - int n_components = 1> + int n_q_points_1d = fe_degree + 1, + typename number = double, + typename VectorType = Vector, + int n_components = 1, + typename VectorizedArrayType = VectorizedArray> class MatrixFreeAdvection { public: - MatrixFreeAdvection(const MatrixFree &data, - const bool zero_within_loop = true, - const unsigned int start_vector_component = 0) + MatrixFreeAdvection(const MatrixFree &data, + const bool zero_within_loop = true, + const unsigned int start_vector_component = 0) : data(data) , zero_within_loop(zero_within_loop) , start_vector_component(start_vector_component) @@ -431,40 +495,47 @@ public: { if (!zero_within_loop) dst = 0; - data.loop(&MatrixFreeAdvection::local_apply, - &MatrixFreeAdvection::local_apply_face, - &MatrixFreeAdvection::local_apply_boundary_face, - this, - dst, - src, - zero_within_loop, - MatrixFree::DataAccessOnFaces::values, - MatrixFree::DataAccessOnFaces::values); + data.loop( + &MatrixFreeAdvection::local_apply, + &MatrixFreeAdvection::local_apply_face, + &MatrixFreeAdvection::local_apply_boundary_face, + this, + dst, + src, + zero_within_loop, + MatrixFree::DataAccessOnFaces::values, + MatrixFree::DataAccessOnFaces::values); } void vmult_add(VectorType &dst, const VectorType &src) const { - data.loop(&MatrixFreeAdvection::local_apply, - &MatrixFreeAdvection::local_apply_face, - &MatrixFreeAdvection::local_apply_boundary_face, - this, - dst, - src, - false, - MatrixFree::DataAccessOnFaces::values, - MatrixFree::DataAccessOnFaces::values); + data.loop( + &MatrixFreeAdvection::local_apply, + &MatrixFreeAdvection::local_apply_face, + &MatrixFreeAdvection::local_apply_boundary_face, + this, + dst, + src, + false, + MatrixFree::DataAccessOnFaces::values, + MatrixFree::DataAccessOnFaces::values); } private: void - local_apply(const MatrixFree & data, - VectorType & dst, - const VectorType & src, + local_apply(const MatrixFree &data, + VectorType & dst, + const VectorType & src, const std::pair &cell_range) const { - FEEvaluation phi( - data, 0, 0, start_vector_component); + FEEvaluation + phi(data, 0, 0, start_vector_component); for (unsigned int cell = cell_range.first; cell < cell_range.second; ++cell) { @@ -480,20 +551,32 @@ private: void local_apply_face( - const MatrixFree & data, - VectorType & dst, - const VectorType & src, - const std::pair &face_range) const + const MatrixFree &data, + VectorType & dst, + const VectorType & src, + const std::pair & face_range) const { - FEFaceEvaluation phi_m( - data, true, 0, 0, start_vector_component); - FEFaceEvaluation phi_p( - data, false, 0, 0, start_vector_component); - typedef typename FEFaceEvaluation::value_type value_type; + FEFaceEvaluation + phi_m(data, true, 0, 0, start_vector_component); + FEFaceEvaluation + phi_p(data, false, 0, 0, start_vector_component); + typedef + typename FEFaceEvaluation::value_type value_type; for (unsigned int face = face_range.first; face < face_range.second; face++) { @@ -506,10 +589,12 @@ private: { value_type u_minus = phi_m.get_value(q), u_plus = phi_p.get_value(q); - const VectorizedArray normal_times_advection = + const VectorizedArrayType normal_times_advection = advection * phi_m.get_normal_vector(q); const value_type flux_times_normal = - make_vectorized_array(0.5) * + make_vectorized_array( + 0.5) * ((u_minus + u_plus) * normal_times_advection + std::abs(normal_times_advection) * (u_minus - u_plus)); phi_m.submit_value(-flux_times_normal, q); @@ -523,20 +608,28 @@ private: void local_apply_boundary_face( - const MatrixFree & data, - VectorType & dst, - const VectorType & src, - const std::pair &face_range) const + const MatrixFree &data, + VectorType & dst, + const VectorType & src, + const std::pair & face_range) const { - FEFaceEvaluation - fe_eval(data, true, 0, 0, start_vector_component); - typedef typename FEFaceEvaluation::value_type value_type; - value_type u_plus; - u_plus = make_vectorized_array(1.3); + FEFaceEvaluation + fe_eval(data, true, 0, 0, start_vector_component); + typedef + typename FEFaceEvaluation::value_type value_type; + value_type u_plus; + u_plus = + make_vectorized_array(1.3); for (unsigned int face = face_range.first; face < face_range.second; face++) { @@ -545,11 +638,13 @@ private: for (unsigned int q = 0; q < fe_eval.n_q_points; ++q) { - value_type u_minus = fe_eval.get_value(q); - const VectorizedArray normal_times_advection = + value_type u_minus = fe_eval.get_value(q); + const VectorizedArrayType normal_times_advection = advection * fe_eval.get_normal_vector(q); const value_type flux_times_normal = - make_vectorized_array(0.5) * + make_vectorized_array( + 0.5) * ((u_minus + u_plus) * normal_times_advection + std::abs(normal_times_advection) * (u_minus - u_plus)); fe_eval.submit_value(-flux_times_normal, q); @@ -559,10 +654,10 @@ private: } } - const MatrixFree & data; - const bool zero_within_loop; - const unsigned int start_vector_component; - Tensor<1, dim, VectorizedArray> advection; + const MatrixFree &data; + const bool zero_within_loop; + const unsigned int start_vector_component; + Tensor<1, dim, VectorizedArrayType> advection; }; @@ -656,7 +751,11 @@ MatrixIntegrator::boundary( -template +template > void do_test(const DoFHandler & dof, const AffineConstraints &constraints, @@ -716,12 +815,13 @@ do_test(const DoFHandler & dof, matrix.vmult(out, in); - MatrixFree mf_data; - const QGauss<1> quad(n_q_points_1d > 0 ? n_q_points_1d : + MatrixFree mf_data; + const QGauss<1> quad(n_q_points_1d > 0 ? n_q_points_1d : dof.get_fe().degree + 1); - typename MatrixFree::AdditionalData data; - data.tasks_parallel_scheme = MatrixFree::AdditionalData::none; - data.tasks_block_size = 3; + typename MatrixFree::AdditionalData data; + data.tasks_parallel_scheme = + MatrixFree::AdditionalData::none; + data.tasks_block_size = 3; data.mapping_update_flags_inner_faces = (update_gradients | update_JxW_values); data.mapping_update_flags_boundary_faces = @@ -729,7 +829,14 @@ do_test(const DoFHandler & dof, mf_data.reinit(mapping, dof, constraints, quad, data); - MatrixFreeTest mf(mf_data); + MatrixFreeTest, + 1, + VectorizedArrayType> + mf(mf_data); mf.vmult(out_dist, in); out_dist -= out; @@ -740,10 +847,18 @@ do_test(const DoFHandler & dof, { mf_data.clear(); data.tasks_parallel_scheme = - MatrixFree::AdditionalData::partition_partition; + MatrixFree::AdditionalData:: + partition_partition; mf_data.reinit(mapping, dof, constraints, quad, data); - MatrixFreeTest mf(mf_data); + MatrixFreeTest, + 1, + VectorizedArrayType> + mf(mf_data); mf.vmult(out_dist, in); out_dist -= out; -- 2.39.5