From 55e6eac2ead44cae17fc7ad67505b075c0eef89d Mon Sep 17 00:00:00 2001
From: Matthias Maier <tamiko@43-1.org>
Date: Thu, 7 Jan 2016 16:14:39 -0600
Subject: [PATCH] Always inline some tensor functions.

This drastically reduces the time in assembly in ASPECT:

DEBUG:
| Assemble Stokes system          |         8 |      38.3s |        27% |
| Assemble temperature system     |         8 |      51.4s |        36% |
| Build Stokes preconditioner     |         3 |      13.8s |       9.7% |
| Build temperature preconditioner|         8 |     0.192s |      0.13% |
| Solve Stokes system             |         8 |      12.3s |       8.6% |

DEBUG WITH PATCH:
| Assemble Stokes system          |         8 |      30.8s |        25% |
| Assemble temperature system     |         8 |      41.7s |        34% |
| Build Stokes preconditioner     |         3 |      12.1s |       9.9% |
| Build temperature preconditioner|         8 |     0.192s |      0.16% |
| Solve Stokes system             |         8 |      12.2s |        10% |

(Run with 16 processors, using the testcase discussed in https://github.com/geodynamics/aspect/pull/654 .)

There are no significant differences in release mode with this patch,
as probably expected.
---
 include/deal.II/base/tensor.h           |  7 ++++---
 include/deal.II/base/tensor_accessors.h | 28 +++++++++++++------------
 2 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/include/deal.II/base/tensor.h b/include/deal.II/base/tensor.h
index 3a240f5d53..81475a71f6 100644
--- a/include/deal.II/base/tensor.h
+++ b/include/deal.II/base/tensor.h
@@ -843,6 +843,7 @@ namespace internal
   namespace TensorSubscriptor
   {
     template <typename ArrayElementType, int dim>
+    inline __attribute__((always_inline))
     ArrayElementType &
     subscript (ArrayElementType *values,
                const unsigned int i,
@@ -868,7 +869,7 @@ namespace internal
 
 
 template <int rank_, int dim, typename Number>
-inline
+inline __attribute__((always_inline))
 typename Tensor<rank_,dim,Number>::value_type &
 Tensor<rank_,dim,Number>::operator[] (const unsigned int i)
 {
@@ -877,7 +878,7 @@ Tensor<rank_,dim,Number>::operator[] (const unsigned int i)
 
 
 template <int rank_, int dim, typename Number>
-inline
+inline __attribute__((always_inline))
 const typename Tensor<rank_,dim,Number>::value_type &
 Tensor<rank_,dim,Number>::operator[] (const unsigned int i) const
 {
@@ -1467,7 +1468,7 @@ operator- (const Tensor<rank,dim,Number> &p, const Tensor<rank,dim,OtherNumber>
  */
 template <int rank_1, int rank_2, int dim,
           typename Number, typename OtherNumber>
-inline
+inline __attribute__((always_inline))
 typename Tensor<rank_1 + rank_2 - 2, dim, typename ProductType<Number, OtherNumber>::type>::tensor_type
 operator * (const Tensor<rank_1, dim, Number> &src1,
             const Tensor<rank_2, dim, OtherNumber> &src2)
diff --git a/include/deal.II/base/tensor_accessors.h b/include/deal.II/base/tensor_accessors.h
index 63b8b88b73..60d3133fbb 100644
--- a/include/deal.II/base/tensor_accessors.h
+++ b/include/deal.II/base/tensor_accessors.h
@@ -183,7 +183,7 @@ namespace TensorAccessors
    * @author Matthias Maier, 2015
    */
   template <int index, int rank, typename T>
-  internal::ReorderedIndexView<index, rank, T>
+  __attribute__((always_inline)) internal::ReorderedIndexView<index, rank, T>
   reordered_index_view(T &t)
   {
 #ifdef DEAL_II_WITH_CXX11
@@ -264,7 +264,7 @@ namespace TensorAccessors
    * @author Matthias Maier, 2015
    */
   template <int no_contr, int rank_1, int rank_2, int dim, typename T1, typename T2, typename T3>
-  void contract(T1 &result, const T2 &left, const T3 &right)
+  __attribute__((always_inline)) void contract(T1 &result, const T2 &left, const T3 &right)
   {
 #ifdef DEAL_II_WITH_CXX11
     static_assert(rank_1 >= no_contr, "The rank of the left tensor must be "
@@ -393,7 +393,7 @@ namespace TensorAccessors
       value_type;
 
       // Recurse by applying index j directly:
-      inline
+      inline __attribute__((always_inline))
       value_type operator[](unsigned int j) const
       {
         return value_type(t_[j]);
@@ -422,7 +422,7 @@ namespace TensorAccessors
 
       typedef StoreIndex<rank - 1, internal::Identity<T> > value_type;
 
-      inline
+      inline __attribute__((always_inline))
       value_type operator[](unsigned int j) const
       {
         return value_type(Identity<T>(t_), j);
@@ -443,7 +443,8 @@ namespace TensorAccessors
 
       typedef typename ReferenceType<typename ValueType<T>::value_type>::type value_type;
 
-      inline value_type operator[](unsigned int j) const
+      inline __attribute__((always_inline))
+      value_type operator[](unsigned int j) const
       {
         return t_[j];
       }
@@ -465,7 +466,7 @@ namespace TensorAccessors
 
       typedef typename ValueType<T>::value_type return_type;
 
-      inline
+      inline __attribute__((always_inline))
       typename ReferenceType<return_type>::type apply(unsigned int j) const
       {
         return t_[j];
@@ -491,7 +492,7 @@ namespace TensorAccessors
 
       typedef StoreIndex<rank - 1, StoreIndex<rank, S> > value_type;
 
-      inline
+      inline __attribute__((always_inline))
       value_type operator[](unsigned int j) const
       {
         return value_type(*this, j);
@@ -523,7 +524,8 @@ namespace TensorAccessors
       typedef typename ValueType<typename S::return_type>::value_type return_type;
       typedef return_type value_type;
 
-      inline return_type &operator[](unsigned int j) const
+      inline __attribute__((always_inline))
+      return_type &operator[](unsigned int j) const
       {
         return s_.apply(j)[i_];
       }
@@ -593,7 +595,7 @@ namespace TensorAccessors
     {
     public:
       template<typename T1, typename T2, typename T3>
-      inline static
+      inline __attribute__((always_inline)) static
       void contract(T1 &result, const T2 &left, const T3 &right)
       {
         for (unsigned int i = 0; i < dim; ++i)
@@ -621,7 +623,7 @@ namespace TensorAccessors
     {
     public:
       template<typename T1, typename T2, typename T3>
-      inline static
+      inline __attribute__((always_inline)) static
       void contract(T1 &result, const T2 &left, const T3 &right)
       {
         for (unsigned int i = 0; i < dim; ++i)
@@ -649,7 +651,7 @@ namespace TensorAccessors
     {
     public:
       template<typename T1, typename T2, typename T3>
-      inline static
+      inline __attribute__((always_inline)) static
       void contract(T1 &result, const T2 &left, const T3 &right)
       {
         result = Contract2<no_contr, dim>::template contract2<T1>(left, right);
@@ -665,7 +667,7 @@ namespace TensorAccessors
     {
     public:
       template<typename T1, typename T2, typename T3>
-      inline static
+      inline __attribute__((always_inline)) static
       T1 contract2(const T2 &left, const T3 &right)
       {
         T1 result = T1();
@@ -683,7 +685,7 @@ namespace TensorAccessors
     {
     public:
       template<typename T1, typename T2, typename T3>
-      inline static
+      inline __attribute__((always_inline)) static
       T1 contract2(const T2 &left, const T3 &right)
       {
         return left * right;
-- 
2.39.5