From: Denis Davydov <davydden@gmail.com>
Date: Mon, 2 Apr 2018 07:13:05 +0000 (+0200)
Subject: introduce LA::d::Vector::add_local() and sadd_local() to avoid to many communication... 
X-Git-Tag: v9.0.0-rc1~244^2
X-Git-Url: https://gitweb.dealii.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=refs%2Fpull%2F6140%2Fhead;p=dealii.git

introduce LA::d::Vector::add_local() and sadd_local() to avoid to many communication in LA::d::BlockVector::mmult()
---

diff --git a/include/deal.II/lac/la_parallel_block_vector.templates.h b/include/deal.II/lac/la_parallel_block_vector.templates.h
index df6ad24706..cb5a2e34b1 100644
--- a/include/deal.II/lac/la_parallel_block_vector.templates.h
+++ b/include/deal.II/lac/la_parallel_block_vector.templates.h
@@ -944,11 +944,20 @@ namespace LinearAlgebra
           // below we make this work gracefully for identity-like matrices in
           // which case the two loops over j won't do any work as A(j,i)==0
           const unsigned int k = std::min(i,m-1);
-          V.block(i).sadd(s,matrix(k,i)*b, this->block(k));
+          V.block(i).sadd_local(s,matrix(k,i)*b, this->block(k));
           for (unsigned int j = 0  ; j < k; j++)
-            V.block(i).add (matrix(j,i)*b, this->block(j));
+            V.block(i).add_local (matrix(j,i)*b, this->block(j));
           for (unsigned int j = k+1; j < m; j++)
-            V.block(i).add (matrix(j,i)*b, this->block(j));
+            V.block(i).add_local (matrix(j,i)*b, this->block(j));
+        }
+
+      if (V.block(0).vector_is_ghosted)
+        {
+          for (unsigned int i = 0; i < n; i++)
+            Assert (V.block(i).vector_is_ghosted,
+                    ExcMessage("All blocks should be either in ghosted state or not."));
+
+          V.update_ghost_values();
         }
     }
 
diff --git a/include/deal.II/lac/la_parallel_vector.h b/include/deal.II/lac/la_parallel_vector.h
index ec5b5677ca..00179449cf 100644
--- a/include/deal.II/lac/la_parallel_vector.h
+++ b/include/deal.II/lac/la_parallel_vector.h
@@ -1060,6 +1060,20 @@ namespace LinearAlgebra
                       << "that this vector can access.");
 
     private:
+
+      /**
+       * Simple addition of a multiple of a vector, i.e. <tt>*this += a*V</tt>
+       * without MPI communication.
+       */
+      void add_local(const Number a, const VectorSpaceVector<Number> &V);
+
+      /**
+       * Scaling and simple addition of a multiple of a vector, i.e. <tt>*this =
+       * s*(*this)+a*V</tt> without MPI communication.
+       */
+      void sadd_local(const Number s, const Number a,
+                      const VectorSpaceVector<Number> &V);
+
       /**
        * Local part of all_zero().
        */
diff --git a/include/deal.II/lac/la_parallel_vector.templates.h b/include/deal.II/lac/la_parallel_vector.templates.h
index 6a1a0c53f9..d9ba0a6621 100644
--- a/include/deal.II/lac/la_parallel_vector.templates.h
+++ b/include/deal.II/lac/la_parallel_vector.templates.h
@@ -839,8 +839,8 @@ namespace LinearAlgebra
 
     template <typename Number>
     void
-    Vector<Number>::add (const Number a,
-                         const VectorSpaceVector<Number> &vv)
+    Vector<Number>::add_local (const Number a,
+                               const VectorSpaceVector<Number> &vv)
     {
       // Downcast. Throws an exception if invalid.
       Assert(dynamic_cast<const Vector<Number> *>(&vv)!=nullptr,
@@ -857,6 +857,16 @@ namespace LinearAlgebra
       internal::VectorOperations::Vectorization_add_av<Number> vector_add(values.get(), v.values.get(), a);
       internal::VectorOperations::parallel_for(vector_add, 0, partitioner->local_size(),
                                                thread_loop_partitioner);
+    }
+
+
+
+    template <typename Number>
+    void
+    Vector<Number>::add (const Number a,
+                         const VectorSpaceVector<Number> &vv)
+    {
+      add_local(a,vv);
 
       if (vector_is_ghosted)
         update_ghost_values();
@@ -930,9 +940,9 @@ namespace LinearAlgebra
 
     template <typename Number>
     void
-    Vector<Number>::sadd (const Number x,
-                          const Number a,
-                          const VectorSpaceVector<Number> &vv)
+    Vector<Number>::sadd_local (const Number x,
+                                const Number a,
+                                const VectorSpaceVector<Number> &vv)
     {
       // Downcast. Throws an exception if invalid.
       Assert(dynamic_cast<const Vector<Number> *>(&vv)!=nullptr,
@@ -946,6 +956,17 @@ namespace LinearAlgebra
       internal::VectorOperations::Vectorization_sadd_xav<Number> vector_sadd(values.get(), v.values.get(), a, x);
       internal::VectorOperations::parallel_for(vector_sadd, 0, partitioner->local_size(),
                                                thread_loop_partitioner);
+    }
+
+
+
+    template <typename Number>
+    void
+    Vector<Number>::sadd (const Number x,
+                          const Number a,
+                          const VectorSpaceVector<Number> &vv)
+    {
+      sadd_local(x,a,vv);
 
       if (vector_is_ghosted)
         update_ghost_values();