void
BlockVector<Number>::compress (::dealii::VectorOperation::values operation)
{
- // start all requests for all blocks before finishing the transfers as
- // this saves repeated synchronizations. In order to avoid conflict with
- // possible other ongoing communication requests (from
- // LA::distributed::Vector that supports unfinished requests), add an
- // arbitrary number 8273 to the communication tag
- for (unsigned int block=0; block<this->n_blocks(); ++block)
- this->block(block).compress_start(block + 8273, operation);
- for (unsigned int block=0; block<this->n_blocks(); ++block)
- this->block(block).compress_finish(operation);
+ const unsigned int n_chunks = (this->n_blocks()+communication_block_size-1)/communication_block_size;
+ for (unsigned int c = 0; c < n_chunks; ++c)
+ {
+ const unsigned int start = c*communication_block_size;
+ const unsigned int end = std::min(start+communication_block_size, this->n_blocks());
+
+ // start all requests for all blocks before finishing the transfers as
+ // this saves repeated synchronizations. In order to avoid conflict with
+ // possible other ongoing communication requests (from
+ // LA::distributed::Vector that supports unfinished requests), add an
+ // arbitrary number 8273 to the communication tag
+ for (unsigned int block=start; block<end; ++block)
+ this->block(block).compress_start(block + 8273 - start, operation);
+ for (unsigned int block=start; block<end; ++block)
+ this->block(block).compress_finish(operation);
+ }
}
void
BlockVector<Number>::update_ghost_values () const
{
- // In order to avoid conflict with possible other ongoing communication
- // requests (from LA::distributed::Vector that supports unfinished
- // requests), add an arbitrary number 9923 to the communication tag
- for (unsigned int block=0; block<this->n_blocks(); ++block)
- this->block(block).update_ghost_values_start(block + 9923);
- for (unsigned int block=0; block<this->n_blocks(); ++block)
- this->block(block).update_ghost_values_finish();
+ const unsigned int n_chunks = (this->n_blocks()+communication_block_size-1)/communication_block_size;
+ for (unsigned int c = 0; c < n_chunks; ++c)
+ {
+ const unsigned int start = c*communication_block_size;
+ const unsigned int end = std::min(start+communication_block_size, this->n_blocks());
+
+ // In order to avoid conflict with possible other ongoing communication
+ // requests (from LA::distributed::Vector that supports unfinished
+ // requests), add an arbitrary number 9923 to the communication tag
+ for (unsigned int block=start; block<end; ++block)
+ this->block(block).update_ghost_values_start(block - start + 9923);
+ for (unsigned int block=start; block<end; ++block)
+ this->block(block).update_ghost_values_finish();
+ }
}
--- /dev/null
+// ---------------------------------------------------------------------
+//
+// Copyright (C) 2011 - 2017 by the deal.II authors
+//
+// This file is part of the deal.II library.
+//
+// The deal.II library is free software; you can use it, redistribute
+// it, and/or modify it under the terms of the GNU Lesser General
+// Public License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+// The full text of the license can be found in the file LICENSE at
+// the top level of the deal.II distribution.
+//
+// ---------------------------------------------------------------------
+
+
+// check ghost handling on parallel block vectors for large
+// number of blocks with split compres_start()/update_ghosts_start().
+// almost copy-paste of parallel_block_vector_02.cc
+
+#include "../tests.h"
+#include <deal.II/base/utilities.h>
+#include <deal.II/base/index_set.h>
+#include <deal.II/lac/la_parallel_block_vector.h>
+#include <deal.II/lac/la_parallel_vector.h>
+#include <iostream>
+#include <vector>
+
+
+void test ()
+{
+ unsigned int myid = Utilities::MPI::this_mpi_process (MPI_COMM_WORLD);
+ unsigned int numproc = Utilities::MPI::n_mpi_processes (MPI_COMM_WORLD);
+
+ if (myid==0) deallog << "numproc=" << numproc << std::endl;
+
+
+ // each processor from processor 1 to 8 owns 2 indices (the other processors
+ // do not own any dof), and all processors are ghosting element 1
+ IndexSet local_owned(std::min(16U,numproc*2));
+ if (myid < 8)
+ local_owned.add_range(myid*2,myid*2+2);
+ IndexSet local_relevant(numproc*2);
+ local_relevant = local_owned;
+ local_relevant.add_range(1,2);
+
+ LinearAlgebra::distributed::Vector<double> v(local_owned, local_relevant,
+ MPI_COMM_WORLD);
+
+ // set local values
+ if (myid < 8)
+ {
+ v(myid*2)=myid*2.0;
+ v(myid*2+1)=myid*2.0+1.0;
+ }
+ v.compress(VectorOperation::insert);
+
+ const unsigned int n_blocks = 107;
+
+ LinearAlgebra::distributed::BlockVector<double> w(n_blocks);
+ for (unsigned int i=0; i<n_blocks; ++i)
+ {
+ w.block(i) = v;
+ w.block(i) *= (i+1);
+ }
+ w.collect_sizes();
+
+ // see if we can access the ghosted entry 1 in each block with the correct
+ // value: the initialization should non have updated the ghost values, so
+ // all other processors except 0 should have zero entry on the global index
+ // 1
+ for (unsigned int i=0; i<n_blocks; ++i)
+ if (myid == 0)
+ {
+ AssertDimension(i+1,(unsigned int)w.block(i)(1));
+ }
+ else
+ {
+ AssertDimension(0,(unsigned int)w.block(i)(1));
+ }
+
+ // import ghost values, all processors should still have i+1
+ w.update_ghost_values();
+ for (unsigned int i=0; i<n_blocks; ++i)
+ AssertDimension(i+1,(unsigned int)w.block(i)(1));
+
+ // zero out ghosts, now all processors except processor 1 should have 0.
+ w.zero_out_ghosts();
+ for (unsigned int i=0; i<n_blocks; ++i)
+ if (myid == 0)
+ {
+ AssertDimension(i+1,(unsigned int)w.block(i)(1));
+ }
+ else
+ {
+ AssertDimension(0,(unsigned int)w.block(i)(1));
+ }
+
+ // create a vector copy that gets the entries from w. First, it should not
+ // have updated the ghosts because it is created from an empty state.
+ LinearAlgebra::distributed::BlockVector<double> x(w);
+ Assert(x.has_ghost_elements() == false, ExcInternalError());
+ for (unsigned int i=0; i<n_blocks; ++i)
+ if (myid == 0)
+ {
+ AssertDimension(i+1,(unsigned int)x.block(i)(1));
+ }
+ else
+ {
+ AssertDimension(0,(unsigned int)x.block(i)(1));
+ }
+
+ // now we zero the vector, which should disable ghost elements
+ x = 0;
+ Assert(x.has_ghost_elements() == false, ExcInternalError());
+
+ // we copy from w (i.e., the same vector but one that does not have ghosts
+ // enabled) -> should not have ghosts enabled
+ x = w;
+ Assert(x.has_ghost_elements() == false, ExcInternalError());
+ for (unsigned int i=0; i<n_blocks; ++i)
+ if (myid == 0)
+ {
+ AssertDimension(i+1,(unsigned int)x.block(i)(1));
+ }
+ else
+ {
+ AssertDimension(0,(unsigned int)x.block(i)(1));
+ }
+
+ x.update_ghost_values();
+ Assert(x.has_ghost_elements() == true, ExcInternalError());
+
+
+ // add something to entry 1 on all processors
+ w(1) += myid+1;
+ w.compress(VectorOperation::add);
+ if (myid == 0)
+ AssertDimension((unsigned int)w(1), 1+(numproc*(numproc+1))/2);
+
+ // add again and check if everything is still correct
+ w(1+v.size()) += myid+1;
+ w.compress(VectorOperation::add);
+ if (myid == 0)
+ AssertDimension((unsigned int)w(1), 1+(numproc*(numproc+1))/2);
+ if (myid == 0)
+ AssertDimension((unsigned int)w(v.size()+1), 2+(numproc*(numproc+1))/2);
+
+ w.update_ghost_values();
+ AssertDimension((unsigned int)w(1), 1+(numproc*(numproc+1))/2);
+ AssertDimension((unsigned int)w(v.size()+1), 2+(numproc*(numproc+1))/2);
+
+ if (myid == 0)
+ deallog << "OK" << std::endl;
+}
+
+
+
+int main (int argc, char **argv)
+{
+ Utilities::MPI::MPI_InitFinalize mpi_initialization (argc, argv, testing_max_num_threads());
+
+ unsigned int myid = Utilities::MPI::this_mpi_process (MPI_COMM_WORLD);
+ deallog.push(Utilities::int_to_string(myid));
+
+ if (myid == 0)
+ {
+ initlog();
+ deallog << std::setprecision(4);
+
+ test();
+ }
+ else
+ test();
+
+}
projects / dealii.git / commitdiff