Fix overflow issue with parallel::distributed::Vector

diff --git a/doc/news/changes.h b/doc/news/changes.h
index b9d8d2e7bfb6cf29a674f668ac74f7f2e77e1398..545f7031d65ec4e34e298ba48851980f50ea1fd0 100644 (file)
--- a/doc/news/changes.h
+++ b/doc/news/changes.h
@@ -437,6 +437,13 @@ inconvenience this causes.
 
 
 <ol>
+  <li> Fixed: parallel::distributed::Vector now detects of the size of MPI
+  messages exceeds 2GB or if the local range exceeds the size of 32-bit
+  integers and throws an exception about unsupported range of operation.
+  <br>
+  (Martin Kronbichler, 2015/11/26)
+  </li>
+
   <li> Fixed: GridGenerator::extract_boundary_mesh() in 3d could generate
   surface cells that did not uniformly had a right- or left-handed coordinate
   system associated with them when viewed from one side of the surface. This

diff --git a/include/deal.II/base/partitioner.h b/include/deal.II/base/partitioner.h
index 129659aaa4ce7dcdc55ab8ed73ad26069306865c..415782de2def42cb2cb35ea9986ee50edf07350c 100644 (file)
--- a/include/deal.II/base/partitioner.h
+++ b/include/deal.II/base/partitioner.h
@@ -441,7 +441,7 @@ namespace Utilities
     {
       AssertIndexRange (local_index, local_size() + n_ghost_indices_data);
       if (local_index < local_size())
-        return local_range_data.first + local_index;
+        return local_range_data.first + types::global_dof_index(local_index);
       else
         return ghost_indices_data.nth_index_in_set (local_index-local_size());
     }

diff --git a/include/deal.II/lac/parallel_vector.h b/include/deal.II/lac/parallel_vector.h
index b00d3b1c76ef3902efd6c2249344a4b3904c1895..a6881ace24fd041374e81a1c68f8314c4623a3c8 100644 (file)
--- a/include/deal.II/lac/parallel_vector.h
+++ b/include/deal.II/lac/parallel_vector.h
@@ -140,6 +140,31 @@ namespace parallel
      * multiple threads. This may or may not be desired when working also with
      * MPI.
      *
+     * <h4>Limitations regarding the vector size</h4>
+     *
+     * This vector class is based on two different number types for indexing.
+     * The so-called global index type encodes the overall size of the vector.
+     * Its type is <code>types::global_dof_index</type>. The largest possible
+     * value is <code>2^32-1</code> or approximately four billion in case 64
+     * bit integers are disabled at configuration of deal.II (default case) or
+     * <code>2^64-1</code> or approximately <code>10^19</code> if 64 bit
+     * integers are enabled (see the glossary entry on
+     * @ref GlobalDoFIndex for further information).
+     *
+     * The second relevant index type is the local index used within one MPI
+     * rank. As opposed to the global index, the implementation assumes
+     * 32-bit unsigned integers unconditionally. In other words, to actually
+     * use a vector with more than four billion entries, you need to use MPI
+     * with more than one rank (which in general is a safe assumption since
+     * four billion entries consume at least 16 GB of memory for floats or 32
+     * GB of memory for doubles). If more than 4 billion local elements are
+     * present, the implementation tries to detect that, which triggers an
+     * exception will be thrown and aborts the code. Note, however, that the
+     * detection of overflow is tricky and the detection mechanism might fail
+     * in some circumstances. Therefore, it is strongly recommended to not
+     * rely on this class to detect the case if local elements are more than
+     * four billion.
+     *
      * @author Katharina Kormann, Martin Kronbichler, 2010, 2011
      */
     template <typename Number>

diff --git a/include/deal.II/lac/parallel_vector.templates.h b/include/deal.II/lac/parallel_vector.templates.h
index 5171807f70f8cf6a3007d1bdb10155722a5d2dc1..a3d585fe851a4594d39acd2faebc558ae6d91f03 100644 (file)
--- a/include/deal.II/lac/parallel_vector.templates.h
+++ b/include/deal.II/lac/parallel_vector.templates.h
@@ -378,6 +378,12 @@ namespace parallel
             import_data = new Number[part.n_import_indices()];
           for (unsigned int i=0; i<n_import_targets; i++)
             {
+              AssertThrow (static_cast<size_type>(part.import_targets()[i].second)*
+                           sizeof(Number) <
+                           static_cast<size_type>(std::numeric_limits<int>::max()),
+                           ExcMessage("Index overflow: Maximum message size in MPI is 2GB. "
+                                      "The number of ghost entries times the size of 'Number' "
+                                      "exceeds this value. This is not supported."));
               MPI_Recv_init (&import_data[current_index_start],
                              part.import_targets()[i].second*sizeof(Number),
                              MPI_BYTE,
@@ -394,6 +400,12 @@ namespace parallel
           current_index_start = part.local_size();
           for (unsigned int i=0; i<n_ghost_targets; i++)
             {
+              AssertThrow (static_cast<size_type>(part.ghost_targets()[i].second)*
+                           sizeof(Number) <
+                           static_cast<size_type>(std::numeric_limits<int>::max()),
+                           ExcMessage("Index overflow: Maximum message size in MPI is 2GB. "
+                                      "The number of ghost entries times the size of 'Number' "
+                                      "exceeds this value. This is not supported."));
               MPI_Send_init (&this->val[current_index_start],
                              part.ghost_targets()[i].second*sizeof(Number),
                              MPI_BYTE,

diff --git a/source/base/partitioner.cc b/source/base/partitioner.cc
index 57c1b96a0fde90094420993487ecc3bcbd3677f1..3e44d8b2a31132acb9c501336fdf75987f4a732b 100644 (file)
--- a/source/base/partitioner.cc
+++ b/source/base/partitioner.cc
@@ -113,6 +113,9 @@ namespace Utilities
                            (locally_owned_indices.nth_index_in_set(0),
                             locally_owned_indices.nth_index_in_set(0) +
                             locally_owned_indices.n_elements());
+      AssertThrow (local_range_data.second-local_range_data.first <
+                   static_cast<types::global_dof_index>(std::numeric_limits<unsigned int>::max()),
+                   ExcMessage("Index overflow: This class supports at most 2^32-1 locally owned vector entries"));
       locally_owned_range_data.set_size (locally_owned_indices.size());
       locally_owned_range_data.add_range (local_range_data.first, local_range_data.second);
       locally_owned_range_data.compress();
@@ -138,6 +141,9 @@ namespace Utilities
         ghost_indices_data.set_size(locally_owned_range_data.size());
       ghost_indices_data.subtract_set (locally_owned_range_data);
       ghost_indices_data.compress();
+      AssertThrow (ghost_indices_data.n_elements() <
+                   static_cast<types::global_dof_index>(std::numeric_limits<unsigned int>::max()),
+                   ExcMessage("Index overflow: This class supports at most 2^32-1 ghost elements"));
       n_ghost_indices_data = ghost_indices_data.n_elements();
 
       have_ghost_indices =