]> https://gitweb.dealii.org/ - dealii.git/commitdiff
unify MPI tags for Partitioner/parallel Vector 9034/head
authorTimo Heister <timo.heister@gmail.com>
Sun, 10 Nov 2019 17:39:10 +0000 (12:39 -0500)
committerTimo Heister <timo.heister@gmail.com>
Wed, 13 Nov 2019 14:41:05 +0000 (09:41 -0500)
- use specified range of MPI tags by Partitioner (and not different tags
by rank)
- limit communication channel in partitioner to 0..200
- limit vector communication channel to 0..100 (using Partitioner)
- use 100..200 for block vectors

include/deal.II/base/mpi_tags.h
include/deal.II/base/partitioner.h
include/deal.II/base/partitioner.templates.h
include/deal.II/lac/la_parallel_block_vector.templates.h
include/deal.II/lac/la_parallel_vector.h
include/deal.II/lac/la_parallel_vector.templates.h

index a9d87f971c3285c17558c213b81c9f833a5bfa71..51fa10b714f621d8dc13bca87af0bf5a73abc24a 100644 (file)
@@ -122,6 +122,15 @@ namespace Utilities
           /// ProcessGrid::ProcessGrid
           process_grid_constructor,
 
+          /// 200 tags for Partitioner::import_from_ghosted_array_start
+          partitioner_import_start,
+          partitioner_import_end = partitioner_import_start + 200,
+
+          /// 200 tags for Partitioner::export_to_ghosted_array_start
+          partitioner_export_start,
+          partitioner_export_end = partitioner_export_start + 200,
+
+
         };
       } // namespace Tags
     }   // namespace internal
index 9362b03e3c1355a21aad01f3053108c0bcefb4a6..aa0b30c08d1021ffd8132ff3af3a8072f6cc2013 100644 (file)
@@ -393,7 +393,8 @@ namespace Utilities
        * @param communication_channel Sets an offset to the MPI_Isend and
        * MPI_Irecv calls that avoids interference with other ongoing
        * export_to_ghosted_array_start() calls on different entries. Typically
-       * handled within the blocks of a block vector.
+       * handled within the blocks of a block vector. Any value less than 200
+       * is a valid value.
        *
        * @param locally_owned_array The array of data from which the data is
        * extracted and sent to the ghost entries on a remote processor.
@@ -464,6 +465,7 @@ namespace Utilities
        * MPI_Irecv calls that avoids interference with other ongoing
        * import_from_ghosted_array_start() calls on different
        * entries. Typically handled within the blocks of a block vector.
+       * Any value less than 200 is a valid value.
        *
        * @param ghost_array The array of ghost data that is sent to a remote
        * owner of the respective index in a vector. Its size must either be
index a06383c16da22c8d6a620efeb2a084327fa1ee53..6c284bb5c2428628c700c7bcbdaf923a4213eb3f 100644 (file)
@@ -47,6 +47,7 @@ namespace Utilities
       std::vector<MPI_Request> &                      requests) const
     {
       AssertDimension(temporary_storage.size(), n_import_indices());
+      AssertIndexRange(communication_channel, 200);
       Assert(ghost_array.size() == n_ghost_indices() ||
                ghost_array.size() == n_ghost_indices_in_larger_set,
              ExcGhostIndexArrayHasWrongSize(ghost_array.size(),
@@ -63,6 +64,12 @@ namespace Utilities
              ExcMessage("Another operation seems to still be running. "
                         "Call update_ghost_values_finish() first."));
 
+      const unsigned int mpi_tag =
+        Utilities::MPI::internal::Tags::partitioner_export_start +
+        communication_channel;
+      Assert(mpi_tag <= Utilities::MPI::internal::Tags::partitioner_export_end,
+             ExcInternalError());
+
       // Need to send and receive the data. Use non-blocking communication,
       // where it is usually less overhead to first initiate the receive and
       // then actually send the data
@@ -90,7 +97,7 @@ namespace Utilities
                       ghost_targets_data[i].second * sizeof(Number),
                       MPI_BYTE,
                       ghost_targets_data[i].first,
-                      ghost_targets_data[i].first + communication_channel,
+                      mpi_tag,
                       communicator,
                       &requests[i]);
           AssertThrowMPI(ierr);
@@ -159,7 +166,7 @@ namespace Utilities
                       import_targets_data[i].second * sizeof(Number),
                       MPI_BYTE,
                       import_targets_data[i].first,
-                      my_pid + communication_channel,
+                      mpi_tag,
                       communicator,
                       &requests[n_ghost_targets + i]);
           AssertThrowMPI(ierr);
@@ -264,6 +271,7 @@ namespace Utilities
       std::vector<MPI_Request> &                requests) const
     {
       AssertDimension(temporary_storage.size(), n_import_indices());
+      AssertIndexRange(communication_channel, 200);
       Assert(ghost_array.size() == n_ghost_indices() ||
                ghost_array.size() == n_ghost_indices_in_larger_set,
              ExcGhostIndexArrayHasWrongSize(ghost_array.size(),
@@ -298,8 +306,11 @@ namespace Utilities
       // where it is generally less overhead to first initiate the receive and
       // then actually send the data
 
-      // set channels in different range from update_ghost_values channels
-      const unsigned int channel = communication_channel + 401;
+      const unsigned int mpi_tag =
+        Utilities::MPI::internal::Tags::partitioner_import_start +
+        communication_channel;
+      Assert(mpi_tag <= Utilities::MPI::internal::Tags::partitioner_import_end,
+             ExcInternalError());
       requests.resize(n_import_targets + n_ghost_targets);
 
       // initiate the receive operations
@@ -318,7 +329,7 @@ namespace Utilities
                       import_targets_data[i].second * sizeof(Number),
                       MPI_BYTE,
                       import_targets_data[i].first,
-                      import_targets_data[i].first + channel,
+                      mpi_tag,
                       communicator,
                       &requests[i]);
           AssertThrowMPI(ierr);
@@ -411,7 +422,7 @@ namespace Utilities
                       ghost_targets_data[i].second * sizeof(Number),
                       MPI_BYTE,
                       ghost_targets_data[i].first,
-                      this_mpi_process() + channel,
+                      mpi_tag,
                       communicator,
                       &requests[n_import_targets + i]);
           AssertThrowMPI(ierr);
index ed780b8b242f296ea4aa5a9d37b2aff1217771c3..663e2421f865997ed0e04154cd3af7a7f4557b3a 100644 (file)
@@ -262,10 +262,11 @@ namespace LinearAlgebra
           // start all requests for all blocks before finishing the transfers as
           // this saves repeated synchronizations. In order to avoid conflict
           // with possible other ongoing communication requests (from
-          // LA::distributed::Vector that supports unfinished requests), add an
-          // arbitrary number 8273 to the communication tag
+          // LA::distributed::Vector that supports unfinished requests), add
+          // 100 to the communication tag (the first 100 can be used by normal
+          // vectors).
           for (unsigned int block = start; block < end; ++block)
-            this->block(block).compress_start(block + 8273 - start, operation);
+            this->block(block).compress_start(block - start + 100, operation);
           for (unsigned int block = start; block < end; ++block)
             this->block(block).compress_finish(operation);
         }
index 40d8bbc2118aeae330bd636dc2c601c3a05f5308..adc4460a78ed1e50733c3ca89252c5ee8c1c6839 100644 (file)
@@ -523,7 +523,8 @@ namespace LinearAlgebra
        * In case this function is called for more than one vector before @p
        * compress_finish() is invoked, it is mandatory to specify a unique
        * communication channel to each such call, in order to avoid several
-       * messages with the same ID that will corrupt this operation.
+       * messages with the same ID that will corrupt this operation. Any
+       * communication channel less than 100 is a valid value.
        */
       void
       compress_start(
@@ -561,6 +562,7 @@ namespace LinearAlgebra
        * update_ghost_values_finish() is invoked, it is mandatory to specify a
        * unique communication channel to each such call, in order to avoid
        * several messages with the same ID that will corrupt this operation.
+       * Any communication channel less than 100 is a valid value.
        */
       void
       update_ghost_values_start(
index 0f465176e8208eb12edac594638326f066289f15..101350981a9df4701191eb7cd07b2e236e0672b7 100644 (file)
@@ -897,10 +897,10 @@ namespace LinearAlgebra
     template <typename Number, typename MemorySpaceType>
     void
     Vector<Number, MemorySpaceType>::compress_start(
-      const unsigned int                counter,
+      const unsigned int                communication_channel,
       ::dealii::VectorOperation::values operation)
     {
-      (void)counter;
+      (void)communication_channel;
       (void)operation;
       Assert(vector_is_ghosted == false,
              ExcMessage("Cannot call compress() on a ghosted vector"));
@@ -972,7 +972,7 @@ namespace LinearAlgebra
         {
           partitioner->import_from_ghosted_array_start(
             operation,
-            counter,
+            communication_channel,
             ArrayView<Number, MemorySpace::CUDA>(
               data.values_dev.get() + partitioner->local_size(),
               partitioner->n_ghost_indices()),
@@ -985,7 +985,7 @@ namespace LinearAlgebra
         {
           partitioner->import_from_ghosted_array_start(
             operation,
-            counter,
+            communication_channel,
             ArrayView<Number, MemorySpace::Host>(
               data.values.get() + partitioner->local_size(),
               partitioner->n_ghost_indices()),
@@ -1076,7 +1076,7 @@ namespace LinearAlgebra
     template <typename Number, typename MemorySpaceType>
     void
     Vector<Number, MemorySpaceType>::update_ghost_values_start(
-      const unsigned int counter) const
+      const unsigned int communication_channel) const
     {
 #ifdef DEAL_II_WITH_MPI
       // nothing to do when we neither have import nor ghost indices.
@@ -1141,7 +1141,7 @@ namespace LinearAlgebra
 #  if !(defined(DEAL_II_COMPILER_CUDA_AWARE) && \
         defined(DEAL_II_MPI_WITH_CUDA_SUPPORT))
       partitioner->export_to_ghosted_array_start<Number, MemorySpace::Host>(
-        counter,
+        communication_channel,
         ArrayView<const Number, MemorySpace::Host>(data.values.get(),
                                                    partitioner->local_size()),
         ArrayView<Number, MemorySpace::Host>(import_data.values.get(),
@@ -1152,7 +1152,7 @@ namespace LinearAlgebra
         update_ghost_values_requests);
 #  else
       partitioner->export_to_ghosted_array_start<Number, MemorySpace::CUDA>(
-        counter,
+        communication_channel,
         ArrayView<const Number, MemorySpace::CUDA>(data.values_dev.get(),
                                                    partitioner->local_size()),
         ArrayView<Number, MemorySpace::CUDA>(import_data.values_dev.get(),
@@ -1164,7 +1164,7 @@ namespace LinearAlgebra
 #  endif
 
 #else
-      (void)counter;
+      (void)communication_channel;
 #endif
     }
 

In the beginning the Universe was created. This has made a lot of people very angry and has been widely regarded as a bad move.

Douglas Adams


Typeset in Trocchi and Trocchi Bold Sans Serif.