From: Timo Heister Date: Sun, 10 Nov 2019 17:39:10 +0000 (-0500) Subject: unify MPI tags for Partitioner/parallel Vector X-Git-Tag: v9.2.0-rc1~876^2 X-Git-Url: https://gitweb.dealii.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=04ec897725cf130c99544d657fcb2b70f5f74835;p=dealii.git unify MPI tags for Partitioner/parallel Vector - use specified range of MPI tags by Partitioner (and not different tags by rank) - limit communication channel in partitioner to 0..200 - limit vector communication channel to 0..100 (using Partitioner) - use 100..200 for block vectors --- diff --git a/include/deal.II/base/mpi_tags.h b/include/deal.II/base/mpi_tags.h index a9d87f971c..51fa10b714 100644 --- a/include/deal.II/base/mpi_tags.h +++ b/include/deal.II/base/mpi_tags.h @@ -122,6 +122,15 @@ namespace Utilities /// ProcessGrid::ProcessGrid process_grid_constructor, + /// 200 tags for Partitioner::import_from_ghosted_array_start + partitioner_import_start, + partitioner_import_end = partitioner_import_start + 200, + + /// 200 tags for Partitioner::export_to_ghosted_array_start + partitioner_export_start, + partitioner_export_end = partitioner_export_start + 200, + + }; } // namespace Tags } // namespace internal diff --git a/include/deal.II/base/partitioner.h b/include/deal.II/base/partitioner.h index 9362b03e3c..aa0b30c08d 100644 --- a/include/deal.II/base/partitioner.h +++ b/include/deal.II/base/partitioner.h @@ -393,7 +393,8 @@ namespace Utilities * @param communication_channel Sets an offset to the MPI_Isend and * MPI_Irecv calls that avoids interference with other ongoing * export_to_ghosted_array_start() calls on different entries. Typically - * handled within the blocks of a block vector. + * handled within the blocks of a block vector. Any value less than 200 + * is a valid value. * * @param locally_owned_array The array of data from which the data is * extracted and sent to the ghost entries on a remote processor. @@ -464,6 +465,7 @@ namespace Utilities * MPI_Irecv calls that avoids interference with other ongoing * import_from_ghosted_array_start() calls on different * entries. Typically handled within the blocks of a block vector. + * Any value less than 200 is a valid value. * * @param ghost_array The array of ghost data that is sent to a remote * owner of the respective index in a vector. Its size must either be diff --git a/include/deal.II/base/partitioner.templates.h b/include/deal.II/base/partitioner.templates.h index a06383c16d..6c284bb5c2 100644 --- a/include/deal.II/base/partitioner.templates.h +++ b/include/deal.II/base/partitioner.templates.h @@ -47,6 +47,7 @@ namespace Utilities std::vector & requests) const { AssertDimension(temporary_storage.size(), n_import_indices()); + AssertIndexRange(communication_channel, 200); Assert(ghost_array.size() == n_ghost_indices() || ghost_array.size() == n_ghost_indices_in_larger_set, ExcGhostIndexArrayHasWrongSize(ghost_array.size(), @@ -63,6 +64,12 @@ namespace Utilities ExcMessage("Another operation seems to still be running. " "Call update_ghost_values_finish() first.")); + const unsigned int mpi_tag = + Utilities::MPI::internal::Tags::partitioner_export_start + + communication_channel; + Assert(mpi_tag <= Utilities::MPI::internal::Tags::partitioner_export_end, + ExcInternalError()); + // Need to send and receive the data. Use non-blocking communication, // where it is usually less overhead to first initiate the receive and // then actually send the data @@ -90,7 +97,7 @@ namespace Utilities ghost_targets_data[i].second * sizeof(Number), MPI_BYTE, ghost_targets_data[i].first, - ghost_targets_data[i].first + communication_channel, + mpi_tag, communicator, &requests[i]); AssertThrowMPI(ierr); @@ -159,7 +166,7 @@ namespace Utilities import_targets_data[i].second * sizeof(Number), MPI_BYTE, import_targets_data[i].first, - my_pid + communication_channel, + mpi_tag, communicator, &requests[n_ghost_targets + i]); AssertThrowMPI(ierr); @@ -264,6 +271,7 @@ namespace Utilities std::vector & requests) const { AssertDimension(temporary_storage.size(), n_import_indices()); + AssertIndexRange(communication_channel, 200); Assert(ghost_array.size() == n_ghost_indices() || ghost_array.size() == n_ghost_indices_in_larger_set, ExcGhostIndexArrayHasWrongSize(ghost_array.size(), @@ -298,8 +306,11 @@ namespace Utilities // where it is generally less overhead to first initiate the receive and // then actually send the data - // set channels in different range from update_ghost_values channels - const unsigned int channel = communication_channel + 401; + const unsigned int mpi_tag = + Utilities::MPI::internal::Tags::partitioner_import_start + + communication_channel; + Assert(mpi_tag <= Utilities::MPI::internal::Tags::partitioner_import_end, + ExcInternalError()); requests.resize(n_import_targets + n_ghost_targets); // initiate the receive operations @@ -318,7 +329,7 @@ namespace Utilities import_targets_data[i].second * sizeof(Number), MPI_BYTE, import_targets_data[i].first, - import_targets_data[i].first + channel, + mpi_tag, communicator, &requests[i]); AssertThrowMPI(ierr); @@ -411,7 +422,7 @@ namespace Utilities ghost_targets_data[i].second * sizeof(Number), MPI_BYTE, ghost_targets_data[i].first, - this_mpi_process() + channel, + mpi_tag, communicator, &requests[n_import_targets + i]); AssertThrowMPI(ierr); diff --git a/include/deal.II/lac/la_parallel_block_vector.templates.h b/include/deal.II/lac/la_parallel_block_vector.templates.h index ed780b8b24..663e2421f8 100644 --- a/include/deal.II/lac/la_parallel_block_vector.templates.h +++ b/include/deal.II/lac/la_parallel_block_vector.templates.h @@ -262,10 +262,11 @@ namespace LinearAlgebra // start all requests for all blocks before finishing the transfers as // this saves repeated synchronizations. In order to avoid conflict // with possible other ongoing communication requests (from - // LA::distributed::Vector that supports unfinished requests), add an - // arbitrary number 8273 to the communication tag + // LA::distributed::Vector that supports unfinished requests), add + // 100 to the communication tag (the first 100 can be used by normal + // vectors). for (unsigned int block = start; block < end; ++block) - this->block(block).compress_start(block + 8273 - start, operation); + this->block(block).compress_start(block - start + 100, operation); for (unsigned int block = start; block < end; ++block) this->block(block).compress_finish(operation); } diff --git a/include/deal.II/lac/la_parallel_vector.h b/include/deal.II/lac/la_parallel_vector.h index 40d8bbc211..adc4460a78 100644 --- a/include/deal.II/lac/la_parallel_vector.h +++ b/include/deal.II/lac/la_parallel_vector.h @@ -523,7 +523,8 @@ namespace LinearAlgebra * In case this function is called for more than one vector before @p * compress_finish() is invoked, it is mandatory to specify a unique * communication channel to each such call, in order to avoid several - * messages with the same ID that will corrupt this operation. + * messages with the same ID that will corrupt this operation. Any + * communication channel less than 100 is a valid value. */ void compress_start( @@ -561,6 +562,7 @@ namespace LinearAlgebra * update_ghost_values_finish() is invoked, it is mandatory to specify a * unique communication channel to each such call, in order to avoid * several messages with the same ID that will corrupt this operation. + * Any communication channel less than 100 is a valid value. */ void update_ghost_values_start( diff --git a/include/deal.II/lac/la_parallel_vector.templates.h b/include/deal.II/lac/la_parallel_vector.templates.h index 0f465176e8..101350981a 100644 --- a/include/deal.II/lac/la_parallel_vector.templates.h +++ b/include/deal.II/lac/la_parallel_vector.templates.h @@ -897,10 +897,10 @@ namespace LinearAlgebra template void Vector::compress_start( - const unsigned int counter, + const unsigned int communication_channel, ::dealii::VectorOperation::values operation) { - (void)counter; + (void)communication_channel; (void)operation; Assert(vector_is_ghosted == false, ExcMessage("Cannot call compress() on a ghosted vector")); @@ -972,7 +972,7 @@ namespace LinearAlgebra { partitioner->import_from_ghosted_array_start( operation, - counter, + communication_channel, ArrayView( data.values_dev.get() + partitioner->local_size(), partitioner->n_ghost_indices()), @@ -985,7 +985,7 @@ namespace LinearAlgebra { partitioner->import_from_ghosted_array_start( operation, - counter, + communication_channel, ArrayView( data.values.get() + partitioner->local_size(), partitioner->n_ghost_indices()), @@ -1076,7 +1076,7 @@ namespace LinearAlgebra template void Vector::update_ghost_values_start( - const unsigned int counter) const + const unsigned int communication_channel) const { #ifdef DEAL_II_WITH_MPI // nothing to do when we neither have import nor ghost indices. @@ -1141,7 +1141,7 @@ namespace LinearAlgebra # if !(defined(DEAL_II_COMPILER_CUDA_AWARE) && \ defined(DEAL_II_MPI_WITH_CUDA_SUPPORT)) partitioner->export_to_ghosted_array_start( - counter, + communication_channel, ArrayView(data.values.get(), partitioner->local_size()), ArrayView(import_data.values.get(), @@ -1152,7 +1152,7 @@ namespace LinearAlgebra update_ghost_values_requests); # else partitioner->export_to_ghosted_array_start( - counter, + communication_channel, ArrayView(data.values_dev.get(), partitioner->local_size()), ArrayView(import_data.values_dev.get(), @@ -1164,7 +1164,7 @@ namespace LinearAlgebra # endif #else - (void)counter; + (void)communication_channel; #endif }