From 20f3649b73525ba21df4122d52274d3f6575f56c Mon Sep 17 00:00:00 2001 From: Jiaqi Zhang Date: Thu, 2 Jun 2022 13:16:29 -0400 Subject: [PATCH] LargeCount MPI_write/read_at --- source/distributed/fully_distributed_tria.cc | 99 +++++++++++--------- 1 file changed, 56 insertions(+), 43 deletions(-) diff --git a/source/distributed/fully_distributed_tria.cc b/source/distributed/fully_distributed_tria.cc index eda037a619..361af19d8b 100644 --- a/source/distributed/fully_distributed_tria.cc +++ b/source/distributed/fully_distributed_tria.cc @@ -16,6 +16,7 @@ #include #include +#include #include #include @@ -539,35 +540,41 @@ namespace parallel dealii::Utilities::pack(construction_data, buffer, false); // Write offsets to file. - unsigned int buffer_size = buffer.size(); + const std::uint64_t buffer_size = buffer.size(); - unsigned int offset = 0; + std::uint64_t offset = 0; - ierr = MPI_Exscan(&buffer_size, - &offset, - 1, - MPI_UNSIGNED, - MPI_SUM, - this->mpi_communicator); + ierr = MPI_Exscan( + &buffer_size, + &offset, + 1, + Utilities::MPI::mpi_type_id_for_type, + MPI_SUM, + this->mpi_communicator); AssertThrowMPI(ierr); // Write offsets to file. - ierr = MPI_File_write_at(fh, - myrank * sizeof(unsigned int), - &buffer_size, - 1, - MPI_UNSIGNED, - MPI_STATUS_IGNORE); + ierr = MPI_File_write_at( + fh, + myrank * sizeof(std::uint64_t), + &buffer_size, + 1, + Utilities::MPI::mpi_type_id_for_type, + MPI_STATUS_IGNORE); AssertThrowMPI(ierr); + // global position in file + const std::uint64_t global_position = + mpisize * sizeof(std::uint64_t) + offset; + // Write buffers to file. - ierr = MPI_File_write_at(fh, - mpisize * sizeof(unsigned int) + - offset, // global position in file - buffer.data(), - buffer.size(), // local buffer - MPI_CHAR, - MPI_STATUS_IGNORE); + ierr = dealii::Utilities::MPI::LargeCount::File_write_at_c( + fh, + global_position, + buffer.data(), + buffer.size(), // local buffer + MPI_CHAR, + MPI_STATUS_IGNORE); AssertThrowMPI(ierr); ierr = MPI_File_close(&fh); @@ -651,35 +658,41 @@ namespace parallel AssertThrowMPI(ierr); // Read offsets from file. - unsigned int buffer_size; - - ierr = MPI_File_read_at(fh, - myrank * sizeof(unsigned int), - &buffer_size, - 1, - MPI_UNSIGNED, - MPI_STATUS_IGNORE); + std::uint64_t buffer_size; + + ierr = MPI_File_read_at( + fh, + myrank * sizeof(std::uint64_t), + &buffer_size, + 1, + Utilities::MPI::mpi_type_id_for_type, + MPI_STATUS_IGNORE); AssertThrowMPI(ierr); - unsigned int offset = 0; + std::uint64_t offset = 0; - ierr = MPI_Exscan(&buffer_size, - &offset, - 1, - MPI_UNSIGNED, - MPI_SUM, - this->mpi_communicator); + ierr = MPI_Exscan( + &buffer_size, + &offset, + 1, + Utilities::MPI::mpi_type_id_for_type, + MPI_SUM, + this->mpi_communicator); AssertThrowMPI(ierr); + // global position in file + const std::uint64_t global_position = + mpisize * sizeof(std::uint64_t) + offset; + // Read buffers from file. std::vector buffer(buffer_size); - ierr = MPI_File_read_at(fh, - mpisize * sizeof(unsigned int) + - offset, // global position in file - buffer.data(), - buffer.size(), // local buffer - MPI_CHAR, - MPI_STATUS_IGNORE); + ierr = dealii::Utilities::MPI::LargeCount::File_read_at_c( + fh, + global_position, + buffer.data(), + buffer.size(), // local buffer + MPI_CHAR, + MPI_STATUS_IGNORE); AssertThrowMPI(ierr); ierr = MPI_File_close(&fh); -- 2.39.5