From 0425c395ba99ec47898e643f2aa861f5e34007df Mon Sep 17 00:00:00 2001 From: bangerth Date: Wed, 29 Oct 2008 13:51:18 +0000 Subject: [PATCH] Restructure the TrilinosTools a bit: Call it System::MPI_InitFinalize because that's what it does, after all. Take all functions that don't really rely on the state of this object out and make them global functions. git-svn-id: https://svn.dealii.org/trunk@17395 0785d39b-7218-0410-832d-ea1e28bc413d --- deal.II/base/include/base/utilities.h | 182 ++++++++++++++------------ deal.II/base/source/utilities.cc | 129 ++++++++++-------- deal.II/examples/step-31/step-31.cc | 2 +- deal.II/examples/step-32/step-32.cc | 40 +++--- 4 files changed, 189 insertions(+), 164 deletions(-) diff --git a/deal.II/base/include/base/utilities.h b/deal.II/base/include/base/utilities.h index 1d93a6ca25..1b5a7ca497 100644 --- a/deal.II/base/include/base/utilities.h +++ b/deal.II/base/include/base/utilities.h @@ -209,31 +209,95 @@ namespace Utilities /** * Return the number of MPI processes - * there exist. If this is a sequential - * job, it returns 1. + * there exist in the given communicator + * object. If this is a sequential job, + * it returns 1. */ unsigned int get_n_mpi_processes (const MPI_Comm &mpi_communicator); /** - * Return the number of the - * present MPI process in the - * space of processes. This will - * be a unique value for each - * process between zero and (less - * than) the number of all + * Return the number of the present MPI + * process in the space of processes + * described by the given + * communicator. This will be a unique + * value for each process between zero + * and (less than) the number of all * processes (given by * get_n_mpi_processes()). - * - * This function will only work if you - * also configured the library to use - * PETSc. If your program uses MPI but - * not PETSc, then this function is - * likely not going to work. */ unsigned int get_this_mpi_process (const MPI_Comm &mpi_communicator); - } + /** + * A class that is used to initialize the + * MPI system at the beginning of a + * program and to shut it down again at + * the end. + * + * If a program uses MPI one would + * typically just create an object of + * this type at the beginning of + * main(). The constructor + * of this class then runs + * MPI_Init() with the given + * arguments. At the end of the program, + * the compiler will invoke the + * destructor of this object which in + * turns calls MPI_Finalize + * to shut down the MPI system. + * + * This class is used in @ref step_32 + * "step-32", for example. + */ + class MPI_InitFinalize + { + public: + /** + * Constructor. Takes the arguments + * from the command line (in case of + * MPI, the number of processes is + * specified there), and sets up a + * respective communicator by calling + * MPI_Init(). This + * constructor can only be called once + * in a program, since MPI cannot be + * initialized twice. + */ + MPI_InitFinalize (int &argc, + char** &argv); + + /** + * Destructor. Calls + * MPI_Finalize() in + * case this class owns the MPI + * process. + */ + ~MPI_InitFinalize(); + + private: + /** + * This flag tells the class + * whether it owns the MPI + * process (i.e., it has been + * constructed using the + * argc/argv input, or it has + * been copied). In the former + * case, the command + * MPI_Finalize() will + * be called at destruction. + */ + const bool owns_mpi; + }; + + /** + * Returns whether deal.II has been + * configured to use MPI and if so + * whether MPI has already been + * initialized using MPI_Init(). + */ + bool program_uses_mpi(); + } + /** * This class provides the basic structures for the use of the Trilinos @@ -248,43 +312,8 @@ namespace Utilities * in the tutorial program @ref step_32 "step-32". */ #ifdef DEAL_II_USE_TRILINOS - class TrilinosTools + namespace Trilinos { - public: - /** - * Constructor. Takes the arguments - * from the command line (in case of - * MPI, the number of processes is - * specified there), and sets up a - * respective communicator by calling - * MPI_Init(). This - * constructor can only be called once - * in a program, since MPI cannot be - * initialized twice. - */ - TrilinosTools (int* argc, - char*** argv); - - /** - * Copy constructor. Takes the - * Trilinos communicator from the - * input object and copies all - * content. Note that the copied - * class cannot own the MPI - * process, and hence, the - * original object needs to be - * around as long as the copy. - */ - TrilinosTools(const TrilinosTools &InputTrilinos); - - /** - * Destructor. Calls - * MPI_Finalize() in - * case this class owns the MPI - * process. - */ - ~TrilinosTools(); - /** * Returns a Trilinos Epetra_Comm * object needed for creation of @@ -299,48 +328,29 @@ namespace Utilities * that encompasses all processes within * this MPI universe. */ - const Epetra_Comm& comm() const; + const Epetra_Comm& comm_world(); - /** - * Returns whether we are using a - * MPI version or a serial - * version of Trilinos. - */ - bool trilinos_uses_mpi() const; - private: /** - * This flag tells the class - * whether it owns the MPI - * process (i.e., it has been - * constructed using the - * argc/argv input, or it has - * been copied). In the former - * case, the command - * MPI_Finalize() will - * be called at destruction. - */ - const bool owns_mpi; - - /** - * This flag tells whether we use - * MPI or not. + * Return the number of MPI processes + * there exist in the given communicator + * object. If this is a sequential job, + * it returns 1. */ - const bool use_mpi; + unsigned int get_n_mpi_processes (const Epetra_Comm &mpi_communicator); /** - * The actual communicator - * object. If we run the program - * in serial, we will have - * another communicator than when - * running in parallel. + * Return the number of the present MPI + * process in the space of processes + * described by the given + * communicator. This will be a unique + * value for each process between zero + * and (less than) the number of all + * processes (given by + * get_n_mpi_processes()). */ -#ifdef DEAL_II_COMPILER_SUPPORTS_MPI - Teuchos::RCP communicator; -#else - Teuchos::RCP communicator; -#endif - }; + unsigned int get_this_mpi_process (const Epetra_Comm &mpi_communicator); + } #endif diff --git a/deal.II/base/source/utilities.cc b/deal.II/base/source/utilities.cc index d8334f4e97..44b74d70dc 100644 --- a/deal.II/base/source/utilities.cc +++ b/deal.II/base/source/utilities.cc @@ -430,8 +430,7 @@ namespace Utilities // ``void'' -- not beautiful, but // helps calming down unwarranted // compiler warnings... - unsigned int get_n_mpi_processes ( - const MPI_Comm &mpi_communicator) + unsigned int get_n_mpi_processes (const MPI_Comm &mpi_communicator) { int n_jobs=1; (void) MPI_Comm_size (mpi_communicator, &n_jobs); @@ -439,15 +438,17 @@ namespace Utilities return n_jobs; } - unsigned int get_this_mpi_process ( - const MPI_Comm &mpi_communicator) + + unsigned int get_this_mpi_process (const MPI_Comm &mpi_communicator) { int rank=0; (void) MPI_Comm_rank (mpi_communicator, &rank); return rank; } + #else + unsigned int get_n_mpi_processes (const MPI_Comm &) { return 1; @@ -458,82 +459,96 @@ namespace Utilities return 0; } #endif - } - - -#ifdef DEAL_II_USE_TRILINOS - TrilinosTools::TrilinosTools (int* argc, char*** argv) - : - owns_mpi (true), + MPI_InitFinalize::MPI_InitFinalize (int &argc, + char** &argv) + : + owns_mpi (true) + { + static bool constructor_has_already_run = false; + Assert (constructor_has_already_run == false, + ExcMessage ("You can only create a single object of this class " + "in a program sinc it initialize the MPI system.")); + #ifdef DEAL_II_COMPILER_SUPPORTS_MPI - use_mpi (true) -#else - use_mpi (false) + int MPI_has_been_started = 0; + MPI_Initialized(&MPI_has_been_started); + AssertThrow (MPI_has_been_started == 0, + ExcMessage ("MPI error. You can only start MPI once!")); + + int mpi_err; + mpi_err = MPI_Init (&argc, &argv); + AssertThrow (mpi_err == 0, + ExcMessage ("MPI could not be initialized.")); #endif - { -#ifdef DEAL_II_COMPILER_SUPPORTS_MPI - int MPI_has_been_started = 0; - MPI_Initialized(&MPI_has_been_started); - AssertThrow (MPI_has_been_started == 0, - ExcMessage ("MPI error. You can only start MPI once!")); - int mpi_err; - mpi_err = MPI_Init (argc, argv); - AssertThrow (mpi_err == 0, - ExcMessage ("MPI could not be initialized.")); + constructor_has_already_run = true; + } - communicator = Teuchos::rcp (new Epetra_MpiComm (MPI_COMM_WORLD), true); -#else - communicator = Teuchos::rcp (new Epetra_SerialComm (), true); -#endif - } + MPI_InitFinalize::~MPI_InitFinalize() + { +#ifdef DEAL_II_COMPILER_SUPPORTS_MPI + int mpi_err = 0; + if (program_uses_mpi() == true && owns_mpi == true) + mpi_err = MPI_Finalize(); - TrilinosTools::TrilinosTools (const TrilinosTools &InputTrilinos) - : - owns_mpi (false), -#ifdef DEAL_II_COMPILER_SUPPORTS_MPI - use_mpi (true), -#else - use_mpi (false), + AssertThrow (mpi_err == 0, + ExcMessage ("An error occurred while calling MPI_Finalize()")); #endif - communicator (&*InputTrilinos.communicator, false) - {} - + } - TrilinosTools::~TrilinosTools() - { - int mpi_err = 0; + bool + program_uses_mpi () + { #ifdef DEAL_II_COMPILER_SUPPORTS_MPI - if (use_mpi == true && owns_mpi == true) - mpi_err = MPI_Finalize(); -#endif + int MPI_has_been_started = 0; + MPI_Initialized(&MPI_has_been_started); - AssertThrow (mpi_err == 0, - ExcMessage ("An error occurred while calling MPI_Finalize()")); + return true && (MPI_has_been_started > 0); +#else + return false; +#endif + } } + - - const Epetra_Comm& - TrilinosTools::comm() const +#ifdef DEAL_II_USE_TRILINOS + + namespace Trilinos { - return *communicator; - } + const Epetra_Comm& + comm_world() + { +#ifdef DEAL_II_COMPILER_SUPPORTS_MPI + static Teuchos::RCP + communicator = Teuchos::rcp (new Epetra_MpiComm (MPI_COMM_WORLD), true); +#else + static Teuchos::RCP + communicator = Teuchos::rcp (new Epetra_SerialComm (), true); +#endif + + return *communicator; + } + unsigned int get_n_mpi_processes (const Epetra_Comm &mpi_communicator) + { + return mpi_communicator.NumProc(); + } + - bool - TrilinosTools::trilinos_uses_mpi () const - { - return use_mpi; + unsigned int get_this_mpi_process (const Epetra_Comm &mpi_communicator) + { + return (unsigned int)mpi_communicator.MyPID(); + } } - + #endif } diff --git a/deal.II/examples/step-31/step-31.cc b/deal.II/examples/step-31/step-31.cc index 110633ffa3..7175b56496 100644 --- a/deal.II/examples/step-31/step-31.cc +++ b/deal.II/examples/step-31/step-31.cc @@ -2850,7 +2850,7 @@ int main (int argc, char *argv[]) { deallog.depth_console (0); - Utilities::TrilinosTools trilinos (&argc, &argv); + Utilities::System::MPI_InitFinalize mpi_initialization (argc, argv); BoussinesqFlowProblem<2> flow_problem; flow_problem.run (); diff --git a/deal.II/examples/step-32/step-32.cc b/deal.II/examples/step-32/step-32.cc index cbe3838b69..20871d259a 100644 --- a/deal.II/examples/step-32/step-32.cc +++ b/deal.II/examples/step-32/step-32.cc @@ -262,7 +262,7 @@ template class BoussinesqFlowProblem { public: - BoussinesqFlowProblem (Utilities::TrilinosTools &trilinos_tools); + BoussinesqFlowProblem (); void run (); private: @@ -295,7 +295,7 @@ class BoussinesqFlowProblem const double old_time_step); - Utilities::TrilinosTools trilinos_tools; + const Epetra_Comm &trilinos_communicator; ConditionalOStream pcout; @@ -347,10 +347,10 @@ class BoussinesqFlowProblem // @sect4{BoussinesqFlowProblem::BoussinesqFlowProblem} template -BoussinesqFlowProblem::BoussinesqFlowProblem (Utilities::TrilinosTools &trilinos_tools) +BoussinesqFlowProblem::BoussinesqFlowProblem () : - trilinos_tools (trilinos_tools), - pcout (std::cout, trilinos_tools.comm().MyPID()==0), + trilinos_communicator (Utilities::Trilinos::comm_world()), + pcout (std::cout, Utilities::Trilinos::get_this_mpi_process(trilinos_communicator)==0), triangulation (Triangulation::maximum_smoothing), @@ -363,7 +363,7 @@ BoussinesqFlowProblem::BoussinesqFlowProblem (Utilities::TrilinosTools &tri temperature_fe (temperature_degree), temperature_dof_handler (triangulation), - temperature_partitioner (0, 0, trilinos_tools.comm()), + temperature_partitioner (0, 0, trilinos_communicator), time_step (0), old_time_step (0), @@ -394,7 +394,7 @@ double BoussinesqFlowProblem::get_maximal_velocity () const cell = stokes_dof_handler.begin_active(), endc = stokes_dof_handler.end(); for (; cell!=endc; ++cell) - if (cell->subdomain_id() == (unsigned int)trilinos_tools.comm().MyPID()) + if (cell->subdomain_id() == Utilities::Trilinos::get_this_mpi_process(trilinos_communicator)) { fe_values.reinit (cell); fe_values.get_function_values (localized_stokes_solution, stokes_values); @@ -562,7 +562,7 @@ void BoussinesqFlowProblem::setup_dofs () std::vector stokes_sub_blocks (dim+1,0); stokes_sub_blocks[dim] = 1; - GridTools::partition_triangulation (trilinos_tools.comm().NumProc(), + GridTools::partition_triangulation (Utilities::Trilinos::get_n_mpi_processes(trilinos_communicator), triangulation); { @@ -617,7 +617,7 @@ void BoussinesqFlowProblem::setup_dofs () std::vector local_dofs (dim+1); DoFTools:: count_dofs_with_subdomain_association (stokes_dof_handler, - trilinos_tools.comm().MyPID(), + Utilities::Trilinos::get_this_mpi_process(trilinos_communicator), local_dofs); unsigned int n_local_velocities = 0; for (unsigned int c=0; c::setup_dofs () const unsigned int n_local_pressures = local_dofs[dim]; - Epetra_Map map_u(n_u, n_local_velocities, 0, trilinos_tools.comm()); + Epetra_Map map_u(n_u, n_local_velocities, 0, trilinos_communicator); stokes_partitioner.push_back (map_u); - Epetra_Map map_p(n_p, n_local_pressures, 0, trilinos_tools.comm()); + Epetra_Map map_p(n_p, n_local_pressures, 0, trilinos_communicator); stokes_partitioner.push_back (map_p); } { @@ -689,9 +689,9 @@ void BoussinesqFlowProblem::setup_dofs () = Epetra_Map (n_T, DoFTools::count_dofs_with_subdomain_association (temperature_dof_handler, - trilinos_tools.comm().MyPID()), + Utilities::Trilinos::get_this_mpi_process(trilinos_communicator)), 0, - trilinos_tools.comm()); + trilinos_communicator); { temperature_mass_matrix.clear (); temperature_stiffness_matrix.clear (); @@ -745,7 +745,7 @@ BoussinesqFlowProblem::assemble_stokes_preconditioner () cell = stokes_dof_handler.begin_active(), endc = stokes_dof_handler.end(); for (; cell!=endc; ++cell) - if (cell->subdomain_id() == (unsigned int)trilinos_tools.comm().MyPID()) + if (cell->subdomain_id() == Utilities::Trilinos::get_this_mpi_process(trilinos_communicator)) { stokes_fe_values.reinit (cell); local_matrix = 0; @@ -866,7 +866,7 @@ void BoussinesqFlowProblem::assemble_stokes_system () temperature_cell = temperature_dof_handler.begin_active(); for (; cell!=endc; ++cell, ++temperature_cell) - if (cell->subdomain_id() == (unsigned int)trilinos_tools.comm().MyPID()) + if (cell->subdomain_id() == Utilities::Trilinos::get_this_mpi_process(trilinos_communicator)) { stokes_fe_values.reinit (cell); temperature_fe_values.reinit (temperature_cell); @@ -965,7 +965,7 @@ void BoussinesqFlowProblem::assemble_temperature_matrix () cell = temperature_dof_handler.begin_active(), endc = temperature_dof_handler.end(); for (; cell!=endc; ++cell) - if (cell->subdomain_id() == (unsigned int)trilinos_tools.comm().MyPID()) + if (cell->subdomain_id() == Utilities::Trilinos::get_this_mpi_process(trilinos_communicator)) { local_mass_matrix = 0; local_stiffness_matrix = 0; @@ -1084,7 +1084,7 @@ void BoussinesqFlowProblem::assemble_temperature_system () stokes_cell = stokes_dof_handler.begin_active(); for (; cell!=endc; ++cell, ++stokes_cell) - if (cell->subdomain_id() == (unsigned int)trilinos_tools.comm().MyPID() ) + if (cell->subdomain_id() == Utilities::Trilinos::get_this_mpi_process(trilinos_communicator) ) { local_rhs = 0; @@ -1357,7 +1357,7 @@ void BoussinesqFlowProblem::output_results () const DataOut data_out; - if (trilinos_tools.comm().MyPID() == 0) + if (Utilities::Trilinos::get_this_mpi_process(trilinos_communicator) == 0) { data_out.attach_dof_handler (joint_dof_handler); @@ -1510,9 +1510,9 @@ int main (int argc, char *argv[]) { deallog.depth_console (0); - Utilities::TrilinosTools trilinos(&argc, &argv); + Utilities::System::MPI_InitFinalize mpi_initialization(argc, argv); - BoussinesqFlowProblem<2> flow_problem (trilinos); + BoussinesqFlowProblem<2> flow_problem; flow_problem.run (); } catch (std::exception &exc) -- 2.39.5