]> https://gitweb.dealii.org/ - dealii-svn.git/commitdiff
Restructure the TrilinosTools a bit: Call it System::MPI_InitFinalize because that...
authorbangerth <bangerth@0785d39b-7218-0410-832d-ea1e28bc413d>
Wed, 29 Oct 2008 13:51:18 +0000 (13:51 +0000)
committerbangerth <bangerth@0785d39b-7218-0410-832d-ea1e28bc413d>
Wed, 29 Oct 2008 13:51:18 +0000 (13:51 +0000)
git-svn-id: https://svn.dealii.org/trunk@17395 0785d39b-7218-0410-832d-ea1e28bc413d

deal.II/base/include/base/utilities.h
deal.II/base/source/utilities.cc
deal.II/examples/step-31/step-31.cc
deal.II/examples/step-32/step-32.cc

index 1d93a6ca25ebf01f900f955291004ced402ddf08..1b5a7ca497be05c8e8607faee027e5371d91142d 100644 (file)
@@ -209,31 +209,95 @@ namespace Utilities
 
                                     /**
                                      * Return the number of MPI processes
-                                     * there exist. If this is a sequential
-                                     * job, it returns 1.
+                                     * there exist in the given communicator
+                                     * object. If this is a sequential job,
+                                     * it returns 1.
                                      */
     unsigned int get_n_mpi_processes (const MPI_Comm &mpi_communicator);
 
                                     /**
-                                     * Return the number of the
-                                     * present MPI process in the
-                                     * space of processes. This will
-                                     * be a unique value for each
-                                     * process between zero and (less
-                                     * than) the number of all
+                                     * Return the number of the present MPI
+                                     * process in the space of processes
+                                     * described by the given
+                                     * communicator. This will be a unique
+                                     * value for each process between zero
+                                     * and (less than) the number of all
                                      * processes (given by
                                      * get_n_mpi_processes()).
-                                     *
-                                     * This function will only work if you
-                                     * also configured the library to use
-                                     * PETSc. If your program uses MPI but
-                                     * not PETSc, then this function is
-                                     * likely not going to work.
                                      */
     unsigned int get_this_mpi_process (const MPI_Comm &mpi_communicator);
-  }
 
 
+                                    /**
+                                     * A class that is used to initialize the
+                                     * MPI system at the beginning of a
+                                     * program and to shut it down again at
+                                     * the end.
+                                     *
+                                     * If a program uses MPI one would
+                                     * typically just create an object of
+                                     * this type at the beginning of
+                                     * <code>main()</code>. The constructor
+                                     * of this class then runs
+                                     * <code>MPI_Init()</code> with the given
+                                     * arguments. At the end of the program,
+                                     * the compiler will invoke the
+                                     * destructor of this object which in
+                                     * turns calls <code>MPI_Finalize</code>
+                                     * to shut down the MPI system.
+                                     *
+                                     * This class is used in @ref step_32
+                                     * "step-32", for example.
+                                     */
+    class MPI_InitFinalize
+    {
+      public:
+                                        /**
+                                         * Constructor. Takes the arguments
+                                         * from the command line (in case of
+                                         * MPI, the number of processes is
+                                         * specified there), and sets up a
+                                         * respective communicator by calling
+                                         * <tt>MPI_Init()</tt>. This
+                                         * constructor can only be called once
+                                         * in a program, since MPI cannot be
+                                         * initialized twice.
+                                         */
+       MPI_InitFinalize (int    &argc,
+                         char** &argv);
+       
+                                        /**
+                                         * Destructor. Calls
+                                         * <tt>MPI_Finalize()</tt> in
+                                         * case this class owns the MPI
+                                         * process.
+                                         */
+       ~MPI_InitFinalize();
+
+      private:
+                                        /**
+                                         * This flag tells the class
+                                         * whether it owns the MPI
+                                         * process (i.e., it has been
+                                         * constructed using the
+                                         * argc/argv input, or it has
+                                         * been copied). In the former
+                                         * case, the command
+                                         * <tt>MPI_Finalize()</tt> will
+                                         * be called at destruction.
+                                         */
+       const bool owns_mpi;
+    };
+
+                                    /**
+                                     * Returns whether deal.II has been
+                                     * configured to use MPI and if so
+                                     * whether MPI has already been
+                                     * initialized using MPI_Init().
+                                     */
+    bool program_uses_mpi();
+  }
+
 
   /**
    * This class provides the basic structures for the use of the Trilinos
@@ -248,43 +312,8 @@ namespace Utilities
    * in the tutorial program @ref step_32 "step-32".
    */
 #ifdef DEAL_II_USE_TRILINOS
-  class TrilinosTools
+  namespace Trilinos
   {
-    public:
-                                    /**
-                                     * Constructor. Takes the arguments
-                                     * from the command line (in case of
-                                     * MPI, the number of processes is
-                                     * specified there), and sets up a
-                                     * respective communicator by calling
-                                     * <tt>MPI_Init()</tt>. This
-                                     * constructor can only be called once
-                                     * in a program, since MPI cannot be
-                                     * initialized twice.
-                                     */
-      TrilinosTools (int*    argc,
-                    char*** argv);
-
-                                    /**
-                                     * Copy constructor. Takes the
-                                     * Trilinos communicator from the
-                                     * input object and copies all
-                                     * content. Note that the copied
-                                     * class cannot own the MPI
-                                     * process, and hence, the
-                                     * original object needs to be
-                                     * around as long as the copy.
-                                     */
-      TrilinosTools(const TrilinosTools &InputTrilinos);
-
-                                    /**
-                                     * Destructor. Calls
-                                     * <tt>MPI_Finalize()</tt> in
-                                     * case this class owns the MPI
-                                     * process.
-                                     */
-      ~TrilinosTools();
-
                                     /**
                                      * Returns a Trilinos Epetra_Comm
                                      * object needed for creation of
@@ -299,48 +328,29 @@ namespace Utilities
                                      * that encompasses all processes within
                                      * this MPI universe.
                                      */
-      const Epetra_Comm& comm() const;
+    const Epetra_Comm& comm_world();
 
-                                    /**
-                                     * Returns whether we are using a
-                                     * MPI version or a serial
-                                     * version of Trilinos.
-                                     */
-      bool trilinos_uses_mpi() const;
 
-    private:
                                     /**
-                                     * This flag tells the class
-                                     * whether it owns the MPI
-                                     * process (i.e., it has been
-                                     * constructed using the
-                                     * argc/argv input, or it has
-                                     * been copied). In the former
-                                     * case, the command
-                                     * <tt>MPI_Finalize()</tt> will
-                                     * be called at destruction.
-                                     */
-      const bool owns_mpi;
-
-                                    /**
-                                     * This flag tells whether we use
-                                     * MPI or not.
+                                     * Return the number of MPI processes
+                                     * there exist in the given communicator
+                                     * object. If this is a sequential job,
+                                     * it returns 1.
                                      */
-      const bool use_mpi;
+    unsigned int get_n_mpi_processes (const Epetra_Comm &mpi_communicator);
 
                                     /**
-                                     * The actual communicator
-                                     * object. If we run the program
-                                     * in serial, we will have
-                                     * another communicator than when
-                                     * running in parallel.
+                                     * Return the number of the present MPI
+                                     * process in the space of processes
+                                     * described by the given
+                                     * communicator. This will be a unique
+                                     * value for each process between zero
+                                     * and (less than) the number of all
+                                     * processes (given by
+                                     * get_n_mpi_processes()).
                                      */
-#ifdef DEAL_II_COMPILER_SUPPORTS_MPI
-      Teuchos::RCP<Epetra_MpiComm> communicator;
-#else
-      Teuchos::RCP<Epetra_SerialComm> communicator;
-#endif
-  };
+    unsigned int get_this_mpi_process (const Epetra_Comm &mpi_communicator);
+  }
 
 #endif
 
index d8334f4e97cd20b65b9462f92649ff515bb38276..44b74d70dc25934c2eb1fa43c03fb3428b9077d6 100644 (file)
@@ -430,8 +430,7 @@ namespace Utilities
                                 // ``void'' -- not beautiful, but
                                 // helps calming down unwarranted
                                 // compiler warnings...
-    unsigned int get_n_mpi_processes (
-      const MPI_Comm &mpi_communicator)
+    unsigned int get_n_mpi_processes (const MPI_Comm &mpi_communicator)
     {
       int n_jobs=1;
       (void) MPI_Comm_size (mpi_communicator, &n_jobs);
@@ -439,15 +438,17 @@ namespace Utilities
       return n_jobs;
     }
 
-    unsigned int get_this_mpi_process (
-      const MPI_Comm &mpi_communicator)
+    
+    unsigned int get_this_mpi_process (const MPI_Comm &mpi_communicator)
     {
       int rank=0;
       (void) MPI_Comm_rank (mpi_communicator, &rank);
       
       return rank;
     }
+    
 #else
+    
     unsigned int get_n_mpi_processes (const MPI_Comm &)
     {
       return 1;
@@ -458,82 +459,96 @@ namespace Utilities
       return 0;
     }
 #endif
-  }
-
-
 
-#ifdef DEAL_II_USE_TRILINOS
 
 
-  TrilinosTools::TrilinosTools (int* argc, char*** argv)
-                      :
-                      owns_mpi (true),
+    MPI_InitFinalize::MPI_InitFinalize (int    &argc,
+                                       char** &argv)
+                   :
+                   owns_mpi (true)
+    {
+      static bool constructor_has_already_run = false;
+      Assert (constructor_has_already_run == false,
+             ExcMessage ("You can only create a single object of this class "
+                         "in a program sinc it initialize the MPI system."));
+      
 #ifdef DEAL_II_COMPILER_SUPPORTS_MPI
-                     use_mpi (true)
-#else
-                     use_mpi (false)
+      int MPI_has_been_started = 0;
+      MPI_Initialized(&MPI_has_been_started);
+      AssertThrow (MPI_has_been_started == 0,
+                  ExcMessage ("MPI error. You can only start MPI once!"));
+
+      int mpi_err;
+      mpi_err = MPI_Init (&argc, &argv);
+      AssertThrow (mpi_err == 0,
+                  ExcMessage ("MPI could not be initialized."));
 #endif
-  {
-#ifdef DEAL_II_COMPILER_SUPPORTS_MPI
-    int MPI_has_been_started = 0;
-    MPI_Initialized(&MPI_has_been_started);
-    AssertThrow (MPI_has_been_started == 0,
-                ExcMessage ("MPI error. You can only start MPI once!"));
 
-    int mpi_err;
-    mpi_err = MPI_Init (argc, argv);
-    AssertThrow (mpi_err == 0,
-                ExcMessage ("MPI could not be initialized."));
+      constructor_has_already_run = true;
+    }
 
-    communicator = Teuchos::rcp (new Epetra_MpiComm (MPI_COMM_WORLD), true);
-#else
-    communicator = Teuchos::rcp (new Epetra_SerialComm (), true);
-#endif
-  }
 
+    MPI_InitFinalize::~MPI_InitFinalize()
+    {
+#ifdef DEAL_II_COMPILER_SUPPORTS_MPI
+      int mpi_err = 0;
 
+      if (program_uses_mpi() == true && owns_mpi == true)
+       mpi_err = MPI_Finalize();
 
-  TrilinosTools::TrilinosTools (const TrilinosTools &InputTrilinos)
-                      :
-                      owns_mpi (false),
-#ifdef DEAL_II_COMPILER_SUPPORTS_MPI
-                     use_mpi (true),
-#else
-                     use_mpi (false),
+      AssertThrow (mpi_err == 0,
+                  ExcMessage ("An error occurred while calling MPI_Finalize()"));
 #endif
-                     communicator (&*InputTrilinos.communicator, false)
-  {}
-
+    }
 
 
-  TrilinosTools::~TrilinosTools()
-  {
-    int mpi_err = 0;
 
+    bool
+    program_uses_mpi ()
+    {
 #ifdef DEAL_II_COMPILER_SUPPORTS_MPI
-    if (use_mpi == true && owns_mpi == true)
-      mpi_err = MPI_Finalize();
-#endif
+      int MPI_has_been_started = 0;
+      MPI_Initialized(&MPI_has_been_started);
 
-    AssertThrow (mpi_err == 0,
-                ExcMessage ("An error occurred while calling MPI_Finalize()"));
+      return true && (MPI_has_been_started > 0);
+#else
+      return false;
+#endif
+    }    
   }
+  
 
-
-  const Epetra_Comm&
-  TrilinosTools::comm() const
+#ifdef DEAL_II_USE_TRILINOS
+  
+  namespace Trilinos
   {
-    return *communicator;
-  }
+    const Epetra_Comm&
+    comm_world()
+    {
+#ifdef DEAL_II_COMPILER_SUPPORTS_MPI
+      static Teuchos::RCP<Epetra_MpiComm>
+       communicator = Teuchos::rcp (new Epetra_MpiComm (MPI_COMM_WORLD), true);
+#else
+      static Teuchos::RCP<Epetra_SerialComm>
+       communicator = Teuchos::rcp (new Epetra_SerialComm (), true);
+#endif
+    
+      return *communicator;
+    }
 
 
+    unsigned int get_n_mpi_processes (const Epetra_Comm &mpi_communicator)
+    {
+      return mpi_communicator.NumProc();
+    }
+    
 
-  bool
-  TrilinosTools::trilinos_uses_mpi () const
-  {
-    return use_mpi;
+    unsigned int get_this_mpi_process (const Epetra_Comm &mpi_communicator)
+    {
+      return (unsigned int)mpi_communicator.MyPID();
+    }
   }
-
+  
 #endif
 
 }
index 110633ffa3297d0c6dfc851b8d3c3ffc528760cc..7175b56496e8325db821aa5be5b85c6da0e386d4 100644 (file)
@@ -2850,7 +2850,7 @@ int main (int argc, char *argv[])
     {
       deallog.depth_console (0);
 
-      Utilities::TrilinosTools trilinos (&argc, &argv);
+      Utilities::System::MPI_InitFinalize mpi_initialization (argc, argv);
 
       BoussinesqFlowProblem<2> flow_problem;
       flow_problem.run ();
index cbe3838b69db026449c1b7b96bf6ec779630957e..20871d259a5cf5dca82702e6ef3236aa12eb696d 100644 (file)
@@ -262,7 +262,7 @@ template <int dim>
 class BoussinesqFlowProblem
 {
   public:
-    BoussinesqFlowProblem (Utilities::TrilinosTools &trilinos_tools);
+    BoussinesqFlowProblem ();
     void run ();
 
   private:
@@ -295,7 +295,7 @@ class BoussinesqFlowProblem
                      const double                        old_time_step);
 
 
-    Utilities::TrilinosTools            trilinos_tools;
+    const Epetra_Comm                  &trilinos_communicator;
     
     ConditionalOStream                  pcout;
 
@@ -347,10 +347,10 @@ class BoussinesqFlowProblem
 
                                 // @sect4{BoussinesqFlowProblem::BoussinesqFlowProblem}
 template <int dim>
-BoussinesqFlowProblem<dim>::BoussinesqFlowProblem (Utilities::TrilinosTools &trilinos_tools)
+BoussinesqFlowProblem<dim>::BoussinesqFlowProblem ()
                 :
-                trilinos_tools (trilinos_tools),
-               pcout (std::cout, trilinos_tools.comm().MyPID()==0),
+                trilinos_communicator (Utilities::Trilinos::comm_world()),
+               pcout (std::cout, Utilities::Trilinos::get_this_mpi_process(trilinos_communicator)==0),
 
                triangulation (Triangulation<dim>::maximum_smoothing),
 
@@ -363,7 +363,7 @@ BoussinesqFlowProblem<dim>::BoussinesqFlowProblem (Utilities::TrilinosTools &tri
                temperature_fe (temperature_degree),
                 temperature_dof_handler (triangulation),
 
-               temperature_partitioner (0, 0, trilinos_tools.comm()),
+               temperature_partitioner (0, 0, trilinos_communicator),
 
                 time_step (0),
                old_time_step (0),
@@ -394,7 +394,7 @@ double BoussinesqFlowProblem<dim>::get_maximal_velocity () const
     cell = stokes_dof_handler.begin_active(),
     endc = stokes_dof_handler.end();
   for (; cell!=endc; ++cell)
-    if (cell->subdomain_id() == (unsigned int)trilinos_tools.comm().MyPID())
+    if (cell->subdomain_id() == Utilities::Trilinos::get_this_mpi_process(trilinos_communicator))
       {
        fe_values.reinit (cell);
        fe_values.get_function_values (localized_stokes_solution, stokes_values);
@@ -562,7 +562,7 @@ void BoussinesqFlowProblem<dim>::setup_dofs ()
   std::vector<unsigned int> stokes_sub_blocks (dim+1,0);
   stokes_sub_blocks[dim] = 1;
 
-  GridTools::partition_triangulation (trilinos_tools.comm().NumProc(), 
+  GridTools::partition_triangulation (Utilities::Trilinos::get_n_mpi_processes(trilinos_communicator), 
                                      triangulation);
 
   {
@@ -617,7 +617,7 @@ void BoussinesqFlowProblem<dim>::setup_dofs ()
     std::vector<unsigned int> local_dofs (dim+1);
     DoFTools::
       count_dofs_with_subdomain_association (stokes_dof_handler,
-                                            trilinos_tools.comm().MyPID(),
+                                            Utilities::Trilinos::get_this_mpi_process(trilinos_communicator),
                                             local_dofs);
     unsigned int n_local_velocities = 0;
     for (unsigned int c=0; c<dim; ++c)
@@ -625,9 +625,9 @@ void BoussinesqFlowProblem<dim>::setup_dofs ()
 
     const unsigned int n_local_pressures = local_dofs[dim];
 
-    Epetra_Map map_u(n_u, n_local_velocities, 0, trilinos_tools.comm());
+    Epetra_Map map_u(n_u, n_local_velocities, 0, trilinos_communicator);
     stokes_partitioner.push_back (map_u);
-    Epetra_Map map_p(n_p, n_local_pressures, 0, trilinos_tools.comm());
+    Epetra_Map map_p(n_p, n_local_pressures, 0, trilinos_communicator);
     stokes_partitioner.push_back (map_p);
   }
   {
@@ -689,9 +689,9 @@ void BoussinesqFlowProblem<dim>::setup_dofs ()
     = Epetra_Map (n_T,
                  DoFTools::count_dofs_with_subdomain_association
                  (temperature_dof_handler,
-                  trilinos_tools.comm().MyPID()),
+                  Utilities::Trilinos::get_this_mpi_process(trilinos_communicator)),
                  0,
-                 trilinos_tools.comm());
+                 trilinos_communicator);
   {
     temperature_mass_matrix.clear ();
     temperature_stiffness_matrix.clear ();
@@ -745,7 +745,7 @@ BoussinesqFlowProblem<dim>::assemble_stokes_preconditioner ()
     cell = stokes_dof_handler.begin_active(),
     endc = stokes_dof_handler.end();
   for (; cell!=endc; ++cell)
-    if (cell->subdomain_id() == (unsigned int)trilinos_tools.comm().MyPID())
+    if (cell->subdomain_id() == Utilities::Trilinos::get_this_mpi_process(trilinos_communicator))
       {
        stokes_fe_values.reinit (cell);
        local_matrix = 0;
@@ -866,7 +866,7 @@ void BoussinesqFlowProblem<dim>::assemble_stokes_system ()
     temperature_cell = temperature_dof_handler.begin_active();
   
   for (; cell!=endc; ++cell, ++temperature_cell)
-    if (cell->subdomain_id() == (unsigned int)trilinos_tools.comm().MyPID())
+    if (cell->subdomain_id() == Utilities::Trilinos::get_this_mpi_process(trilinos_communicator))
       {
        stokes_fe_values.reinit (cell);
        temperature_fe_values.reinit (temperature_cell);
@@ -965,7 +965,7 @@ void BoussinesqFlowProblem<dim>::assemble_temperature_matrix ()
     cell = temperature_dof_handler.begin_active(),
     endc = temperature_dof_handler.end();
   for (; cell!=endc; ++cell)
-    if (cell->subdomain_id() == (unsigned int)trilinos_tools.comm().MyPID())
+    if (cell->subdomain_id() == Utilities::Trilinos::get_this_mpi_process(trilinos_communicator))
       {
        local_mass_matrix = 0;
        local_stiffness_matrix = 0;
@@ -1084,7 +1084,7 @@ void BoussinesqFlowProblem<dim>::assemble_temperature_system ()
     stokes_cell = stokes_dof_handler.begin_active();
 
   for (; cell!=endc; ++cell, ++stokes_cell)
-    if (cell->subdomain_id() == (unsigned int)trilinos_tools.comm().MyPID() )
+    if (cell->subdomain_id() == Utilities::Trilinos::get_this_mpi_process(trilinos_communicator) )
       {
        local_rhs = 0;
   
@@ -1357,7 +1357,7 @@ void BoussinesqFlowProblem<dim>::output_results ()  const
 
   DataOut<dim> data_out;
 
-  if (trilinos_tools.comm().MyPID() == 0)
+  if (Utilities::Trilinos::get_this_mpi_process(trilinos_communicator) == 0)
     {
       data_out.attach_dof_handler (joint_dof_handler);
 
@@ -1510,9 +1510,9 @@ int main (int argc, char *argv[])
     {
       deallog.depth_console (0);
       
-      Utilities::TrilinosTools trilinos(&argc, &argv);
+      Utilities::System::MPI_InitFinalize mpi_initialization(argc, argv);
 
-      BoussinesqFlowProblem<2> flow_problem (trilinos);
+      BoussinesqFlowProblem<2> flow_problem;
       flow_problem.run ();
     }
   catch (std::exception &exc)

In the beginning the Universe was created. This has made a lot of people very angry and has been widely regarded as a bad move.

Douglas Adams


Typeset in Trocchi and Trocchi Bold Sans Serif.