]> https://gitweb.dealii.org/ - dealii-svn.git/commitdiff
Wrote a TrilinosTools class that makes the use of Trilinos a bit easier.
authorkronbichler <kronbichler@0785d39b-7218-0410-832d-ea1e28bc413d>
Fri, 24 Oct 2008 10:36:50 +0000 (10:36 +0000)
committerkronbichler <kronbichler@0785d39b-7218-0410-832d-ea1e28bc413d>
Fri, 24 Oct 2008 10:36:50 +0000 (10:36 +0000)
git-svn-id: https://svn.dealii.org/trunk@17332 0785d39b-7218-0410-832d-ea1e28bc413d

deal.II/base/include/base/utilities.h
deal.II/base/source/utilities.cc
deal.II/examples/step-31/step-31.cc
deal.II/examples/step-32/step-32.cc

index 439cda2b0e082822f85031f3511ed9edc0a05559..bc86d128c8a8fccf1d8a764efdc32216fdf7eb19 100644 (file)
 typedef int MPI_Comm;
 #endif
 
+#ifdef DEAL_II_USE_TRILINOS
+#  include <Teuchos_RCP.hpp>
+#  include <Epetra_Comm.h>
+#  ifdef DEAL_II_COMPILER_SUPPORTS_MPI
+#    include <Epetra_MpiComm.h>
+#  else
+#    include <Epetra_SerialComm.h>
+#  endif
+#endif
+
 DEAL_II_NAMESPACE_OPEN
 
 
@@ -222,6 +232,108 @@ namespace Utilities
                                      */
     unsigned int get_this_mpi_process (const MPI_Comm &mpi_communicator);
   }
+
+
+
+  /**
+   * This class provides the basic structures for the use of the
+   * Trilinos classes such as matrices, vectors, and
+   * preconditioners. The most important function in this class is
+   * <tt>comm()</tt>, which is needed for the initialization of
+   * Trilinos Epetra_Maps, which design the parallel distribution of
+   * vectors and matrices. Moreover, this class provides a unified
+   * interface to both serial and parallel implementations of
+   * Trilinos, sets up the MPI communicator in case the programs are
+   * run in parallel, and correctly terminates all processes when the
+   * destructor is called. An example usage of this class is shown in
+   * @ref step_32 step-32.
+   */
+#ifdef DEAL_II_USE_TRILINOS
+  class TrilinosTools
+  {
+    public:
+                                    /**
+                                     * Constructor. Takes the
+                                     * arguments from the command
+                                     * line (in case of MPI, the
+                                     * number of processes is
+                                     * specified there), and sets up
+                                     * a respective communicator by
+                                     * calling <tt>MPI_Init()</tt>.
+                                     */
+      TrilinosTools (int*    argc,
+                    char*** argv);
+
+                                    /**
+                                     * Copy constructor. Takes the
+                                     * Trilinos communicator from the
+                                     * input object and copies all
+                                     * content. Note that the copied
+                                     * class cannot own the MPI
+                                     * process, and hence, the
+                                     * original object needs to be
+                                     * around as long as the copy.
+                                     */
+      TrilinosTools(const TrilinosTools &InputTrilinos);
+
+                                    /**
+                                     * Destructor. Calls
+                                     * <tt>MPI_Finalize()</tt> in
+                                     * case this class owns the MPI
+                                     * process.
+                                     */
+      ~TrilinosTools();
+
+                                    /**
+                                     * Returns a Trilinos Epetra_Comm
+                                     * object needed for creation of
+                                     * Epetra_Maps.
+                                     */
+      const Epetra_Comm& comm() const;
+
+                                    /**
+                                     * Returns whether we are using a
+                                     * MPI version or a serial
+                                     * version of Trilinos.
+                                     */
+      bool trilinos_uses_mpi() const;
+
+    private:
+                                    /**
+                                     * This flag tells the class
+                                     * whether it owns the MPI
+                                     * process (i.e., it has been
+                                     * constructed using the
+                                     * argc/argv input, or it has
+                                     * been copied). In the former
+                                     * case, the command
+                                     * <tt>MPI_Finalize()</tt> will
+                                     * be called at destruction.
+                                     */
+      const bool owns_mpi;
+
+                                    /**
+                                     * This flag tells whether we use
+                                     * MPI or not.
+                                     */
+      const bool use_mpi;
+
+                                    /**
+                                     * The actual communicator
+                                     * object. If we run the program
+                                     * in serial, we will have
+                                     * another communicator than when
+                                     * running in parallel.
+                                     */
+#ifdef DEAL_II_COMPILER_SUPPORTS_MPI
+      Teuchos::RCP<Epetra_MpiComm> communicator;
+#else
+      Teuchos::RCP<Epetra_SerialComm> communicator;
+#endif
+  };
+
+#endif
+
 }
 
 
index 23076cd993e91ea8fc71e2a871024b7988b70a33..f65386d5e3774e6ea8bb8fc1bf8fdcc9904b0911 100644 (file)
@@ -459,7 +459,83 @@ namespace Utilities
     }
 #endif
   }
-  
+
+
+
+#ifdef DEAL_II_USE_TRILINOS
+
+
+  TrilinosTools::TrilinosTools (int* argc, char*** argv)
+                      :
+                      owns_mpi (true),
+#ifdef DEAL_II_COMPILER_SUPPORTS_MPI
+                     use_mpi (true)
+#else
+                     use_mpi (false)
+#endif
+  {
+#ifdef DEAL_II_COMPILER_SUPPORTS_MPI
+    int MPI_has_been_started = 0;
+    MPI_Initialized(&MPI_has_been_started);
+    Assert (MPI_has_been_started == 0,
+           ExcMessage ("MPI error. You can only start MPI once!"));
+
+    int mpi_err;
+    mpi_err = MPI_Init (argc, argv);
+    Assert (mpi_err == 0,
+           ExcMessage ("MPI could not be initialized."));
+
+    communicator = Teuchos::rcp (new Epetra_MpiComm (MPI_COMM_WORLD), true);
+#else
+    communicator = Teuchos::rcp (new Epetra_SerialComm (MPI_COMM_WORLD), true);
+#endif
+  }
+
+
+
+  TrilinosTools::TrilinosTools (const TrilinosTools &InputTrilinos)
+                      :
+                      owns_mpi (false),
+#ifdef DEAL_II_COMPILER_SUPPORTS_MPI
+                     use_mpi (true),
+#else
+                     use_mpi (false),
+#endif
+                     communicator (&*InputTrilinos.communicator, false)
+  {}
+
+
+
+  TrilinosTools::~TrilinosTools()
+  {
+    int mpi_err = 0;
+
+#ifdef DEAL_II_COMPILER_SUPPORTS_MPI
+    if (use_mpi == true && owns_mpi == true)
+      mpi_err = MPI_Finalize();
+#endif
+
+    Assert (mpi_err == 0,
+           ExcMessage ("An error occurred while calling MPI_Finalize()"));
+  }
+
+
+  const Epetra_Comm&
+  TrilinosTools::comm() const
+  {
+    return *communicator;
+  }
+
+
+
+  bool
+  TrilinosTools::trilinos_uses_mpi () const
+  {
+    return use_mpi;
+  }
+
+#endif
+
 }
 
 DEAL_II_NAMESPACE_CLOSE
index e1ca873c0a176b10bc1dd14f4013bb6e0ea6ad9a..7bea85ab703abaa6bd931ce54fff6974278adbf5 100644 (file)
@@ -2744,17 +2744,12 @@ void BoussinesqFlowProblem<dim>::run ()
                                   // in serial).
 int main (int argc, char *argv[])
 {
-#ifdef DEAL_II_COMPILER_SUPPORTS_MPI
-  MPI_Init (&argc,&argv);
-#else
-  (void)argc;
-  (void)argv;
-#endif
-  
   try
     {
       deallog.depth_console (0);
 
+      Utilities::TrilinosTools trilinos (&argc, &argv);
+
       BoussinesqFlowProblem<2> flow_problem;
       flow_problem.run ();
     }
@@ -2782,10 +2777,6 @@ int main (int argc, char *argv[])
                 << std::endl;
       return 1;
     }
-    
-#ifdef DEAL_II_COMPILER_SUPPORTS_MPI
-  MPI_Finalize();
-#endif
 
   return 0;
 }
index 55426bf18a3105b1abda272416fd6e4eb93b4b2d..01b0ebd959eb9ef796594167320fef0a27d78e4e 100644 (file)
 #include <numerics/error_estimator.h>
 #include <numerics/solution_transfer.h>
 
-#ifdef DEAL_II_COMPILER_SUPPORTS_MPI
-#  include <Epetra_MpiComm.h>
-#else
-#  include <Epetra_SerialComm.h>
-#endif
-
 #include <Epetra_Map.h>
 
 #include <fstream>
@@ -264,7 +258,7 @@ template <int dim>
 class BoussinesqFlowProblem
 {
   public:
-    BoussinesqFlowProblem ();
+    BoussinesqFlowProblem (Utilities::TrilinosTools &trilinos_tools);
     void run ();
 
   private:
@@ -297,11 +291,7 @@ class BoussinesqFlowProblem
                      const double                        old_time_step);
 
 
-#ifdef DEAL_II_COMPILER_SUPPORTS_MPI
-    Epetra_MpiComm                      trilinos_communicator;
-#else
-    Epetra_SerialComm                   trilinos_communicator;
-#endif
+    Utilities::TrilinosTools            trilinos_tools;
     
     ConditionalOStream                  pcout;
 
@@ -353,12 +343,10 @@ class BoussinesqFlowProblem
 
                                 // @sect4{BoussinesqFlowProblem::BoussinesqFlowProblem}
 template <int dim>
-BoussinesqFlowProblem<dim>::BoussinesqFlowProblem ()
+BoussinesqFlowProblem<dim>::BoussinesqFlowProblem (Utilities::TrilinosTools &trilinos_tools)
                 :
-#ifdef DEAL_II_COMPILER_SUPPORTS_MPI
-               trilinos_communicator (MPI_COMM_WORLD),
-#endif
-               pcout (std::cout, trilinos_communicator.MyPID()==0),
+                trilinos_tools (trilinos_tools),
+               pcout (std::cout, trilinos_tools.comm().MyPID()==0),
 
                triangulation (Triangulation<dim>::maximum_smoothing),
 
@@ -371,7 +359,7 @@ BoussinesqFlowProblem<dim>::BoussinesqFlowProblem ()
                temperature_fe (temperature_degree),
                 temperature_dof_handler (triangulation),
 
-               temperature_partitioner (0, 0, trilinos_communicator),
+               temperature_partitioner (0, 0, trilinos_tools.comm()),
 
                 time_step (0),
                old_time_step (0),
@@ -401,7 +389,7 @@ double BoussinesqFlowProblem<dim>::get_maximal_velocity () const
     cell = stokes_dof_handler.begin_active(),
     endc = stokes_dof_handler.end();
   for (; cell!=endc; ++cell)
-    if (cell->subdomain_id() == (unsigned int)trilinos_communicator.MyPID())
+    if (cell->subdomain_id() == (unsigned int)trilinos_tools.comm().MyPID())
       {
        fe_values.reinit (cell);
        fe_values.get_function_values (localized_stokes_solution, stokes_values);
@@ -568,7 +556,7 @@ void BoussinesqFlowProblem<dim>::setup_dofs ()
   std::vector<unsigned int> stokes_sub_blocks (dim+1,0);
   stokes_sub_blocks[dim] = 1;
 
-  GridTools::partition_triangulation (trilinos_communicator.NumProc(), 
+  GridTools::partition_triangulation (trilinos_tools.comm().NumProc(), 
                                      triangulation);
 
   {
@@ -623,7 +611,7 @@ void BoussinesqFlowProblem<dim>::setup_dofs ()
     std::vector<unsigned int> local_dofs (dim+1);
     DoFTools::
       count_dofs_with_subdomain_association (stokes_dof_handler,
-                                            trilinos_communicator.MyPID(),
+                                            trilinos_tools.comm().MyPID(),
                                             local_dofs);
     unsigned int n_local_velocities = 0;
     for (unsigned int c=0; c<dim; ++c)
@@ -631,9 +619,9 @@ void BoussinesqFlowProblem<dim>::setup_dofs ()
 
     const unsigned int n_local_pressures = local_dofs[dim];
 
-    Epetra_Map map_u(n_u, n_local_velocities, 0, trilinos_communicator);
+    Epetra_Map map_u(n_u, n_local_velocities, 0, trilinos_tools.comm());
     stokes_partitioner.push_back (map_u);
-    Epetra_Map map_p(n_p, n_local_pressures, 0, trilinos_communicator);
+    Epetra_Map map_p(n_p, n_local_pressures, 0, trilinos_tools.comm());
     stokes_partitioner.push_back (map_p);
   }
   {
@@ -695,9 +683,9 @@ void BoussinesqFlowProblem<dim>::setup_dofs ()
     = Epetra_Map (n_T,
                  DoFTools::count_dofs_with_subdomain_association
                  (temperature_dof_handler,
-                  trilinos_communicator.MyPID()),
+                  trilinos_tools.comm().MyPID()),
                  0,
-                 trilinos_communicator);
+                 trilinos_tools.comm());
   {
     temperature_mass_matrix.clear ();
     temperature_stiffness_matrix.clear ();
@@ -751,7 +739,7 @@ BoussinesqFlowProblem<dim>::assemble_stokes_preconditioner ()
     cell = stokes_dof_handler.begin_active(),
     endc = stokes_dof_handler.end();
   for (; cell!=endc; ++cell)
-    if (cell->subdomain_id() == (unsigned int)trilinos_communicator.MyPID())
+    if (cell->subdomain_id() == (unsigned int)trilinos_tools.comm().MyPID())
       {
        stokes_fe_values.reinit (cell);
        local_matrix = 0;
@@ -876,7 +864,7 @@ void BoussinesqFlowProblem<dim>::assemble_stokes_system ()
     temperature_cell = temperature_dof_handler.begin_active();
   
   for (; cell!=endc; ++cell, ++temperature_cell)
-    if (cell->subdomain_id() == (unsigned int)trilinos_communicator.MyPID())
+    if (cell->subdomain_id() == (unsigned int)trilinos_tools.comm().MyPID())
       {
        stokes_fe_values.reinit (cell);
        temperature_fe_values.reinit (temperature_cell);
@@ -977,7 +965,7 @@ void BoussinesqFlowProblem<dim>::assemble_temperature_matrix ()
     cell = temperature_dof_handler.begin_active(),
     endc = temperature_dof_handler.end();
   for (; cell!=endc; ++cell)
-    if (cell->subdomain_id() == (unsigned int)trilinos_communicator.MyPID())
+    if (cell->subdomain_id() == (unsigned int)trilinos_tools.comm().MyPID())
       {
        local_mass_matrix = 0;
        local_stiffness_matrix = 0;
@@ -1094,7 +1082,7 @@ void BoussinesqFlowProblem<dim>::assemble_temperature_system ()
     stokes_cell = stokes_dof_handler.begin_active();
 
   for (; cell!=endc; ++cell, ++stokes_cell)
-    if (cell->subdomain_id() == (unsigned int)trilinos_communicator.MyPID() )
+    if (cell->subdomain_id() == (unsigned int)trilinos_tools.comm().MyPID() )
       {
        local_rhs = 0;
   
@@ -1363,7 +1351,7 @@ void BoussinesqFlowProblem<dim>::output_results ()  const
 
   DataOut<dim> data_out;
 
-  if (trilinos_communicator.MyPID() == 0)
+  if (trilinos_tools.comm().MyPID() == 0)
     {
       data_out.attach_dof_handler (joint_dof_handler);
 
@@ -1513,20 +1501,13 @@ void BoussinesqFlowProblem<dim>::run ()
                                 // @sect3{The <code>main</code> function}
 int main (int argc, char *argv[])
 {
-#ifdef DEAL_II_COMPILER_SUPPORTS_MPI
-  MPI_Init (&argc,&argv);
-#else
-  (void)argc;
-  (void)argv;
-#endif
-
-  //sleep (20);
-
   try
     {
       deallog.depth_console (0);
+      
+      Utilities::TrilinosTools trilinos(&argc, &argv);
 
-      BoussinesqFlowProblem<2> flow_problem;
+      BoussinesqFlowProblem<2> flow_problem (trilinos);
       flow_problem.run ();
     }
   catch (std::exception &exc)
@@ -1554,9 +1535,5 @@ int main (int argc, char *argv[])
       return 1;
     }
 
-#ifdef DEAL_II_COMPILER_SUPPORTS_MPI
-  MPI_Finalize();
-#endif
-
   return 0;
 }

In the beginning the Universe was created. This has made a lot of people very angry and has been widely regarded as a bad move.

Douglas Adams


Typeset in Trocchi and Trocchi Bold Sans Serif.