Routine to set number of threads of TBB. MPI_InitFinalize by default sets it to 1.

git-svn-id: https://svn.dealii.org/trunk@29045 0785d39b-7218-0410-832d-ea1e28bc413d

diff --git a/deal.II/doc/news/changes.h b/deal.II/doc/news/changes.h
index c81a48e3400b93930b72d3a05886a625ebba52c8..4115f219e3235638486e71edf888fe8b156c9342 100644 (file)
--- a/deal.II/doc/news/changes.h
+++ b/deal.II/doc/news/changes.h
@@ -42,6 +42,13 @@ this function.
 
 
 <ol>
+  <li> New: The number of threads used by deal.II/TBB can now be limited at
+  run time. Using MPI based code using PETSc/Trilinos no longer requires you
+  to compile the library without threads. See MPI_InitFinalize and
+  MultithreadInfo::set_thread_limit for details.
+  </li>
+  (Timo Heister, 2013/03/26)
+
   <li> New: The results section of step-36 now explains how to use ARPACK
   as an alternative to SLEPc as eigenvalue solver.
   <br>

diff --git a/deal.II/include/deal.II/base/mpi.h b/deal.II/include/deal.II/base/mpi.h
index 1b1098f12d3750a1aa0cbed67ac2e4b2c1a723ed..821ad1e4d3b8d28c743474533a62723d3977cbf8 100644 (file)
--- a/deal.II/include/deal.II/base/mpi.h
+++ b/deal.II/include/deal.II/base/mpi.h
@@ -272,7 +272,9 @@ namespace Utilities
      * A class that is used to initialize the
      * MPI system at the beginning of a
      * program and to shut it down again at
-     * the end.
+     * the end. It also allows you to control
+     * the number threads used in each MPI
+     * task.
      *
      * If deal.II is configured with PETSc,
      * the library will also be initialized
@@ -307,10 +309,23 @@ namespace Utilities
        * constructor can only be called once
        * in a program, since MPI cannot be
        * initialized twice.
+       *
+       * This constructor sets max_num_threads
+       * to 1 (see other constructor).
        */
       MPI_InitFinalize (int    &argc,
-                        char ** &argv);
+                        char ** &argv) /*DEAL_II_DEPRECATED*/;
 
+      /**
+       * Initialize MPI (and optionally PETSc)
+       * and set the number of threads used by deal.II (and TBB) to the given
+       * parameter. If set to numbers::invalid_unsigned_int, the number
+       * of threads is determined by TBB. When in doubt, set this value
+       * to 1.
+       */
+      MPI_InitFinalize (int    &argc,
+                        char ** &argv,
+                        unsigned int max_num_threads);
       /**
        * Destructor. Calls
        * <tt>MPI_Finalize()</tt> in
@@ -332,6 +347,14 @@ namespace Utilities
        * be called at destruction.
        */
       const bool owns_mpi;
+
+
+      /**
+       * Called by the constructors.
+       */
+      void do_init(int    &argc,
+          char ** &argv,
+          unsigned int max_num_threads);
     };
 
     namespace internal

diff --git a/deal.II/include/deal.II/base/multithread_info.h b/deal.II/include/deal.II/base/multithread_info.h
index df4b59829ad114bdf31b156846525eb66673c34f..181926b82f48d42c659241a3949cba37181fef64 100644 (file)
--- a/deal.II/include/deal.II/base/multithread_info.h
+++ b/deal.II/include/deal.II/base/multithread_info.h
@@ -15,6 +15,7 @@
 
 
 #include <deal.II/base/config.h>
+#include <deal.II/base/types.h>
 #include <deal.II/base/exceptions.h>
 
 DEAL_II_NAMESPACE_OPEN
@@ -101,6 +102,28 @@ public:
    */
   static std::size_t memory_consumption ();
 
+  /**
+   * Sets the maximum number of threads to be used
+   * to the minimum of the environment variable DEAL_II_NUM_THREADS
+   * and the given parameter (if not the default value). This
+   * affects the initialization of the TBB. If neither is given, the
+   * default from TBB is used (based on the number of cores in the system).
+   *
+   * This routine is called automatically by MPI_InitFinalize. Due to
+   * limitations in the way TBB can be controlled, only the first call to this
+   * method will have any effect. Use the parameter of the MPI_InitFinalize
+   * if you have an MPI based code.
+   */
+  void set_thread_limit(const unsigned int max_threads = numbers::invalid_unsigned_int);
+
+
+  /**
+   * Returns if the TBB is running using a single thread either
+   * because of thread affinity or because it is set via a call
+   * to set_thread_limit. This is used in the PETScWrappers to
+   * avoid using the interface that is not thread-safe.
+   */
+  bool is_running_single_threaded();
   /**
    * Exception
    */
@@ -119,6 +142,11 @@ private:
    * one.
    */
   static unsigned int get_n_cpus();
+
+  /**
+   * variable representing the maximum number of threads
+   */
+  unsigned int n_max_threads;
 };
 
 

diff --git a/deal.II/source/base/mpi.cc b/deal.II/source/base/mpi.cc
index 4d1a83c771582687e32ab1041308b5ba15a66e22..edc8c03da9e4c1184303e9689afa2d7f96fef9ba 100644 (file)
--- a/deal.II/source/base/mpi.cc
+++ b/deal.II/source/base/mpi.cc
@@ -15,6 +15,7 @@
 #include <deal.II/base/utilities.h>
 #include <deal.II/base/exceptions.h>
 #include <deal.II/lac/vector_memory.h>
+#include <deal.II/base/multithread_info.h>
 
 #include <cstddef>
 #include <iostream>
@@ -306,11 +307,31 @@ namespace Utilities
 
 
 
+    MPI_InitFinalize::MPI_InitFinalize (int    &argc,
+                       char ** &argv,
+                       unsigned int max_num_threads)
+    :
+        owns_mpi (true)
+    {
+      do_init(argc, argv, max_num_threads);
+    }
+
+
+
 
     MPI_InitFinalize::MPI_InitFinalize (int    &argc,
                                         char ** &argv)
       :
       owns_mpi (true)
+    {
+      do_init(argc, argv, 1);
+    }
+
+
+    void
+    MPI_InitFinalize::do_init(int    &argc,
+              char ** &argv,
+              unsigned int max_num_threads)
     {
       static bool constructor_has_already_run = false;
       Assert (constructor_has_already_run == false,
@@ -348,6 +369,9 @@ namespace Utilities
 #endif
 
       constructor_has_already_run = true;
+
+      //set maximum number of threads:
+      multithread_info.set_thread_limit(max_num_threads);
     }
 
 

diff --git a/deal.II/source/base/multithread_info.cc b/deal.II/source/base/multithread_info.cc
index 78ef8a56eaa0f56c93472496845f95cd69a7395c..f871f7bf4778ac7b3bbee019aacbae9b67cc6b4c 100644 (file)
--- a/deal.II/source/base/multithread_info.cc
+++ b/deal.II/source/base/multithread_info.cc
@@ -13,6 +13,7 @@
 
 
 #include <deal.II/base/multithread_info.h>
+#include <deal.II/base/utilities.h>
 
 #ifdef HAVE_UNISTD_H
 #  include <unistd.h>
@@ -23,9 +24,13 @@
 #  include <sys/sysctl.h>
 #endif
 
+#ifdef DEAL_II_WITH_THREADS
 
-DEAL_II_NAMESPACE_OPEN
+#  include <deal.II/base/thread_management.h>
+#  include <tbb/task_scheduler_init.h>
+#endif
 
+DEAL_II_NAMESPACE_OPEN
 
 #ifdef DEAL_II_WITH_THREADS
 
@@ -92,6 +97,31 @@ unsigned int MultithreadInfo::get_n_cpus()
 
 #  endif
 
+void MultithreadInfo::set_thread_limit(const unsigned int max_threads)
+{
+  unsigned int max_threads_env = numbers::invalid_unsigned_int;
+  char* penv;
+  penv = getenv ("DEAL_II_NUM_THREADS");
+
+  if (penv!=NULL)
+    max_threads_env = Utilities::string_to_int(std::string(penv));
+
+  n_max_threads = std::min(max_threads, max_threads_env);
+  if (n_max_threads == numbers::invalid_unsigned_int)
+    n_max_threads = tbb::task_scheduler_init::default_num_threads();
+  else
+  {
+      static tbb::task_scheduler_init dummy (n_max_threads);
+  }
+}
+
+bool MultithreadInfo::is_running_single_threaded()
+{
+  if (n_max_threads == numbers::invalid_unsigned_int)
+    n_max_threads = tbb::task_scheduler_init::default_num_threads();
+  return n_max_threads == 1;
+}
+
 
 #else                            // not in MT mode
 
@@ -100,6 +130,15 @@ unsigned int MultithreadInfo::get_n_cpus()
   return 1;
 }
 
+bool MultithreadInfo::is_running_single_threaded()
+{
+  return true;
+}
+
+void MultithreadInfo::set_thread_limit(const unsigned int)
+{
+}
+
 #endif