From 229cedde2d056a4435b70cd561e65853f28951da Mon Sep 17 00:00:00 2001 From: heister Date: Tue, 26 Mar 2013 20:16:53 +0000 Subject: [PATCH] Routine to set number of threads of TBB. MPI_InitFinalize by default sets it to 1. git-svn-id: https://svn.dealii.org/trunk@29045 0785d39b-7218-0410-832d-ea1e28bc413d --- deal.II/doc/news/changes.h | 7 ++++ deal.II/include/deal.II/base/mpi.h | 27 +++++++++++- .../include/deal.II/base/multithread_info.h | 28 +++++++++++++ deal.II/source/base/mpi.cc | 24 +++++++++++ deal.II/source/base/multithread_info.cc | 41 ++++++++++++++++++- 5 files changed, 124 insertions(+), 3 deletions(-) diff --git a/deal.II/doc/news/changes.h b/deal.II/doc/news/changes.h index c81a48e340..4115f219e3 100644 --- a/deal.II/doc/news/changes.h +++ b/deal.II/doc/news/changes.h @@ -42,6 +42,13 @@ this function.
    +
  1. New: The number of threads used by deal.II/TBB can now be limited at + run time. Using MPI based code using PETSc/Trilinos no longer requires you + to compile the library without threads. See MPI_InitFinalize and + MultithreadInfo::set_thread_limit for details. +
    2. + (Timo Heister, 2013/03/26) +
  2. New: The results section of step-36 now explains how to use ARPACK as an alternative to SLEPc as eigenvalue solver.
    diff --git a/deal.II/include/deal.II/base/mpi.h b/deal.II/include/deal.II/base/mpi.h index 1b1098f12d..821ad1e4d3 100644 --- a/deal.II/include/deal.II/base/mpi.h +++ b/deal.II/include/deal.II/base/mpi.h @@ -272,7 +272,9 @@ namespace Utilities * A class that is used to initialize the * MPI system at the beginning of a * program and to shut it down again at - * the end. + * the end. It also allows you to control + * the number threads used in each MPI + * task. * * If deal.II is configured with PETSc, * the library will also be initialized @@ -307,10 +309,23 @@ namespace Utilities * constructor can only be called once * in a program, since MPI cannot be * initialized twice. + * + * This constructor sets max_num_threads + * to 1 (see other constructor). */ MPI_InitFinalize (int &argc, - char ** &argv); + char ** &argv) /*DEAL_II_DEPRECATED*/; + /** + * Initialize MPI (and optionally PETSc) + * and set the number of threads used by deal.II (and TBB) to the given + * parameter. If set to numbers::invalid_unsigned_int, the number + * of threads is determined by TBB. When in doubt, set this value + * to 1. + */ + MPI_InitFinalize (int &argc, + char ** &argv, + unsigned int max_num_threads); /** * Destructor. Calls * MPI_Finalize() in @@ -332,6 +347,14 @@ namespace Utilities * be called at destruction. */ const bool owns_mpi; + + + /** + * Called by the constructors. + */ + void do_init(int &argc, + char ** &argv, + unsigned int max_num_threads); }; namespace internal diff --git a/deal.II/include/deal.II/base/multithread_info.h b/deal.II/include/deal.II/base/multithread_info.h index df4b59829a..181926b82f 100644 --- a/deal.II/include/deal.II/base/multithread_info.h +++ b/deal.II/include/deal.II/base/multithread_info.h @@ -15,6 +15,7 @@ #include +#include #include DEAL_II_NAMESPACE_OPEN @@ -101,6 +102,28 @@ public: */ static std::size_t memory_consumption (); + /** + * Sets the maximum number of threads to be used + * to the minimum of the environment variable DEAL_II_NUM_THREADS + * and the given parameter (if not the default value). This + * affects the initialization of the TBB. If neither is given, the + * default from TBB is used (based on the number of cores in the system). + * + * This routine is called automatically by MPI_InitFinalize. Due to + * limitations in the way TBB can be controlled, only the first call to this + * method will have any effect. Use the parameter of the MPI_InitFinalize + * if you have an MPI based code. + */ + void set_thread_limit(const unsigned int max_threads = numbers::invalid_unsigned_int); + + + /** + * Returns if the TBB is running using a single thread either + * because of thread affinity or because it is set via a call + * to set_thread_limit. This is used in the PETScWrappers to + * avoid using the interface that is not thread-safe. + */ + bool is_running_single_threaded(); /** * Exception */ @@ -119,6 +142,11 @@ private: * one. */ static unsigned int get_n_cpus(); + + /** + * variable representing the maximum number of threads + */ + unsigned int n_max_threads; }; diff --git a/deal.II/source/base/mpi.cc b/deal.II/source/base/mpi.cc index 4d1a83c771..edc8c03da9 100644 --- a/deal.II/source/base/mpi.cc +++ b/deal.II/source/base/mpi.cc @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -306,11 +307,31 @@ namespace Utilities + MPI_InitFinalize::MPI_InitFinalize (int &argc, + char ** &argv, + unsigned int max_num_threads) + : + owns_mpi (true) + { + do_init(argc, argv, max_num_threads); + } + + + MPI_InitFinalize::MPI_InitFinalize (int &argc, char ** &argv) : owns_mpi (true) + { + do_init(argc, argv, 1); + } + + + void + MPI_InitFinalize::do_init(int &argc, + char ** &argv, + unsigned int max_num_threads) { static bool constructor_has_already_run = false; Assert (constructor_has_already_run == false, @@ -348,6 +369,9 @@ namespace Utilities #endif constructor_has_already_run = true; + + //set maximum number of threads: + multithread_info.set_thread_limit(max_num_threads); } diff --git a/deal.II/source/base/multithread_info.cc b/deal.II/source/base/multithread_info.cc index 78ef8a56ea..f871f7bf47 100644 --- a/deal.II/source/base/multithread_info.cc +++ b/deal.II/source/base/multithread_info.cc @@ -13,6 +13,7 @@ #include +#include #ifdef HAVE_UNISTD_H # include @@ -23,9 +24,13 @@ # include #endif +#ifdef DEAL_II_WITH_THREADS -DEAL_II_NAMESPACE_OPEN +# include +# include +#endif +DEAL_II_NAMESPACE_OPEN #ifdef DEAL_II_WITH_THREADS @@ -92,6 +97,31 @@ unsigned int MultithreadInfo::get_n_cpus() # endif +void MultithreadInfo::set_thread_limit(const unsigned int max_threads) +{ + unsigned int max_threads_env = numbers::invalid_unsigned_int; + char* penv; + penv = getenv ("DEAL_II_NUM_THREADS"); + + if (penv!=NULL) + max_threads_env = Utilities::string_to_int(std::string(penv)); + + n_max_threads = std::min(max_threads, max_threads_env); + if (n_max_threads == numbers::invalid_unsigned_int) + n_max_threads = tbb::task_scheduler_init::default_num_threads(); + else + { + static tbb::task_scheduler_init dummy (n_max_threads); + } +} + +bool MultithreadInfo::is_running_single_threaded() +{ + if (n_max_threads == numbers::invalid_unsigned_int) + n_max_threads = tbb::task_scheduler_init::default_num_threads(); + return n_max_threads == 1; +} + #else // not in MT mode @@ -100,6 +130,15 @@ unsigned int MultithreadInfo::get_n_cpus() return 1; } +bool MultithreadInfo::is_running_single_threaded() +{ + return true; +} + +void MultithreadInfo::set_thread_limit(const unsigned int) +{ +} + #endif -- 2.39.5