From db5c930487c1a9fe25874d58336d52639f958632 Mon Sep 17 00:00:00 2001 From: Daniel Arndt Date: Mon, 3 Jun 2024 08:42:27 -0400 Subject: [PATCH] Initialize Kokkos after setting threads --- source/base/init_finalize.cc | 68 ++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/source/base/init_finalize.cc b/source/base/init_finalize.cc index 6556242339..2043b1d534 100644 --- a/source/base/init_finalize.cc +++ b/source/base/init_finalize.cc @@ -107,40 +107,6 @@ InitFinalize::InitFinalize(int &argc, (void)ierr; #endif - // Initialize Kokkos - if (static_cast(libraries & InitializeLibrary::Kokkos)) - { - // argv has argc+1 elements and the last one is a nullptr. For appending - // one element we thus create a new argv by copying the first argc - // elements, append the new option, and then a nullptr. - // - // We do get in trouble, though, if a user program is called with - // '--help' as a command line argument. This '--help' gets passed on to - // Kokkos, which promptly responds with a lengthy message that the user - // likely did not intend. As a consequence, filter out this specific - // flag. - std::vector argv_new; - for (auto *const arg : make_array_view(&argv[0], &argv[0] + argc)) - if (strcmp(arg, "--help") != 0) - argv_new.push_back(arg); - - std::stringstream threads_flag; -#if KOKKOS_VERSION >= 30700 - threads_flag << "--kokkos-num-threads=" << MultithreadInfo::n_threads(); -#else - threads_flag << "--kokkos-threads=" << MultithreadInfo::n_threads(); -#endif - const std::string threads_flag_string = threads_flag.str(); - argv_new.push_back(const_cast(threads_flag_string.c_str())); - argv_new.push_back(nullptr); - - // The first argument in Kokkos::initialize is of type int&. Hence, we - // need to define a new variable to pass to it (instead of using argc+1 - // inline). - int argc_new = argv_new.size() - 1; - Kokkos::initialize(argc_new, argv_new.data()); - } - // we are allowed to call MPI_Init ourselves and PETScInitialize will // detect this. This allows us to use MPI_Init_thread instead. #ifdef DEAL_II_WITH_PETSC @@ -299,6 +265,40 @@ InitFinalize::InitFinalize(int &argc, MultithreadInfo::set_thread_limit(n_threads); } + // Initialize Kokkos + if (static_cast(libraries & InitializeLibrary::Kokkos)) + { + // argv has argc+1 elements and the last one is a nullptr. For appending + // one element we thus create a new argv by copying the first argc + // elements, append the new option, and then a nullptr. + // + // We do get in trouble, though, if a user program is called with + // '--help' as a command line argument. This '--help' gets passed on to + // Kokkos, which promptly responds with a lengthy message that the user + // likely did not intend. As a consequence, filter out this specific + // flag. + std::vector argv_new; + for (auto *const arg : make_array_view(&argv[0], &argv[0] + argc)) + if (strcmp(arg, "--help") != 0) + argv_new.push_back(arg); + + std::stringstream threads_flag; +#if KOKKOS_VERSION >= 30700 + threads_flag << "--kokkos-num-threads=" << MultithreadInfo::n_threads(); +#else + threads_flag << "--kokkos-threads=" << MultithreadInfo::n_threads(); +#endif + const std::string threads_flag_string = threads_flag.str(); + argv_new.push_back(const_cast(threads_flag_string.c_str())); + argv_new.push_back(nullptr); + + // The first argument in Kokkos::initialize is of type int&. Hence, we + // need to define a new variable to pass to it (instead of using argc+1 + // inline). + int argc_new = argv_new.size() - 1; + Kokkos::initialize(argc_new, argv_new.data()); + } + // As a final step call the at_mpi_init() signal handler. signals.at_mpi_init(); } -- 2.39.5