From: Daniel Arndt <daniel.arndt@iwr.uni-heidelberg.de> Date: Thu, 21 Mar 2019 07:15:44 +0000 (+0100) Subject: Fix problem with GrowingVectorMemory for LinearAlgebra::distributed::Vector<Number... X-Git-Tag: v9.1.0-rc1~246^2 X-Git-Url: https://gitweb.dealii.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=refs%2Fpull%2F7847%2Fhead;p=dealii.git Fix problem with GrowingVectorMemory for LinearAlgebra::distributed::Vector<Number, MemorySpace::CUDA> --- diff --git a/source/base/mpi.cc b/source/base/mpi.cc index b8cd45e7c2..de4204d548 100644 --- a/source/base/mpi.cc +++ b/source/base/mpi.cc @@ -720,6 +720,18 @@ namespace Utilities } #endif +// There is a similar issue with CUDA: The destructor of static objects might +// run after the CUDA driver is unloaded. Hence, also release all memory +// related to CUDA vectors. +#ifdef DEAL_II_WITH_CUDA + GrowingVectorMemory< + LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA>>:: + release_unused_memory(); + GrowingVectorMemory< + LinearAlgebra::distributed::Vector<float, MemorySpace::CUDA>>:: + release_unused_memory(); +#endif + #ifdef DEAL_II_WITH_P4EST // now end p4est and libsc // Note: p4est has no finalize function diff --git a/source/lac/CMakeLists.txt b/source/lac/CMakeLists.txt index 973837219e..ac29c1f6ed 100644 --- a/source/lac/CMakeLists.txt +++ b/source/lac/CMakeLists.txt @@ -60,6 +60,14 @@ SET(_separate_src sparse_matrix_inst2.cc ) +# Add CUDA wrapper files +IF(DEAL_II_WITH_CUDA) + SET(_separate_src + ${_separate_src} + vector_memory.cu + ) +ENDIF() + SET(_inst affine_constraints.inst.in block_sparse_matrix.inst.in diff --git a/source/lac/vector_memory.cc b/source/lac/vector_memory.cc index 3fa5affd9b..e760248a58 100644 --- a/source/lac/vector_memory.cc +++ b/source/lac/vector_memory.cc @@ -14,7 +14,6 @@ // --------------------------------------------------------------------- #include <deal.II/lac/block_vector.h> -#include <deal.II/lac/cuda_vector.h> #include <deal.II/lac/la_parallel_block_vector.h> #include <deal.II/lac/la_parallel_vector.h> #include <deal.II/lac/la_vector.h> @@ -31,45 +30,21 @@ DEAL_II_NAMESPACE_OPEN #include "vector_memory.inst" -#ifdef DEAL_II_WITH_CUDA -template class VectorMemory<LinearAlgebra::CUDAWrappers::Vector<float>>; -template class VectorMemory<LinearAlgebra::CUDAWrappers::Vector<double>>; -template class GrowingVectorMemory<LinearAlgebra::CUDAWrappers::Vector<float>>; -template class GrowingVectorMemory<LinearAlgebra::CUDAWrappers::Vector<double>>; -# ifdef DEAL_II_WITH_MPI -template class VectorMemory< - LinearAlgebra::distributed::Vector<float, MemorySpace::CUDA>>; -template class VectorMemory< - LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA>>; -template class GrowingVectorMemory< - LinearAlgebra::distributed::Vector<float, MemorySpace::CUDA>>; -template class GrowingVectorMemory< - LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA>>; -# endif -#endif - namespace internal { namespace GrowingVectorMemoryImplementation { +#ifdef DEAL_II_WITH_CUDA + void + release_all_unused_cuda_memory(); +#endif + void release_all_unused_memory() { #include "vector_memory_release.inst" - #ifdef DEAL_II_WITH_CUDA - dealii::GrowingVectorMemory<dealii::LinearAlgebra::CUDAWrappers::Vector< - float>>::release_unused_memory(); - dealii::GrowingVectorMemory<dealii::LinearAlgebra::CUDAWrappers::Vector< - double>>::release_unused_memory(); -# ifdef DEAL_II_WITH_MPI - dealii::GrowingVectorMemory< - dealii::LinearAlgebra::distributed::Vector<float, MemorySpace::CUDA>>:: - release_unused_memory(); - dealii::GrowingVectorMemory< - dealii::LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA>>:: - release_unused_memory(); -# endif + release_all_unused_cuda_memory(); #endif } } // namespace GrowingVectorMemoryImplementation diff --git a/source/lac/vector_memory.cu b/source/lac/vector_memory.cu new file mode 100644 index 0000000000..147c6ac8c5 --- /dev/null +++ b/source/lac/vector_memory.cu @@ -0,0 +1,57 @@ +// --------------------------------------------------------------------- +// +// Copyright (C) 2019 by the deal.II authors +// +// This file is part of the deal.II library. +// +// The deal.II library is free software; you can use it, redistribute +// it, and/or modify it under the terms of the GNU Lesser General +// Public License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// The full text of the license can be found in the file LICENSE.md at +// the top level directory of deal.II. +// +// --------------------------------------------------------------------- + +#include <deal.II/lac/cuda_vector.h> +#include <deal.II/lac/la_parallel_vector.h> +#include <deal.II/lac/vector_memory.templates.h> + + +DEAL_II_NAMESPACE_OPEN + +template class VectorMemory<LinearAlgebra::CUDAWrappers::Vector<float>>; +template class VectorMemory<LinearAlgebra::CUDAWrappers::Vector<double>>; +template class GrowingVectorMemory<LinearAlgebra::CUDAWrappers::Vector<float>>; +template class GrowingVectorMemory<LinearAlgebra::CUDAWrappers::Vector<double>>; +template class VectorMemory< + LinearAlgebra::distributed::Vector<float, MemorySpace::CUDA>>; +template class VectorMemory< + LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA>>; +template class GrowingVectorMemory< + LinearAlgebra::distributed::Vector<float, MemorySpace::CUDA>>; +template class GrowingVectorMemory< + LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA>>; + +namespace internal +{ + namespace GrowingVectorMemoryImplementation + { + void + release_all_unused_cuda_memory() + { + dealii::GrowingVectorMemory<dealii::LinearAlgebra::CUDAWrappers::Vector< + float>>::release_unused_memory(); + dealii::GrowingVectorMemory<dealii::LinearAlgebra::CUDAWrappers::Vector< + double>>::release_unused_memory(); + dealii::GrowingVectorMemory< + dealii::LinearAlgebra::distributed::Vector<float, MemorySpace::CUDA>>:: + release_unused_memory(); + dealii::GrowingVectorMemory< + dealii::LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA>>:: + release_unused_memory(); + } + } // namespace GrowingVectorMemoryImplementation +} // namespace internal + +DEAL_II_NAMESPACE_CLOSE diff --git a/tests/cuda/vector_memory_01.cu b/tests/cuda/vector_memory_01.cu new file mode 100644 index 0000000000..00e086f827 --- /dev/null +++ b/tests/cuda/vector_memory_01.cu @@ -0,0 +1,76 @@ +// --------------------------------------------------------------------- +// +// Copyright (C) 2007 - 2017 by the deal.II authors +// +// This file is part of the deal.II library. +// +// The deal.II library is free software; you can use it, redistribute +// it, and/or modify it under the terms of the GNU Lesser General +// Public License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// The full text of the license can be found in the file LICENSE.md at +// the top level directory of deal.II. +// +// --------------------------------------------------------------------- + + +// Test that we can successfully fill a GrowingVectorMemory pool +// with LinearAlgebra::distributed::Vector<Number, MemorySpace::CUDA> objects. +// Partially copied from lac/vector_memory.cc + + +#include <deal.II/base/exceptions.h> + +#include <deal.II/lac/la_parallel_vector.h> +#include <deal.II/lac/vector_memory.h> + +#include "../tests.h" + + +using namespace dealii; + +template <typename VectorType> +void +test_stat() +{ + GrowingVectorMemory<VectorType> mem(1, true); + VectorType * v1 = mem.alloc(); + VectorType * v2 = mem.alloc(); + VectorType * v3 = mem.alloc(); + VectorType * v4 = mem.alloc(); + VectorType * v5 = mem.alloc(); + VectorType * v6 = mem.alloc(); + v1->reinit(5); + v2->reinit(5); + v3->reinit(5); + v4->reinit(5); + v5->reinit(5); + v6->reinit(5); + mem.free(v1); + mem.free(v2); + mem.free(v3); + mem.free(v4); + mem.free(v5); + mem.free(v6); + v1 = mem.alloc(); + mem.free(v1); + v1 = mem.alloc(); + mem.free(v1); + v1 = mem.alloc(); + mem.free(v1); + v1 = mem.alloc(); + mem.free(v1); +} + + +int +main(int argc, char *argv[]) +{ + initlog(); + Utilities::MPI::MPI_InitFinalize mpi_init(argc, argv, 1); + + test_stat<LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA>>(); + test_stat<LinearAlgebra::distributed::Vector<float, MemorySpace::CUDA>>(); + + return 0; +} diff --git a/tests/cuda/vector_memory_01.output b/tests/cuda/vector_memory_01.output new file mode 100644 index 0000000000..408e05b837 --- /dev/null +++ b/tests/cuda/vector_memory_01.output @@ -0,0 +1,5 @@ + +DEAL::GrowingVectorMemory:Overall allocated vectors: 10 +DEAL::GrowingVectorMemory:Maximum allocated vectors: 6 +DEAL::GrowingVectorMemory:Overall allocated vectors: 10 +DEAL::GrowingVectorMemory:Maximum allocated vectors: 6 diff --git a/tests/cuda/vector_memory_02.cu b/tests/cuda/vector_memory_02.cu new file mode 100644 index 0000000000..8725653d55 --- /dev/null +++ b/tests/cuda/vector_memory_02.cu @@ -0,0 +1,60 @@ +// --------------------------------------------------------------------- +// +// Copyright (C) 2007 - 2017 by the deal.II authors +// +// This file is part of the deal.II library. +// +// The deal.II library is free software; you can use it, redistribute +// it, and/or modify it under the terms of the GNU Lesser General +// Public License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// The full text of the license can be found in the file LICENSE.md at +// the top level directory of deal.II. +// +// --------------------------------------------------------------------- + + +// Test that memory leaks are detected correctly for a GrowingVectorMemory pool +// with LinearAlgebra::distributed<Number, MemorySpace::CUDA> objects. +// Partially copied from lac/vector_memory.cc + + +#include <deal.II/base/exceptions.h> + +#include <deal.II/lac/la_parallel_vector.h> +#include <deal.II/lac/vector_memory.h> + +#include "../tests.h" + + +using namespace dealii; + +template <typename VectorType> +void +test_leak() +{ + GrowingVectorMemory<VectorType> mem; + VectorType * v = mem.alloc(); + v->reinit(5); +} + +int +main(int argc, char *argv[]) +{ + initlog(); + Utilities::MPI::MPI_InitFinalize mpi_init(argc, argv, 1); + deal_II_exceptions::disable_abort_on_exception(); + + try + { + test_leak< + LinearAlgebra::distributed::Vector<double, MemorySpace::CUDA>>(); + test_leak<LinearAlgebra::distributed::Vector<float, MemorySpace::CUDA>>(); + } + catch (const StandardExceptions::ExcMemoryLeak &e) + { + deallog << "Exception: " << e.get_exc_name() << std::endl; + } + + return 0; +} diff --git a/tests/cuda/vector_memory_02.output b/tests/cuda/vector_memory_02.output new file mode 100644 index 0000000000..bdfb84b98d --- /dev/null +++ b/tests/cuda/vector_memory_02.output @@ -0,0 +1,23 @@ + +DEAL::Exception: StandardExceptions::ExcMemoryLeak(current_alloc) +DEAL:: +-------------------------------------------------------- +An error occurred in file <vector_memory.templates.h> in function + dealii::GrowingVectorMemory<VectorType>::~GrowingVectorMemory() [with VectorType = dealii::LinearAlgebra::distributed::Vector<double, dealii::MemorySpace::CUDA>] +The violated condition was: + current_alloc == 0 +Additional information: + Destroying memory handler while 1 objects are still allocated. +-------------------------------------------------------- + +DEAL::Exception: StandardExceptions::ExcMemoryLeak(current_alloc) +DEAL:: +-------------------------------------------------------- +An error occurred in file <vector_memory.templates.h> in function + dealii::GrowingVectorMemory<VectorType>::~GrowingVectorMemory() [with VectorType = dealii::LinearAlgebra::distributed::Vector<float, dealii::MemorySpace::CUDA>] +The violated condition was: + current_alloc == 0 +Additional information: + Destroying memory handler while 1 objects are still allocated. +-------------------------------------------------------- +