From: Daniel Arndt Date: Thu, 21 Mar 2019 07:15:44 +0000 (+0100) Subject: Fix problem with GrowingVectorMemory for LinearAlgebra::distributed::Vector --- diff --git a/source/base/mpi.cc b/source/base/mpi.cc index b8cd45e7c2..de4204d548 100644 --- a/source/base/mpi.cc +++ b/source/base/mpi.cc @@ -720,6 +720,18 @@ namespace Utilities } #endif +// There is a similar issue with CUDA: The destructor of static objects might +// run after the CUDA driver is unloaded. Hence, also release all memory +// related to CUDA vectors. +#ifdef DEAL_II_WITH_CUDA + GrowingVectorMemory< + LinearAlgebra::distributed::Vector>:: + release_unused_memory(); + GrowingVectorMemory< + LinearAlgebra::distributed::Vector>:: + release_unused_memory(); +#endif + #ifdef DEAL_II_WITH_P4EST // now end p4est and libsc // Note: p4est has no finalize function diff --git a/source/lac/CMakeLists.txt b/source/lac/CMakeLists.txt index 973837219e..ac29c1f6ed 100644 --- a/source/lac/CMakeLists.txt +++ b/source/lac/CMakeLists.txt @@ -60,6 +60,14 @@ SET(_separate_src sparse_matrix_inst2.cc ) +# Add CUDA wrapper files +IF(DEAL_II_WITH_CUDA) + SET(_separate_src + ${_separate_src} + vector_memory.cu + ) +ENDIF() + SET(_inst affine_constraints.inst.in block_sparse_matrix.inst.in diff --git a/source/lac/vector_memory.cc b/source/lac/vector_memory.cc index 3fa5affd9b..e760248a58 100644 --- a/source/lac/vector_memory.cc +++ b/source/lac/vector_memory.cc @@ -14,7 +14,6 @@ // --------------------------------------------------------------------- #include -#include #include #include #include @@ -31,45 +30,21 @@ DEAL_II_NAMESPACE_OPEN #include "vector_memory.inst" -#ifdef DEAL_II_WITH_CUDA -template class VectorMemory>; -template class VectorMemory>; -template class GrowingVectorMemory>; -template class GrowingVectorMemory>; -# ifdef DEAL_II_WITH_MPI -template class VectorMemory< - LinearAlgebra::distributed::Vector>; -template class VectorMemory< - LinearAlgebra::distributed::Vector>; -template class GrowingVectorMemory< - LinearAlgebra::distributed::Vector>; -template class GrowingVectorMemory< - LinearAlgebra::distributed::Vector>; -# endif -#endif - namespace internal { namespace GrowingVectorMemoryImplementation { +#ifdef DEAL_II_WITH_CUDA + void + release_all_unused_cuda_memory(); +#endif + void release_all_unused_memory() { #include "vector_memory_release.inst" - #ifdef DEAL_II_WITH_CUDA - dealii::GrowingVectorMemory>::release_unused_memory(); - dealii::GrowingVectorMemory>::release_unused_memory(); -# ifdef DEAL_II_WITH_MPI - dealii::GrowingVectorMemory< - dealii::LinearAlgebra::distributed::Vector>:: - release_unused_memory(); - dealii::GrowingVectorMemory< - dealii::LinearAlgebra::distributed::Vector>:: - release_unused_memory(); -# endif + release_all_unused_cuda_memory(); #endif } } // namespace GrowingVectorMemoryImplementation diff --git a/source/lac/vector_memory.cu b/source/lac/vector_memory.cu new file mode 100644 index 0000000000..147c6ac8c5 --- /dev/null +++ b/source/lac/vector_memory.cu @@ -0,0 +1,57 @@ +// --------------------------------------------------------------------- +// +// Copyright (C) 2019 by the deal.II authors +// +// This file is part of the deal.II library. +// +// The deal.II library is free software; you can use it, redistribute +// it, and/or modify it under the terms of the GNU Lesser General +// Public License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// The full text of the license can be found in the file LICENSE.md at +// the top level directory of deal.II. +// +// --------------------------------------------------------------------- + +#include +#include +#include + + +DEAL_II_NAMESPACE_OPEN + +template class VectorMemory>; +template class VectorMemory>; +template class GrowingVectorMemory>; +template class GrowingVectorMemory>; +template class VectorMemory< + LinearAlgebra::distributed::Vector>; +template class VectorMemory< + LinearAlgebra::distributed::Vector>; +template class GrowingVectorMemory< + LinearAlgebra::distributed::Vector>; +template class GrowingVectorMemory< + LinearAlgebra::distributed::Vector>; + +namespace internal +{ + namespace GrowingVectorMemoryImplementation + { + void + release_all_unused_cuda_memory() + { + dealii::GrowingVectorMemory>::release_unused_memory(); + dealii::GrowingVectorMemory>::release_unused_memory(); + dealii::GrowingVectorMemory< + dealii::LinearAlgebra::distributed::Vector>:: + release_unused_memory(); + dealii::GrowingVectorMemory< + dealii::LinearAlgebra::distributed::Vector>:: + release_unused_memory(); + } + } // namespace GrowingVectorMemoryImplementation +} // namespace internal + +DEAL_II_NAMESPACE_CLOSE diff --git a/tests/cuda/vector_memory_01.cu b/tests/cuda/vector_memory_01.cu new file mode 100644 index 0000000000..00e086f827 --- /dev/null +++ b/tests/cuda/vector_memory_01.cu @@ -0,0 +1,76 @@ +// --------------------------------------------------------------------- +// +// Copyright (C) 2007 - 2017 by the deal.II authors +// +// This file is part of the deal.II library. +// +// The deal.II library is free software; you can use it, redistribute +// it, and/or modify it under the terms of the GNU Lesser General +// Public License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// The full text of the license can be found in the file LICENSE.md at +// the top level directory of deal.II. +// +// --------------------------------------------------------------------- + + +// Test that we can successfully fill a GrowingVectorMemory pool +// with LinearAlgebra::distributed::Vector objects. +// Partially copied from lac/vector_memory.cc + + +#include + +#include +#include + +#include "../tests.h" + + +using namespace dealii; + +template +void +test_stat() +{ + GrowingVectorMemory mem(1, true); + VectorType * v1 = mem.alloc(); + VectorType * v2 = mem.alloc(); + VectorType * v3 = mem.alloc(); + VectorType * v4 = mem.alloc(); + VectorType * v5 = mem.alloc(); + VectorType * v6 = mem.alloc(); + v1->reinit(5); + v2->reinit(5); + v3->reinit(5); + v4->reinit(5); + v5->reinit(5); + v6->reinit(5); + mem.free(v1); + mem.free(v2); + mem.free(v3); + mem.free(v4); + mem.free(v5); + mem.free(v6); + v1 = mem.alloc(); + mem.free(v1); + v1 = mem.alloc(); + mem.free(v1); + v1 = mem.alloc(); + mem.free(v1); + v1 = mem.alloc(); + mem.free(v1); +} + + +int +main(int argc, char *argv[]) +{ + initlog(); + Utilities::MPI::MPI_InitFinalize mpi_init(argc, argv, 1); + + test_stat>(); + test_stat>(); + + return 0; +} diff --git a/tests/cuda/vector_memory_01.output b/tests/cuda/vector_memory_01.output new file mode 100644 index 0000000000..408e05b837 --- /dev/null +++ b/tests/cuda/vector_memory_01.output @@ -0,0 +1,5 @@ + +DEAL::GrowingVectorMemory:Overall allocated vectors: 10 +DEAL::GrowingVectorMemory:Maximum allocated vectors: 6 +DEAL::GrowingVectorMemory:Overall allocated vectors: 10 +DEAL::GrowingVectorMemory:Maximum allocated vectors: 6 diff --git a/tests/cuda/vector_memory_02.cu b/tests/cuda/vector_memory_02.cu new file mode 100644 index 0000000000..8725653d55 --- /dev/null +++ b/tests/cuda/vector_memory_02.cu @@ -0,0 +1,60 @@ +// --------------------------------------------------------------------- +// +// Copyright (C) 2007 - 2017 by the deal.II authors +// +// This file is part of the deal.II library. +// +// The deal.II library is free software; you can use it, redistribute +// it, and/or modify it under the terms of the GNU Lesser General +// Public License as published by the Free Software Foundation; either +// version 2.1 of the License, or (at your option) any later version. +// The full text of the license can be found in the file LICENSE.md at +// the top level directory of deal.II. +// +// --------------------------------------------------------------------- + + +// Test that memory leaks are detected correctly for a GrowingVectorMemory pool +// with LinearAlgebra::distributed objects. +// Partially copied from lac/vector_memory.cc + + +#include + +#include +#include + +#include "../tests.h" + + +using namespace dealii; + +template +void +test_leak() +{ + GrowingVectorMemory mem; + VectorType * v = mem.alloc(); + v->reinit(5); +} + +int +main(int argc, char *argv[]) +{ + initlog(); + Utilities::MPI::MPI_InitFinalize mpi_init(argc, argv, 1); + deal_II_exceptions::disable_abort_on_exception(); + + try + { + test_leak< + LinearAlgebra::distributed::Vector>(); + test_leak>(); + } + catch (const StandardExceptions::ExcMemoryLeak &e) + { + deallog << "Exception: " << e.get_exc_name() << std::endl; + } + + return 0; +} diff --git a/tests/cuda/vector_memory_02.output b/tests/cuda/vector_memory_02.output new file mode 100644 index 0000000000..bdfb84b98d --- /dev/null +++ b/tests/cuda/vector_memory_02.output @@ -0,0 +1,23 @@ + +DEAL::Exception: StandardExceptions::ExcMemoryLeak(current_alloc) +DEAL:: +-------------------------------------------------------- +An error occurred in file in function + dealii::GrowingVectorMemory::~GrowingVectorMemory() [with VectorType = dealii::LinearAlgebra::distributed::Vector] +The violated condition was: + current_alloc == 0 +Additional information: + Destroying memory handler while 1 objects are still allocated. +-------------------------------------------------------- + +DEAL::Exception: StandardExceptions::ExcMemoryLeak(current_alloc) +DEAL:: +-------------------------------------------------------- +An error occurred in file in function + dealii::GrowingVectorMemory::~GrowingVectorMemory() [with VectorType = dealii::LinearAlgebra::distributed::Vector] +The violated condition was: + current_alloc == 0 +Additional information: + Destroying memory handler while 1 objects are still allocated. +-------------------------------------------------------- +